From 4d1c4f6a22a9be839a22f09b086d3e00e2d06518 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 1 Jul 2019 10:56:06 +0200 Subject: [PATCH 1/7] Release 0.28.0 --- Makefile | 3 ++- NEWS.md | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 330033f..6afa010 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.27.2 +EXTVERSION = 0.28.0 SED = sed AWK = awk @@ -99,6 +99,7 @@ UPGRADABLE = \ 0.27.0 \ 0.27.1 \ 0.27.2 \ + 0.28.0 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index 10e9aef..8461371 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +0.28.0 (2019-07-01) +* New function CDB_SyncTable (#355) + 0.27.2 (2019-06-21) * Improvements and fixes in Ghost tables functions (#360) From 1028b24333ee01e14e8d58b2610dcc3084759922 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 1 Jul 2019 14:48:42 +0200 Subject: [PATCH 2/7] Fix cartodb schema references --- scripts-available/CDB_SyncTable.sql | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts-available/CDB_SyncTable.sql b/scripts-available/CDB_SyncTable.sql index 521495a..c7979c3 100644 --- a/scripts-available/CDB_SyncTable.sql +++ b/scripts-available/CDB_SyncTable.sql @@ -3,9 +3,9 @@ Sample usage: - SELECT cartodb._CDB_GetColumns('public.films'); + SELECT @extschema@._CDB_GetColumns('public.films'); */ -CREATE OR REPLACE FUNCTION cartodb._CDB_GetColumns(src_table REGCLASS) +CREATE OR REPLACE FUNCTION @extschema@._CDB_GetColumns(src_table REGCLASS) RETURNS SETOF NAME AS $$ SELECT @@ -35,9 +35,9 @@ $$ LANGUAGE sql STABLE PARALLEL UNSAFE; Example of usage: - SELECT cartodb.__CDB_GetUpdateSetClause('{the_geom, id, elevation}', 'changed'); + SELECT @extschema@.__CDB_GetUpdateSetClause('{the_geom, id, elevation}', 'changed'); */ -CREATE OR REPLACE FUNCTION cartodb.__CDB_GetUpdateSetClause(colnames TEXT[], update_source TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_GetUpdateSetClause(colnames TEXT[], update_source TEXT) RETURNS TEXT AS $$ DECLARE @@ -58,10 +58,10 @@ $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; Example of usage: - SELECT cartodb.__CDB_GenerateUniqueName('src_sync'); --> src_sync_718794_120106 + SELECT @extschema@.__CDB_GenerateUniqueName('src_sync'); --> src_sync_718794_120106 */ -CREATE OR REPLACE FUNCTION cartodb.__CDB_GenerateUniqueName(prefix TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_GenerateUniqueName(prefix TEXT) RETURNS NAME AS $$ SELECT format('%s_%s_%s', prefix, txid_current(), (random()*1000000)::int)::NAME; @@ -81,7 +81,7 @@ $$ LANGUAGE sql VOLATILE PARALLEL UNSAFE; SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}'); */ -CREATE OR REPLACE FUNCTION cartodb.CDB_SyncTable(src_table REGCLASS, dst_schema REGNAMESPACE, dst_table NAME, skip_cols NAME[] = '{}') +CREATE OR REPLACE FUNCTION @extschema@.CDB_SyncTable(src_table REGCLASS, dst_schema REGNAMESPACE, dst_table NAME, skip_cols NAME[] = '{}') RETURNS void AS $$ DECLARE @@ -112,11 +112,11 @@ BEGIN skip_cols := skip_cols || '{cartodb_id}'; -- Get the list of columns from the source table, excluding skip_cols - SELECT ARRAY(SELECT quote_ident(c) FROM cartodb._CDB_GetColumns(src_table) as c EXCEPT SELECT unnest(skip_cols)) INTO colnames; + SELECT ARRAY(SELECT quote_ident(c) FROM @extschema@._CDB_GetColumns(src_table) as c EXCEPT SELECT unnest(skip_cols)) INTO colnames; quoted_colnames := array_to_string(colnames, ','); - src_hash_table_name := cartodb.__CDB_GenerateUniqueName('src_sync'); - dst_hash_table_name := cartodb.__CDB_GenerateUniqueName('dst_sync'); + src_hash_table_name := @extschema@.__CDB_GenerateUniqueName('src_sync'); + dst_hash_table_name := @extschema@.__CDB_GenerateUniqueName('dst_sync'); EXECUTE format('CREATE TEMP TABLE %I(cartodb_id BIGINT, hash TEXT) ON COMMIT DROP', src_hash_table_name); EXECUTE format('CREATE TEMP TABLE %I(cartodb_id BIGINT, hash TEXT) ON COMMIT DROP', dst_hash_table_name); @@ -159,7 +159,7 @@ BEGIN -- Deal with modified rows: ids in source and dest but different hashes t := clock_timestamp(); - update_set_clause := cartodb.__CDB_GetUpdateSetClause(colnames, 'changed'); + update_set_clause := @extschema@.__CDB_GetUpdateSetClause(colnames, 'changed'); EXECUTE format(' UPDATE %1$s dst SET %2$s FROM ( From 69cc56b58934ddfa9fa92c6abc77e407ece9b0fc Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 1 Jul 2019 15:13:48 +0200 Subject: [PATCH 3/7] Remove schema from public function examples --- scripts-available/CDB_SyncTable.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_SyncTable.sql b/scripts-available/CDB_SyncTable.sql index c7979c3..9d6453e 100644 --- a/scripts-available/CDB_SyncTable.sql +++ b/scripts-available/CDB_SyncTable.sql @@ -77,8 +77,8 @@ $$ LANGUAGE sql VOLATILE PARALLEL UNSAFE; Sample usage: - SELECT cartodb.CDB_SyncTable('radar_stations', 'public', 'syncdest'); - SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}'); + SELECT CDB_SyncTable('radar_stations', 'public', 'syncdest'); + SELECT CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}'); */ CREATE OR REPLACE FUNCTION @extschema@.CDB_SyncTable(src_table REGCLASS, dst_schema REGNAMESPACE, dst_table NAME, skip_cols NAME[] = '{}') From 9f414938ba7ee7daa5469ff4cf08057416e804e6 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 1 Jul 2019 16:41:40 +0200 Subject: [PATCH 4/7] Document CDB_SyncTable --- doc/CDB_SyncTable.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 doc/CDB_SyncTable.md diff --git a/doc/CDB_SyncTable.md b/doc/CDB_SyncTable.md new file mode 100644 index 0000000..7069747 --- /dev/null +++ b/doc/CDB_SyncTable.md @@ -0,0 +1,42 @@ +Synchronize two tables. This function will synchronize a *destination* table with a *source* table. +The idea is that the *destination* is a replica of *source* and *source* has been subject to +modifications that are to be applied to *destination*. + +This will be achieved by deleting the rows in the destination not present +in the source, inserting rows of the source not in the destination and updating modified rows. +If the destination table does not exist it will be created and all the rows of the source inserted into it. + +Both tables must have a consistent `cartodb_id` primary key column which will be used to match +the source and destination rows. + +Note that both tables do not necessaryly become identical after the synchronization, since additional columns +may have been added to the destination; those columns will not be altered by the synchronization. + +In addition some source columns may be skipped by listing them in the optional last argument; such columns +will not be updated in the destination, so if they are present in it their values won't be altered. + + +#### Using the function + +Import some data using COPY FROM into a temporary table, then synchronize a table with the data and +finally delete the temporary table. This could be used import and update some data periodically while +allowing to add columns to the data that will be preserved across updates. + + +```sql +CREATE tmp_pois(cartodb_id int, name text, type text, longitude double precision, latitude double precision); +COPY tmp_pois FROM '/tmp/pois.csv'; +SELECT CDB_SyncTable('tmp_pois', 'public', 'pois'); +DROP TABLE tmp_pois; +``` + +#### Arguments + +``` +CDB_SyncTable(src_table, dst_schema, dst_table, skip_cols) +``` + +* **src_table** REGCLASS the source data for the synchronization +* **dst_scgena** REGNAMESPACE the destination schema +* **dst_table** NAME the destination table to be updated +* **skip_cols** NAME[] an array of column names, empty by default, which will be skipped From f2dae651b3733d21d7a720b430f4f449fa92059c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 1 Jul 2019 16:58:24 +0200 Subject: [PATCH 5/7] More complete sync table example --- doc/CDB_SyncTable.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/CDB_SyncTable.md b/doc/CDB_SyncTable.md index 7069747..08d7bbe 100644 --- a/doc/CDB_SyncTable.md +++ b/doc/CDB_SyncTable.md @@ -22,14 +22,28 @@ Import some data using COPY FROM into a temporary table, then synchronize a tabl finally delete the temporary table. This could be used import and update some data periodically while allowing to add columns to the data that will be preserved across updates. - ```sql -CREATE tmp_pois(cartodb_id int, name text, type text, longitude double precision, latitude double precision); +CREATE tmp_pois(cartodb_id int, name text, type text, longitude double precision, latitude double precision, rank int); COPY tmp_pois FROM '/tmp/pois.csv'; SELECT CDB_SyncTable('tmp_pois', 'public', 'pois'); DROP TABLE tmp_pois; ``` +Now we could perform some changes to the `pois` to maintain our own ranking: + +```sql +UPDATE pois SET rank = random()*4 + 1; +``` + +Then, if the source were updated at `/tmp/pois.csv` we could synchronize with it while maintaining our `rank` values with: + +```sql +CREATE tmp_pois(cartodb_id int, name text, type text, longitude double precision, latitude double precision, rank int); +COPY tmp_pois FROM '/tmp/pois.csv'; +SELECT CDB_SyncTable('tmp_pois', 'public', 'pois', '{rank}'); +DROP TABLE tmp_pois; +``` + #### Arguments ``` From 0e1aeb0a76eca616b49428b33b390063eaaa3802 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 1 Jul 2019 17:00:22 +0200 Subject: [PATCH 6/7] Minor copy edit --- doc/CDB_SyncTable.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/CDB_SyncTable.md b/doc/CDB_SyncTable.md index 08d7bbe..f1b5696 100644 --- a/doc/CDB_SyncTable.md +++ b/doc/CDB_SyncTable.md @@ -35,7 +35,7 @@ Now we could perform some changes to the `pois` to maintain our own ranking: UPDATE pois SET rank = random()*4 + 1; ``` -Then, if the source were updated at `/tmp/pois.csv` we could synchronize with it while maintaining our `rank` values with: +Then, if the source were updated at `/tmp/pois.csv` we could synchronize with it while preserving our `rank` values with: ```sql CREATE tmp_pois(cartodb_id int, name text, type text, longitude double precision, latitude double precision, rank int); From 057695361d64ed61e7d140063becdc286a85eaa3 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 1 Jul 2019 17:49:42 +0200 Subject: [PATCH 7/7] Fix typo --- doc/CDB_SyncTable.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/CDB_SyncTable.md b/doc/CDB_SyncTable.md index f1b5696..9447dd0 100644 --- a/doc/CDB_SyncTable.md +++ b/doc/CDB_SyncTable.md @@ -9,7 +9,7 @@ If the destination table does not exist it will be created and all the rows of t Both tables must have a consistent `cartodb_id` primary key column which will be used to match the source and destination rows. -Note that both tables do not necessaryly become identical after the synchronization, since additional columns +Note that both tables do not necessarily become identical after the synchronization, since additional columns may have been added to the destination; those columns will not be altered by the synchronization. In addition some source columns may be skipped by listing them in the optional last argument; such columns