Merge pull request #74 from CartoDB/73-fix-timestamps

Fix conversion of strings to datetimes #73

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.5.2
+EXTVERSION = 0.7.2
 
 SED = sed
 
@@ -31,6 +31,11 @@ UPGRADABLE = \
   0.4.1 \
   0.5.0 \
   0.5.1 \
+  0.5.2 \
+  0.5.3 \
+  0.6.0 \
+  0.7.0 \
+  0.7.1 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -91,14 +96,14 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	for f in $(REGRESS_OLD); do \
     tn=`basename $${f} .sql`; \
     of=sql/test/$${tn}.sql; \
-    echo '\\set ECHO off' > $${of}; \
+    echo '\\set ECHO none' > $${of}; \
     echo '\\a' >> $${of}; \
     echo '\\t' >> $${of}; \
     echo '\\set QUIET off' >> $${of}; \
     cat $${f} | \
       $(SED) -e 's/public\./cartodb./g' >> $${of}; \
     exp=expected/test/$${tn}.out; \
-    echo '\\set ECHO off' > $${exp}; \
+    echo '\\set ECHO none' > $${exp}; \
     cat test/$${tn}_expect >> $${exp}; \
   done
 

NEWS.md

@@ -1,3 +1,25 @@
+0.7.2 (2015-03-03)
+------------------
+* Fix conversion of strings to datetime
+
+0.7.1 (2015-02-27)
+------------------
+* Revert quota checks to `pg_total_relation_size`
+
+0.7.0 (2015-02-19)
+------------------
+* Adds CDB_ZoomFromScale function
+
+0.6.0 (2015-02-19)
+------------------
+* Select permission in CDB_TableMetadata no longer granted to public
+* New function to upsert the updated_at in CDB_TableMetadata for a regclass
+
+0.5.3 (2015-02-17)
+------------------
+* Fixed security problem related with system tables
+* Changed quota checks to use `pg_relation_size` instead of `pg_total_relation_size`
+
 0.5.2 (2015-01-29)
 ------------------
 * Improvement: make CDB_UserDataSize functions much faster.

expected/test_ddl_triggers.out

@@ -32,6 +32,7 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 ----------------------
 -- CREATE TABLE
 ----------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 NOTICE:  trigger "track_updates" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t3" does not exist, skipping
@@ -55,6 +56,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)) as age
@@ -64,6 +66,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
  c.t3    |   0
 (1 row)
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
@@ -91,6 +94,7 @@ from c.t4;
           1 | t   | t    |          | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now() - updated_at)) as age
@@ -103,6 +107,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
 ----------------------------
 -- ALTER TABLE RENAME COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -131,6 +136,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -140,6 +146,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
  c.t3    |     0
 (1 row)
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -168,6 +175,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 |         | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -180,6 +188,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE DROP COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -208,6 +217,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 |         | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -220,6 +230,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE ADD COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -239,6 +250,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 |         |          |    
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs

scripts-available/CDB_QueryTables.sql

@@ -16,9 +16,9 @@ BEGIN
 
   FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
 
-    IF NOT ( rec.q ilike 'select %' or rec.q ilike 'with %' ) THEN
-      --RAISE WARNING 'Skipping %', rec.q;
-      CONTINUE;
+    IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
+        --RAISE WARNING 'Skipping %', rec.q;
+        CONTINUE;
     END IF;
 
     BEGIN

scripts-available/CDB_StringToDate.sql

@@ -1,8 +1,8 @@
 -- Convert string to date
 --
 CREATE OR REPLACE FUNCTION CDB_StringToDate(input character varying)
-RETURNS date AS $$
-DECLARE output DATE;
+RETURNS TIMESTAMP AS $$
+DECLARE output TIMESTAMP;
 BEGIN
     BEGIN
         output := input::date;

scripts-available/CDB_TableMetadata.sql

@@ -5,9 +5,9 @@ CREATE TABLE IF NOT EXISTS
     updated_at timestamp with time zone not null default now()
   );
 
--- Anyone can see this, but updates are only possible trough
--- the security definer trigger
-GRANT SELECT ON public.CDB_TableMetadata TO public;
+-- No one can see this
+-- Updates are only possible trough the security definer trigger
+-- GRANT SELECT ON public.CDB_TableMetadata TO public;
 
 --
 -- Trigger logging updated_at in the CDB_TableMetadata
@@ -118,3 +118,21 @@ CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
 ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
  _CDB_TableMetadata_Updated();
 
+
+-- similar to TOUCH(1) in unix filesystems but for table in cdb_tablemetadata
+CREATE OR REPLACE FUNCTION public.CDB_TableMetadataTouch(tablename regclass)
+    RETURNS void AS
+    $$
+    BEGIN
+        WITH upsert AS (
+            UPDATE public.cdb_tablemetadata
+            SET updated_at = NOW()
+            WHERE tabname = tablename
+            RETURNING *
+        )
+        INSERT INTO public.cdb_tablemetadata (tabname, updated_at)
+            SELECT tablename, NOW()
+            WHERE NOT EXISTS (SELECT * FROM upsert);
+    END;
+    $$
+LANGUAGE 'plpgsql' VOLATILE STRICT;

scripts-available/CDB_ZoomFromScale.sql

@@ -0,0 +1,31 @@
+CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
+BEGIN
+  CASE
+    WHEN scaleDenominator > 1000000000 THEN RETURN 0;
+    WHEN scaleDenominator <= 1000000000 AND scaleDenominator > 500000000 THEN RETURN 1;
+    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 2;
+    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 3;
+    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
+    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
+    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
+    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
+    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
+    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
+    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
+    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
+    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
+    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
+    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
+    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
+    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
+    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
+    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
+    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
+    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
+    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
+    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
+    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
+    WHEN scaleDenominator <= 100 THEN RETURN 23;
+  END CASE;
+END
+$$ LANGUAGE plpgsql IMMUTABLE;

scripts-enabled/230-CDB_ZoomFromScale.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_ZoomFromScale.sql

sql/test_ddl_triggers.sql

@@ -18,7 +18,7 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 ----------------------
 -- CREATE TABLE
 ----------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 
 select
@@ -28,11 +28,13 @@ select
  i
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)) as age
 FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
@@ -41,6 +43,8 @@ select
  NOW() - updated_at < '1 secs' as "u<1s",
  the_geom, the_geom_webmercator
 from c.t4;
+
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now() - updated_at)) as age
@@ -49,6 +53,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
 ----------------------------
 -- ALTER TABLE RENAME COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc;
@@ -60,11 +65,13 @@ select
  i, webmerc
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
 FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
 
@@ -75,6 +82,7 @@ select
  i, webmerc, webmerc2
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -83,7 +91,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE DROP COLUMN
 ----------------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 drop column the_geom_webmercator;
 
@@ -94,6 +102,7 @@ select
  i, webmerc, webmerc2
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -102,7 +111,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE ADD COLUMN
 ----------------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 add column id2 int;
 
@@ -113,6 +122,7 @@ select
  i, webmerc, webmerc2, id2
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs

test/CDB_CartodbfyTableTest.sql

@@ -1,5 +1,5 @@
 SET client_min_messages TO error;
-\set VERBOSITY terse;
+\set VERBOSITY default
 
 CREATE OR REPLACE FUNCTION CDB_CartodbfyTableCheck(tabname regclass, label text)
 RETURNS text AS

test/CDB_QueryTablesTest.sql

@@ -31,3 +31,7 @@ create table sc.test (a int);
 insert into sc.test values (1);
 WITH inp AS ( select 'select * from sc.test'::text as q )
  SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'SELECT
+* FROM geometry_columns'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;

test/CDB_QueryTablesTest_expect

@@ -13,3 +13,5 @@ CREATE SCHEMA
 CREATE TABLE
 INSERT 0 1
 select * from sc.test|{sc.test}
+SELECT
+* FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_namespace,pg_catalog.pg_type}

test/CDB_QuotaTest.sql

@@ -1,5 +1,6 @@
-set client_min_messages to ERROR;
-\set VERBOSITY terse
+set client_min_messages to error;
+\set VERBOSITY default
+
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
@@ -9,8 +10,10 @@ INSERT INTO big VALUES (1); -- allowed, check runs before
 INSERT INTO big VALUES (1); -- disallowed, quota exceeds before
 SELECT CDB_SetUserQuotaInBytes(0);
 SELECT CDB_CartodbfyTable('big');
-INSERT INTO big SELECT generate_series(1,1024);
-SELECT CDB_SetUserQuotaInBytes(8);
+INSERT INTO big SELECT generate_series(1,2048);
+INSERT INTO big SELECT generate_series(1,2048);
+INSERT INTO big SELECT generate_series(1,2048);
+SELECT CDB_SetUserQuotaInBytes(2);
 INSERT INTO big VALUES (1);
 SELECT CDB_SetUserQuotaInBytes(0);
 INSERT INTO big VALUES (1);

test/CDB_QuotaTest_expect

@@ -5,9 +5,11 @@ INSERT 0 1
 ERROR:  Quota exceeded by 3.9990234375KB
 0
 
-INSERT 0 1024
-8
-ERROR:  Quota exceeded by 123.9921875KB
+INSERT 0 2048
+INSERT 0 2048
+INSERT 0 2048
+2
+ERROR:  Quota exceeded by 567.998046875KB
 0
 INSERT 0 1
 DROP TABLE

test/extension/run_at_cartodb_schema.sql

@@ -1,3 +1,4 @@
 SET SCHEMA 'cartodb';
 \i scripts-available/CDB_Quota.sql
+\i scripts-available/CDB_TableMetadata.sql
 SET SCHEMA 'public';

test/extension/test.sh

@@ -75,6 +75,15 @@ function sql() {
             set_failed
         fi
     fi
+
+    if [[ "$3" == "should-not" ]]
+    then
+        if [[ "${RESULT}" == "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' did not expect '${RESULT}'"
+            set_failed
+        fi
+    fi
 }
 
 
@@ -253,7 +262,7 @@ function run_tests() {
 # Tests quota checking taking into account both geom and raster tables
 function test_quota_for_each_user() {
     # Normal tables add 4096 bytes
-    # Raster tables with overview constraints add 16384 bytes
+    # Raster tables no longer add anything so also count as 4096
 
     sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
     sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
@@ -277,6 +286,42 @@ function test_quota_for_each_user() {
     sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
 }
 
+function test_cdb_tablemetadatatouch() {
+    sql "CREATE TABLE touch_example (a int)"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should ''
+    sql "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Another call doesn't fail
+    sql "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Works with qualified tables
+    sql "SELECT CDB_TableMetadataTouch('public.touch_example');"
+    sql "SELECT CDB_TableMetadataTouch('public.\"touch_example\"');"
+    sql "SELECT CDB_TableMetadataTouch('\"public\".touch_example');"
+    sql "SELECT CDB_TableMetadataTouch('\"public\".\"touch_example\"');"
+
+    #### test tear down
+    sql 'DROP TABLE touch_example;'
+}
+
+function test_cdb_tablemetadatatouch_fails_for_unexistent_table() {
+    sql postgres "SELECT CDB_TableMetadataTouch('unexistent_example');" fails
+}
+
+function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
+    sql "CREATE TABLE touch_example (a int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');" fails
+
+    sql postgres "GRANT ALL ON CDB_TableMetadata TO cdb_testmember_1;"
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@