Merge pull request #91 from CartoDB/upgrade-version-0-7-4

Update Makefile and NEWS.md for new version

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.5.2
+EXTVERSION = 0.7.4
 
 SED = sed
 
@@ -31,6 +31,13 @@ UPGRADABLE = \
   0.4.1 \
   0.5.0 \
   0.5.1 \
+  0.5.2 \
+  0.5.3 \
+  0.6.0 \
+  0.7.0 \
+  0.7.1 \
+  0.7.2 \
+  0.7.3 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -91,14 +98,14 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	for f in $(REGRESS_OLD); do \
     tn=`basename $${f} .sql`; \
     of=sql/test/$${tn}.sql; \
-    echo '\\set ECHO off' > $${of}; \
+    echo '\\set ECHO none' > $${of}; \
     echo '\\a' >> $${of}; \
     echo '\\t' >> $${of}; \
     echo '\\set QUIET off' >> $${of}; \
     cat $${f} | \
       $(SED) -e 's/public\./cartodb./g' >> $${of}; \
     exp=expected/test/$${tn}.out; \
-    echo '\\set ECHO off' > $${exp}; \
+    echo '\\set ECHO none' > $${exp}; \
     cat test/$${tn}_expect >> $${exp}; \
   done
 

NEWS.md

@@ -1,3 +1,39 @@
+0.7.4 (2015-06-29)
+------------------
+* Adds new function CDB_QueryTablesText that can deal with "schema.table_name"
+  longer than 63 chars.
+* Adds a set of statistical functions:
+  - CDB_DistType
+  - CDB_DistinctMeasure
+  - CDB_EqualIntervalBins
+
+0.7.3 (2015-03-03)
+------------------
+* Fix upgrade of CDB_StringToDate function
+* Add a test for to validate CDB_TableMetadataTouch usage with OID
+
+0.7.2 (2015-03-03)
+------------------
+* Fix conversion of strings to datetime
+
+0.7.1 (2015-02-27)
+------------------
+* Revert quota checks to `pg_total_relation_size`
+
+0.7.0 (2015-02-19)
+------------------
+* Adds CDB_ZoomFromScale function
+
+0.6.0 (2015-02-19)
+------------------
+* Select permission in CDB_TableMetadata no longer granted to public
+* New function to upsert the updated_at in CDB_TableMetadata for a regclass
+
+0.5.3 (2015-02-17)
+------------------
+* Fixed security problem related with system tables
+* Changed quota checks to use `pg_relation_size` instead of `pg_total_relation_size`
+
 0.5.2 (2015-01-29)
 ------------------
 * Improvement: make CDB_UserDataSize functions much faster.

expected/test_ddl_triggers.out

@@ -32,6 +32,7 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 ----------------------
 -- CREATE TABLE
 ----------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 NOTICE:  trigger "track_updates" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t3" does not exist, skipping
@@ -55,6 +56,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)) as age
@@ -64,6 +66,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
  c.t3    |   0
 (1 row)
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
@@ -91,6 +94,7 @@ from c.t4;
           1 | t   | t    |          | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now() - updated_at)) as age
@@ -103,6 +107,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
 ----------------------------
 -- ALTER TABLE RENAME COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -131,6 +136,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -140,6 +146,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
  c.t3    |     0
 (1 row)
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -168,6 +175,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 |         | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -180,6 +188,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE DROP COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -208,6 +217,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 |         | 
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -220,6 +230,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE ADD COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -239,6 +250,7 @@ from c.t3;
           1 | t   | t    |          |                      | 1 |         |          |    
 (1 row)
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs

scripts-available/CDB_DistType.sql

@@ -0,0 +1,122 @@
+--
+-- CDB_DistType classifies the histograms of a column into
+-- one of the basic types listed by Galtung: http://druedin.com/2012/12/08/galtungs-ajus-system/
+-- 
+-- Future improvements:
+--    variable number of bins (7 is baked in right now)
+--    catch the number of items to ensure that the sample is large enough
+--
+-- Refs:
+--    1. width_bucket/histograms: http://tapoueh.org/blog/2014/02/21-PostgreSQL-histogram
+--    2. R implementation: https://github.com/cran/agrmt
+
+CREATE OR REPLACE FUNCTION CDB_DistType ( in_array NUMERIC[] ) RETURNS text as $$
+DECLARE
+    element_count INT4;
+    minv numeric;
+    maxv numeric;
+    bins numeric[];
+    freqs numeric[];
+    ajus INT[];
+    freq INT4;
+    signature text;
+    i INT := 1;
+BEGIN
+    SELECT min(e), max(e), count(e) INTO minv, maxv, element_count FROM ( SELECT unnest(in_array) e ) x;
+
+    IF abs(maxv - minv) < 1e-7 THEN -- if max and min are nearly equal, call if 'F' (make relative to maxv?)
+        signature = 'F';
+    ELSE
+        -- Calculate bins and count in bins
+        EXECUTE 'WITH stats as (
+            SELECT min(e) as minv,
+                   max(e) as maxv,
+                   count(e) as total
+            FROM (SELECT unnest($1) e) x
+            WHERE e is not null
+        ),
+        hist as (
+            SELECT width_bucket(e, s.minv, s.maxv, 7) bucket,
+                   count(*) freq
+            FROM (SELECT unnest($1) e) x, stats s
+            WHERE e is not null
+            GROUP BY 1
+            ORDER BY 1
+        )
+        SELECT array_agg(round(100.0 * hist.freq::numeric / stats.total::numeric,1)) freqs,
+               array_agg(hist.bucket) buckets
+        FROM hist, stats'
+        INTO freqs, bins
+        USING in_array;
+
+        LOOP
+            IF i < 7 THEN
+                ajus[i] = CASE WHEN freqs[i] > freqs[i+1] THEN -1
+                               WHEN abs(freqs[i] - freqs[i+1]) <= 0.05 THEN 0
+                               ELSE 1 END;
+            ELSE
+                EXIT;
+            END IF;
+            i := i + 1;
+        END LOOP;
+
+        signature = _CDB_DistTypeClassify(ajus);
+    END IF;
+
+    RETURN signature;
+END;
+$$ language plpgsql IMMUTABLE;
+
+-- Classify data into AJUSFL
+
+CREATE OR REPLACE FUNCTION _CDB_DistTypeClassify ( in_array INT[] ) RETURNS text as $$
+DECLARE
+    element_count INT4;
+    maxv numeric;
+    minv numeric;
+    uniques INT[];
+    type text;
+BEGIN
+    SELECT max(e), min(e) INTO maxv, minv FROM ( SELECT unnest(in_array) e ) x;
+
+    IF (maxv = 0 AND minv = 0) THEN
+        type = 'F';
+    ELSIF maxv < 1 THEN
+        type = 'L';
+    ELSIF minv > -1 THEN
+        type = 'J';
+    ELSE
+        -- Get distinct elements ordered by original position
+        EXECUTE 'WITH b AS (
+            SELECT a
+            FROM (SELECT unnest($1) a) x
+        ),
+        c AS (
+            SELECT a, row_number() OVER () r
+            FROM b
+        ),
+        d AS (
+            SELECT DISTINCT a
+            FROM c
+        ),
+        e AS (
+            SELECT a FROM d ORDER BY (
+                SELECT r FROM c WHERE d.a = c.a ORDER BY r ASC LIMIT 1
+            ) ASC)
+        SELECT array_agg(a) FROM e'
+        INTO uniques
+        USING in_array;
+
+        -- Decide if it's an A, U, or other
+        IF (uniques = ARRAY[1,-1] OR uniques = ARRAY[1,0,-1] OR uniques = ARRAY[1,-1,0] OR uniques = ARRAY[0,1,-1]) THEN
+            type = 'A';
+        ELSIF (uniques = ARRAY[-1,1] OR uniques = ARRAY[-1,0,1] OR uniques = ARRAY[-1,1,0] OR uniques = ARRAY[0,-1,1]) THEN
+            type = 'U';
+        ELSE
+            type = 'S';
+        END IF;
+    END IF;
+
+    RETURN type;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_DistinctMeasure.sql

@@ -0,0 +1,46 @@
+--
+-- CDB_DistinctMeasure 
+--     calculates the fraction of rows in the 10 most common distinct categories
+--     returns true if the number of rows in these 10 categories is >= 0.9 * total number of rows
+-- 
+-- 
+
+CREATE OR REPLACE FUNCTION CDB_DistinctMeasure ( in_array text[], threshold numeric DEFAULT null ) RETURNS numeric as $$
+DECLARE
+    element_count INT4;
+    maxval numeric;
+    passes numeric;
+BEGIN
+    SELECT count(e) INTO element_count FROM ( SELECT unnest(in_array) e ) x;
+
+    -- count number of occurrences per bin
+    -- calculate the normalized cumulative sum
+    -- return the max value: which corresponds nth entry 
+    -- for n <= 10 depending on # of distinct values
+    EXECUTE 'WITH a As (
+              SELECT
+                count(*) cnt
+              FROM
+                (SELECT * FROM unnest($2) e ) x
+              WHERE e is not null
+              GROUP BY e
+              ORDER BY cnt DESC
+            ),
+            b As (
+              SELECT
+                sum(cnt) OVER (ORDER BY cnt DESC) / $1 As cumsum
+              FROM a
+              LIMIT 10
+            )
+            SELECT max(cumsum) maxval FROM b'
+            INTO maxval
+            USING element_count, in_array;
+    IF threshold is null THEN
+        passes = maxval;
+    ELSE
+        passes = CASE WHEN (maxval >= threshold) THEN 1 ELSE 0 END;
+    END IF;
+
+    RETURN passes;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_EqualIntervalBins.sql

@@ -0,0 +1,37 @@
+--
+-- Calculate the equal interval bins for a given column
+--
+-- @param in_array A numeric array of numbers to determine the best
+--                   to determine the bin boundary
+--
+-- @param breaks The number of bins you want to find.
+--  
+--
+-- Returns: upper edges of bins
+-- 
+--
+
+CREATE OR REPLACE FUNCTION CDB_EqualIntervalBins ( in_array NUMERIC[], breaks INT ) RETURNS NUMERIC[] as $$
+DECLARE 
+    diff numeric;
+    min_val numeric;
+    max_val numeric;
+    tmp_val numeric;
+    i INT := 1;
+    reply numeric[];
+BEGIN
+    SELECT min(e), max(e) INTO min_val, max_val FROM ( SELECT unnest(in_array) e ) x WHERE e IS NOT NULL;
+    diff = (max_val - min_val) / breaks::numeric;
+    LOOP
+        IF i < breaks THEN
+            tmp_val = min_val + i::numeric * diff;
+            reply = array_append(reply, tmp_val);
+            i := i+1;
+        ELSE
+            reply = array_append(reply, max_val);
+            EXIT;
+        END IF;
+    END LOOP;
+    RETURN reply;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_QuantileBins.sql

@@ -15,18 +15,29 @@ DECLARE
     i INT := 1; 
     reply numeric[]; 
 BEGIN 
-    -- get our unique values
-    SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e GROUP BY e ORDER BY e ASC) x;
-    -- get the total size of our row
-    element_count := array_upper(in_array, 1) - array_lower(in_array, 1); 
+    -- sort our values
+    SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e ORDER BY e ASC) x;
+    -- get the total size of our data
+    element_count := array_length(in_array, 1); 
     break_size :=  element_count::numeric / breaks;
     -- slice our bread
     LOOP  
-        IF i > breaks THEN  EXIT;  END IF;  
-        SELECT e INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 1 OFFSET round(break_size * i)) x;
+        IF i < breaks THEN
+            IF break_size * i % 1 > 0 THEN
+                SELECT e INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 1 OFFSET ceil(break_size * i) - 1) x;
+            ELSE
+                SELECT avg(e) INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 2 OFFSET ceil(break_size * i) - 1 ) x;
+            END IF;
+        ELSIF i = breaks THEN
+            -- select the last value
+            SELECT max(e) INTO tmp_val FROM ( SELECT unnest(in_array) e ) x;
+        ELSE
+            EXIT;
+        END IF;
+
         reply = array_append(reply, tmp_val);
-        i := i+1; 
-    END LOOP; 
-    RETURN reply; 
+        i := i+1;
+    END LOOP;
+    RETURN reply;
 END; 
-$$ language plpgsql IMMUTABLE;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_QueryTables.sql

@@ -2,12 +2,12 @@
 --
 -- Requires PostgreSQL 9.x+
 --
-CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
-RETURNS name[]
+CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
+RETURNS text[]
 AS $$
 DECLARE
   exp XML;
-  tables NAME[];
+  tables text[];
   rec RECORD;
   rec2 RECORD;
 BEGIN
@@ -16,9 +16,9 @@ BEGIN
 
   FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
 
-    IF NOT ( rec.q ilike 'select %' or rec.q ilike 'with %' ) THEN
-      --RAISE WARNING 'Skipping %', rec.q;
-      CONTINUE;
+    IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
+        --RAISE WARNING 'Skipping %', rec.q;
+        CONTINUE;
     END IF;
 
     BEGIN
@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x)::name as p, unnest(s)::name as sc from inp
+      SELECT unnest(x) as p, unnest(s) as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p)::name);
+      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;
@@ -65,3 +65,14 @@ BEGIN
   return tables;
 END
 $$ LANGUAGE 'plpgsql' VOLATILE STRICT;
+
+
+-- Keep CDB_QueryTables with same signature for backwards compatibility.
+-- It should probably be removed in the future.
+CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
+RETURNS name[]
+AS $$
+BEGIN
+  RETURN CDB_QueryTablesText(query)::name[];
+END
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

scripts-available/CDB_StringToDate.sql

@@ -1,8 +1,9 @@
 -- Convert string to date
 --
+DROP FUNCTION IF EXISTS CDB_StringToDate(character varying);
 CREATE OR REPLACE FUNCTION CDB_StringToDate(input character varying)
-RETURNS date AS $$
-DECLARE output DATE;
+RETURNS TIMESTAMP AS $$
+DECLARE output TIMESTAMP;
 BEGIN
     BEGIN
         output := input::date;

scripts-available/CDB_TableMetadata.sql

@@ -5,9 +5,9 @@ CREATE TABLE IF NOT EXISTS
     updated_at timestamp with time zone not null default now()
   );
 
--- Anyone can see this, but updates are only possible trough
--- the security definer trigger
-GRANT SELECT ON public.CDB_TableMetadata TO public;
+-- No one can see this
+-- Updates are only possible trough the security definer trigger
+-- GRANT SELECT ON public.CDB_TableMetadata TO public;
 
 --
 -- Trigger logging updated_at in the CDB_TableMetadata
@@ -118,3 +118,21 @@ CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
 ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
  _CDB_TableMetadata_Updated();
 
+
+-- similar to TOUCH(1) in unix filesystems but for table in cdb_tablemetadata
+CREATE OR REPLACE FUNCTION public.CDB_TableMetadataTouch(tablename regclass)
+    RETURNS void AS
+    $$
+    BEGIN
+        WITH upsert AS (
+            UPDATE public.cdb_tablemetadata
+            SET updated_at = NOW()
+            WHERE tabname = tablename
+            RETURNING *
+        )
+        INSERT INTO public.cdb_tablemetadata (tabname, updated_at)
+            SELECT tablename, NOW()
+            WHERE NOT EXISTS (SELECT * FROM upsert);
+    END;
+    $$
+LANGUAGE 'plpgsql' VOLATILE STRICT;

scripts-available/CDB_ZoomFromScale.sql

@@ -0,0 +1,31 @@
+CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
+BEGIN
+  CASE
+    WHEN scaleDenominator > 1000000000 THEN RETURN 0;
+    WHEN scaleDenominator <= 1000000000 AND scaleDenominator > 500000000 THEN RETURN 1;
+    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 2;
+    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 3;
+    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
+    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
+    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
+    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
+    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
+    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
+    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
+    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
+    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
+    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
+    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
+    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
+    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
+    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
+    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
+    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
+    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
+    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
+    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
+    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
+    WHEN scaleDenominator <= 100 THEN RETURN 23;
+  END CASE;
+END
+$$ LANGUAGE plpgsql IMMUTABLE;

scripts-enabled/230-CDB_ZoomFromScale.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_ZoomFromScale.sql

scripts-enabled/240-CDB_EqualIntervalBins.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_EqualIntervalBins.sql

scripts-enabled/CDB_DistType.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_DistType.sql

scripts-enabled/CDB_DistinctMeasure.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_DistinctMeasure.sql

sql/test_ddl_triggers.sql

@@ -18,7 +18,7 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 ----------------------
 -- CREATE TABLE
 ----------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 
 select
@@ -28,11 +28,13 @@ select
  i
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)) as age
 FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
@@ -41,6 +43,8 @@ select
  NOW() - updated_at < '1 secs' as "u<1s",
  the_geom, the_geom_webmercator
 from c.t4;
+
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now() - updated_at)) as age
@@ -49,6 +53,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
 ----------------------------
 -- ALTER TABLE RENAME COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc;
@@ -60,11 +65,13 @@ select
  i, webmerc
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
 FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
 
@@ -75,6 +82,7 @@ select
  i, webmerc, webmerc2
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -83,7 +91,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE DROP COLUMN
 ----------------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 drop column the_geom_webmercator;
 
@@ -94,6 +102,7 @@ select
  i, webmerc, webmerc2
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -102,7 +111,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE ADD COLUMN
 ----------------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 add column id2 int;
 
@@ -113,6 +122,7 @@ select
  i, webmerc, webmerc2, id2
 from c.t3;
 
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs

test/CDB_CartodbfyTableTest.sql

@@ -1,5 +1,5 @@
 SET client_min_messages TO error;
-\set VERBOSITY terse;
+\set VERBOSITY default
 
 CREATE OR REPLACE FUNCTION CDB_CartodbfyTableCheck(tabname regclass, label text)
 RETURNS text AS

test/CDB_DistTypeTest.sql

@@ -0,0 +1,4 @@
+WITH data AS (
+    SELECT pow(x,3)::numeric x FROM generate_series(-100,100) x
+    ) 
+SELECT CDB_DistType(array_agg(x)) FROM data

test/CDB_DistTypeTest_expect

@@ -0,0 +1 @@
+A

test/CDB_DistinctMeasureTest.sql

@@ -0,0 +1,20 @@
+-- a - j add up to 89%, k-m add up to 11%
+WITH a As (
+	SELECT (
+		repeat('a',12) || 
+		repeat('b',11) || 
+		repeat('c',11) || 
+		repeat('d',10) || 
+		repeat('e',10) || 
+		repeat('f',9) || 
+		repeat('g',8) || 
+		repeat('h',7) || 
+		repeat('i',6) || 
+		repeat('j',5) || 
+		repeat('k',4) || 
+		repeat('l',4) || 
+		repeat('m',3)
+		)::text AS x 
+	) 
+
+SELECT CDB_DistinctMeasure(string_to_array(x,null),0.90) from a

test/CDB_DistinctMeasureTest_expect

@@ -0,0 +1 @@
+0

test/CDB_EqualIntervalBinsTest.sql

@@ -0,0 +1,5 @@
+WITH data AS (
+    SELECT array_agg(x::numeric) s FROM generate_series(1,300) x 
+        WHERE x % 5 != 0 AND x % 7 != 0
+    ) 
+SELECT round(unnest(CDB_EqualIntervalBins(s, 7)),7) FROM data

test/CDB_EqualIntervalBinsTest_expect

@@ -0,0 +1,7 @@
+43.5714286
+86.1428571
+128.7142857
+171.2857143
+213.8571429
+256.4285714
+299.0000000

test/CDB_QuantileBinsTest_expect

@@ -1,7 +1,7 @@
-16
+13
 29
 43
 57
 71
-83
+86
 99

test/CDB_QueryTablesTest.sql

@@ -31,3 +31,7 @@ create table sc.test (a int);
 insert into sc.test values (1);
 WITH inp AS ( select 'select * from sc.test'::text as q )
  SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'SELECT
+* FROM geometry_columns'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;

test/CDB_QueryTablesTest_expect

@@ -4,12 +4,18 @@ CREATE table "my'tab;le" as select 1|{}
 SELECT a.oid, b.oid FROM pg_class a, pg_class b|{pg_catalog.pg_class}
 SELECT 1 as col1; select 2 as col2|{}
 WARNING:  CDB_QueryTables cannot explain query: select 1 from nonexistant (42P01: relation "nonexistant" does not exist)
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 ERROR:  relation "nonexistant" does not exist
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 begin; select * from pg_class; commit;|{pg_catalog.pg_class}
 WARNING:  CDB_QueryTables cannot explain query: select * from test (42P01: relation "test" does not exist)
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 ERROR:  relation "test" does not exist
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 WITH a AS (select * from pg_class) select * from a|{pg_catalog.pg_class}
 CREATE SCHEMA
 CREATE TABLE
 INSERT 0 1
 select * from sc.test|{sc.test}
+SELECT
+* FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_namespace,pg_catalog.pg_type}

test/CDB_QuotaTest.sql

@@ -1,5 +1,6 @@
-set client_min_messages to ERROR;
-\set VERBOSITY terse
+set client_min_messages to error;
+\set VERBOSITY default
+
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
@@ -9,8 +10,10 @@ INSERT INTO big VALUES (1); -- allowed, check runs before
 INSERT INTO big VALUES (1); -- disallowed, quota exceeds before
 SELECT CDB_SetUserQuotaInBytes(0);
 SELECT CDB_CartodbfyTable('big');
-INSERT INTO big SELECT generate_series(1,1024);
-SELECT CDB_SetUserQuotaInBytes(8);
+INSERT INTO big SELECT generate_series(1,2048);
+INSERT INTO big SELECT generate_series(1,2048);
+INSERT INTO big SELECT generate_series(1,2048);
+SELECT CDB_SetUserQuotaInBytes(2);
 INSERT INTO big VALUES (1);
 SELECT CDB_SetUserQuotaInBytes(0);
 INSERT INTO big VALUES (1);

test/CDB_QuotaTest_expect

@@ -5,9 +5,11 @@ INSERT 0 1
 ERROR:  Quota exceeded by 3.9990234375KB
 0
 
-INSERT 0 1024
-8
-ERROR:  Quota exceeded by 123.9921875KB
+INSERT 0 2048
+INSERT 0 2048
+INSERT 0 2048
+2
+ERROR:  Quota exceeded by 567.998046875KB
 0
 INSERT 0 1
 DROP TABLE

test/extension/run_at_cartodb_schema.sql

@@ -1,3 +1,4 @@
 SET SCHEMA 'cartodb';
 \i scripts-available/CDB_Quota.sql
+\i scripts-available/CDB_TableMetadata.sql
 SET SCHEMA 'public';

test/extension/test.sh

@@ -75,6 +75,15 @@ function sql() {
             set_failed
         fi
     fi
+
+    if [[ "$3" == "should-not" ]]
+    then
+        if [[ "${RESULT}" == "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' did not expect '${RESULT}'"
+            set_failed
+        fi
+    fi
 }
 
 
@@ -253,7 +262,7 @@ function run_tests() {
 # Tests quota checking taking into account both geom and raster tables
 function test_quota_for_each_user() {
     # Normal tables add 4096 bytes
-    # Raster tables with overview constraints add 16384 bytes
+    # Raster tables no longer add anything so also count as 4096
 
     sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
     sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
@@ -277,6 +286,57 @@ function test_quota_for_each_user() {
     sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
 }
 
+function test_cdb_tablemetadatatouch() {
+    sql "CREATE TABLE touch_example (a int)"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should ''
+    sql "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Another call doesn't fail
+    sql "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Works with qualified tables
+    sql "SELECT CDB_TableMetadataTouch('public.touch_example');"
+    sql "SELECT CDB_TableMetadataTouch('public.\"touch_example\"');"
+    sql "SELECT CDB_TableMetadataTouch('\"public\".touch_example');"
+    sql "SELECT CDB_TableMetadataTouch('\"public\".\"touch_example\"');"
+
+    # Works with OID
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+    TABLE_OID=`${CMD} -U postgres ${DATABASE} -c "SELECT attrelid FROM pg_attribute WHERE attrelid = 'touch_example'::regclass limit 1;" -A -t`
+
+    # quoted OID works
+    sql "SELECT CDB_TableMetadataTouch('${TABLE_OID}');"
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+
+    # non quoted OID works
+    sql "SELECT CDB_TableMetadataTouch(${TABLE_OID});"
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+
+    #### test tear down
+    sql 'DROP TABLE touch_example;'
+}
+
+function test_cdb_tablemetadatatouch_fails_for_unexistent_table() {
+    sql postgres "SELECT CDB_TableMetadataTouch('unexistent_example');" fails
+}
+
+function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
+    sql "CREATE TABLE touch_example (a int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');" fails
+
+    sql postgres "GRANT ALL ON CDB_TableMetadata TO cdb_testmember_1;"
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@