Merge pull request #327 from CartoDB/test_for_325

Added test for the case that arose in #325

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.20.0
+EXTVERSION = 0.22.0
 
 SED = sed
 AWK = awk
@@ -84,6 +84,8 @@ UPGRADABLE = \
   0.19.1 \
   0.19.2 \
   0.20.0 \
+  0.21.0 \
+  0.22.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -124,8 +126,10 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	cat cartodb_version.sql >> $@
 ifeq ($(PG_PARALLEL), 0)
 # Remove PARALLEL in aggregates and functions
+	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
 	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' -i $@
+		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
+	mv $(TMPFILE) $@
 endif
 
 $(EXTENSION)--unpackaged--$(EXTVERSION).sql: $(EXTENSION)--$(EXTVERSION).sql util/create_from_unpackaged.sh Makefile

NEWS.md

@@ -1,3 +1,12 @@
+0.22.0 (2018-03-22)
+* Fix: allow older ogr2ogr to work in -append mode (#319,#325)
+* Refactors CDB_QuantileBins to rely on PostgreSQL function `percentile_disc` #316
+
+0.21.0 (2018-02-15)
+* Add optional parameter to limit the number of cells in grid-generation functions #322
+* Fix: grant usage on cartodb_id sequence when sharing read write #323
+* Fix: Change sed in-place for tmpfiles 524319
+
 0.20.0 (2017-11-08)
 * Added VOLATILITY and PARALLEL categories to all functions
 

scripts-available/CDB_CartodbfyTable.sql

@@ -347,7 +347,7 @@ $$ LANGUAGE PLPGSQL;
 -- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
 --
 --  CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
---    
+--
 --     Main function, calls the following functions, with a little
 --     logic before the table re-write to avoid re-writing if the table
 --     already has all the necessary columns in place.
@@ -468,7 +468,7 @@ $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL SAFE;
 
 -- Find out if the table already has a usable primary key
 -- If the table has both a usable key and usable geometry
--- we can no-op on the table copy and just ensure that the 
+-- we can no-op on the table copy and just ensure that the
 -- indexes and triggers are in place
 DROP FUNCTION IF EXISTS _CDB_Has_Usable_Primary_ID(reloid REGCLASS);
 CREATE OR REPLACE FUNCTION _CDB_Has_Usable_Primary_ID(reloid REGCLASS)
@@ -560,7 +560,7 @@ BEGIN
     -- Is there another integer suitable primary key already?
     SELECT a.attname
     INTO rec
-    FROM pg_class c 
+    FROM pg_class c
     JOIN pg_attribute a ON a.attrelid = c.oid
     JOIN pg_type t ON a.atttypid = t.oid
     LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
@@ -659,12 +659,12 @@ DECLARE
   rv RECORD;
 
   const RECORD;
-  
+
   has_geom BOOLEAN := false;
   has_mercgeom BOOLEAN := false;
   has_geom_name TEXT;
   has_mercgeom_name TEXT;
-  
+
   -- In case 'the_geom' is a text column
   text_geom_column BOOLEAN := false;
   text_geom_column_name TEXT := '';
@@ -685,12 +685,12 @@ BEGIN
   FOR r1 IN
     SELECT * FROM _cdb_geom_candidate_columns(reloid)
   LOOP
-  
+
     RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('checking column ''%s''', r1.attname);
 
     -- Name collision: right name (the_geom, the_geomwebmercator?)  but wrong type...
     IF r1.typname != 'geometry' AND r1.attname = r1.desired_attname THEN
-    
+
       -- Maybe it's a geometry column hiding in a text column?
       IF r1.typname IN ('text','varchar','char') THEN
 
@@ -710,14 +710,14 @@ BEGIN
           END IF;
         -- Nope, the text in the column can't be converted into geometry
         -- so rename it out of the way
-        EXCEPTION 
+        EXCEPTION
           WHEN others THEN
             IF SQLERRM = 'parse error - invalid geometry' THEN
               text_geom_column := false;
               str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid);
               sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str);
               PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom');
-              RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', 
+              RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %',
                 Format('Text column %s is not convertible to geometry, renamed to %s', r1.attname, str);
             ELSE
               RAISE EXCEPTION 'CDB(_CDB_Has_Usable_Geom) UNEXPECTED ERROR';
@@ -729,7 +729,7 @@ BEGIN
         str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid);
         sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str);
         PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom');
-        RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', 
+        RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %',
           Format('%s is the wrong type, renamed to %s', r1.attname, str);
       END IF;
 
@@ -749,20 +749,20 @@ BEGIN
           has_mercgeom := true;
           has_mercgeom_name := r1.attname;
         END IF;
-        
+
       -- If it's an unknown SRID, we need to know that too
       ELSIF r1.srid = 0 THEN
-        
+
         -- Unknown SRID, we'll have to fill it in later
         text_geom_column_srid := true;
-        
+
       END IF;
-      
+
     END IF;
-    
+
   END LOOP;
 
-  SELECT 
+  SELECT
     -- If table is perfect (no transforms required), return TRUE!
     has_geom AND has_mercgeom AS has_usable_geoms,
     -- If the geometry column is hiding in a text field, return enough info to deal w/ it.
@@ -772,9 +772,9 @@ BEGIN
     INTO rv;
 
   RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('returning %s', rv);
-  
+
   RETURN rv;
-    
+
 END;
 $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE;
 
@@ -782,7 +782,7 @@ $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE;
 -- Create a copy of the table. Assumes that the "Has usable" functions
 -- have already been run, so that if there is a 'cartodb_id' column, it is
 -- a "good" one, and the same for the geometry columns. If all the required
--- columns are in place already, it no-ops and just renames the table to 
+-- columns are in place already, it no-ops and just renames the table to
 -- the destination if necessary.
 CREATE OR REPLACE FUNCTION _CDB_Rewrite_Table(reloid REGCLASS, destschema TEXT DEFAULT NULL)
 RETURNS BOOLEAN
@@ -791,6 +791,7 @@ DECLARE
 
   relname TEXT;
   relschema TEXT;
+  relseq TEXT;
 
   destoid REGCLASS;
   destname TEXT;
@@ -853,7 +854,7 @@ BEGIN
   -- transformation of the table, we can just ensure proper
   -- indexes are in place and apply a rename
   SELECT *
-  FROM _CDB_Has_Usable_Geom(reloid) 
+  FROM _CDB_Has_Usable_Geom(reloid)
   INTO STRICT gc;
 
   -- If geom is the wrong name, just rename it.
@@ -898,6 +899,21 @@ BEGIN
 
   -- We must rewrite, so here we go...
 
+  -- Our desired PK sequence name
+
+  -- We are going to drop the source table when we're done anyways
+  -- but it's possible the source PK sequence is living in a name we would like to use
+  -- so we check to see if that's the case, and rename it out of the way
+  IF has_usable_primary_key AND has_usable_pk_sequence THEN
+    -- See if the existing sequence is squatting on our preferred name
+    destseq := Format('%s_%s_seq', relname, const.pkey);
+    SELECT pg_catalog.pg_get_serial_sequence(Format('%I.%I', relschema, relname), const.pkey)
+      INTO relseq;
+    -- If it's the name we want, then rename it
+    IF relseq IS NOT NULL AND relseq = Format('%I.%I', destschema, destseq) THEN
+      PERFORM _CDB_SQL(Format('ALTER SEQUENCE %s RENAME TO %I', relseq, Format('tmp_%s', destseq)), '_CDB_Rewrite_Table');
+    END IF;
+  END IF;
 
   -- Put the primary key sequence in the right schema
   -- If the new table is not moving, better ensure the sequence name
@@ -931,16 +947,16 @@ BEGIN
 
     -- Arg, this "geometry" column is actually text!!
     -- OK, we tested back in our geometry column research that it could
-    -- be safely cast to geometry, so let's do that.  
+    -- be safely cast to geometry, so let's do that.
     IF gc.text_geom_column THEN
 
       WITH t AS (
-        SELECT 
+        SELECT
           a.attname,
           CASE WHEN NOT gc.text_geom_column_srid THEN 'ST_SetSRID(' ELSE '' END AS missing_srid_start,
           CASE WHEN NOT gc.text_geom_column_srid THEN ',4326)' ELSE '' END AS missing_srid_end
-        FROM pg_class c 
-        JOIN pg_attribute a ON a.attrelid = c.oid 
+        FROM pg_class c
+        JOIN pg_attribute a ON a.attrelid = c.oid
         JOIN pg_type t ON a.atttypid = t.oid
         WHERE c.oid = reloid
         AND t.typname IN ('text','varchar','char')
@@ -954,7 +970,7 @@ BEGIN
             || t.missing_srid_start || t.attname || '::geometry' || t.missing_srid_end
             || ',4326)::Geometry(GEOMETRY,4326) AS '
             || const.geomcol
-            || ', cartodb.CDB_TransformToWebmercator(' 
+            || ', cartodb.CDB_TransformToWebmercator('
             || t.missing_srid_start || t.attname || '::geometry' || t.missing_srid_end
             || ')::Geometry(GEOMETRY,3857) AS '
             || const.mercgeomcol,
@@ -967,19 +983,19 @@ BEGIN
         -- better be found.
         RAISE EXCEPTION 'CDB(_CDB_Rewrite_Table): Text column % is missing!', gc.text_geom_column_name;
       ELSE
-        sql := sql || geom_transform_sql;  
+        sql := sql || geom_transform_sql;
       END IF;
 
     -- There is at least one true geometry column in here, we'll
-    -- reproject that into the projections we need.  
+    -- reproject that into the projections we need.
     ELSE
 
       -- Find the column we are going to be working with (the first
       -- column with type "geometry")
       SELECT a.attname
       INTO rec
-      FROM pg_class c 
-      JOIN pg_attribute a ON a.attrelid = c.oid 
+      FROM pg_class c
+      JOIN pg_attribute a ON a.attrelid = c.oid
       JOIN pg_type t ON a.atttypid = t.oid
       WHERE c.oid = reloid
       AND t.typname = 'geometry'
@@ -989,16 +1005,16 @@ BEGIN
       LIMIT 1;
 
       -- The SRID could be undeclared at the table level, but still
-      -- exist in the geometries themselves. We first find our geometry 
+      -- exist in the geometries themselves. We first find our geometry
       -- column and read the first SRID off it it, if there is a row
       -- to read.
       IF FOUND THEN
-        EXECUTE Format('SELECT ST_SRID(%s) AS srid FROM %s LIMIT 1', rec.attname, reloid::text) 
+        EXECUTE Format('SELECT ST_SRID(%s) AS srid FROM %s LIMIT 1', rec.attname, reloid::text)
         INTO geom_srid;
       ELSE
         geom_srid := 0;
       END IF;
-  
+
       -- The geometry columns weren't in the right projection,
       -- so we need to find the first decent geometry column
       -- in the table and wrap it in two transforms, one to 4326
@@ -1006,13 +1022,13 @@ BEGIN
       -- ignore it when we build the list of other columns to
       -- add to the output table
       WITH t AS (
-        SELECT 
-          a.attname, 
-          postgis_typmod_type(a.atttypmod) AS geomtype, 
+        SELECT
+          a.attname,
+          postgis_typmod_type(a.atttypmod) AS geomtype,
           CASE WHEN postgis_typmod_srid(a.atttypmod) = 0 AND srid.srid = 0 THEN 'ST_SetSRID(' ELSE '' END AS missing_srid_start,
           CASE WHEN postgis_typmod_srid(a.atttypmod) = 0 AND srid.srid = 0 THEN ',4326)' ELSE '' END AS missing_srid_end
-        FROM pg_class c 
-        JOIN pg_attribute a ON a.attrelid = c.oid 
+        FROM pg_class c
+        JOIN pg_attribute a ON a.attrelid = c.oid
         JOIN pg_type t ON a.atttypid = t.oid,
         ( SELECT geom_srid AS srid ) AS srid
         WHERE c.oid = reloid
@@ -1022,22 +1038,22 @@ BEGIN
         ORDER BY a.attnum
         LIMIT 1
       )
-      SELECT ', ST_Transform(' 
+      SELECT ', ST_Transform('
             || t.missing_srid_start || t.attname || t.missing_srid_end
             || ',4326)::Geometry(GEOMETRY,4326) AS '
             || const.geomcol
-            || ', cartodb.CDB_TransformToWebmercator(' 
+            || ', cartodb.CDB_TransformToWebmercator('
             || t.missing_srid_start || t.attname || t.missing_srid_end
             || ')::Geometry(GEOMETRY,3857) AS '
             || const.mercgeomcol,
             t.attname
       INTO geom_transform_sql, geom_column_source
       FROM t;
-  
+
       IF NOT FOUND THEN
-        -- If there are no geometry columns, we continue making a 
+        -- If there are no geometry columns, we continue making a
         -- non-spatial table. This is important for folks who want
-        -- their tables to invalidate the SQL API 
+        -- their tables to invalidate the SQL API
         -- cache on update/insert/delete.
         geom_column_source := '';
         sql := sql || ',NULL::geometry(Geometry,4326) AS ' || const.geomcol;
@@ -1057,8 +1073,8 @@ BEGIN
     ',' || array_to_string(array_agg(Format('%I',a.attname)),',') AS column_name_sql,
     Count(*) AS count
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   JOIN pg_type t ON a.atttypid = t.oid
   WHERE c.oid = reloid
   AND a.attnum > 0
@@ -1103,11 +1119,11 @@ BEGIN
   sql := Format('ALTER SEQUENCE %s OWNED BY %s.%s', destseq, copyname, const.pkey);
   PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table');
 
-  
+
   -- We just made a copy, so we can drop the original now
   sql := Format('DROP TABLE %s', reloid::text);
   PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table');
-  
+
   -- If the table is being created by a SECURITY DEFINER function
   -- make sure the user is set back to the user who is connected
   IF current_user != session_user THEN
@@ -1116,7 +1132,7 @@ BEGIN
     sql := Format('ALTER SEQUENCE IF EXISTS %s OWNER TO %s', destseq, session_user);
     PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table');
   END IF;
-  
+
   -- If we used a temporary destination table
   -- we can now rename it into place
   IF destschema = relschema THEN
@@ -1155,15 +1171,15 @@ BEGIN
   FROM pg_class c
   WHERE c.oid = reloid;
 
-  -- Is there already a primary key on this table for 
+  -- Is there already a primary key on this table for
   -- a column other than our chosen primary key?
   SELECT ci.relname AS pkey
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
   JOIN pg_class ci ON i.indexrelid = ci.oid
-  WHERE c.oid = reloid 
+  WHERE c.oid = reloid
   AND NOT a.attisdropped
   AND a.attname != const.pkey
   AND i.indisprimary;
@@ -1180,40 +1196,40 @@ BEGIN
   -- Is the default primary key flagged as primary?
   SELECT a.attname
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
   JOIN pg_class ci ON ci.oid = i.indexrelid
-  WHERE attnum > 0 
+  WHERE attnum > 0
   AND c.oid = reloid
   AND a.attname = const.pkey
   AND i.indisprimary
   AND i.indisunique
   AND NOT attisdropped;
-  
+
   -- No primary key? Add one.
   IF NOT FOUND THEN
     sql := Format('ALTER TABLE %s ADD PRIMARY KEY (%s)', reloid::text, const.pkey);
     PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes');
   END IF;
-  
-  -- Add geometry indexes to all "special geometry columns" that 
+
+  -- Add geometry indexes to all "special geometry columns" that
   -- don't have one (either have no index at all, or have a non-GIST index)
-  FOR rec IN 
+  FOR rec IN
     SELECT a.attname, n.nspname
-    FROM pg_class c 
+    FROM pg_class c
     JOIN pg_namespace n ON n.oid = c.relnamespace
-    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0 
+    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0
     LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
     WHERE NOT attisdropped
     AND a.attname IN (const.geomcol, const.mercgeomcol)
     AND c.oid = reloid
     AND i.indexrelid IS NULL
-    UNION 
+    UNION
     SELECT a.attname, n.nspname
-    FROM pg_class c 
+    FROM pg_class c
     JOIN pg_namespace n ON n.oid = c.relnamespace
-    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0 
+    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0
     JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
     JOIN pg_class ci ON ci.oid = i.indexrelid
     JOIN pg_am am ON ci.relam = am.oid
@@ -1225,7 +1241,7 @@ BEGIN
     sql := Format('CREATE INDEX ON %s USING GIST (%s)', reloid::text, rec.attname);
     PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes');
   END LOOP;
-    
+
   RETURN true;
 
 END;
@@ -1236,7 +1252,7 @@ CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
 RETURNS REGCLASS
 AS $$
 DECLARE
-  
+
   is_raster BOOLEAN;
   relname TEXT;
   relschema TEXT;
@@ -1245,13 +1261,13 @@ DECLARE
   destname TEXT;
 
   rec RECORD;
-  
+
 BEGIN
 
   -- Save the raw schema/table names for later
   SELECT n.nspname, c.relname, c.relname
   INTO STRICT relschema, relname, destname
-  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid 
+  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
   WHERE c.oid = reloid;
 
   PERFORM cartodb._CDB_check_prerequisites(destschema, reloid);
@@ -1259,7 +1275,7 @@ BEGIN
   -- Check destination schema exists
   -- Throws an exception of there is no matching schema
   IF destschema IS NOT NULL THEN
-  
+
     SELECT n.nspname
     INTO rec FROM pg_namespace n WHERE n.nspname = destschema;
     IF NOT FOUND THEN
@@ -1282,7 +1298,7 @@ BEGIN
     PERFORM cartodb._CDB_create_raster_triggers(destschema, reloid);
 
   ELSE
-    
+
     -- Rewrite (or rename) the table to the new location
     PERFORM _CDB_Rewrite_Table(reloid, destschema);
 
@@ -1291,7 +1307,7 @@ BEGIN
 
     -- Add indexes to the destination table, as necessary
     PERFORM _CDB_Add_Indexes(destoid);
-  
+
     -- Add triggers to the destination table, as necessary
     PERFORM _CDB_create_triggers(destschema, destoid);
 

scripts-available/CDB_Hexagon.sql

@@ -10,6 +10,10 @@ AS $$
     WHERE path[1] % 2 != 0
 $$ LANGUAGE 'sql' IMMUTABLE STRICT PARALLEL SAFE;
 
+
+-- In older versions of the extension, CDB_HexagonGrid had a different signature
+DROP FUNCTION IF EXISTS cartodb.CDB_HexagonGrid(GEOMETRY, FLOAT8, GEOMETRY);
+
 --
 -- Fill given extent with an hexagonal coverage
 --
@@ -25,10 +29,13 @@ $$ LANGUAGE 'sql' IMMUTABLE STRICT PARALLEL SAFE;
 --               If omitted the origin will be 0,0.
 --               The parameter is checked for having the same SRID
 --               as the extent.
---  
 --
+-- @param maxcells Optional maximum number of grid cells to generate;
+--                 if the grid requires more cells to cover the extent
+--                 and exception will occur.
+----
 -- DROP FUNCTION IF EXISTS CDB_HexagonGrid(ext GEOMETRY, side FLOAT8);
-CREATE OR REPLACE FUNCTION CDB_HexagonGrid(ext GEOMETRY, side FLOAT8, origin GEOMETRY DEFAULT NULL)
+CREATE OR REPLACE FUNCTION CDB_HexagonGrid(ext GEOMETRY, side FLOAT8, origin GEOMETRY DEFAULT NULL, maxcells INTEGER DEFAULT 512*512)
 RETURNS SETOF GEOMETRY
 AS $$
 DECLARE
@@ -107,6 +114,12 @@ BEGIN
     vstartary := ARRAY[ vstart - (vstep/2.0), vstart ];
   END IF;
 
+  If maxcells IS NOT NULL AND maxcells > 0 THEN
+    IF CEIL((CEIL((vend-vstart)/(vstep/2.0)) * CEIL((hend-hstart)/(hstep*2.0/3.0)))/3.0)::integer > maxcells THEN
+      RAISE EXCEPTION 'The requested grid is too big to be rendered';
+    END IF;
+  END IF;
+
   vstartidx := abs(hskip)%2;
 
   RAISE DEBUG 'vstartary: % : %', vstartary[1], vstartary[2];

scripts-available/CDB_Organizations.sql

@@ -111,6 +111,7 @@ AS $$
 BEGIN
     EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
     EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
+    EXECUTE 'GRANT USAGE, SELECT ON SEQUENCE ' || pg_catalog.pg_get_serial_sequence(Format('%I.%I', from_schema, table_name), 'cartodb_id') || ' TO "' || to_role_name || '"';
 END
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 

scripts-available/CDB_QuantileBins.sql

@@ -5,39 +5,14 @@
 --            bins based on the Quantile method.
 --
 -- @param breaks The number of bins you want to find.
---  
 --
-CREATE OR REPLACE FUNCTION CDB_QuantileBins ( in_array NUMERIC[], breaks INT) RETURNS NUMERIC[] as $$ 
-DECLARE 
-    element_count INT4; 
-    break_size numeric;
-    tmp_val numeric; 
-    i INT := 1; 
-    reply numeric[]; 
-BEGIN 
-    -- sort our values
-    SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e ORDER BY e ASC) x;
-    -- get the total size of our data
-    element_count := array_length(in_array, 1); 
-    break_size :=  element_count::numeric / breaks;
-    -- slice our bread
-    LOOP  
-        IF i < breaks THEN
-            IF break_size * i % 1 > 0 THEN
-                SELECT e INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 1 OFFSET ceil(break_size * i) - 1) x;
-            ELSE
-                SELECT avg(e) INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 2 OFFSET ceil(break_size * i) - 1 ) x;
-            END IF;
-        ELSIF i = breaks THEN
-            -- select the last value
-            SELECT max(e) INTO tmp_val FROM ( SELECT unnest(in_array) e ) x;
-        ELSE
-            EXIT;
-        END IF;
-
-        reply = array_append(reply, tmp_val);
-        i := i+1;
-    END LOOP;
-    RETURN reply;
-END; 
-$$ language plpgsql IMMUTABLE STRICT PARALLEL SAFE;
+--
+CREATE OR REPLACE FUNCTION CDB_QuantileBins(in_array numeric[], breaks int)
+RETURNS numeric[]
+AS $$
+  SELECT
+    percentile_disc(Array(SELECT generate_series(1, breaks) / breaks::numeric))
+    WITHIN GROUP (ORDER BY x ASC) AS p
+  FROM
+    unnest(in_array) AS x;
+$$ language SQL IMMUTABLE STRICT PARALLEL SAFE;

scripts-available/CDB_RectangleGrid.sql

@@ -1,3 +1,6 @@
+-- In older versions of the extension, CDB_RectangleGrid had a different signature
+DROP FUNCTION IF EXISTS cartodb.CDB_RectangleGrid(GEOMETRY, FLOAT8, FLOAT8, GEOMETRY);
+
 --
 -- Fill given extent with a rectangular coverage
 --
@@ -14,9 +17,12 @@
 --               If omitted the origin will be 0,0.
 --               The parameter is checked for having the same SRID
 --               as the extent.
---  
 --
-CREATE OR REPLACE FUNCTION CDB_RectangleGrid(ext GEOMETRY, width FLOAT8, height FLOAT8, origin GEOMETRY DEFAULT NULL)
+-- @param maxcells Optional maximum number of grid cells to generate;
+--                 if the grid requires more cells to cover the extent
+--                 and exception will occur.
+--
+CREATE OR REPLACE FUNCTION CDB_RectangleGrid(ext GEOMETRY, width FLOAT8, height FLOAT8, origin GEOMETRY DEFAULT NULL, maxcells INTEGER DEFAULT 512*512)
 RETURNS SETOF GEOMETRY
 AS $$
 DECLARE
@@ -79,6 +85,12 @@ BEGIN
   --RAISE DEBUG 'hend: %', hend;
   --RAISE DEBUG 'vend: %', vend;
 
+  If maxcells IS NOT NULL AND maxcells > 0 THEN
+    IF ((hend - hstart)/hstep * (vend - vstart)/vstep)::integer > maxcells THEN
+        RAISE EXCEPTION 'The requested grid is too big to be rendered';
+    END IF;
+  END IF;
+
   x := hstart;
   WHILE x < hend LOOP -- over X
     y := vstart;

test/CDB_CartodbfyTableTest.sql

@@ -385,6 +385,11 @@ BEGIN;
     $$;
 ROLLBACK;
 
+-- Long table name could cause possible sequence rename collision #325
+CREATE TABLE "wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8" ( cartodb_id serial primary key );
+SELECT CDB_CartodbfyTableCheck('wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8'::REGCLASS, 'Long table name could cause sequence collision while renaming #325');
+DROP TABLE "wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8";
+
 -- TODO: table with existing custom-triggered the_geom
 
 DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text);

test/CDB_CartodbfyTableTest_expect

@@ -150,5 +150,8 @@ SET
 BEGIN
 DO
 ROLLBACK
+CREATE TABLE
+Long table name could cause sequence collision while renaming #325 cartodbfied fine
+DROP TABLE
 DROP FUNCTION
 DROP FUNCTION

test/CDB_HexagonTest.sql

@@ -1,3 +1,6 @@
+set client_min_messages to error;
+\set VERBOSITY TERSE
+
 -- Check correctness of an hexagons grid
 --
 -- Cells must have no overlaps and have a number of
@@ -45,3 +48,9 @@ WITH
  0.002 as radius ),
  grid AS ( SELECT CDB_HexagonGrid(env, radius) AS cell from params)
 SELECT '#160', count(cell) > 23000 from grid;
+
+-- Check small grids are generated...
+SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10);
+
+-- But large grids produce an error
+SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1);

test/CDB_HexagonTest_expect

@@ -1,2 +1,5 @@
+SET
 9|63|count / npoints
 #160|t
+3886
+ERROR:  The requested grid is too big to be rendered

test/CDB_JenksBinsTest.sql

@@ -1,11 +1,23 @@
 WITH data AS (
-    SELECT array_agg(x::numeric) s FROM generate_series(1,300) x
-        WHERE x % 5 != 0 AND x % 7 != 0
-    ) 
-SELECT unnest(CDB_JenksBins(s, 7)) FROM data;
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 20.1, 19.9]::numeric[] AS s
+)
+-- expectation is: 1, 5, 10, 15, 20
+-- TODO: fix cdb_jenksbins to match ^^
+SELECT round(unnest(CDB_JenksBins(s, 5))) FROM data;
 
 WITH data_nulls AS (
-    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,300) x
-        WHERE x % 5 != 0 AND x % 7 != 0
-    )
-SELECT unnest(CDB_JenksBins(s, 7)) FROM data_nulls;
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 null, null,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 null, null,
+                 20.1, 19.9]::numeric[] AS s
+)
+-- expectation is: 1, 5, 10, 15, 20
+-- TODO: fix cdb_jenksbins to match ^^
+SELECT round(unnest(CDB_JenksBins(s, 5))) FROM data_nulls;

test/CDB_JenksBinsTest_expect

@@ -1,14 +1,10 @@
-43
-86
-129
-172
-213
-257
-299
-37
-51
-97
-157
-213
-241
+1
+5
+10
+20
+20
+1
+5
+10
+20
 

test/CDB_QuantileBinsTest.sql

@@ -1,11 +1,17 @@
 WITH data AS (
-    SELECT array_agg(x::numeric) s FROM generate_series(1,100) x
-        WHERE x % 5 != 0 AND x % 7 != 0
+    SELECT array_agg(x::numeric) AS s
+    FROM generate_series(0, 99) AS x
     ) 
-SELECT unnest(CDB_QuantileBins(s, 7)) FROM data;
+SELECT unnest(CDB_QuantileBins(s, 10))
+  FROM data;
 
 WITH data_nulls AS (
-    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x
-        WHERE x % 5 != 0 AND x % 7 != 0
+    SELECT array_agg(x::numeric) AS s
+      FROM (
+        SELECT x FROM generate_series(0, 99) AS x
+        UNION ALL
+        SELECT null AS x FROM generate_series(1, 10) AS x
+        ) _wrap
     )
-SELECT unnest(CDB_QuantileBins(s, 7)) FROM data_nulls;
+SELECT unnest(CDB_QuantileBins(s, 10))
+  FROM data_nulls;

test/CDB_QuantileBinsTest_expect

@@ -1,14 +1,20 @@
-13
+9
+19
 29
-43
-57
-71
-86
+39
+49
+59
+69
+79
+89
 99
+9
+19
 29
-57
-87
-
-
-
+39
+49
+59
+69
+79
+89
 99

test/CDB_RectangleTest.sql

@@ -0,0 +1,8 @@
+set client_min_messages to error;
+\set VERBOSITY TERSE
+
+-- Check small grids are generated...
+SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10, 10);
+
+-- But large grids produce an error
+SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1, 1);

test/CDB_RectangleTest_expect

@@ -0,0 +1,3 @@
+SET
+10000
+ERROR:  The requested grid is too big to be rendered

test/organization/test.sh

@@ -315,6 +315,8 @@ function test_member_1_cannot_grant_read_write_permission_to_other_schema_than_i
 }
 
 function test_member_2_can_write_to_member_1_table_after_write_permission_is_added() {
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id SERIAL NOT NULL UNIQUE;"
+
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
     sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
     sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10
@@ -322,6 +324,8 @@ function test_member_2_can_write_to_member_1_table_after_write_permission_is_add
     sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;'
     sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
     sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
+
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;"
 }
 
 function test_member_1_removes_access_and_member_2_can_no_longer_query_the_table() {
@@ -332,10 +336,14 @@ function test_member_1_removes_access_and_member_2_can_no_longer_query_the_table
 }
 
 function test_member_1_removes_access_and_member_2_can_no_longer_write_to_the_table() {
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id SERIAL NOT NULL UNIQUE;"
+
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
     sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
     sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' fails
+
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;"
 }
 
 function test_giving_permissions_to_two_tables_and_removing_from_first_table_should_not_remove_from_second() {
@@ -418,7 +426,7 @@ function test_cdb_usertables_should_work_with_orgusers() {
     sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)"
     sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);"
     sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser"
-    
+
     sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"