Update version and news

[wip] New cartodbfy function (overwrites CDB_CartodbfyTable)

CONTRIBUTING.md

@@ -19,7 +19,7 @@ in which those scripts are loaded.
 
 Scripts would be best coded in a way to be usable both for creation
 and upgrade of the objects. This means using CREATE OR REPLACE for
-the functions, and whatever it takes to check existance of any previous
+the functions, and whatever it takes to check existence of any previous
 version of objects in other cases.
 
 When used as an extension (probably always from version 0.2.0 onwards)
@@ -27,8 +27,8 @@ all the objects will be installed in a "cartodb" schema. Take this into
 account to fully-qualify internal calls to avoid (possibly dangerous)
 name clashes.
 
-Every new feature (as well as bugfixes) should come with a testcase,
-see next session.
+Every new feature (as well as bugfixes) should come with a test case,
+see next section.
 
 Writing testcases
 -----------------
@@ -49,13 +49,16 @@ also installs migration scripts to go from "V" to "V"next and from "V"next
 to "V". Example to upgrade a 0.2.0dev version:
 
 ```sql
-ALTER EXTENSION cartodb UPDATE TO '0.2.0devnext';
+ALTER EXTENSION cartodb UPDATE TO '0.2.0next';
 ALTER EXTENSION cartodb UPDATE TO '0.2.0dev';
 ```
-
 Starting with 0.2.0, the in-place reload can be done with an ad-hoc function:
 
 ```sql
 SELECT cartodb.cdb_extension_reload();
 ```
 
+A useful query:
+```sql
+SELECT * FROM pg_extension_update_paths('cartodb') WHERE path IS NOT NULL AND source = cdb_version();
+```

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.3.6
+EXTVERSION = 0.9.1
 
 SED = sed
 
@@ -26,6 +26,24 @@ UPGRADABLE = \
   0.3.3 \
   0.3.4 \
   0.3.5 \
+  0.3.6 \
+  0.4.0 \
+  0.4.1 \
+  0.5.0 \
+  0.5.1 \
+  0.5.2 \
+  0.5.3 \
+  0.6.0 \
+  0.7.0 \
+  0.7.1 \
+  0.7.2 \
+  0.7.3 \
+  0.7.4 \
+  0.8.0 \
+  0.8.1 \
+  0.8.2 \
+  0.9.0 \
+  0.9.1 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -36,7 +54,6 @@ UPGRADES = \
      $(SED) 's/$$/--$(EXTVERSION).sql/' | \
      $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g')
 
-REV=$(shell git describe)
 GITDIR=$(shell test -d .git && echo '.git' || cat .git | $(SED) 's/^gitdir: //')
 
 DATA_built = \
@@ -78,7 +95,7 @@ $(EXTENSION).control: $(EXTENSION).control.in Makefile
 	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION) $(REV)/' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
 legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
@@ -87,21 +104,24 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	for f in $(REGRESS_OLD); do \
     tn=`basename $${f} .sql`; \
     of=sql/test/$${tn}.sql; \
-    echo '\\set ECHO off' > $${of}; \
+    echo '\\set ECHO none' > $${of}; \
     echo '\\a' >> $${of}; \
     echo '\\t' >> $${of}; \
     echo '\\set QUIET off' >> $${of}; \
     cat $${f} | \
       $(SED) -e 's/public\./cartodb./g' >> $${of}; \
     exp=expected/test/$${tn}.out; \
-    echo '\\set ECHO off' > $${exp}; \
+    echo '\\set ECHO none' > $${exp}; \
     cat test/$${tn}_expect >> $${exp}; \
   done
 
 test_organization:
 	bash test/organization/test.sh
 
+test_extension_new:
+	bash test/extension/test.sh
+
 legacy_tests: legacy_regress 
 
-installcheck: legacy_tests test_organization
+installcheck: legacy_tests test_extension_new test_organization
 

NEWS

@@ -1,77 +0,0 @@
-0.3.6 (2014-08-11)
-------------------
-Dummy release to solve some issues with cdb branch/tag
-
-0.3.5 (2014-08-11)
-------------------
-Inverting priority of CDB_CheckQuota qmax so gies more priority to existing user quota function over parameter value.
-
-0.3.4 (2014-08-01)
-------------------
-Fixes issue with schemas in CDB_QueryTables
-
-0.3.3 (2014-07-30)
-------------------
-* Splitting of CartodbfyTable method in subfunctions to be able to call in fragments and evade timeouts on hot zones
-
-0.3.2 (2014-07-28)
-------------------
-* Make 0.3.0dev version upgradeable
-
-0.3.1 (2014-07-22)
-------------------
-* Dummy version. We start using semantic versioning
-
-0.3.0 (2014-07-15)
-------------------
-* Permission management functions
-* Adapt functions to use schemas
-
-0.2.1 - 2014-06-11
-------------------
-
-Enhancements:
-
- - Do not force re-cartodbfication on CREATE FROM unpackaged
- - Drop useless DEFAULT specification in plpgsql variable declarations
- - List plpythonu requirement first, to get pg_catalog scanned before public
-
-Bug fixes:
-
- - Do not add unique index on cartodb_id if already a primary key (#38)
-
-0.2.0 - 2014-06-09
-------------------
-
-Important changes:
-
- - This release adds dependency on "plpythonu" extension
- - Roles are not created anymore, previously private functions
-   for table information extraction (CDB_UserTables, CDB_TableIndexes,
-   CDB_ColumnNames, CDB_ColumnType) will now be callable by anyone while
-   only returning information about tables over which the calling user
-   has SELECT privilege (#36)
-
-Bug fixes:
-
- - Fix recursive trigger on create table (#32)
- - Ensure cartodb_id uses an associated sequence (#33)
- - Fully qualify call to cdb_disable_ddl_hooks from cdb_enable_ddl_hooks
- - Fully qualify call to CDB_UserDataSize from quota trigger
- - Fully qualify call to CDB_TransformToWebmercator from CDB_CartodbfyTable
- - Fix potential infinite loop in CDB_CartodbfyTable
- - Fix potential infinite loop in CDB_QueryStatements
-
-Enhancements:
-
- - Include revision info in cdb_version() output (#34)
-
-New features:
-
- - Add a cdb_extension_reload() function
-
-
-0.1.0 - 2014-05-23
-------------------
-
-Initial release

NEWS.md

@@ -0,0 +1,172 @@
+0.9.1 (2015-08-19)
+------------------
+* Fix for transformation to webmercator in corner cases [#116](https://github.com/CartoDB/cartodb-postgresql/issues/116)
+
+0.9.0 (2015-08-19)
+------------------
+* Re-implementation of `CDB_CartodbfyTable` functions
+  - The signature of the main function changes to
+    ```
+    FUNCTION CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
+    RETURNS REGCLASS
+    ```
+    - The `destschema` does not need to match the origin schema of `reloid`
+	- It returns the `regclass` of the cartodbfy'ed table, if it needs to be rewritten.
+  - There are many optimizations
+  - The columns `created_at` and `updated_at` will no longer be added
+* Fix for CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` #110
+* Review test to validate permissions in public tables [#112](https://github.com/CartoDB/cartodb-postgresql/pull/112)
+
+0.8.3 (2015-08-14)
+------------------
+* Fixes CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` [#108](https://github.com/CartoDB/cartodb-postgresql/issues/108)
+
+0.8.2 (2015-07-27)
+------------------
+* Fix for CDB_UserTables returning wrong listings when publicuser is used
+
+0.8.1 (2015-06-30)
+------------------
+* Fix for [#95](https://github.com/CartoDB/cartodb-postgresql/issues/95) *cdb_usertables should return public tables when the user is publicuser*
+
+0.8.0 (2015-06-30)
+------------------
+* Adds new function CDB_QueryTablesText that can deal with "schema.table_name"
+  longer than 63 chars.
+* Adds a set of statistical functions:
+  - CDB_DistType
+  - CDB_DistinctMeasure
+  - CDB_EqualIntervalBins
+* Fix for CDB_UserTables returns 0 entries for multiuser accounts [#64](https://github.com/CartoDB/cartodb-postgresql/issues/64)
+
+0.7.4 (2015-06-29)
+------------------
+Dummy transitional version.
+
+0.7.3 (2015-03-03)
+------------------
+* Fix upgrade of CDB_StringToDate function
+* Add a test for to validate CDB_TableMetadataTouch usage with OID
+
+0.7.2 (2015-03-03)
+------------------
+* Fix conversion of strings to datetime
+
+0.7.1 (2015-02-27)
+------------------
+* Revert quota checks to `pg_total_relation_size`
+
+0.7.0 (2015-02-19)
+------------------
+* Adds CDB_ZoomFromScale function
+
+0.6.0 (2015-02-19)
+------------------
+* Select permission in CDB_TableMetadata no longer granted to public
+* New function to upsert the updated_at in CDB_TableMetadata for a regclass
+
+0.5.3 (2015-02-17)
+------------------
+* Fixed security problem related with system tables
+* Changed quota checks to use `pg_relation_size` instead of `pg_total_relation_size`
+
+0.5.2 (2015-01-29)
+------------------
+* Improvement: make CDB_UserDataSize functions much faster.
+
+0.5.1 (2014-11-21)
+------------------
+* Bugfix: Quota check and some organization permissions functions were not properly escaping table name.
+
+0.5.0 (2014-11-03)
+------------------
+* Support of raster tables for cartodbfication
+* Modified quota functions: vector tables stay the same, raster tables count as full size (as have no
+  the_geom + the_geom_webmercator combo) and raster overviews are not counted
+
+0.4.1 (2014-09-21)
+------------------
+* Bugfix for Cartodbfication: Set primary key of the table if not already present (e.g. tables created from SQL API)
+
+0.4.0 (2014-08-27)
+------------------
+Added CDB_Math_Mode function
+Changes in versioning: no revision is attached so it no longer uses `git describe` for the version.
+
+0.3.6 (2014-08-11)
+------------------
+Dummy release to solve some issues with cdb branch/tag
+
+0.3.5 (2014-08-11)
+------------------
+Inverting priority of CDB_CheckQuota qmax so gies more priority to existing user quota function over parameter value.
+
+0.3.4 (2014-08-01)
+------------------
+Fixes issue with schemas in CDB_QueryTables
+
+0.3.3 (2014-07-30)
+------------------
+* Splitting of CartodbfyTable method in subfunctions to be able to call in fragments and evade timeouts on hot zones
+
+0.3.2 (2014-07-28)
+------------------
+* Make 0.3.0dev version upgradeable
+
+0.3.1 (2014-07-22)
+------------------
+* Dummy version. We start using semantic versioning
+
+0.3.0 (2014-07-15)
+------------------
+* Permission management functions
+* Adapt functions to use schemas
+
+0.2.1 - 2014-06-11
+------------------
+
+Enhancements:
+
+ - Do not force re-cartodbfication on CREATE FROM unpackaged
+ - Drop useless DEFAULT specification in plpgsql variable declarations
+ - List plpythonu requirement first, to get pg_catalog scanned before public
+
+Bug fixes:
+
+ - Do not add unique index on cartodb_id if already a primary key (#38)
+
+0.2.0 - 2014-06-09
+------------------
+
+Important changes:
+
+ - This release adds dependency on "plpythonu" extension
+ - Roles are not created anymore, previously private functions
+   for table information extraction (CDB_UserTables, CDB_TableIndexes,
+   CDB_ColumnNames, CDB_ColumnType) will now be callable by anyone while
+   only returning information about tables over which the calling user
+   has SELECT privilege (#36)
+
+Bug fixes:
+
+ - Fix recursive trigger on create table (#32)
+ - Ensure cartodb_id uses an associated sequence (#33)
+ - Fully qualify call to cdb_disable_ddl_hooks from cdb_enable_ddl_hooks
+ - Fully qualify call to CDB_UserDataSize from quota trigger
+ - Fully qualify call to CDB_TransformToWebmercator from CDB_CartodbfyTable
+ - Fix potential infinite loop in CDB_CartodbfyTable
+ - Fix potential infinite loop in CDB_QueryStatements
+
+Enhancements:
+
+ - Include revision info in cdb_version() output (#34)
+
+New features:
+
+ - Add a cdb_extension_reload() function
+
+
+0.1.0 - 2014-05-23
+------------------
+
+Initial release

README.md

@@ -11,7 +11,7 @@ See https://github.com/CartoDB/cartodb/wiki/CartoDB-PostgreSQL-extension
 Dependencies
 ------------
 
- * PostgreSQL 9.3+ (with plpythonu extension)
+ * PostgreSQL 9.3+ (with plpythonu extension and xml support)
  * [PostGIS extension](http://postgis.net) 
  * [Schema triggers extension]
    (https://bitbucket.org/malloclabs/pg_schema_triggers)
@@ -20,12 +20,16 @@ Dependencies
 Install
 -------
 
- make all install
+```sh
+make all install
+```
 
 Test installation
 -----------------
 
- make installcheck
+```sh
+make installcheck
+```
 
 NOTE: if ``test_ddl_triggers`` fails it's likely due to an incomplete
       installation of schema_triggers: you need to add ``schema_triggers.so``
@@ -33,6 +37,8 @@ NOTE: if ``test_ddl_triggers`` fails it's likely due to an incomplete
 
 NOTE: you need to run the installcheck as a superuser, use PGUSER
       env variable if needed, like: PGUSER=postgres make installcheck
+      
+NOTE: the tests need to run against a **clean postgres instance**, if you have some roles already created test will likely fail due `publicuser` not being dropped.
 
 Enable database
 ---------------

doc/CDB_ColumnNames.md

@@ -0,0 +1,14 @@
+Retrieve all column names in a particular table
+
+#### Using the function
+
+```sql
+SELECT CDB_ColumnNames('table_name')
+--- Returns a set of rows with column names
+```
+
+#### Arguments
+
+CDB_ColumnNames(table_name)
+
+* **table_name** text

doc/CDB_ColumnType.md

@@ -0,0 +1,15 @@
+Returns a column type for any column in a table
+
+#### Using the function
+
+```sql
+SELECT CDB_ColumnType('column_name','table_name')
+--- Returns a set of rows with column types
+```
+
+#### Arguments
+
+CDB_ColumnType(column_name, table_name)
+
+* **column_name** text
+* **table_name** text

doc/CDB_HeadsTailsBins.md

@@ -0,0 +1,21 @@
+Find the breaks for N categories in a numerical column based on the [Heads/Tails optimization](http://arxiv.org/pdf/1209.2801v1.pdf). Below, Heads/Tails used to color based on the area of the polygons.
+
+![headtails](https://f.cloud.github.com/assets/370259/140655/6eebb918-7228-11e2-89fa-149745f25d34.png)
+
+#### Using the function
+
+We can determine the 7 most optimal breaks in a column of numerical data as follows, 
+
+```sql
+SELECT CDB_HeadsTailsBins(array_agg(numeric_column), 7) FROM table_name
+-- Results in an ordered array like, {7824,23492,52696,233857,666089,1001709,1638094}
+-- Each break happens up to, and equal, to a number: 
+-- (bin1 is less than or equal to 7824, bin2 is less than or equal to 23492, etc.)
+```
+
+#### Arguments
+
+CDB_HeadsTailsBins(in_array, breaks)
+
+* **in_array** numeric[]. A NUMERIC array of values.
+* **breaks** int. The number of categories you want to create

doc/CDB_HexagonGrid.md

@@ -0,0 +1,43 @@
+Fill given extent with an hexagonal coverage
+
+#### Using the function
+
+Create a hexagonal grid from a polygon geometry. For example, take the geometry
+
+```sql
+  ST_SetSRID(
+    ST_Envelope(
+      ST_Collect(
+        ST_MakePoint(10000000,-10000000),
+        ST_MakePoint(-10000000,10000000)
+      )
+     ),
+     3857)
+```
+
+We can create a grid as follows, 
+
+```sql
+SELECT CDB_HexagonGrid(
+  ST_SetSRID(
+    ST_Envelope(
+      ST_Collect(
+        ST_MakePoint(10000000,-10000000),
+        ST_MakePoint(-10000000,10000000)
+      )
+     ),
+     3857),
+   1000000) the_geom_webmercator
+```
+
+Which will look something like this,
+
+![grid tile](http://i.imgur.com/4rZXGMb.png)
+
+#### Arguments
+
+CDB_HexagonGrid(ext, side, origin)
+
+* **ext** geometry. Extent to fill. Only hexagons with center point falling inside the extent (or at the lower or leftmost edge) will be emitted. The returned hexagons will have the same SRID as this extent.
+* **side** float. Side measure for the hexagon. Maximum diameter will be 2 * side. Measure is in the same projection as **ext**
+* **origin** OPTIONAL geometry. Optional origin to allow for exact tiling. If omitted the origin will be 0,0. The parameter is checked for having the same SRID as the extent.

doc/CDB_JenksBins.md

@@ -0,0 +1,23 @@
+Find the breaks for N categories in a numerical column based on the [Jenks optimization](http://en.wikipedia.org/wiki/Jenks_natural_breaks_optimization). Below, Jenks used to color based on the area of the polygons.
+
+![Jenks](https://f.cloud.github.com/assets/370259/140093/b64a9382-7210-11e2-81a4-c65cce3c885e.png)
+
+#### Using the function
+
+We can determine the 7 most optimal breaks in a column of numerical data as follows, 
+
+```sql
+SELECT CDB_JenksBins(array_agg(numeric_column), 7) FROM table_name
+-- Results in an ordered array like, {0,73,2568,9408,29411,768230,1638094}
+-- Each break happens up to, and equal, to a number: 
+-- (bin1 is less than or equal to 0, bin2 is less than or equal to 73, etc.)
+```
+
+#### Arguments
+
+CDB_JenksBins(in_array, breaks, invert)
+
+* **in_array** numeric[]. A NUMERIC array of values.
+* **breaks** int. The number of categories you want to create
+* **iterations** OPTIONAL int. The number of iterations used for calculating breaks.
+* **invert** OPTIONAL boolean. Flips whether you receive top down breaks or bottom up breaks. Default is top down (so, <=). Bottom up would give you values that define the lower-end start of a bin (so >=).

doc/CDB_MakeHexagon.md

@@ -0,0 +1,21 @@
+Return an Hexagon with given center and side (or maximal radius)
+
+#### Using the function
+
+Running the following SQL
+
+```sql
+SELECT CDB_MakeHexagon(ST_MakePoint(0,0),10000000)
+```
+
+Would give you back a single hexagon geometry, 
+
+![hexagon](http://i.imgur.com/6jeGStb.png)
+
+
+#### Arguments
+
+CDB_MakeHexagon(center, radius)
+
+* **center** geometry
+* **radius** float. Radius of hexagon measured in same projection as **center**

doc/CDB_QuantileBins.md

@@ -0,0 +1,21 @@
+Find the breaks for N categories in a numerical column based on the [Quantile bins]. Below, the quantile method is used to determine color based on the area of the polygons.
+
+![qunatile](https://f.cloud.github.com/assets/370259/140714/932ed0e6-722b-11e2-9807-ffbd0fddb9ac.png)
+
+#### Using the function
+
+We can determine the 7 most optimal breaks in a column of numerical data as follows, 
+
+```sql
+SELECT CDB_QuantileBins(array_agg(numeric_column), 7) FROM table_name
+-- Results in an ordered array like, {80,2281,7162,17652,39730,91077,1638094}
+-- Each break happens up to, and equal, to a number: 
+-- (bin1 is less than or equal to 80, bin2 is less than or equal to 2281, etc.)
+```
+
+#### Arguments
+
+CDB_QuantileBins(in_array, breaks)
+
+* **in_array** numeric[]. A NUMERIC array of values.
+* **breaks** int. The number of categories you want to create

doc/CDB_RectangleGrid.md

@@ -0,0 +1,46 @@
+Fill given extent with a rectangular coverage
+
+#### Using the function
+
+Create a rectangular grid from a polygon geometry. For example, take the geometry
+
+```sql
+  ST_SetSRID(
+    ST_Envelope(
+      ST_Collect(
+        ST_MakePoint(10000000,-10000000),
+        ST_MakePoint(-10000000,10000000)
+      )
+     ),
+     3857)
+```
+
+We can create a grid as follows, 
+
+```sql
+SELECT CDB_RectangleGrid(
+  ST_SetSRID(
+    ST_Envelope(
+      ST_Collect(
+        ST_MakePoint(10000000,-10000000),
+        ST_MakePoint(-10000000,10000000)
+      )
+     ),
+     3857),
+   1000000,
+   1000000
+) the_geom_webmercator
+```
+
+Which will look something like this,
+
+![rect grid](http://i.imgur.com/HuhOJRs.png)
+
+#### Arguments
+
+CDB_RectangleGrid(ext, width, height, origin)
+
+* **ext** geometry. Extent to fill. Only rectangles with center point falling inside the extent (or at the lower or leftmost edge) will be emitted. The returned hexagons will have the same SRID as this extent.
+* **width** float. Width of each rectangle. Measure is in the same projection as **ext**
+* **height** float. Height of each rectangle. Measure is in the same projection as **ext**
+* **origin** OPTIONAL geometry. Optional origin to allow for exact tiling. If omitted the origin will be 0,0. The parameter is checked for having the same SRID as the extent.

doc/CDB_SetUserQuotaInBytes.md

@@ -0,0 +1,11 @@
+Sets user quota in bytes (superuser only)
+
+#### Using the function
+
+```sql
+SELECT CDB_SetUserQuotaInBytes(10485760);
+--- Returns the previously set quota.
+--- Use 0 to disable quota.
+```
+
+REF: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_Quota.sql

doc/CDB_TransformToWebmercator.md

@@ -0,0 +1,44 @@
+Function to "safely" transform to webmercator. This function is most useful for rendering custom geometries using the CartoDB tiler. Often, transforming a projection like WGS84 can cause issues with extents beyond what are actually valid in webmercator, this attempts to fix those issues.
+
+#### Using the function
+
+Using a box that is nearly the full globe,
+
+```sql
+ST_SetSRID(
+  ST_Envelope(
+    ST_Collect(
+      ST_MakePoint(-180,60),
+      ST_MakePoint(180,-60)
+    )
+   ),
+   4326
+)
+```
+
+We can then convert it to a renderable webmercator geometry.
+
+```sql
+SELECT CDB_TransformToWebmercator(
+ ST_SetSRID(
+  ST_Envelope(
+    ST_Collect(
+      ST_MakePoint(-10,60),
+      ST_MakePoint(300,-60)
+    )
+   ),
+   4326
+ )
+)
+```
+
+Would give you back a single valid rectangle in webmercator. Since a longitude of 300 would convert to an unallowed webmercator coordinate, it gets clipped first. Valid extent is WGS84 (-180, -89, 180, 89)
+
+![valid geom](http://i.imgur.com/EFdXiqt.png)
+
+
+#### Arguments
+
+CDB_TransformToWebmercator(geom)
+
+* **geom** geometry

doc/CDB_UserTables.md

@@ -0,0 +1,11 @@
+List the name of available tables (only the usable ones)
+
+#### Using the function
+
+```sql
+--- Returns a row for each table having given permission with the table name
+--- Currently accepted permissions are: 'public', 'private' or 'all'
+SELECT CDB_UserTables(perms)
+```
+
+REF: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_UserTables.sql

doc/CDB_XYZ_Extent.md

@@ -0,0 +1,22 @@
+Determine the spatial extent of a tile based on the tile's XYZ coordinate.
+
+#### Using the function
+
+Take a common tile with coordinates x=3, y=2, z=2,
+
+![2/3/2](https://viz2.cartodb.com/tiles/quantile_breaks/2/3/2.png)
+
+To determine its extent you would run,
+
+```sql
+SELECT CDB_XYZ_Extent(3,2,2)
+--- Returns a WKB polygon in Webmercator (SRID 3857)
+```
+
+#### Arguments
+
+CDB_XYZ_Extent(x,y,z)
+
+* **x** integer
+* **y** integer
+* **z** integer

doc/CDB_XYZ_Resolution.md

@@ -0,0 +1,20 @@
+Return pixel resolution of tiles at a given zoom level
+
+#### Using the function
+
+Take a common tile with zoom, z=2,
+
+![2/3/2](https://viz2.cartodb.com/tiles/quantile_breaks/2/3/2.png)
+
+To determine the resolution of these pixels,
+
+```sql
+SELECT CDB_XYZ_Resolution(2)
+--- Returns a float, 39135.7587890625
+```
+
+#### Arguments
+
+CDB_XYZ_Resolution(z)
+
+* **z** integer

doc/CartoDB-PLpgSQL.md

@@ -0,0 +1,38 @@
+INTRODUCTION
+============
+
+CartoDB uses a number of custom [PLpgSQL](http://www.postgresql.org/docs/8.3/static/plpgsql.html) functions to perform a few magical things. Those functions are accessible to users on CartoDB as well, so we would like to document what they are and what they do here.
+
+## Spatial functions
+
+[CDB_HexagonGrid](CDB_HexagonGrid) - create hexagonal grid from extent and size
+
+[CDB_MakeHexagon](CDB_MakeHexagon) - make a hexagon with given center and side
+
+[CDB_RectangleGrid](CDB_RectangleGrid) - fill given extent with a rectangular coverage
+
+##### Tile based
+
+[CDB_XYZ_Extent](CDB_XYZ_Extent) - Find the extent of a tile by XYZ
+
+[CDB_XYZ_Resolution](CDB_XYZ_Resolution) - Find the pixel resolution of tiles
+
+[CDB_TransformToWebmercator](CDB_TransformToWebmercator) - Convert a geometry to valid webmercator
+
+## Statistical functions
+
+[CDB_JenksBins](CDB_JenksBins) - Find breaks in an array of numbers using Jenks method
+
+[CDB_HeadsTailsBins](CDB_HeadsTailsBins) - Find breaks in an array of numbers using Heads/Tails method
+
+[CDB_QuantileBins](CDB_QuantileBins) - Find quantile breaks in an array of numbers
+
+## System functions
+
+[CDB_UserTables](CDB_UserTables) - Get a list of all tables in your account
+
+[[CDB_SetUserQuotaInBytes]] - Set maximum user quota in bytes
+
+column names - now returned in JSON response
+
+column types - now returned in JSON response

doc/CartoDB-user-table.md

@@ -0,0 +1,29 @@
+A "cartodb" user table is a table with a well-known set of fields and a well-known set of triggers attached on. 
+
+The fields are:
+
+ - `cartodb_id`, a numerical primary key of serial type
+ - `created_at`, timestamp with timezone not null default now()
+ - `updated_at`, timestamp with timezone not null default now()
+ - `the_geom`, geometry, GiST indexed, constrained (see below)
+ - `the_geom_webmercator`, geometry, GiST indexed, constrained (see below)
+
+The values of "the_geom" and "the_geom_webmercator" must match these constraints:
+
+ - Only POINT, MULTILINE, MULTIPOLYGON types ? Maybe UNCONSTRAINED
+ - Only 2 dimensions ? Maybe UNCONSTRAINED
+ - SRID=4326 for the_geom and SRID=3857 for the_geom_webmercator
+
+The triggers are:
+
+ - `track_updates` after modifying statement updates cdb_tablemetadata
+ - `test_quota` before changing statement to forbid if overquota
+ - `test_quota_per_row` before changing row to forbod if overquota (checked on a probabilistic basis)
+ - `update_the_geom_webmercator` before insert or update row to maintain the_geom_webmercator
+ - `update_updated_at_trigger` before update row to maintain updated_at
+
+Some conversions will be attempted to perform upon cartodbfication when certain fields appear:
+
+ - `cartodb_id`: If found type TEXT will be attempted to cast
+ - `created_at`: If found type TEXT will be attempted to cast
+ - `updated_at`: If found type TEXT will be attempted to cast

doc/README.md

@@ -0,0 +1,48 @@
+# Contents
+
+* [CartoDB-user-table](CartoDB-user-table.md)
+* [CartoDB-PLpgSQL](CartoDB-PLpgSQL.md)
+* [CDB_ColumnNames](CDB_ColumnNames.md)
+* [CDB_ColumnType](CDB_ColumnType.md)
+* [CDB_HeadsTailsBins](CDB_HeadsTailsBins.md)
+* [CDB_HexagonGrid](CDB_HexagonGrid.md)
+* [CDB_JenksBins](CDB_JenksBins.md)
+* [CDB_MakeHexagon](CDB_MakeHexagon.md)
+* [CDB_QuantileBins](CDB_QuantileBins.md)
+* [CDB_RectangleGrid](CDB_RectangleGrid.md)
+* [CDB_SetUserQuotaInBytes](CDB_SetUserQuotaInBytes.md)
+* [CDB_TransformToWebmercator](CDB_TransformToWebmercator.md)
+* [CDB_UserTables](CDB_UserTables.md)
+* [CDB_XYZ_Extent](CDB_XYZ_Extent.md)
+* [CDB_XYZ_Resolution](CDB_XYZ_Resolution.md)
+
+The CartoDB PostgreSQL extension is a module to load into each CartoDB user database to perform cartodb-specific security and functionality checks.
+
+# Checks
+
+No user other than the superuser should be allowed to change `statement_timeout` for the session (SET statement_timeout disallowed).
+
+User tables need to match certain structure criteria (See [[CartoDB-user-table]]) so the extension should provide a mean to enforce such structure everytime an attempt to change structure is encountered. 
+
+# Events
+
+The events we want some function to be called upon are:
+
+| event                  | arguments to handler function        | function duty                    | OK* |
+|------------------------|--------------------------------------|----------------------------------|-----|
+| SET variable           | name of variable                     | forbid changing some vars        |     |
+| RENAME table           | old and new name + oid of the table  | flush cache                      |     |
+| ADD/DROP/ALTER column  | oid of the table                     | flush cache, cartodbfy, upd meta |  Y  |
+| DISABLE/DROP trigger   | oid of table, name of trigger        | cartodbfy or forbid              |     |
+| DROP table             | oid of the table                     | flush cache and metadata         |  Y  |
+| CREATE table           | oid of the table                     | cartodby, upd metadata           |  Y  |
+| GRANT                  | oid of table, privilege, role        | flush cache, upd metadata        |
+| REVOKE                 | oid of table, privilege, role        | flush cache, upd metadata        |
+
+
+* event available by installing https://github.com/CartoDB/pg_schema_triggers
+
+At this stage we don't need more than this, but the number of events and the number of arguments passed to the handler function may expand in the future, so the extension should be written in a way to easily allow that.
+
+Some of the handler will need to act _after_ the statement is completed (CREATE TABLE, for example).
+

expected/test_ddl_triggers.out

@@ -19,6 +19,14 @@ NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 (1 row)
 
 create schema c;
+SELECT CDB_SetUserQuotaInBytes('c', 0);
+ cdb_setuserquotainbytes 
+-------------------------
+                       0
+(1 row)
+
+DROP USER IF EXISTS cartodb_postgresql_unpriv_user;
+NOTICE:  role "cartodb_postgresql_unpriv_user" does not exist, skipping
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -26,10 +34,10 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 ----------------------
 -- CREATE TABLE
 ----------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 NOTICE:  trigger "track_updates" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t3" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "test_quota" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "test_quota_per_row" for table "c.t3" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
@@ -38,17 +46,7 @@ NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i 
-------------+-----+------+----------+----------------------+---
-          1 | t   | t    |          |                      | 1
-(1 row)
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)) as age
@@ -58,33 +56,21 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
  c.t3    |   0
 (1 row)
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
 NOTICE:  trigger "track_updates" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t4" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "test_quota" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "test_quota_per_row" for table "c.t4" does not exist, skipping
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Existing cartodb_id field does not have an associated sequence, renaming
-NOTICE:  Trying to recover data from _cartodb_id0 column
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t4) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator 
-------------+-----+------+----------+----------------------
-          1 | t   | t    |          | 
-(1 row)
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now() - updated_at)) as age
@@ -97,6 +83,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
 ----------------------------
 -- ALTER TABLE RENAME COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -104,27 +91,13 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc 
-------------+-----+------+----------+----------------------+---+---------
-          1 | t   | t    |          |                      | 1 | 
-(1 row)
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -134,6 +107,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
  c.t3    |     0
 (1 row)
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -141,27 +115,13 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -174,6 +134,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE DROP COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -181,27 +142,13 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 drop column the_geom_webmercator;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -214,6 +161,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE ADD COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
  pg_sleep 
 ----------
@@ -222,17 +170,7 @@ select pg_sleep(.1);
 
 alter table c.t3 add column id2 int;
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 | id2 
-------------+-----+------+----------+----------------------+---+---------+----------+-----
-          1 | t   | t    |          |                      | 1 |         |          |    
-(1 row)
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -247,12 +185,13 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 RESET SESSION AUTHORIZATION;
 drop schema c cascade;
-NOTICE:  drop cascades to 2 other objects
+NOTICE:  drop cascades to 3 other objects
 select count(*) from CDB_TableMetadata;
  count 
 -------
      0
 (1 row)
 
-DROP USER cartodb_postgresql_unpriv_user;
+DROP OWNED BY cartodb_postgresql_unpriv_user;
+DROP ROLE cartodb_postgresql_unpriv_user;
 DROP FUNCTION _CDB_UserQuotaInBytes();

scripts-available/CDB_DDLTriggers.sql

@@ -4,14 +4,23 @@ CREATE OR REPLACE FUNCTION cartodb.cdb_handle_create_table ()
 RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
+  rel RECORD;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_relation_create_eventinfo();
 
   -- We're only interested in real relations
   IF (event_info.new).relkind != 'r' THEN RETURN; END IF;
 
-  RAISE DEBUG 'Relation % of kind % created in namespace oid %',
-	 event_info.relation, (event_info.new).relkind, (event_info.new).relnamespace;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
+
+  RAISE DEBUG 'Relation % of kind % created in table % namespace % (oid %)',
+	 event_info.relation, rel.relkind, rel.relname, rel.nspname, rel.relnamespace;
 
   -- We don't want to react to alters triggered by superuser,
   IF current_setting('is_superuser') = 'on' THEN
@@ -22,15 +31,15 @@ BEGIN
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
   -- CDB_CartodbfyTable must not create tables, or infinite loop will happen
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   RAISE DEBUG 'Inserting into cartodb.CDB_TableMetadata';
 
   -- Add entry to CDB_TableMetadata (should CartodbfyTable do this?)
-  INSERT INTO cartodb.CDB_TableMetadata(tabname,updated_at) 
-  VALUES (event_info.relation, now());
+  INSERT INTO cartodb.CDB_TableMetadata(tabname, updated_at) 
+  VALUES (newtable, now());
 
 END; $$;
 -- }
@@ -63,13 +72,19 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_column_alter_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % altered by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.old).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;
@@ -82,12 +97,12 @@ BEGIN
 
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
+  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW(), tabname = newtable
   WHERE tabname = event_info.relation; 
 
 END; $$;
@@ -100,13 +115,19 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_column_drop_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % drop by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.old).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;
@@ -117,15 +138,16 @@ BEGIN
     RETURN;
   END IF;
 
+
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
-  WHERE tabname = event_info.relation; 
+  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW(), tabname = newtable
+  WHERE tabname = event_info.relation;
 
 END; $$;
 -- }
@@ -140,10 +162,15 @@ DECLARE
 BEGIN
   event_info := schema_triggers.get_column_add_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % added by % (superuser? %) in relation % of kind %',
-	 (event_info.new).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.new).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;

scripts-available/CDB_DistType.sql

@@ -0,0 +1,122 @@
+--
+-- CDB_DistType classifies the histograms of a column into
+-- one of the basic types listed by Galtung: http://druedin.com/2012/12/08/galtungs-ajus-system/
+-- 
+-- Future improvements:
+--    variable number of bins (7 is baked in right now)
+--    catch the number of items to ensure that the sample is large enough
+--
+-- Refs:
+--    1. width_bucket/histograms: http://tapoueh.org/blog/2014/02/21-PostgreSQL-histogram
+--    2. R implementation: https://github.com/cran/agrmt
+
+CREATE OR REPLACE FUNCTION CDB_DistType ( in_array NUMERIC[] ) RETURNS text as $$
+DECLARE
+    element_count INT4;
+    minv numeric;
+    maxv numeric;
+    bins numeric[];
+    freqs numeric[];
+    ajus INT[];
+    freq INT4;
+    signature text;
+    i INT := 1;
+BEGIN
+    SELECT min(e), max(e), count(e) INTO minv, maxv, element_count FROM ( SELECT unnest(in_array) e ) x;
+
+    IF abs(maxv - minv) < 1e-7 THEN -- if max and min are nearly equal, call if 'F' (make relative to maxv?)
+        signature = 'F';
+    ELSE
+        -- Calculate bins and count in bins
+        EXECUTE 'WITH stats as (
+            SELECT min(e) as minv,
+                   max(e) as maxv,
+                   count(e) as total
+            FROM (SELECT unnest($1) e) x
+            WHERE e is not null
+        ),
+        hist as (
+            SELECT width_bucket(e, s.minv, s.maxv, 7) bucket,
+                   count(*) freq
+            FROM (SELECT unnest($1) e) x, stats s
+            WHERE e is not null
+            GROUP BY 1
+            ORDER BY 1
+        )
+        SELECT array_agg(round(100.0 * hist.freq::numeric / stats.total::numeric,1)) freqs,
+               array_agg(hist.bucket) buckets
+        FROM hist, stats'
+        INTO freqs, bins
+        USING in_array;
+
+        LOOP
+            IF i < 7 THEN
+                ajus[i] = CASE WHEN freqs[i] > freqs[i+1] THEN -1
+                               WHEN abs(freqs[i] - freqs[i+1]) <= 0.05 THEN 0
+                               ELSE 1 END;
+            ELSE
+                EXIT;
+            END IF;
+            i := i + 1;
+        END LOOP;
+
+        signature = _CDB_DistTypeClassify(ajus);
+    END IF;
+
+    RETURN signature;
+END;
+$$ language plpgsql IMMUTABLE;
+
+-- Classify data into AJUSFL
+
+CREATE OR REPLACE FUNCTION _CDB_DistTypeClassify ( in_array INT[] ) RETURNS text as $$
+DECLARE
+    element_count INT4;
+    maxv numeric;
+    minv numeric;
+    uniques INT[];
+    type text;
+BEGIN
+    SELECT max(e), min(e) INTO maxv, minv FROM ( SELECT unnest(in_array) e ) x;
+
+    IF (maxv = 0 AND minv = 0) THEN
+        type = 'F';
+    ELSIF maxv < 1 THEN
+        type = 'L';
+    ELSIF minv > -1 THEN
+        type = 'J';
+    ELSE
+        -- Get distinct elements ordered by original position
+        EXECUTE 'WITH b AS (
+            SELECT a
+            FROM (SELECT unnest($1) a) x
+        ),
+        c AS (
+            SELECT a, row_number() OVER () r
+            FROM b
+        ),
+        d AS (
+            SELECT DISTINCT a
+            FROM c
+        ),
+        e AS (
+            SELECT a FROM d ORDER BY (
+                SELECT r FROM c WHERE d.a = c.a ORDER BY r ASC LIMIT 1
+            ) ASC)
+        SELECT array_agg(a) FROM e'
+        INTO uniques
+        USING in_array;
+
+        -- Decide if it's an A, U, or other
+        IF (uniques = ARRAY[1,-1] OR uniques = ARRAY[1,0,-1] OR uniques = ARRAY[1,-1,0] OR uniques = ARRAY[0,1,-1]) THEN
+            type = 'A';
+        ELSIF (uniques = ARRAY[-1,1] OR uniques = ARRAY[-1,0,1] OR uniques = ARRAY[-1,1,0] OR uniques = ARRAY[0,-1,1]) THEN
+            type = 'U';
+        ELSE
+            type = 'S';
+        END IF;
+    END IF;
+
+    RETURN type;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_DistinctMeasure.sql

@@ -0,0 +1,46 @@
+--
+-- CDB_DistinctMeasure 
+--     calculates the fraction of rows in the 10 most common distinct categories
+--     returns true if the number of rows in these 10 categories is >= 0.9 * total number of rows
+-- 
+-- 
+
+CREATE OR REPLACE FUNCTION CDB_DistinctMeasure ( in_array text[], threshold numeric DEFAULT null ) RETURNS numeric as $$
+DECLARE
+    element_count INT4;
+    maxval numeric;
+    passes numeric;
+BEGIN
+    SELECT count(e) INTO element_count FROM ( SELECT unnest(in_array) e ) x;
+
+    -- count number of occurrences per bin
+    -- calculate the normalized cumulative sum
+    -- return the max value: which corresponds nth entry 
+    -- for n <= 10 depending on # of distinct values
+    EXECUTE 'WITH a As (
+              SELECT
+                count(*) cnt
+              FROM
+                (SELECT * FROM unnest($2) e ) x
+              WHERE e is not null
+              GROUP BY e
+              ORDER BY cnt DESC
+            ),
+            b As (
+              SELECT
+                sum(cnt) OVER (ORDER BY cnt DESC) / $1 As cumsum
+              FROM a
+              LIMIT 10
+            )
+            SELECT max(cumsum) maxval FROM b'
+            INTO maxval
+            USING element_count, in_array;
+    IF threshold is null THEN
+        passes = maxval;
+    ELSE
+        passes = CASE WHEN (maxval >= threshold) THEN 1 ELSE 0 END;
+    END IF;
+
+    RETURN passes;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_EqualIntervalBins.sql

@@ -0,0 +1,37 @@
+--
+-- Calculate the equal interval bins for a given column
+--
+-- @param in_array A numeric array of numbers to determine the best
+--                   to determine the bin boundary
+--
+-- @param breaks The number of bins you want to find.
+--  
+--
+-- Returns: upper edges of bins
+-- 
+--
+
+CREATE OR REPLACE FUNCTION CDB_EqualIntervalBins ( in_array NUMERIC[], breaks INT ) RETURNS NUMERIC[] as $$
+DECLARE 
+    diff numeric;
+    min_val numeric;
+    max_val numeric;
+    tmp_val numeric;
+    i INT := 1;
+    reply numeric[];
+BEGIN
+    SELECT min(e), max(e) INTO min_val, max_val FROM ( SELECT unnest(in_array) e ) x WHERE e IS NOT NULL;
+    diff = (max_val - min_val) / breaks::numeric;
+    LOOP
+        IF i < breaks THEN
+            tmp_val = min_val + i::numeric * diff;
+            reply = array_append(reply, tmp_val);
+            i := i+1;
+        ELSE
+            reply = array_append(reply, max_val);
+            EXIT;
+        END IF;
+    END LOOP;
+    RETURN reply;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_Math.sql

@@ -0,0 +1,26 @@
+-- CartoDB Math SQL functions
+
+
+-- Mode
+-- https://wiki.postgresql.org/wiki/Aggregate_Mode
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Math_final_mode(anyarray)
+  RETURNS anyelement AS
+$BODY$
+    SELECT a
+    FROM unnest($1) a
+    GROUP BY 1 
+    ORDER BY COUNT(1) DESC, 1
+    LIMIT 1;
+$BODY$
+LANGUAGE 'sql' IMMUTABLE;
+
+DROP AGGREGATE IF EXISTS cartodb.CDB_Math_Mode(anyelement);
+
+CREATE AGGREGATE cartodb.CDB_Math_Mode(anyelement) (
+  SFUNC=array_append,
+  STYPE=anyarray,
+  FINALFUNC=_CDB_Math_final_mode,
+  INITCOND='{}'
+);
+

scripts-available/CDB_Organizations.sql

@@ -35,7 +35,7 @@ FUNCTION cartodb.CDB_Organization_Add_Table_Read_Permission(from_schema text, ta
 AS $$
 BEGIN
     EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
-    EXECUTE 'GRANT SELECT ON "' || from_schema || '".' || table_name || ' TO "' || to_role_name || '"';
+    EXECUTE 'GRANT SELECT ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
@@ -54,7 +54,7 @@ FUNCTION cartodb.CDB_Organization_Add_Table_Read_Write_Permission(from_schema te
 AS $$
 BEGIN
     EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
-    EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '".' || table_name || ' TO "' || to_role_name || '"';
+    EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
@@ -73,7 +73,7 @@ FUNCTION cartodb.CDB_Organization_Remove_Access_Permission(from_schema text, tab
     RETURNS void
 AS $$
 BEGIN
-    EXECUTE 'REVOKE ALL PRIVILEGES ON TABLE "' || from_schema || '".' || table_name || ' FROM "' || to_role_name || '"';
+    EXECUTE 'REVOKE ALL PRIVILEGES ON TABLE "' || from_schema || '"."' || table_name || '" FROM "' || to_role_name || '"';
     -- EXECUTE 'REVOKE USAGE ON SCHEMA ' || from_schema || ' FROM "' || to_role_name || '"';
     -- We need to revoke usage on schema only if we are revoking privileges from the last table where to_role_name has
     -- any permission granted within the schema from_schema

scripts-available/CDB_QuantileBins.sql

@@ -15,18 +15,29 @@ DECLARE
     i INT := 1; 
     reply numeric[]; 
 BEGIN 
-    -- get our unique values
-    SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e GROUP BY e ORDER BY e ASC) x;
-    -- get the total size of our row
-    element_count := array_upper(in_array, 1) - array_lower(in_array, 1); 
+    -- sort our values
+    SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e ORDER BY e ASC) x;
+    -- get the total size of our data
+    element_count := array_length(in_array, 1); 
     break_size :=  element_count::numeric / breaks;
     -- slice our bread
     LOOP  
-        IF i > breaks THEN  EXIT;  END IF;  
-        SELECT e INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 1 OFFSET round(break_size * i)) x;
+        IF i < breaks THEN
+            IF break_size * i % 1 > 0 THEN
+                SELECT e INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 1 OFFSET ceil(break_size * i) - 1) x;
+            ELSE
+                SELECT avg(e) INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 2 OFFSET ceil(break_size * i) - 1 ) x;
+            END IF;
+        ELSIF i = breaks THEN
+            -- select the last value
+            SELECT max(e) INTO tmp_val FROM ( SELECT unnest(in_array) e ) x;
+        ELSE
+            EXIT;
+        END IF;
+
         reply = array_append(reply, tmp_val);
-        i := i+1; 
-    END LOOP; 
-    RETURN reply; 
+        i := i+1;
+    END LOOP;
+    RETURN reply;
 END; 
-$$ language plpgsql IMMUTABLE;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_QueryTables.sql

@@ -2,12 +2,12 @@
 --
 -- Requires PostgreSQL 9.x+
 --
-CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
-RETURNS name[]
+CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
+RETURNS text[]
 AS $$
 DECLARE
   exp XML;
-  tables NAME[];
+  tables text[];
   rec RECORD;
   rec2 RECORD;
 BEGIN
@@ -16,9 +16,9 @@ BEGIN
 
   FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
 
-    IF NOT ( rec.q ilike 'select %' or rec.q ilike 'with %' ) THEN
-      --RAISE WARNING 'Skipping %', rec.q;
-      CONTINUE;
+    IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
+        --RAISE WARNING 'Skipping %', rec.q;
+        CONTINUE;
     END IF;
 
     BEGIN
@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x)::name as p, unnest(s)::name as sc from inp
+      SELECT unnest(x) as p, unnest(s) as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p)::name);
+      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;
@@ -65,3 +65,14 @@ BEGIN
   return tables;
 END
 $$ LANGUAGE 'plpgsql' VOLATILE STRICT;
+
+
+-- Keep CDB_QueryTables with same signature for backwards compatibility.
+-- It should probably be removed in the future.
+CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
+RETURNS name[]
+AS $$
+BEGIN
+  RETURN CDB_QueryTablesText(query)::name[];
+END
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

scripts-available/CDB_Quota.sql

@@ -1,21 +1,62 @@
+CREATE OR REPLACE FUNCTION cartodb._CDB_total_relation_size(_schema_name TEXT, _table_name TEXT)
+RETURNS bigint AS
+$$
+DECLARE relation_size bigint := 0;
+BEGIN
+  BEGIN
+    SELECT pg_total_relation_size(format('"%s"."%s"', _schema_name, _table_name)) INTO relation_size;
+  EXCEPTION
+    WHEN undefined_table OR OTHERS THEN
+      RAISE NOTICE 'cartodb._CDB_total_relation_size(''%'', ''%'') caught error: % (%)', _schema_name, _table_name, SQLERRM, SQLSTATE;
+  END;
+  RETURN relation_size;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;
+
 -- Return the estimated size of user data. Used for quota checking.
 CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
 RETURNS bigint AS
 $$
-  -- TODO: double check this query. Maybe use CDB_TableMetadata for lookup ?
-  --       also, it's "table_name" sounds sensible to search_path
-  --
-  -- NOTE: division by 2 is an hack for the_geom_webmercator
-  --
-  SELECT coalesce(int8(sum(pg_total_relation_size(schema_name || '.' || table_name)) / 2), 0)
-    AS quota
-  FROM information_schema.tables
-  WHERE table_catalog = current_database() AND table_schema = schema_name
+DECLARE
+  total_size INT8;
+BEGIN
+  WITH raster_tables AS (
+    SELECT o_table_name, r_table_name FROM raster_overviews
+      WHERE o_table_schema = schema_name AND o_table_catalog = current_database()
+  ),
+  user_tables AS (
+    SELECT table_name FROM information_schema.tables
+      WHERE table_catalog = current_database() AND table_schema = schema_name
         AND table_name != 'spatial_ref_sys'
         AND table_name != 'cdb_tablemetadata'
-        AND table_type = 'BASE TABLE';
+        AND table_type = 'BASE TABLE'
+  ),
+  table_cat AS (
+    SELECT
+      table_name,
+      EXISTS(select * from raster_tables where o_table_name = table_name) AS is_overview,
+      EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
+    FROM user_tables
+  ),
+  sizes AS (
+    SELECT COALESCE(INT8(SUM(cartodb._CDB_total_relation_size(schema_name, table_name)))) table_size,
+      CASE
+        WHEN is_overview THEN 0
+	WHEN is_raster THEN 1
+	ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator
+      END AS multiplier FROM table_cat GROUP BY is_overview, is_raster
+  )
+  SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes;
+
+  IF total_size IS NOT NULL THEN
+    RETURN total_size;
+  ELSE
+    RETURN 0;
+  END IF;
+END;
 $$
-LANGUAGE 'sql' VOLATILE;
+LANGUAGE 'plpgsql' VOLATILE;
 
 
 -- Return the estimated size of user data. Used for quota checking.

scripts-available/CDB_StringToDate.sql

@@ -1,8 +1,9 @@
 -- Convert string to date
 --
+DROP FUNCTION IF EXISTS CDB_StringToDate(character varying);
 CREATE OR REPLACE FUNCTION CDB_StringToDate(input character varying)
-RETURNS date AS $$
-DECLARE output DATE;
+RETURNS TIMESTAMP AS $$
+DECLARE output TIMESTAMP;
 BEGIN
     BEGIN
         output := input::date;

scripts-available/CDB_TableMetadata.sql

@@ -5,9 +5,9 @@ CREATE TABLE IF NOT EXISTS
     updated_at timestamp with time zone not null default now()
   );
 
--- Anyone can see this, but updates are only possible trough
--- the security definer trigger
-GRANT SELECT ON public.CDB_TableMetadata TO public;
+-- No one can see this
+-- Updates are only possible trough the security definer trigger
+-- GRANT SELECT ON public.CDB_TableMetadata TO public;
 
 --
 -- Trigger logging updated_at in the CDB_TableMetadata
@@ -118,3 +118,21 @@ CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
 ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
  _CDB_TableMetadata_Updated();
 
+
+-- similar to TOUCH(1) in unix filesystems but for table in cdb_tablemetadata
+CREATE OR REPLACE FUNCTION public.CDB_TableMetadataTouch(tablename regclass)
+    RETURNS void AS
+    $$
+    BEGIN
+        WITH upsert AS (
+            UPDATE public.cdb_tablemetadata
+            SET updated_at = NOW()
+            WHERE tabname = tablename
+            RETURNING *
+        )
+        INSERT INTO public.cdb_tablemetadata (tabname, updated_at)
+            SELECT tablename, NOW()
+            WHERE NOT EXISTS (SELECT * FROM upsert);
+    END;
+    $$
+LANGUAGE 'plpgsql' VOLATILE STRICT;

scripts-available/CDB_UserTables.sql

@@ -5,35 +5,22 @@
 --
 -- Currently accepted permissions are: 'public', 'private' or 'all'
 --
+DROP FUNCTION IF EXISTS cdb_usertables(text);
 CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all')
-RETURNS SETOF information_schema.sql_identifier
+RETURNS SETOF name
 AS $$
-  WITH usertables AS ( 
-    -- TODO: query CDB_TableMetadata for this ?
-    -- See http://github.com/CartoDB/cartodb/issues/254#issuecomment-26044777
-    SELECT table_name as t
-    FROM information_schema.tables
-    WHERE
-         table_type='BASE TABLE'
-     AND table_schema='public'
-     AND table_name NOT IN (
-      'cdb_tablemetadata',
-      'spatial_ref_sys'
-     )
-  ), perms AS (
-    SELECT t, has_table_privilege('public', 'public'||'.'||t, 'SELECT') as p
-    FROM usertables
-  )
-  SELECT t FROM perms
-  WHERE (
-    p = CASE WHEN $1 = 'private' THEN false
-                 WHEN $1 = 'public' THEN true
-                 ELSE not p -- none
-            END
-    OR $1 = 'all'
-  )
-  AND has_table_privilege('public'||'.'||t, 'SELECT')
-   ;
+
+SELECT c.relname 
+FROM pg_class c 
+JOIN pg_namespace n ON n.oid = c.relnamespace
+WHERE c.relkind = 'r' 
+AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
+AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology')
+AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
+         WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')
+         WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT')
+         ELSE false END;
+
 $$ LANGUAGE 'sql';
 
 -- This is to migrate from pre-0.2.0 version

scripts-available/CDB_ZoomFromScale.sql

@@ -0,0 +1,31 @@
+CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
+BEGIN
+  CASE
+    WHEN scaleDenominator > 1000000000 THEN RETURN 0;
+    WHEN scaleDenominator <= 1000000000 AND scaleDenominator > 500000000 THEN RETURN 1;
+    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 2;
+    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 3;
+    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
+    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
+    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
+    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
+    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
+    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
+    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
+    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
+    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
+    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
+    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
+    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
+    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
+    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
+    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
+    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
+    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
+    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
+    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
+    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
+    WHEN scaleDenominator <= 100 THEN RETURN 23;
+  END CASE;
+END
+$$ LANGUAGE plpgsql IMMUTABLE;

scripts-enabled/220-CDB_Math.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Math.sql

scripts-enabled/230-CDB_ZoomFromScale.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_ZoomFromScale.sql

scripts-enabled/240-CDB_EqualIntervalBins.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_EqualIntervalBins.sql

scripts-enabled/CDB_DistType.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_DistType.sql

scripts-enabled/CDB_DistinctMeasure.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_DistinctMeasure.sql

sql/test_ddl_triggers.sql

@@ -8,6 +8,9 @@ SELECT cartodb.cdb_enable_ddl_hooks();
 
 create schema c;
 
+SELECT CDB_SetUserQuotaInBytes('c', 0);
+
+DROP USER IF EXISTS cartodb_postgresql_unpriv_user;
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -16,29 +19,21 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 ----------------------
 -- CREATE TABLE
 ----------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)) as age
 FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
+
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now() - updated_at)) as age
@@ -47,32 +42,22 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
 ----------------------------
 -- ALTER TABLE RENAME COLUMN
 ----------------------------
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
 FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -81,17 +66,11 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE DROP COLUMN
 ----------------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 drop column the_geom_webmercator;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -100,17 +79,11 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 -- ALTER TABLE ADD COLUMN
 ----------------------------
-
+SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 add column id2 int;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
-
+RESET SESSION AUTHORIZATION;
 select
  tabname::text,
  round(extract('secs' from now()  - updated_at)*10) as agecs
@@ -124,5 +97,6 @@ RESET SESSION AUTHORIZATION;
 drop schema c cascade;
 select count(*) from CDB_TableMetadata;
 
-DROP USER cartodb_postgresql_unpriv_user;
+DROP OWNED BY cartodb_postgresql_unpriv_user;
+DROP ROLE cartodb_postgresql_unpriv_user;
 DROP FUNCTION _CDB_UserQuotaInBytes();

sql/test_setup.sql

@@ -7,4 +7,4 @@ RETURNS void AS $$
 BEGIN
   RAISE NOTICE 'cdb_invalidate_varnish(%) called', table_name;
 END;
-$$ LANGUAGE 'plpgsql';
+$$ LANGUAGE 'plpgsql';

test/CDB_CartodbfyTableTest.sql

@@ -1,5 +1,5 @@
 SET client_min_messages TO error;
-\set VERBOSITY terse;
+\set VERBOSITY default
 
 CREATE OR REPLACE FUNCTION CDB_CartodbfyTableCheck(tabname regclass, label text)
 RETURNS text AS
@@ -12,6 +12,7 @@ DECLARE
   tmp INTEGER;
   ogc_geom geometry_columns; -- old the_geom record in geometry_columns
   ogc_merc geometry_columns; -- old the_geom_webmercator record in geometry_columns
+  tabtext TEXT;
 BEGIN
 
   -- Save current constraints on geometry columns, if any
@@ -30,24 +31,17 @@ BEGIN
     END IF;
   END LOOP;
 
+  tabtext := Format('%s.%s','public',tabname);
+  RAISE NOTICE 'CARTODBFYING % !!!!', tabtext;
   PERFORM CDB_CartodbfyTable('public', tabname);
+  tabname := tabtext::regclass;
 
   sql := 'INSERT INTO ' || tabname::text || '(the_geom) values ( CDB_LatLng(2,1) ) RETURNING cartodb_id';
   EXECUTE sql INTO STRICT id;
-  sql := 'SELECT created_at,updated_at,the_geom_webmercator FROM '
+  sql := 'SELECT the_geom_webmercator FROM '
     || tabname::text || ' WHERE cartodb_id = ' || id;
   EXECUTE sql INTO STRICT rec;
 
-  -- Check created_at and updated_at at creation time
-  lag = rec.created_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'created_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-  lag = rec.updated_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'updated_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-
   -- Check the_geom_webmercator trigger
   IF round(st_x(rec.the_geom_webmercator)) != 111319 THEN
     RAISE EXCEPTION 'the_geom_webmercator X is % (expecting 111319)', round(st_x(rec.the_geom_webmercator));
@@ -84,7 +78,7 @@ BEGIN
         rec.f_geometry_column, rec.srid, rec.expsrid;
     END IF;
     -- Check TYPE constraint didn't change
-    IF rec.type != rec.exptype THEN
+    IF (rec.type != 'GEOMETRY') AND (rec.type != 'POINT') THEN
       RAISE EXCEPTION 'type of % in geometry_columns is %, expected %',
         rec.f_geometry_column, rec.type, rec.exptype;
     END IF;
@@ -115,10 +109,10 @@ BEGIN
   -- Check null constraint on cartodb_id, created_at, updated_at
   SELECT count(*) FROM pg_attribute a, pg_class c WHERE c.oid = tabname::oid
     AND a.attrelid = c.oid AND NOT a.attisdropped AND a.attname in
-      ( 'cartodb_id', 'created_at', 'updated_at' )
+      ( 'cartodb_id' )
     AND NOT a.attnotnull INTO strict tmp;
   IF tmp > 0 THEN
-      RAISE EXCEPTION 'cartodb_id or created_at or updated_at are missing not-null constraint';
+      RAISE EXCEPTION 'cartodb_id is missing not-null constraint';
   END IF;
 
   -- Cleanup
@@ -170,29 +164,19 @@ SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
 SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
 DROP TABLE t;
 
--- table with existing updated_at and created_at fields ot type text
-CREATE TABLE t AS SELECT NOW()::text as created_at,
-                         NOW()::text as updated_at,
-                         NOW() as reftime;
-SELECT CDB_CartodbfyTableCheck('t', 'text timestamps');
-SELECT extract(secs from reftime-created_at),
-       extract(secs from reftime-updated_at) FROM t;
-CREATE VIEW v AS SELECT * FROM t;
-SELECT CDB_CartodbfyTableCheck('t', 'cartodbfied with view');
-DROP VIEW v;
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text
+-- CREATE TABLE t AS SELECT 10::text as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
+-- select cartodb_id/2 FROM t;
+-- DROP TABLE t;
 
--- table with existing cartodb_id field of type text
-CREATE TABLE t AS SELECT 10::text as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
-select cartodb_id/2 FROM t; 
-DROP TABLE t;
-
--- table with existing cartodb_id field of type text not casting
-CREATE TABLE t AS SELECT 'nan' as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
-select cartodb_id,_cartodb_id0 FROM t; 
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text not casting
+-- CREATE TABLE t AS SELECT 'nan' as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
+-- select cartodb_id,_cartodb_id0 FROM t;
+-- DROP TABLE t;
 
 -- table with existing cartodb_id field of type int4 not sequenced
 CREATE TABLE t AS SELECT 1::int4 as cartodb_id;
@@ -208,22 +192,6 @@ WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
 AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
 DROP TABLE t;
 
--- table with existing the_geom and created_at and containing null values
--- Really, a test for surviving an longstanding PostgreSQL bug:
--- http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
-CREATE TABLE t (
-    the_geom geometry(Geometry,4326),
-    created_at timestamptz,
-    updated_at timestamptz
-);
-COPY t (the_geom, created_at, updated_at) FROM stdin;
-0106000020E610000001000000010300000001000000050000009EB8244146435BC017B65E062AD343409EB8244146435BC0F51AF6E2708044400B99891683765AC0F51AF6E2708044400B99891683765AC017B65E062AD343409EB8244146435BC017B65E062AD34340	2012-06-06 21:59:08	2013-06-10 20:17:20
-0106000020E61000000100000001030000000100000005000000DA7763431A1A5CC0FBCEE869313C3A40DA7763431A1A5CC09C1B8F55BC494440F9F4A9C7993356C09C1B8F55BC494440F9F4A9C7993356C0FBCEE869313C3A40DA7763431A1A5CC0FBCEE869313C3A40	2012-06-06 21:59:08	2013-06-10 20:17:20
-\N	\N	\N
-\.
-SELECT CDB_CartodbfyTableCheck('t', 'null geom and timestamp values');
-DROP TABLE t;
-
 -- TODO: table with existing custom-triggered the_geom
 
 DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text);

test/CDB_CartodbfyTableTest_expect

@@ -2,8 +2,8 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
-CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(schema_name, reloid)"
-PL/pgSQL function cdb_cartodbfytable(text,regclass) line 6 at PERFORM
+CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(destschema, reloid)"
+PL/pgSQL function cdb_cartodbfytable(text,regclass) line 21 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE
@@ -28,21 +28,6 @@ trigger-protected the_geom cartodbfied fine
 extent|BOX(1 1,2 2)
 DROP TABLE
 SELECT 1
-text timestamps cartodbfied fine
-0|0
-CREATE VIEW
-cartodbfied with view cartodbfied fine
-DROP VIEW
-DROP TABLE
-SELECT 1
-text cartodb_id cartodbfied fine
-5
-DROP TABLE
-SELECT 1
-uncasting text cartodb_id cartodbfied fine
-1|nan
-DROP TABLE
-SELECT 1
 unsequenced cartodb_id cartodbfied fine
 1
 DROP TABLE
@@ -50,8 +35,5 @@ CREATE TABLE
 cartodb_id serial primary key cartodbfied fine
 t_pkey|cartodb_id
 DROP TABLE
-CREATE TABLE
-null geom and timestamp values cartodbfied fine
-DROP TABLE
 DROP FUNCTION
 DROP FUNCTION

test/CDB_DistTypeTest.sql

@@ -0,0 +1,4 @@
+WITH data AS (
+    SELECT pow(x,3)::numeric x FROM generate_series(-100,100) x
+    ) 
+SELECT CDB_DistType(array_agg(x)) FROM data

test/CDB_DistTypeTest_expect

@@ -0,0 +1 @@
+A

test/CDB_DistinctMeasureTest.sql

@@ -0,0 +1,20 @@
+-- a - j add up to 89%, k-m add up to 11%
+WITH a As (
+	SELECT (
+		repeat('a',12) || 
+		repeat('b',11) || 
+		repeat('c',11) || 
+		repeat('d',10) || 
+		repeat('e',10) || 
+		repeat('f',9) || 
+		repeat('g',8) || 
+		repeat('h',7) || 
+		repeat('i',6) || 
+		repeat('j',5) || 
+		repeat('k',4) || 
+		repeat('l',4) || 
+		repeat('m',3)
+		)::text AS x 
+	) 
+
+SELECT CDB_DistinctMeasure(string_to_array(x,null),0.90) from a

test/CDB_DistinctMeasureTest_expect

@@ -0,0 +1 @@
+0

test/CDB_EqualIntervalBinsTest.sql

@@ -0,0 +1,5 @@
+WITH data AS (
+    SELECT array_agg(x::numeric) s FROM generate_series(1,300) x 
+        WHERE x % 5 != 0 AND x % 7 != 0
+    ) 
+SELECT round(unnest(CDB_EqualIntervalBins(s, 7)),7) FROM data

test/CDB_EqualIntervalBinsTest_expect

@@ -0,0 +1,7 @@
+43.5714286
+86.1428571
+128.7142857
+171.2857143
+213.8571429
+256.4285714
+299.0000000

test/CDB_MathTest.sql

@@ -0,0 +1,4 @@
+
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1]) a;

test/CDB_MathTest_expect

@@ -0,0 +1,3 @@
+2
+1
+1

test/CDB_QuantileBinsTest_expect

@@ -1,7 +1,7 @@
-16
+13
 29
 43
 57
 71
-83
+86
 99

test/CDB_QueryTablesTest.sql

@@ -31,3 +31,9 @@ create table sc.test (a int);
 insert into sc.test values (1);
 WITH inp AS ( select 'select * from sc.test'::text as q )
  SELECT q, CDB_QueryTables(q) from inp;
+DROP TABLE sc.test;
+DROP SCHEMA sc;
+
+WITH inp AS ( select 'SELECT
+* FROM geometry_columns'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;

test/CDB_QueryTablesTest_expect

@@ -4,12 +4,20 @@ CREATE table "my'tab;le" as select 1|{}
 SELECT a.oid, b.oid FROM pg_class a, pg_class b|{pg_catalog.pg_class}
 SELECT 1 as col1; select 2 as col2|{}
 WARNING:  CDB_QueryTables cannot explain query: select 1 from nonexistant (42P01: relation "nonexistant" does not exist)
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 ERROR:  relation "nonexistant" does not exist
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 begin; select * from pg_class; commit;|{pg_catalog.pg_class}
 WARNING:  CDB_QueryTables cannot explain query: select * from test (42P01: relation "test" does not exist)
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 ERROR:  relation "test" does not exist
+CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 WITH a AS (select * from pg_class) select * from a|{pg_catalog.pg_class}
 CREATE SCHEMA
 CREATE TABLE
 INSERT 0 1
 select * from sc.test|{sc.test}
+DROP TABLE
+DROP SCHEMA
+SELECT
+* FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_namespace,pg_catalog.pg_type}

test/CDB_QuotaTest.sql

@@ -1,19 +1,27 @@
-set client_min_messages to ERROR;
-\set VERBOSITY terse
+set client_min_messages to error;
+\set VERBOSITY default
+
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
       EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
-INSERT INTO big VALUES (1); -- disallowed, quota exceeds before
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
 SELECT CDB_SetUserQuotaInBytes(0);
 SELECT CDB_CartodbfyTable('big');
-INSERT INTO big SELECT generate_series(1,1024);
-SELECT CDB_SetUserQuotaInBytes(8);
-INSERT INTO big VALUES (1);
+INSERT INTO big SELECT generate_series(2049,4096);
+INSERT INTO big SELECT generate_series(4097,6144);
+INSERT INTO big SELECT generate_series(6145,8192);
+-- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
+SELECT CDB_UserDataSize();
+SELECT cartodb._CDB_total_relation_size('public', 'big');
+SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
+-- END Test for #108
+SELECT CDB_SetUserQuotaInBytes(2);
+INSERT INTO big VALUES (8193);
 SELECT CDB_SetUserQuotaInBytes(0);
-INSERT INTO big VALUES (1);
+INSERT INTO big VALUES (8194);
 DROP TABLE big;
 set client_min_messages to NOTICE;
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -4,10 +4,15 @@ CREATE TRIGGER
 INSERT 0 1
 ERROR:  Quota exceeded by 3.9990234375KB
 0
-
-INSERT 0 1024
-8
-ERROR:  Quota exceeded by 103.9921875KB
+big
+INSERT 0 2048
+INSERT 0 2048
+INSERT 0 2048
+454656
+909312
+0
+2
+ERROR:  Quota exceeded by 443.998046875KB
 0
 INSERT 0 1
 DROP TABLE

test/CDB_UserTablesTest.sql

@@ -1,11 +1,19 @@
-create table pub(a int);
-create table prv(a int);
-GRANT SELECT ON TABLE pub TO public;
-REVOKE SELECT ON TABLE prv FROM public;
+CREATE ROLE publicuser LOGIN;
+CREATE TABLE pub(a int);
+CREATE TABLE prv(a int);
+GRANT SELECT ON TABLE pub TO publicuser;
+REVOKE SELECT ON TABLE prv FROM publicuser;
 SELECT CDB_UserTables() ORDER BY 1;
 SELECT 'all',CDB_UserTables('all') ORDER BY 2;
 SELECT 'public',CDB_UserTables('public') ORDER BY 2;
 SELECT 'private',CDB_UserTables('private') ORDER BY 2;
 SELECT '--unsupported--',CDB_UserTables('--unsupported--') ORDER BY 2;
-drop table pub;
-drop table prv;
+-- now tests with public user
+\c contrib_regression publicuser
+SELECT 'all_publicuser',CDB_UserTables('all') ORDER BY 2;
+SELECT 'public_publicuser',CDB_UserTables('public') ORDER BY 2;
+SELECT 'private_publicuser',CDB_UserTables('private') ORDER BY 2;
+\c contrib_regression postgres
+DROP TABLE pub;
+DROP TABLE prv;
+DROP ROLE publicuser;

test/CDB_UserTablesTest_expect

@@ -1,3 +1,4 @@
+CREATE ROLE
 CREATE TABLE
 CREATE TABLE
 GRANT
@@ -8,5 +9,10 @@ all|prv
 all|pub
 public|pub
 private|prv
+You are now connected to database "contrib_regression" as user "publicuser".
+all_publicuser|pub
+public_publicuser|pub
+You are now connected to database "contrib_regression" as user "postgres".
 DROP TABLE
 DROP TABLE
+DROP ROLE

test/CDB_XYZ_ExtentTest.sql

@@ -5,5 +5,5 @@ inp AS ( select z0.z, r1.r as x, r2.r as y FROM zoom z0, range r1, range r2 WHER
 ext AS ( select x,y,z,CDB_XYZ_Extent(x,y,z) as g from inp )
 select X::text || ',' || Y::text || ',' || Z::text as xyz,
        st_xmin(g), st_xmax(g), st_ymin(g), st_ymax(g)
-       from ext;
+       from ext order by xyz;
 

test/CDB_XYZ_ExtentTest_expect

@@ -1,13 +1,13 @@
 0,0,0|-20037508.5|20037508.5|-20037508.5|20037508.5
 0,0,1|-20037508.5|0|0|20037508.5
-0,1,1|-20037508.5|0|-20037508.5|0
-1,0,1|0|20037508.5|0|20037508.5
-1,1,1|0|20037508.5|-20037508.5|0
 0,0,2|-20037508.5|-10018754.25|10018754.25|20037508.5
+0,1,1|-20037508.5|0|-20037508.5|0
 0,1,2|-20037508.5|-10018754.25|0|10018754.25
 0,2,2|-20037508.5|-10018754.25|-10018754.25|0
 0,3,2|-20037508.5|-10018754.25|-20037508.5|-10018754.25
+1,0,1|0|20037508.5|0|20037508.5
 1,0,2|-10018754.25|0|10018754.25|20037508.5
+1,1,1|0|20037508.5|-20037508.5|0
 1,1,2|-10018754.25|0|0|10018754.25
 1,2,2|-10018754.25|0|-10018754.25|0
 1,3,2|-10018754.25|0|-20037508.5|-10018754.25

test/extension/run_at_cartodb_schema.sql

@@ -0,0 +1,4 @@
+SET SCHEMA 'cartodb';
+\i scripts-available/CDB_Quota.sql
+\i scripts-available/CDB_TableMetadata.sql
+SET SCHEMA 'public';

test/extension/test.sh

@@ -0,0 +1,344 @@
+#!/bin/sh
+
+#
+# Tests for the extension since version 0.5.0. They don't replace SQL based ones, for now need to run both
+#
+
+# It is expected that you run this script as a PostgreSQL superuser, for example:
+#
+#   PGUSER=postgres bash ./test.sh
+#
+
+DATABASE=test_extension
+CMD='echo psql'
+CMD=psql
+
+OK=0
+PARTIALOK=0
+
+function set_failed() {
+    OK=1
+    PARTIALOK=1
+}
+
+
+function clear_partial_result() {
+    PARTIALOK=0
+}
+
+
+function sql() {
+    local ROLE
+    local QUERY
+    if [[ $# -ge 2 ]]
+    then
+        ROLE="$1"
+        QUERY="$2"
+    else
+        QUERY="$1"
+    fi
+
+    if [ -n "${ROLE}" ]; then
+      log_debug "Executing query '${QUERY}' as ${ROLE}"
+      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+    else
+      log_debug "Executing query '${QUERY}'"
+      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+    fi
+    CODERESULT=$?
+
+    echo ${RESULT}
+    echo
+
+    if [[ ${CODERESULT} -ne 0 ]]
+    then
+        echo -n "FAILED TO EXECUTE QUERY: "
+        log_warning "${QUERY}"
+        if [[ "$3" != "fails" ]]
+        then
+            log_error "${QUERY}"
+            set_failed
+        fi
+    else
+        if [[ "$3" == "fails" ]]
+        then
+            log_error "QUERY: '${QUERY}' was expected to fail and it did not fail"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should" ]]
+    then
+        if [[ "${RESULT}" != "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' expected result '${4}' but got '${RESULT}'"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should-not" ]]
+    then
+        if [[ "${RESULT}" == "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' did not expect '${RESULT}'"
+            set_failed
+        fi
+    fi
+}
+
+
+function log_info()
+{
+    echo
+    echo
+    echo
+    _log "1;34m" "$1"
+}
+
+function log_error() {
+    _log "1;31m" "$1"
+}
+
+function log_debug() {
+    _log "1;32m" "> $1"
+}
+
+function log_warning() {
+    _log "0;33m" "$1"
+}
+
+function _log() {
+    echo -e "\033[$1$2\033[0m"
+}
+
+# '############################ HELPERS #############################'
+function create_role_and_schema() {
+    local ROLE=$1
+    sql "CREATE ROLE ${ROLE} LOGIN;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO ${ROLE};"
+    sql "CREATE SCHEMA ${ROLE} AUTHORIZATION ${ROLE};"
+    sql "SELECT cartodb.CDB_Organization_Create_Member('${ROLE}');"
+}
+
+
+function drop_role_and_schema() {
+    local ROLE=$1
+    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
+    sql "DROP ROLE \"${ROLE}\";"
+}
+
+
+function create_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} ( a int );"
+}
+
+
+function create_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local RASTER_COL="the_raster_webmercator"
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${OVERVIEW_TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "SELECT AddOverviewConstraints('${ROLE}','${OVERVIEW_TABLENAME}','${RASTER_COL}','${ROLE}','${TABLENAME}','${RASTER_COL}',2);"
+}
+
+function drop_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "drop_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+
+    sql ${ROLE} "DROP TABLE ${ROLE}.${OVERVIEW_TABLENAME};"
+    sql ${ROLE} "DROP TABLE ${ROLE}.${TABLENAME};"
+}
+
+
+function setup() {
+    ${CMD} -c "CREATE DATABASE ${DATABASE}"
+    sql "CREATE SCHEMA cartodb;"
+    sql "GRANT USAGE ON SCHEMA cartodb TO public;"
+    sql "CREATE EXTENSION postgis;"
+
+    log_info "########################### BOOTSTRAP ###########################"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
+    # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
+    ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
+
+
+    log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_testmember_1
+    create_role_and_schema cdb_testmember_2
+
+    create_table cdb_testmember_1 foo
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4), (5), (6);'
+    sql cdb_testmember_1 'SELECT * FROM cdb_testmember_1.foo;'
+
+    create_table cdb_testmember_2 bar
+    sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3);'
+    sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+}
+
+function tear_down() {
+    log_info "########################### USER TEAR DOWN ###########################"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
+
+    sql "DROP SCHEMA cartodb CASCADE"
+
+    log_info "########################### TEAR DOWN ###########################"
+    sql 'DROP SCHEMA cdb_testmember_1;'
+    sql 'DROP SCHEMA cdb_testmember_2;'
+
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
+
+    sql 'DROP ROLE cdb_testmember_1;'
+    sql 'DROP ROLE cdb_testmember_2;'
+
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
+
+function run_tests() {
+    local FAILED_TESTS=()
+
+    local TESTS
+    if [[ $# -ge 1 ]]
+    then
+        TESTS="$@"
+    else
+        TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
+    fi
+    for t in ${TESTS}
+    do
+        echo "####################################################################"
+        echo "#"
+        echo "# Running: ${t}"
+        echo "#"
+        echo "####################################################################"
+        clear_partial_result
+        setup
+        eval ${t}
+        if [[ ${PARTIALOK} -ne 0 ]]
+        then
+            FAILED_TESTS+=(${t})
+        fi
+        tear_down
+    done
+    if [[ ${OK} -ne 0 ]]
+    then
+        echo
+        log_error "The following tests are failing:"
+        printf -- '\t%s\n' "${FAILED_TESTS[@]}"
+    fi
+}
+
+
+#################################################### TESTS GO HERE ####################################################
+
+
+# Tests quota checking taking into account both geom and raster tables
+function test_quota_for_each_user() {
+    # Normal tables add 4096 bytes
+    # Raster tables no longer add anything so also count as 4096
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+
+    create_raster_table cdb_testmember_1 raster_1
+    create_raster_table cdb_testmember_2 raster_2
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 20480
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    create_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 36864
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    drop_raster_table cdb_testmember_1 raster_1
+    drop_raster_table cdb_testmember_2 raster_2
+    drop_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+}
+
+function test_cdb_tablemetadatatouch() {
+    sql "CREATE TABLE touch_example (a int)"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should ''
+    sql "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Another call doesn't fail
+    sql "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Works with qualified tables
+    sql "SELECT CDB_TableMetadataTouch('public.touch_example');"
+    sql "SELECT CDB_TableMetadataTouch('public.\"touch_example\"');"
+    sql "SELECT CDB_TableMetadataTouch('\"public\".touch_example');"
+    sql "SELECT CDB_TableMetadataTouch('\"public\".\"touch_example\"');"
+
+    # Works with OID
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+    TABLE_OID=`${CMD} -U postgres ${DATABASE} -c "SELECT attrelid FROM pg_attribute WHERE attrelid = 'touch_example'::regclass limit 1;" -A -t`
+
+    # quoted OID works
+    sql "SELECT CDB_TableMetadataTouch('${TABLE_OID}');"
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+
+    # non quoted OID works
+    sql "SELECT CDB_TableMetadataTouch(${TABLE_OID});"
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+
+    #### test tear down
+    sql 'DROP TABLE touch_example;'
+}
+
+function test_cdb_tablemetadatatouch_fails_for_unexistent_table() {
+    sql postgres "SELECT CDB_TableMetadataTouch('unexistent_example');" fails
+}
+
+function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
+    sql "CREATE TABLE touch_example (a int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');" fails
+
+    sql postgres "GRANT ALL ON CDB_TableMetadata TO cdb_testmember_1;"
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
+}
+
+#################################################### TESTS END HERE ####################################################
+
+run_tests $@
+
+exit ${OK}

test/organization/test.sh

@@ -142,6 +142,8 @@ function setup() {
     log_info "############################# SETUP #############################"
     create_role_and_schema cdb_testmember_1
     create_role_and_schema cdb_testmember_2
+    sql "CREATE ROLE publicuser LOGIN;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO publicuser;"
 
     create_table cdb_testmember_1 foo
     sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4), (5);'
@@ -168,9 +170,11 @@ function tear_down() {
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM publicuser;"
 
     sql 'DROP ROLE cdb_testmember_1;'
     sql 'DROP ROLE cdb_testmember_2;'
+    sql 'DROP ROLE publicuser;'
 
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
@@ -346,6 +350,50 @@ function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
 }
 
+function test_cdb_usertables_should_work_with_orgusers() {
+
+    # This test validates the changes proposed in https://github.com/CartoDB/cartodb/pull/5021
+
+    # create tables
+    sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)"
+    sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);"
+    sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser"
+    
+    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+
+
+    # this is what we need to make public tables available in CDB_UserTables
+    sql postgres "grant publicuser to cdb_testmember_1;"
+    sql postgres "grant publicuser to cdb_testmember_2;"
+
+
+    # this is required to enable select from other schema
+    sql postgres "GRANT USAGE ON SCHEMA cdb_testmember_1 TO publicuser";
+
+
+    # test CDB_UserTables with publicuser
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_UserTables.sql
+
+    sql publicuser "SELECT count(*) FROM CDB_UserTables('all')" should 1
+    sql publicuser "SELECT count(*) FROM CDB_UserTables('public')" should 1
+    sql publicuser "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    sql publicuser "SELECT * FROM CDB_UserTables('all')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('private')" should ""
+    # the following tests are for https://github.com/CartoDB/cartodb-postgresql/issues/98
+    # cdb_testmember_2 is already owner of `bar` table
+    sql cdb_testmember_2 "select string_agg(t,',') from (select cdb_usertables('all') t order by t) as s" should "bar,test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('private')" should "bar"
+
+    # test cdb_testmember_2 can select from cdb_testmember_1's public table
+    sql cdb_testmember_2 "SELECT * FROM cdb_testmember_1.test_perms_pub" should 1
+
+    sql cdb_testmember_1 "DROP TABLE test_perms_pub"
+    sql cdb_testmember_1 "DROP TABLE test_perms_priv"
+}
+
+
 #################################################### TESTS END HERE ####################################################