diff --git a/NEWS.md b/NEWS.md index 74bbe860..fb2d8ea2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,16 +1,44 @@ # Changelog -## 6.1.1 +## 6.2.0 Released 2018-mm-dd New features: - CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5 - +- Upgrades Windshaft to [4.8.1](https://github.com/CartoDB/Windshaft/blob/4.8.1/NEWS.md#version-481) which includes: + - Update internal deps. + - A fix in mapnik-vector-tile to avoid grouping together properties with the same value but a different type. + - Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible). + - MVT: Disable simplify_distance to avoid multiple simplifications. + - Fix a bug with zero length lines not being rendered when using the marker symbolizer. +- Upgrades Camshaft to [0.61.10](https://github.com/CartoDB/camshaft/releases/tag/0.61.10): + - Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses. +- Update other deps: + - body-parser: 1.18.3 + - cartodb-psql: 0.11.0 + - dot: 1.1.2 + - express: 4.16.3 + - lru-cache: 4.1.3 + - node-statsd: 0.1.1, + - queue-async: 1.1.0 + - request: 2.87.0 + - semver: 5.5.0 + - step: 1.0.0 + - yargs: 11.1.0 +- Update devel deps: + - istanbul: 0.4.5 + - jshint: 2.9.5 + - mocha: 3.5.3 + - moment: 2.22.1 + - nock: 9.2.6 + - strftime: 0.10.0 +- Optional instantiation metadata stats (https://github.com/CartoDB/Windshaft-cartodb/pull/952) Bug Fixes: - Validates tile coordinates (z/x/y) from request params to be a valid integer value. -- Upgrades Windshaft to 4.7.1, which includes a fix in mapnik-vector-tile to avoid grouping together properties with the same value but different type. - +- Static maps fails for unsupported formats +- Handling errors extracting the column type on dataviews +- Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens ## 6.1.0 Released 2018-04-16 diff --git a/app.js b/app.js index 1dd465f0..f35d580e 100755 --- a/app.js +++ b/app.js @@ -100,8 +100,6 @@ if ( global.environment.log_filename ) { global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); -global.environment.api_hostname = require('os').hostname().split('.')[0]; - // Include cartodb_windshaft only _after_ the "global" variable is set // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 var cartodbWindshaft = require('./lib/cartodb/server'); diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 842f55d2..d6345b82 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -15,16 +15,49 @@ var config = { // Base URLs for the APIs // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // @@ -67,32 +100,18 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS - type: "postgis", user: "publicuser", password: "public", host: '127.0.0.1', port: 5432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - /* experimental - geometry_field: "the_geom", - extent: "-180,-90,180,90", - srid: 4326, - */ - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -111,15 +130,7 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + usePostGIS: false }, mapnik: { // The size of the pool of internal mapnik backend @@ -173,6 +184,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit @@ -186,25 +220,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, @@ -231,16 +246,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 771ed2b5..d601dbc5 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -15,16 +15,49 @@ var config = { // Base URLs for the APIs // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // @@ -67,26 +100,18 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - max_size: 500 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -167,6 +192,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit @@ -180,26 +228,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, @@ -226,16 +254,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index dd2d892c..76260b8f 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -15,16 +15,49 @@ var config = { // Base URLs for the APIs // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/maps/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "/api/v1/maps" is the the new API, - // "/tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // @@ -67,26 +100,18 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -105,15 +130,7 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + usePostGIS: false }, mapnik: { // The size of the pool of internal mapnik backend @@ -167,6 +184,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit @@ -180,26 +220,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, @@ -226,16 +246,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 4d2c2d27..0dd475fe 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -16,15 +16,48 @@ var config = { // Base URLs for the APIs // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // @@ -72,20 +105,14 @@ var config = { password: "public", host: '127.0.0.1', port: 5432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: '' ,mapnik_tile_format: 'png8:m=h' @@ -166,6 +193,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit @@ -179,26 +229,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, @@ -222,16 +252,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js new file mode 100644 index 00000000..b0f42f03 --- /dev/null +++ b/lib/cartodb/api/api-router.js @@ -0,0 +1,319 @@ +const { Router: router } = require('express'); + +const RedisPool = require('redis-mpool'); +const cartodbRedis = require('cartodb-redis'); + +const windshaft = require('windshaft'); + +const PgConnection = require('../backends/pg_connection'); +const AnalysisBackend = require('../backends/analysis'); +const AnalysisStatusBackend = require('../backends/analysis-status'); +const DataviewBackend = require('../backends/dataview'); +const TemplateMaps = require('../backends/template_maps.js'); +const PgQueryRunner = require('../backends/pg_query_runner'); +const StatsBackend = require('../backends/stats'); +const AuthBackend = require('../backends/auth'); + +const UserLimitsBackend = require('../backends/user-limits'); +const OverviewsMetadataBackend = require('../backends/overviews-metadata'); +const FilterStatsApi = require('../backends/filter-stats'); +const TablesExtentBackend = require('../backends/tables-extent'); + +const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables'); +const SurrogateKeysCache = require('../cache/surrogate_keys_cache'); +const VarnishHttpCacheBackend = require('../cache/backend/varnish_http'); +const FastlyCacheBackend = require('../cache/backend/fastly'); +const NamedMapProviderCache = require('../cache/named_map_provider_cache'); +const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); + +const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); +const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter'); +const MapConfigBufferSizeAdapter = require('../models/mapconfig/adapter/mapconfig-buffer-size-adapter'); +const AnalysisMapConfigAdapter = require('../models/mapconfig/adapter/analysis-mapconfig-adapter'); +const MapConfigOverviewsAdapter = require('../models/mapconfig/adapter/mapconfig-overviews-adapter'); +const TurboCartoAdapter = require('../models/mapconfig/adapter/turbo-carto-adapter'); +const DataviewsWidgetsAdapter = require('../models/mapconfig/adapter/dataviews-widgets-adapter'); +const AggregationMapConfigAdapter = require('../models/mapconfig/adapter/aggregation-mapconfig-adapter'); +const MapConfigAdapter = require('../models/mapconfig/adapter'); + +const ResourceLocator = require('../models/resource-locator'); +const LayergroupMetadata = require('../utils/layergroup-metadata'); +const RendererStatsReporter = require('../stats/reporter/renderer'); + +const initializeStatusCode = require('./middlewares/initialize-status-code'); +const logger = require('./middlewares/logger'); +const bodyParser = require('body-parser'); +const servedByHostHeader = require('./middlewares/served-by-host-header'); +const stats = require('./middlewares/stats'); +const lzmaMiddleware = require('./middlewares/lzma'); +const cors = require('./middlewares/cors'); +const user = require('./middlewares/user'); +const sendResponse = require('./middlewares/send-response'); +const syntaxError = require('./middlewares/syntax-error'); +const errorMiddleware = require('./middlewares/error-middleware'); + +const MapRouter = require('./map/map-router'); +const TemplateRouter = require('./template/template-router'); + +module.exports = class ApiRouter { + constructor ({ serverOptions, environmentOptions }) { + this.serverOptions = serverOptions; + + const redisOptions = Object.assign({ + name: 'windshaft-server', + unwatchOnRelease: false, + noReadyCheck: true + }, environmentOptions.redis); + + const redisPool = new RedisPool(redisOptions); + + redisPool.on('status', function (status) { + var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.'; + global.statsClient.gauge(keyPrefix + 'count', status.count); + global.statsClient.gauge(keyPrefix + 'unused', status.unused); + global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); + }); + + const metadataBackend = cartodbRedis({ pool: redisPool }); + const pgConnection = new PgConnection(metadataBackend); + + const mapStore = new windshaft.storage.MapStore({ + pool: redisPool, + expire_time: serverOptions.grainstore.default_layergroup_ttl + }); + + const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); + + const rendererCacheOpts = Object.assign({ + ttl: 60000, // 60 seconds TTL by default + statsInterval: 60000 // reports stats every milliseconds defined here + }, serverOptions.renderCache || {}); + + const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); + const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); + rendererStatsReporter.start(); + + const tileBackend = new windshaft.backend.Tile(rendererCache); + const attributesBackend = new windshaft.backend.Attributes(); + const previewBackend = new windshaft.backend.Preview(rendererCache); + const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); + const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + + const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions); + const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends); + const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache }); + + const analysisStatusBackend = new AnalysisStatusBackend(); + const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); + const dataviewBackend = new DataviewBackend(analysisBackend); + const statsBackend = new StatsBackend(); + + const userLimitsBackend = new UserLimitsBackend(metadataBackend, { + limits: { + cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, + render: serverOptions.renderer.mapnik.limits.render || 0, + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled + } + }); + const authBackend = new AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps); + + const layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); + + if (process.env.NODE_ENV === 'test') { + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + } + + const pgQueryRunner = new PgQueryRunner(pgConnection); + const overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); + + const filterStatsBackend = new FilterStatsApi(pgQueryRunner); + const tablesExtentBackend = new TablesExtentBackend(pgQueryRunner); + + const mapConfigAdapter = new MapConfigAdapter( + new MapConfigNamedLayersAdapter(templateMaps, pgConnection), + new MapConfigBufferSizeAdapter(), + new SqlWrapMapConfigAdapter(), + new DataviewsWidgetsAdapter(), + new AnalysisMapConfigAdapter(analysisBackend), + new AggregationMapConfigAdapter(pgConnection), + new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend), + new TurboCartoAdapter() + ); + + const resourceLocator = new ResourceLocator(global.environment); + const layergroupMetadata = new LayergroupMetadata(resourceLocator); + + const namedMapProviderCache = new NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + layergroupAffectedTablesCache + ); + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); + }); + + const collaborators = { + analysisStatusBackend, + attributesBackend, + dataviewBackend, + previewBackend, + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache, + templateMaps, + mapBackend, + metadataBackend, + mapConfigAdapter, + statsBackend, + layergroupMetadata, + namedMapProviderCache, + tablesExtentBackend + }; + + this.mapRouter = new MapRouter({ collaborators }); + this.templateRouter = new TemplateRouter({ collaborators }); + } + + register (app) { + // FIXME: we need a better way to reset cache while running tests + if (process.env.NODE_ENV === 'test') { + app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; + } + + Object.keys(this.serverOptions.routes).forEach(apiVersion => { + const routes = this.serverOptions.routes[apiVersion]; + + const apiRouter = router({ mergeParams: true }); + + apiRouter.use(logger(this.serverOptions)); + apiRouter.use(initializeStatusCode()); + apiRouter.use(bodyParser.json()); + apiRouter.use(servedByHostHeader()); + apiRouter.use(stats({ + enabled: this.serverOptions.useProfiler, + statsClient: global.statsClient + })); + apiRouter.use(lzmaMiddleware()); + apiRouter.use(cors()); + apiRouter.use(user()); + + this.templateRouter.register(apiRouter, routes.template.paths); + this.mapRouter.register(apiRouter, routes.map.paths); + + apiRouter.use(sendResponse()); + apiRouter.use(syntaxError()); + apiRouter.use(errorMiddleware()); + + const apiPaths = routes.paths; + + apiPaths.forEach(path => app.use(path, apiRouter)); + }); + } +}; + + +function createTemplateMaps ({ redisPool, surrogateKeysCache }) { + const templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates + }); + + function invalidateNamedMap (owner, templateName) { + var startTime = Date.now(); + surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { + var logMessage = JSON.stringify({ + username: owner, + type: 'named_map_invalidation', + elapsed: Date.now() - startTime, + error: !!err ? JSON.stringify(err.message) : undefined + }); + if (err) { + global.logger.warn(logMessage); + } else { + global.logger.info(logMessage); + } + }); + } + + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, invalidateNamedMap); + }); + + return templateMaps; +} + +function createSurrogateKeysCacheBackends(serverOptions) { + var cacheBackends = []; + + if (serverOptions.varnish_purge_enabled) { + cacheBackends.push( + new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) + ); + } + + if (serverOptions.fastly && + !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { + cacheBackends.push( + new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) + ); + } + + return cacheBackends; +} + +const timeoutErrorTilePath = __dirname + '/../../../assets/render-timeout-fallback.png'; +const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); + +function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) { + var onTileErrorStrategy; + if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) { + onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { + + function isRenderTimeoutError (err) { + return err.message === 'Render timed out'; + } + + function isDatasourceTimeoutError (err) { + return err.message && err.message.match(/canceling statement due to statement timeout/i); + } + + function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); + } + + function isRasterFormat (format) { + return format === 'png' || format === 'jpg'; + } + + if (isTimeoutError(err) && isRasterFormat(format)) { + return callback(null, timeoutErrorTile, { + 'Content-Type': 'image/png', + }, {}); + } else { + return callback(err, tile, headers, stats); + } + }; + } + + const rendererFactory = new windshaft.renderer.Factory({ + onTileErrorStrategy: onTileErrorStrategy, + mapnik: { + redisPool: redisPool, + grainstore: serverOptions.grainstore, + mapnik: serverOptions.renderer.mapnik + }, + http: serverOptions.renderer.http, + mvt: serverOptions.renderer.mvt, + torque: serverOptions.renderer.torque + }); + + return rendererFactory; +} diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/api/map/analyses-catalog-controller.js similarity index 67% rename from lib/cartodb/controllers/analyses.js rename to lib/cartodb/api/map/analyses-catalog-controller.js index 2d89a091..2fae5ac2 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -1,44 +1,39 @@ const PSQL = require('cartodb-psql'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const credentials = require('../middleware/credentials'); -const authorize = require('../middleware/authorize'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const rateLimit = require('../middleware/rate-limit'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const authorize = require('../middlewares/authorize'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const cacheControlHeader = require('../middleware/cache-control-header'); -const sendResponse = require('../middleware/send-response'); -const dbParamsFromResLocals = require('../utils/database-params'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const dbParamsFromResLocals = require('../../utils/database-params'); -function AnalysesController(pgConnection, authApi, userLimitsApi) { - this.pgConnection = pgConnection; - this.authApi = authApi; - this.userLimitsApi = userLimitsApi; -} +module.exports = class AnalysesController { + constructor (pgConnection, authBackend, userLimitsBackend) { + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } -module.exports = AnalysesController; + register (mapRouter) { + mapRouter.get('/analyses/catalog', this.middlewares()); + } -AnalysesController.prototype.register = function (app) { - const { base_url_mapconfig: mapconfigBasePath } = app; - - app.get( - `${mapconfigBasePath}/analyses/catalog`, - cors(), - user(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), - cleanUpQueryParams(), - createPGClient(), - getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), - getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), - prepareResponse(), - cacheControlHeader({ ttl: 10, revalidate: true }), - sendResponse(), - unauthorizedError() - ); + middlewares () { + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), + cleanUpQueryParams(), + createPGClient(), + getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), + getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), + prepareResponse(), + cacheControlHeader({ ttl: 10, revalidate: true }), + unauthorizedError() + ]; + } }; function createPGClient () { @@ -107,6 +102,7 @@ function prepareResponse () { return -1; }); + res.statusCode = 200; res.body = { catalog: analysisCatalog }; next(); diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js new file mode 100644 index 00000000..41acf819 --- /dev/null +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -0,0 +1,59 @@ +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const dbParamsFromResLocals = require('../../utils/database-params'); + +module.exports = class AnalysisLayergroupController { + constructor (analysisStatusBackend, pgConnection, userLimitsBackend, authBackend) { + this.analysisStatusBackend = analysisStatusBackend; + this.pgConnection = pgConnection; + this.userLimitsBackend = userLimitsBackend; + this.authBackend = authBackend; + } + + register (mapRouter) { + mapRouter.get('/:token/analysis/node/:nodeId', this.middlewares()); + } + + middlewares () { + return [ + layergroupToken(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), + cleanUpQueryParams(), + analysisNodeStatus(this.analysisStatusBackend) + ]; + } +}; + +function analysisNodeStatus (analysisStatusBackend) { + return function analysisNodeStatusMiddleware(req, res, next) { + const { nodeId } = req.params; + const dbParams = dbParamsFromResLocals(res.locals); + + analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET NODE STATUS'; + return next(err); + } + + res.set({ + 'Cache-Control': 'public,max-age=5', + 'Last-Modified': new Date().toUTCString() + }); + + res.statusCode = 200; + res.body = nodeStatus; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/anonymous-map-controller.js b/lib/cartodb/api/map/anonymous-map-controller.js new file mode 100644 index 00000000..6bbacd37 --- /dev/null +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -0,0 +1,212 @@ +const windshaft = require('windshaft'); +const MapConfig = windshaft.model.MapConfig; +const Datasource = windshaft.model.Datasource; +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const initProfiler = require('../middlewares/init-profiler'); +const checkJsonContentType = require('../middlewares/check-json-content-type'); +const incrementMapViewCount = require('../middlewares/increment-map-view-count'); +const augmentLayergroupData = require('../middlewares/augment-layergroup-data'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup'); +const layerStats = require('../middlewares/layer-stats'); +const layergroupIdHeader = require('../middlewares/layergroup-id-header'); +const layergroupMetadata = require('../middlewares/layergroup-metadata'); +const mapError = require('../middlewares/map-error'); +const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +module.exports = class AnonymousMapController { + /** + * @param {AuthBackend} authBackend + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @param {MapBackend} mapBackend + * @param metadataBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsBackend} userLimitsBackend + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigAdapter} mapConfigAdapter + * @param {StatsBackend} statsBackend + * @constructor + */ + constructor ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTables = layergroupAffectedTables; + this.mapConfigAdapter = mapConfigAdapter; + this.statsBackend = statsBackend; + this.authBackend = authBackend; + this.layergroupMetadata = layergroupMetadata; + } + + register (mapRouter) { + mapRouter.options('/'); + mapRouter.get('/', this.middlewares()); + mapRouter.post('/', this.middlewares()); + } + + middlewares () { + const isTemplateInstantiation = false; + const useTemplateHash = false; + const includeQuery = true; + const label = 'ANONYMOUS LAYERGROUP'; + const addContext = true; + + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), + cleanUpQueryParams(['aggregation']), + initProfiler(isTemplateInstantiation), + checkJsonContentType(), + checkCreateLayergroup(), + prepareAdapterMapConfig(this.mapConfigAdapter), + createLayergroup ( + this.mapBackend, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTables + ), + incrementMapViewCount(this.metadataBackend), + augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), + lastUpdatedTimeLayergroup(), + layerStats(this.pgConnection, this.statsBackend), + layergroupIdHeader(this.templateMaps, useTemplateHash), + layergroupMetadata(this.layergroupMetadata, includeQuery), + mapError({ label, addContext }) + ]; + } +}; + +function checkCreateLayergroup () { + return function checkCreateLayergroupMiddleware (req, res, next) { + if (req.method === 'GET') { + const { config } = req.query; + + if (!config) { + return next(new Error('layergroup GET needs a "config" parameter')); + } + + try { + req.body = JSON.parse(config); + } catch (err) { + return next(err); + } + } + + req.profiler.done('checkCreateLayergroup'); + return next(); + }; +} + +function prepareAdapterMapConfig (mapConfigAdapter) { + return function prepareAdapterMapConfigMiddleware(req, res, next) { + const requestMapConfig = req.body; + + const { user, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + const context = { + analysisConfiguration: { + user, + db: { + host: dbhost, + port: dbport, + dbname: dbname, + user: dbuser, + pass: dbpassword + }, + batch: { + username: user, + apiKey: api_key + } + } + }; + + mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => { + req.profiler.done('anonymous.getMapConfig'); + if (err) { + return next(err); + } + + req.body = requestMapConfig; + res.locals.context = context; + + next(); + }); + }; +} + +function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) { + return function createLayergroupMiddleware (req, res, next) { + const requestMapConfig = req.body; + + const { context } = res.locals; + const { user, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = { + cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport + }; + + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapConfig = new MapConfig(requestMapConfig, datasource); + + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsBackend, + pgConnection, + affectedTablesCache, + params + ); + + res.locals.mapConfig = mapConfig; + res.locals.analysesResults = context.analysesResults; + + const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport }; + + mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => { + req.profiler.done('createLayergroup'); + if (err) { + return next(err); + } + + res.statusCode = 200; + res.body = layergroup; + res.locals.mapConfigProvider = mapConfigProvider; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/api/map/attributes-layergroup-controller.js similarity index 54% rename from lib/cartodb/controllers/layergroup/attributes.js rename to lib/cartodb/api/map/attributes-layergroup-controller.js index 07ae5da1..2723ebd0 100644 --- a/lib/cartodb/controllers/layergroup/attributes.js +++ b/lib/cartodb/api/map/attributes-layergroup-controller.js @@ -1,54 +1,50 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); -module.exports = class AttribitesController { +module.exports = class AttributesLayergroupController { constructor ( attributesBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ) { this.attributesBackend = attributesBackend; this.pgConnection = pgConnection; this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; + this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; } - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; + register (mapRouter) { + mapRouter.get('/:token/:layer/attributes/:fid', this.middlewares()); + } - app.get( - `${mapConfigBasePath}/:token/:layer/attributes/:fid`, - cors(), - user(), + middlewares () { + return [ layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), cleanUpQueryParams(), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), @@ -56,9 +52,8 @@ module.exports = class AttribitesController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); + lastModifiedHeader() + ]; } }; @@ -85,6 +80,7 @@ function getFeatureAttributes (attributesBackend) { return next(err); } + res.statusCode = 200; res.body = tile; next(); diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js new file mode 100644 index 00000000..5fd666ea --- /dev/null +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -0,0 +1,142 @@ +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); + +const ALLOWED_DATAVIEW_QUERY_PARAMS = [ + 'filters', // json + 'own_filter', // 0, 1 + 'no_filters', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'offset', // number + 'q', // widgets search + 'categories', // number +]; + +module.exports = class DataviewLayergroupController { + constructor ( + dataviewBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ) { + this.dataviewBackend = dataviewBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authBackend = authBackend; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (mapRouter) { + // Undocumented/non-supported API endpoint methods. + // Use at your own peril. + + mapRouter.get('/:token/dataview/:dataviewName', this.middlewares({ + action: 'get', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW + })); + + mapRouter.get('/:token/:layer/widget/:dataviewName', this.middlewares({ + action: 'get', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW + })); + + mapRouter.get('/:token/dataview/:dataviewName/search', this.middlewares({ + action: 'search', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH + })); + + mapRouter.get('/:token/:layer/widget/:dataviewName/search', this.middlewares({ + action: 'search', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH + })); + } + + middlewares ({ action, rateLimitGroup }) { + return [ + layergroupToken(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, rateLimitGroup), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + action === 'search' ? dataviewSearch(this.dataviewBackend) : getDataview(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } +}; + +function getDataview (dataviewBackend) { + return function getDataviewMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW'; + return next(err); + } + + res.statusCode = 200; + res.body = dataview; + + next(); + }); + }; +} + +function dataviewSearch (dataviewBackend) { + return function dataviewSearchMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW SEARCH'; + return next(err); + } + + res.statusCode = 200; + res.body = searchResult; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js new file mode 100644 index 00000000..46e2be5e --- /dev/null +++ b/lib/cartodb/api/map/map-router.js @@ -0,0 +1,129 @@ +const { Router: router } = require('express'); + +const AnalysisLayergroupController = require('./analysis-layergroup-controller'); +const AttributesLayergroupController = require('./attributes-layergroup-controller'); +const DataviewLayergroupController = require('./dataview-layergroup-controller'); +const PreviewLayergroupController = require('./preview-layergroup-controller'); +const TileLayergroupController = require('./tile-layergroup-controller'); +const AnonymousMapController = require('./anonymous-map-controller'); +const PreviewTemplateController = require('./preview-template-controller'); +const AnalysesCatalogController = require('./analyses-catalog-controller'); + +module.exports = class MapRouter { + constructor ({ collaborators }) { + const { + analysisStatusBackend, + attributesBackend, + dataviewBackend, + previewBackend, + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache, + templateMaps, + mapBackend, + metadataBackend, + mapConfigAdapter, + statsBackend, + layergroupMetadata, + namedMapProviderCache, + tablesExtentBackend + } = collaborators; + + this.analysisLayergroupController = new AnalysisLayergroupController( + analysisStatusBackend, + pgConnection, + userLimitsBackend, + authBackend + ); + + this.attributesLayergroupController = new AttributesLayergroupController( + attributesBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.dataviewLayergroupController = new DataviewLayergroupController( + dataviewBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.previewLayergroupController = new PreviewLayergroupController( + previewBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.tileLayergroupController = new TileLayergroupController( + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.anonymousMapController = new AnonymousMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ); + + this.previewTemplateController = new PreviewTemplateController( + namedMapProviderCache, + previewBackend, + surrogateKeysCache, + tablesExtentBackend, + metadataBackend, + pgConnection, + authBackend, + userLimitsBackend + ); + + this.analysesController = new AnalysesCatalogController( + pgConnection, + authBackend, + userLimitsBackend + ); + } + + register (apiRouter, mapPaths) { + const mapRouter = router({ mergeParams: true }); + + this.analysisLayergroupController.register(mapRouter); + this.attributesLayergroupController.register(mapRouter); + this.dataviewLayergroupController.register(mapRouter); + this.previewLayergroupController.register(mapRouter); + this.tileLayergroupController.register(mapRouter); + this.anonymousMapController.register(mapRouter); + this.previewTemplateController.register(mapRouter); + this.analysesController.register(mapRouter); + + mapPaths.forEach(path => apiRouter.use(path, mapRouter)); + } +}; diff --git a/lib/cartodb/controllers/layergroup/static.js b/lib/cartodb/api/map/preview-layergroup-controller.js similarity index 53% rename from lib/cartodb/controllers/layergroup/static.js rename to lib/cartodb/api/map/preview-layergroup-controller.js index 1aa9cdeb..975d3f56 100644 --- a/lib/cartodb/controllers/layergroup/static.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -1,94 +1,85 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); -const layergroupToken = require('../../middleware/layergroup-token'); -const coordinates = require('../../middleware/coordinates'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); +const layergroupToken = require('../middlewares/layergroup-token'); +const coordinates = require('../middlewares/coordinates'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const noop = require('../middlewares/noop'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const checkStaticImageFormat = require('../middlewares/check-static-image-format'); -module.exports = class StaticController { +module.exports = class PreviewLayergroupController { constructor ( previewBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ) { this.previewBackend = previewBackend; this.pgConnection = pgConnection; this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; + this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; } - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; + register (mapRouter) { + mapRouter.get('/static/center/:token/:z/:lat/:lng/:width/:height.:format', this.middlewares({ + validateZoom: true, + previewType: 'centered' + })); + mapRouter.get('/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', this.middlewares({ + validateZoom: false, + previewType: 'bbox' + })); + } + + middlewares ({ validateZoom, previewType }) { const forcedFormat = 'png'; - app.get( - `${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, - cors(), - user(), - layergroupToken(), - coordinates({ z: true, x: false, y: false }), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), - cleanUpQueryParams(['layer']), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache, - forcedFormat - ), - getPreviewImageByCenter(this.previewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); + let getPreviewImage; - app.get( - `${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, - cors(), - user(), + if (previewType === 'centered') { + getPreviewImage = getPreviewImageByCenter; + } + + if (previewType === 'bbox') { + getPreviewImage = getPreviewImageByBoundingBox; + } + + return [ layergroupToken(), + validateZoom ? coordinates({ z: true, x: false, y: false }) : noop(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), cleanUpQueryParams(['layer']), + checkStaticImageFormat(), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache, forcedFormat ), - getPreviewImageByBoundingBox(this.previewBackend), + getPreviewImage(this.previewBackend), cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); + lastModifiedHeader() + ]; } }; @@ -120,6 +111,7 @@ function getPreviewImageByCenter (previewBackend) { res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + res.statusCode = 200; res.body = image; next(); @@ -155,6 +147,7 @@ function getPreviewImageByBoundingBox (previewBackend) { res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + res.statusCode = 200; res.body = image; next(); diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/api/map/preview-template-controller.js similarity index 57% rename from lib/cartodb/controllers/named_maps.js rename to lib/cartodb/api/map/preview-template-controller.js index c9656de0..321bb882 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -1,17 +1,14 @@ -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const coordinates = require('../middleware/coordinates'); -const credentials = require('../middleware/credentials'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const authorize = require('../middleware/authorize'); -const cacheControlHeader = require('../middleware/cache-control-header'); -const cacheChannelHeader = require('../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../middleware/last-modified-header'); -const sendResponse = require('../middleware/send-response'); -const vectorError = require('../middleware/vector-error'); -const rateLimit = require('../middleware/rate-limit'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const namedMapProvider = require('../middlewares/named-map-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const checkStaticImageFormat = require('../middlewares/check-static-image-format'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const DEFAULT_ZOOM_CENTER = { @@ -26,123 +23,60 @@ function numMapper(n) { return +n; } -function NamedMapsController ( - namedMapProviderCache, - tileBackend, - previewBackend, - surrogateKeysCache, - tablesExtentApi, - metadataBackend, - pgConnection, - authApi, - userLimitsApi -) { - this.namedMapProviderCache = namedMapProviderCache; - this.tileBackend = tileBackend; - this.previewBackend = previewBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.tablesExtentApi = tablesExtentApi; - this.metadataBackend = metadataBackend; - this.pgConnection = pgConnection; - this.authApi = authApi; - this.userLimitsApi = userLimitsApi; -} +module.exports = class PreviewTemplateController { + constructor ( + namedMapProviderCache, + previewBackend, + surrogateKeysCache, + tablesExtentBackend, + metadataBackend, + pgConnection, + authBackend, + userLimitsBackend + ) { + this.namedMapProviderCache = namedMapProviderCache; + this.previewBackend = previewBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.tablesExtentBackend = tablesExtentBackend; + this.metadataBackend = metadataBackend; + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } -module.exports = NamedMapsController; + register (mapRouter) { + mapRouter.get('/static/named/:template_id/:width/:height.:format', this.middlewares()); + } -NamedMapsController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app; - - app.get( - `${templateBasePath}/:template_id/:layer/:z/:x/:y.(:format)`, - cors(), - user(), - coordinates(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), - cleanUpQueryParams(), - getNamedMapProvider({ - namedMapProviderCache: this.namedMapProviderCache, - label: 'NAMED_MAP_TILE' - }), - getTile({ - tileBackend: this.tileBackend, - label: 'NAMED_MAP_TILE' - }), - setContentTypeHeader(), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse(), - vectorError() - ); - - app.get( - `${mapconfigBasePath}/static/named/:template_id/:width/:height.:format`, - cors(), - user(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), - cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), - getNamedMapProvider({ - namedMapProviderCache: this.namedMapProviderCache, - label: 'STATIC_VIZ_MAP', forcedFormat: 'png' - }), - getTemplate({ label: 'STATIC_VIZ_MAP' }), - prepareLayerFilterFromPreviewLayers({ - namedMapProviderCache: this.namedMapProviderCache, - label: 'STATIC_VIZ_MAP' - }), - getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), - getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), - setContentTypeHeader(), - incrementMapViews({ metadataBackend: this.metadataBackend }), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); + middlewares () { + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), + cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), + checkStaticImageFormat(), + namedMapProvider({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'STATIC_VIZ_MAP', forcedFormat: 'png' + }), + getTemplate({ label: 'STATIC_VIZ_MAP' }), + prepareLayerFilterFromPreviewLayers({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'STATIC_VIZ_MAP' + }), + getStaticImageOptions({ tablesExtentBackend: this.tablesExtentBackend }), + getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), + setContentTypeHeader(), + incrementMapViews({ metadataBackend: this.metadataBackend }), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } }; -function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) { - return function getNamedMapProviderMiddleware (req, res, next) { - const { user, token, cache_buster, api_key } = res.locals; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const { template_id, layer: layerFromParams, z, x, y, format } = req.params; - const { layer: layerFromQuery } = req.query; - - const params = { - user, token, cache_buster, api_key, - dbuser, dbname, dbpassword, dbhost, dbport, - template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format - }; - - if (forcedFormat) { - params.format = forcedFormat; - params.layer = params.layer || 'all'; - } - - const { config, auth_token } = req.query; - - namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { - if (err) { - err.label = label; - return next(err); - } - - res.locals.mapConfigProvider = namedMapProvider; - - next(); - }); - }; -} - function getTemplate ({ label }) { return function getTemplateMiddleware (req, res, next) { const { mapConfigProvider } = res.locals; @@ -209,33 +143,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) }; } -function getTile ({ tileBackend, label }) { - return function getTileMiddleware (req, res, next) { - const { mapConfigProvider } = res.locals; - const { layer, z, x, y, format } = req.params; - const params = { layer, z, x, y, format }; - - tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => { - req.profiler.add(stats); - req.profiler.done('render-' + format); - - if (err) { - err.label = label; - return next(err); - } - - if (headers) { - res.set(headers); - } - - res.body = tile; - - next(); - }); - }; -} - -function getStaticImageOptions ({ tablesExtentApi }) { +function getStaticImageOptions ({ tablesExtentBackend }) { return function getStaticImageOptionsMiddleware(req, res, next) { const { user, mapConfigProvider, template } = res.locals; const { zoom, lon, lat, bbox } = req.query; @@ -261,7 +169,7 @@ function getStaticImageOptions ({ tablesExtentApi }) { return next(); } - tablesExtentApi.getBounds(user, tables, (err, bounds) => { + tablesExtentBackend.getBounds(user, tables, (err, bounds) => { if (err) { return next(); } @@ -365,6 +273,7 @@ function getImage({ previewBackend, label }) { res.set(headers); } + res.statusCode = 200; res.body = image; next(); @@ -384,6 +293,7 @@ function getImage({ previewBackend, label }) { res.set(headers); } + res.statusCode = 200; res.body = image; next(); diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js new file mode 100644 index 00000000..311fec45 --- /dev/null +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -0,0 +1,170 @@ +const layergroupToken = require('../middlewares/layergroup-token'); +const coordinates = require('../middlewares/coordinates'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const vectorError = require('../middlewares/vector-error'); + +const SUPPORTED_FORMATS = { + grid_json: true, + json_torque: true, + torque_json: true, + png: true, + png32: true, + mvt: true +}; + +module.exports = class TileLayergroupController { + constructor ( + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ) { + this.tileBackend = tileBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authBackend = authBackend; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (mapRouter) { + // REGEXP: doesn't match with `val` + const not = (val) => `(?!${val})([^\/]+?)`; + + // Sadly the path that matches 1 also matches with 2 so we need to tell to express + // that performs only the middlewares of the first path that matches + // for that we use one array to group all paths. + mapRouter.get([ + `/:token/:z/:x/:y@:scale_factor?x.:format`, // 1 + `/:token/:z/:x/:y.:format`, // 2 + `/:token${not('static')}/:layer/:z/:x/:y.(:format)` + ], this.middlewares()); + } + + middlewares () { + return [ + layergroupToken(), + coordinates(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError() + ]; + } +}; + +function parseFormat (format = '') { + const prettyFormat = format.replace('.', '_'); + return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; +} + +function getStatusCode(tile, format){ + return tile.length === 0 && format === 'mvt' ? 204 : 200; +} + +function getTile (tileBackend) { + return function getTileMiddleware (req, res, next) { + req.profiler.start(`windshaft.${req.params.layer ? 'maplayer_tile' : 'map_tile'}`); + + const { mapConfigProvider } = res.locals; + const { token } = res.locals; + const { layer, z, x, y, format } = req.params; + + const params = { token, layer, z, x, y, format }; + + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { + req.profiler.add(stats); + + if (err) { + return next(err); + } + + if (headers) { + res.set(headers); + } + + const formatStat = parseFormat(req.params.format); + + res.statusCode = getStatusCode(tile, formatStat); + res.body = tile; + + next(); + }); + }; +} + +function incrementSuccessMetrics (statsClient) { + return function incrementSuccessMetricsMiddleware (req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.success'); + statsClient.increment(`windshaft.tiles.${formatStat}.success`); + + next(); + }; +} + +function incrementErrorMetrics (statsClient) { + return function incrementErrorMetricsMiddleware (err, req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.error'); + statsClient.increment(`windshaft.tiles.${formatStat}.error`); + + next(err); + }; +} + +function tileError () { + return function tileErrorMiddleware (err, req, res, next) { + if (err.message === 'Tile does not exist' && req.params.format === 'mvt') { + res.statusCode = 204; + return next(); + } + + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + + // Rewrite mapnik parsing errors to start with layer number + const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); + + if (matches) { + errMsg = `style${matches[2]}: ${matches[1]}`; + } + + err.message = errMsg; + err.label = 'TILE RENDER'; + + next(err); + }; +} diff --git a/lib/cartodb/api/middlewares/augment-layergroup-data.js b/lib/cartodb/api/middlewares/augment-layergroup-data.js new file mode 100644 index 00000000..65a6702d --- /dev/null +++ b/lib/cartodb/api/middlewares/augment-layergroup-data.js @@ -0,0 +1,14 @@ +const _ = require('underscore'); + +module.exports = function augmentLayergroupData () { + return function augmentLayergroupDataMiddleware (req, res, next) { + const layergroup = res.body; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + _.extend(layergroup, global.environment.serverMetadata); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/authorize.js b/lib/cartodb/api/middlewares/authorize.js similarity index 78% rename from lib/cartodb/middleware/authorize.js rename to lib/cartodb/api/middlewares/authorize.js index a1323fa9..54cab1c4 100644 --- a/lib/cartodb/middleware/authorize.js +++ b/lib/cartodb/api/middlewares/authorize.js @@ -1,6 +1,6 @@ -module.exports = function authorize (authApi) { +module.exports = function authorize (authBackend) { return function authorizeMiddleware (req, res, next) { - authApi.authorize(req, res, (err, authorized) => { + authBackend.authorize(req, res, (err, authorized) => { req.profiler.done('authorize'); if (err) { diff --git a/lib/cartodb/middleware/cache-channel-header.js b/lib/cartodb/api/middlewares/cache-channel-header.js similarity index 100% rename from lib/cartodb/middleware/cache-channel-header.js rename to lib/cartodb/api/middlewares/cache-channel-header.js diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/api/middlewares/cache-control-header.js similarity index 100% rename from lib/cartodb/middleware/cache-control-header.js rename to lib/cartodb/api/middlewares/cache-control-header.js diff --git a/lib/cartodb/api/middlewares/check-json-content-type.js b/lib/cartodb/api/middlewares/check-json-content-type.js new file mode 100644 index 00000000..05aa6f26 --- /dev/null +++ b/lib/cartodb/api/middlewares/check-json-content-type.js @@ -0,0 +1,11 @@ +module.exports = function checkJsonContentType () { + return function checkJsonContentTypeMiddleware(req, res, next) { + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('POST data must be of type application/json')); + } + + req.profiler.done('checkJsonContentTypeMiddleware'); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/check-static-image-format.js b/lib/cartodb/api/middlewares/check-static-image-format.js new file mode 100644 index 00000000..9b9c8b24 --- /dev/null +++ b/lib/cartodb/api/middlewares/check-static-image-format.js @@ -0,0 +1,11 @@ +const VALID_IMAGE_FORMATS = ['png', 'jpg']; + +module.exports = function checkStaticImageFormat () { + return function checkStaticImageFormatMiddleware (req, res, next) { + if(!VALID_IMAGE_FORMATS.includes(req.params.format)) { + return next(new Error(`Unsupported image format "${req.params.format}"`)); + } + + next(); + }; +}; diff --git a/lib/cartodb/middleware/clean-up-query-params.js b/lib/cartodb/api/middlewares/clean-up-query-params.js similarity index 100% rename from lib/cartodb/middleware/clean-up-query-params.js rename to lib/cartodb/api/middlewares/clean-up-query-params.js diff --git a/lib/cartodb/middleware/coordinates.js b/lib/cartodb/api/middlewares/coordinates.js similarity index 100% rename from lib/cartodb/middleware/coordinates.js rename to lib/cartodb/api/middlewares/coordinates.js diff --git a/lib/cartodb/api/middlewares/cors.js b/lib/cartodb/api/middlewares/cors.js new file mode 100644 index 00000000..12db26ac --- /dev/null +++ b/lib/cartodb/api/middlewares/cors.js @@ -0,0 +1,18 @@ +module.exports = function cors () { + return function corsMiddleware (req, res, next) { + const headers = [ + 'X-Requested-With', + 'X-Prototype-Version', + 'X-CSRF-Token' + ]; + + if (req.method === 'OPTIONS') { + headers.push('Content-Type'); + } + + res.set("Access-Control-Allow-Origin", "*"); + res.set("Access-Control-Allow-Headers", headers.join(', ')); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/credentials.js b/lib/cartodb/api/middlewares/credentials.js similarity index 100% rename from lib/cartodb/middleware/credentials.js rename to lib/cartodb/api/middlewares/credentials.js diff --git a/lib/cartodb/middleware/db-conn-setup.js b/lib/cartodb/api/middlewares/db-conn-setup.js similarity index 100% rename from lib/cartodb/middleware/db-conn-setup.js rename to lib/cartodb/api/middlewares/db-conn-setup.js diff --git a/lib/cartodb/middleware/error-middleware.js b/lib/cartodb/api/middlewares/error-middleware.js similarity index 100% rename from lib/cartodb/middleware/error-middleware.js rename to lib/cartodb/api/middlewares/error-middleware.js diff --git a/lib/cartodb/api/middlewares/increment-map-view-count.js b/lib/cartodb/api/middlewares/increment-map-view-count.js new file mode 100644 index 00000000..acf241fd --- /dev/null +++ b/lib/cartodb/api/middlewares/increment-map-view-count.js @@ -0,0 +1,16 @@ +module.exports = function incrementMapViewCount (metadataBackend) { + return function incrementMapViewCountMiddleware(req, res, next) { + const { mapConfig, user } = res.locals; + + // Error won't blow up, just be logged. + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { + req.profiler.done('incMapviewCount'); + + if (err) { + global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); + } + + next(); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/init-profiler.js b/lib/cartodb/api/middlewares/init-profiler.js new file mode 100644 index 00000000..6d0be211 --- /dev/null +++ b/lib/cartodb/api/middlewares/init-profiler.js @@ -0,0 +1,9 @@ +module.exports = function initProfiler (isTemplateInstantiation) { + const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; + + return function initProfilerMiddleware (req, res, next) { + req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); + req.profiler.done(`${operation}.initProfilerMiddleware`); + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/initialize-status-code.js b/lib/cartodb/api/middlewares/initialize-status-code.js new file mode 100644 index 00000000..ed6b1805 --- /dev/null +++ b/lib/cartodb/api/middlewares/initialize-status-code.js @@ -0,0 +1,9 @@ +module.exports = function initializeStatusCode () { + return function initializeStatusCodeMiddleware (req, res, next) { + if (req.method !== 'OPTIONS') { + res.statusCode = 404; + } + + next(); + }; +}; diff --git a/lib/cartodb/middleware/last-modified-header.js b/lib/cartodb/api/middlewares/last-modified-header.js similarity index 100% rename from lib/cartodb/middleware/last-modified-header.js rename to lib/cartodb/api/middlewares/last-modified-header.js diff --git a/lib/cartodb/api/middlewares/last-updated-time-layergroup.js b/lib/cartodb/api/middlewares/last-updated-time-layergroup.js new file mode 100644 index 00000000..f883fc35 --- /dev/null +++ b/lib/cartodb/api/middlewares/last-updated-time-layergroup.js @@ -0,0 +1,39 @@ +module.exports = function setLastUpdatedTimeToLayergroup () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { mapConfigProvider, analysesResults } = res.locals; + const layergroup = res.body; + + mapConfigProvider.createAffectedTables((err, affectedTables) => { + if (err) { + return next(err); + } + + if (!affectedTables) { + return next(); + } + + var lastUpdateTime = affectedTables.getLastUpdatedAt(); + + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + next(); + }); + }; +}; + +function getLastUpdatedTime(analysesResults, lastUpdateTime) { + if (!Array.isArray(analysesResults)) { + return lastUpdateTime; + } + return analysesResults.reduce(function(lastUpdateTime, analysis) { + return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { + var nodeUpdatedAtDate = node.getUpdatedAt(); + var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; + return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; + }, lastUpdateTime); + }, lastUpdateTime); +} diff --git a/lib/cartodb/api/middlewares/layer-stats.js b/lib/cartodb/api/middlewares/layer-stats.js new file mode 100644 index 00000000..e6ca00e6 --- /dev/null +++ b/lib/cartodb/api/middlewares/layer-stats.js @@ -0,0 +1,26 @@ +module.exports = function setLayerStats (pgConnection, statsBackend) { + return function setLayerStatsMiddleware(req, res, next) { + const { user, mapConfig } = res.locals; + const layergroup = res.body; + + pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); + } + + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { + if (err) { + return next(err); + } + + if (layersStats.length > 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + + next(); + }); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/layergroup-id-header.js b/lib/cartodb/api/middlewares/layergroup-id-header.js new file mode 100644 index 00000000..8141d6a6 --- /dev/null +++ b/lib/cartodb/api/middlewares/layergroup-id-header.js @@ -0,0 +1,15 @@ +module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash) { + return function setLayergroupIdHeaderMiddleware (req, res, next) { + const { user, template } = res.locals; + const layergroup = res.body; + + if (useTemplateHash) { + var templateHash = templateMaps.fingerPrint(template).substring(0, 8); + layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + } + + res.set('X-Layergroup-Id', layergroup.layergroupid); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/layergroup-metadata.js b/lib/cartodb/api/middlewares/layergroup-metadata.js new file mode 100644 index 00000000..301089df --- /dev/null +++ b/lib/cartodb/api/middlewares/layergroup-metadata.js @@ -0,0 +1,14 @@ +module.exports = function setMetadataToLayergroup (layergroupMetadata, includeQuery) { + return function setMetadataToLayergroupMiddleware (req, res, next) { + const { user, mapConfig, analysesResults = [], context } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); + layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/layergroup-token.js b/lib/cartodb/api/middlewares/layergroup-token.js similarity index 93% rename from lib/cartodb/middleware/layergroup-token.js rename to lib/cartodb/api/middlewares/layergroup-token.js index 1a32e413..0e6553d2 100644 --- a/lib/cartodb/middleware/layergroup-token.js +++ b/lib/cartodb/api/middlewares/layergroup-token.js @@ -1,4 +1,4 @@ -const LayergroupToken = require('../models/layergroup-token'); +const LayergroupToken = require('../../models/layergroup-token'); const authErrorMessageTemplate = function (signer, user) { return `Cannot use map signature of user "${signer}" on db of user "${user}"`; }; diff --git a/lib/cartodb/api/middlewares/logger.js b/lib/cartodb/api/middlewares/logger.js new file mode 100644 index 00000000..ccd51d45 --- /dev/null +++ b/lib/cartodb/api/middlewares/logger.js @@ -0,0 +1,22 @@ +module.exports = function logger (options) { + if (!global.log4js || !options.log_format) { + return function dummyLoggerMiddleware (req, res, next) { + next(); + }; + } + + const opts = { + level: 'info', + // Allowing for unbuffered logging is mainly + // used to avoid hanging during unit testing. + // TODO: provide an explicit teardown function instead, + // releasing any event handler or timer set by + // this component. + buffer: !options.unbuffered_logging, + // optional log format + format: options.log_format + }; + const logger = global.log4js.getLogger(); + + return global.log4js.connectLogger(logger, opts); +}; diff --git a/lib/cartodb/middleware/lzma.js b/lib/cartodb/api/middlewares/lzma.js similarity index 100% rename from lib/cartodb/middleware/lzma.js rename to lib/cartodb/api/middlewares/lzma.js diff --git a/lib/cartodb/api/middlewares/map-error.js b/lib/cartodb/api/middlewares/map-error.js new file mode 100644 index 00000000..6d87a93d --- /dev/null +++ b/lib/cartodb/api/middlewares/map-error.js @@ -0,0 +1,35 @@ +module.exports = function mapError (options) { + const { addContext = false, label = 'MAPS CONTROLLER' } = options; + + return function mapErrorMiddleware (err, req, res, next) { + req.profiler.done('error'); + const { mapConfig } = res.locals; + + if (addContext) { + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; + } + + err.label = label; + + next(err); + }; +}; + +function populateError(err, mapConfig) { + var error = new Error(err.message); + error.http_status = err.http_status; + + if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { + error.http_status = 400; + } + + error.type = 'layer'; + error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; + error.layer = { + id: mapConfig.getLayerId(err.layerIndex), + index: err.layerIndex, + type: mapConfig.layerType(err.layerIndex) + }; + + return error; +} diff --git a/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js b/lib/cartodb/api/middlewares/map-store-map-config-provider.js similarity index 86% rename from lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js rename to lib/cartodb/api/middlewares/map-store-map-config-provider.js index 7d64cb01..58cdb79e 100644 --- a/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js +++ b/lib/cartodb/api/middlewares/map-store-map-config-provider.js @@ -1,8 +1,8 @@ -const MapStoreMapConfigProvider = require('../../../models/mapconfig/provider/map-store-provider'); +const MapStoreMapConfigProvider = require('../../models/mapconfig/provider/map-store-provider'); module.exports = function createMapStoreMapConfigProvider ( mapStore, - userLimitsApi, + userLimitsBackend, pgConnection, affectedTablesCache, forcedFormat = null @@ -26,7 +26,7 @@ module.exports = function createMapStoreMapConfigProvider ( res.locals.mapConfigProvider = new MapStoreMapConfigProvider( mapStore, user, - userLimitsApi, + userLimitsBackend, pgConnection, affectedTablesCache, params diff --git a/lib/cartodb/api/middlewares/named-map-provider.js b/lib/cartodb/api/middlewares/named-map-provider.js new file mode 100644 index 00000000..08f0cad5 --- /dev/null +++ b/lib/cartodb/api/middlewares/named-map-provider.js @@ -0,0 +1,32 @@ +module.exports = function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) { + return function getNamedMapProviderMiddleware (req, res, next) { + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, layer: layerFromParams, z, x, y, format } = req.params; + const { layer: layerFromQuery } = req.query; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format + }; + + if (forcedFormat) { + params.format = forcedFormat; + params.layer = params.layer || 'all'; + } + + const { config, auth_token } = req.query; + + namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { + if (err) { + err.label = label; + return next(err); + } + + res.locals.mapConfigProvider = namedMapProvider; + + next(); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/noop.js b/lib/cartodb/api/middlewares/noop.js new file mode 100644 index 00000000..16ceabbe --- /dev/null +++ b/lib/cartodb/api/middlewares/noop.js @@ -0,0 +1,5 @@ +module.exports = function noop () { + return function noopMiddleware (req, res, next) { + next(); + }; +}; diff --git a/lib/cartodb/middleware/rate-limit.js b/lib/cartodb/api/middlewares/rate-limit.js similarity index 92% rename from lib/cartodb/middleware/rate-limit.js rename to lib/cartodb/api/middlewares/rate-limit.js index b3c6d7fe..55062d76 100644 --- a/lib/cartodb/middleware/rate-limit.js +++ b/lib/cartodb/api/middlewares/rate-limit.js @@ -19,13 +19,13 @@ const RATE_LIMIT_ENDPOINTS_GROUPS = { NAMED_TILES: 'named_tiles' }; -function rateLimit(userLimitsApi, endpointGroup = null) { +function rateLimit(userLimitsBackend, endpointGroup = null) { if (!isRateLimitEnabled(endpointGroup)) { return function rateLimitDisabledMiddleware(req, res, next) { next(); }; } return function rateLimitMiddleware(req, res, next) { - userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) { + userLimitsBackend.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) { if (err) { return next(err); } diff --git a/lib/cartodb/middleware/send-response.js b/lib/cartodb/api/middlewares/send-response.js similarity index 89% rename from lib/cartodb/middleware/send-response.js rename to lib/cartodb/api/middlewares/send-response.js index 469cf0a7..ec13c5bf 100644 --- a/lib/cartodb/middleware/send-response.js +++ b/lib/cartodb/api/middlewares/send-response.js @@ -2,7 +2,7 @@ module.exports = function sendResponse () { return function sendResponseMiddleware (req, res) { req.profiler.done('res'); - res.status(res.statusCode || 200); + res.status(res.statusCode); if (Buffer.isBuffer(res.body)) { return res.send(res.body); diff --git a/lib/cartodb/api/middlewares/served-by-host-header.js b/lib/cartodb/api/middlewares/served-by-host-header.js new file mode 100644 index 00000000..809100a2 --- /dev/null +++ b/lib/cartodb/api/middlewares/served-by-host-header.js @@ -0,0 +1,11 @@ +const os = require('os'); + +module.exports = function servedByHostHeader () { + const hostname = os.hostname().split('.')[0]; + + return function servedByHostHeaderMiddleware (req, res, next) { + res.set('X-Served-By-Host', hostname); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/api/middlewares/stats.js similarity index 93% rename from lib/cartodb/middleware/stats.js rename to lib/cartodb/api/middlewares/stats.js index 83ff3054..53d2cf9a 100644 --- a/lib/cartodb/middleware/stats.js +++ b/lib/cartodb/api/middlewares/stats.js @@ -1,4 +1,4 @@ -const Profiler = require('../stats/profiler_proxy'); +const Profiler = require('../../stats/profiler_proxy'); const debug = require('debug')('windshaft:cartodb:stats'); const onHeaders = require('on-headers'); diff --git a/lib/cartodb/middleware/surrogate-key-header.js b/lib/cartodb/api/middlewares/surrogate-key-header.js similarity index 84% rename from lib/cartodb/middleware/surrogate-key-header.js rename to lib/cartodb/api/middlewares/surrogate-key-header.js index 51cec1c1..d2a3243c 100644 --- a/lib/cartodb/middleware/surrogate-key-header.js +++ b/lib/cartodb/api/middlewares/surrogate-key-header.js @@ -1,5 +1,5 @@ -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); +const NamedMapsCacheEntry = require('../../cache/model/named_maps_entry'); +const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) { return function setSurrogateKeyHeaderMiddleware(req, res, next) { diff --git a/lib/cartodb/api/middlewares/syntax-error.js b/lib/cartodb/api/middlewares/syntax-error.js new file mode 100644 index 00000000..ad4df4e8 --- /dev/null +++ b/lib/cartodb/api/middlewares/syntax-error.js @@ -0,0 +1,10 @@ +module.exports = function syntaxError () { + return function syntaxErrorMiddleware (err, req, res, next) { + if (err.name === 'SyntaxError') { + err.http_status = 400; + err.message = `${err.name}: ${err.message}`; + } + + next(err); + }; +}; diff --git a/lib/cartodb/middleware/user.js b/lib/cartodb/api/middlewares/user.js similarity index 79% rename from lib/cartodb/middleware/user.js rename to lib/cartodb/api/middlewares/user.js index 9c7968bc..551fa001 100644 --- a/lib/cartodb/middleware/user.js +++ b/lib/cartodb/api/middlewares/user.js @@ -1,4 +1,4 @@ -const CdbRequest = require('../models/cdb_request'); +const CdbRequest = require('../../models/cdb_request'); module.exports = function user () { const cdbRequest = new CdbRequest(); diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/api/middlewares/vector-error.js similarity index 95% rename from lib/cartodb/middleware/vector-error.js rename to lib/cartodb/api/middlewares/vector-error.js index 8268ec1f..7e2ec75f 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/api/middlewares/vector-error.js @@ -1,5 +1,5 @@ const fs = require('fs'); -const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt'); +const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../../assets/render-timeout-fallback.mvt'); module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/api/template/admin-template-controller.js similarity index 53% rename from lib/cartodb/controllers/named_maps_admin.js rename to lib/cartodb/api/template/admin-template-controller.js index 3305dd46..4a727d23 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/api/template/admin-template-controller.js @@ -1,95 +1,92 @@ -const { templateName } = require('../backends/template_maps'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const credentials = require('../middleware/credentials'); -const rateLimit = require('../middleware/rate-limit'); +const { templateName } = require('../../backends/template_maps'); +const credentials = require('../middlewares/credentials'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const sendResponse = require('../middleware/send-response'); -/** - * @param {AuthApi} authApi - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @constructor - */ -function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) { - this.authApi = authApi; - this.templateMaps = templateMaps; - this.userLimitsApi = userLimitsApi; -} +module.exports = class AdminTemplateController { + /** + * @param {AuthBackend} authBackend + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @constructor + */ + constructor (authBackend, templateMaps, userLimitsBackend) { + this.authBackend = authBackend; + this.templateMaps = templateMaps; + this.userLimitsBackend = userLimitsBackend; + } -module.exports = NamedMapsAdminController; + register (templateRouter) { + templateRouter.options(`/:template_id`); -NamedMapsAdminController.prototype.register = function (app) { - const { base_url_templated: templateBasePath } = app; + templateRouter.post('/', this.middlewares({ + action: 'create', + label: 'POST TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE + })); - app.post( - `${templateBasePath}/`, - cors(), - user(), - credentials(), - checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), - authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), - createTemplate({ templateMaps: this.templateMaps }), - sendResponse() - ); + templateRouter.put('/:template_id', this.middlewares({ + action: 'update', + label: 'PUT TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE + })); - app.put( - `${templateBasePath}/:template_id`, - cors(), - user(), - credentials(), - checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), - authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), - updateTemplate({ templateMaps: this.templateMaps }), - sendResponse() - ); + templateRouter.get('/:template_id', this.middlewares({ + action: 'get', + label: 'GET TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET + })); - app.get( - `${templateBasePath}/:template_id`, - cors(), - user(), - credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), - retrieveTemplate({ templateMaps: this.templateMaps }), - sendResponse() - ); + templateRouter.delete('/:template_id', this.middlewares({ + action: 'delete', + label: 'DELETE TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE + })); - app.delete( - `${templateBasePath}/:template_id`, - cors(), - user(), - credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), - destroyTemplate({ templateMaps: this.templateMaps }), - sendResponse() - ); + templateRouter.get('/', this.middlewares({ + action: 'list', + label: 'GET TEMPLATE LIST', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST + })); + } - app.get( - `${templateBasePath}/`, - cors(), - user(), - credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), - listTemplates({ templateMaps: this.templateMaps }), - sendResponse() - ); + middlewares ({ action, label, rateLimitGroup }) { + let template; - app.options( - `${templateBasePath}/:template_id`, - cors('Content-Type') - ); + if (action === 'create') { + template = createTemplate; + } + + if (action === 'update') { + template = updateTemplate; + } + + if (action === 'get') { + template = retrieveTemplate; + } + + if (action === 'delete') { + template = destroyTemplate; + } + + if (action === 'list') { + template = listTemplates; + } + + return [ + credentials(), + authorizedByAPIKey({ authBackend: this.authBackend, action, label }), + rateLimit(this.userLimitsBackend, rateLimitGroup), + checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), + template({ templateMaps: this.templateMaps }) + ]; + } }; -function checkContentType ({ action, label }) { +function checkContentType ({ label }) { return function checkContentTypeMiddleware (req, res, next) { - if (!req.is('application/json')) { - const error = new Error(`template ${action} data must be of type application/json`); + if ((req.method === 'POST' || req.method === 'PUT') && !req.is('application/json')) { + const error = new Error(`${req.method} template data must be of type application/json`); error.label = label; return next(error); } @@ -98,11 +95,11 @@ function checkContentType ({ action, label }) { }; } -function authorizedByAPIKey ({ authApi, action, label }) { +function authorizedByAPIKey ({ authBackend, action, label }) { return function authorizedByAPIKeyMiddleware (req, res, next) { const { user } = res.locals; - authApi.authorizedByAPIKey(user, res, (err, authenticated, apikey) => { + authBackend.authorizedByAPIKey(user, res, (err, authenticated, apikey) => { if (err) { return next(err); } @@ -138,6 +135,7 @@ function createTemplate ({ templateMaps }) { return next(err); } + res.statusCode = 200; res.body = { template_id: templateId }; next(); @@ -156,6 +154,7 @@ function updateTemplate ({ templateMaps }) { return next(err); } + res.statusCode = 200; res.body = { template_id: templateId }; next(); @@ -184,6 +183,7 @@ function retrieveTemplate ({ templateMaps }) { // so we remove it before returning to the user delete template.auth_id; + res.statusCode = 200; res.body = { template }; next(); @@ -222,6 +222,7 @@ function listTemplates ({ templateMaps }) { return next(err); } + res.statusCode = 200; res.body = { template_ids: templateIds }; next(); diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js new file mode 100644 index 00000000..0af1ad65 --- /dev/null +++ b/lib/cartodb/api/template/named-template-controller.js @@ -0,0 +1,213 @@ +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const initProfiler = require('../middlewares/init-profiler'); +const checkJsonContentType = require('../middlewares/check-json-content-type'); +const incrementMapViewCount = require('../middlewares/increment-map-view-count'); +const augmentLayergroupData = require('../middlewares/augment-layergroup-data'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup'); +const layerStats = require('../middlewares/layer-stats'); +const layergroupIdHeader = require('../middlewares/layergroup-id-header'); +const layergroupMetadata = require('../middlewares/layergroup-metadata'); +const mapError = require('../middlewares/map-error'); +const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); +const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +module.exports = class NamedMapController { + /** + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @param {MapBackend} mapBackend + * @param metadataBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsBackend} userLimitsBackend + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigAdapter} mapConfigAdapter + * @param {StatsBackend} statsBackend + * @param {AuthBackend} authBackend + * @param layergroupMetadata + * @constructor + */ + constructor ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTables = layergroupAffectedTables; + this.mapConfigAdapter = mapConfigAdapter; + this.statsBackend = statsBackend; + this.authBackend = authBackend; + this.layergroupMetadata = layergroupMetadata; + } + + register (templateRouter) { + templateRouter.get('/:template_id/jsonp', this.middlewares()); + templateRouter.post('/:template_id', this.middlewares()); + } + + middlewares () { + const isTemplateInstantiation = true; + const useTemplateHash = true; + const includeQuery = false; + const label = 'NAMED MAP LAYERGROUP'; + const addContext = false; + + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), + cleanUpQueryParams(['aggregation']), + initProfiler(isTemplateInstantiation), + checkJsonContentType(), + checkInstantiteLayergroup(), + getTemplate( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsBackend, + this.mapConfigAdapter, + this.layergroupAffectedTables + ), + instantiateLayergroup( + this.mapBackend, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTables + ), + incrementMapViewCount(this.metadataBackend), + augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), + lastUpdatedTimeLayergroup(), + layerStats(this.pgConnection, this.statsBackend), + layergroupIdHeader(this.templateMaps ,useTemplateHash), + layergroupMetadata(this.layergroupMetadata, includeQuery), + mapError({ label, addContext }) + ]; + } +}; + +function checkInstantiteLayergroup () { + return function checkInstantiteLayergroupMiddleware(req, res, next) { + if (req.method === 'GET') { + const { callback, config } = req.query; + + if (callback === undefined || callback.length === 0) { + return next(new Error('callback parameter should be present and be a function name')); + } + + if (config) { + try { + req.body = JSON.parse(config); + } catch(e) { + return next(new Error('Invalid config parameter, should be a valid JSON')); + } + } + } + + req.profiler.done('checkInstantiteLayergroup'); + + return next(); + }; +} + +function getTemplate ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + affectedTablesCache +) { + return function getTemplateMiddleware (req, res, next) { + const templateParams = req.body; + const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id } = req.params; + const { auth_token } = req.query; + + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + const mapConfigProvider = new NamedMapMapConfigProvider( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + affectedTablesCache, + user, + template_id, + templateParams, + auth_token, + params + ); + + mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { + req.profiler.done('named.getMapConfig'); + if (err) { + return next(err); + } + + res.locals.mapConfig = mapConfig; + res.locals.rendererParams = rendererParams; + res.locals.mapConfigProvider = mapConfigProvider; + + next(); + }); + }; +} + +function instantiateLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) { + return function instantiateLayergroupMiddleware (req, res, next) { + const { user, mapConfig, rendererParams } = res.locals; + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsBackend, + pgConnection, + affectedTablesCache, + rendererParams + ); + + mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { + req.profiler.done('createLayergroup'); + if (err) { + return next(err); + } + + res.statusCode = 200; + res.body = layergroup; + + const { mapConfigProvider } = res.locals; + + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js new file mode 100644 index 00000000..4234bb22 --- /dev/null +++ b/lib/cartodb/api/template/template-router.js @@ -0,0 +1,64 @@ +const { Router: router } = require('express'); + +const NamedMapController = require('./named-template-controller'); +const AdminTemplateController = require('./admin-template-controller'); +const TileTemplateController = require('./tile-template-controller'); + +module.exports = class TemplateRouter { + constructor ({ collaborators }) { + const { + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata, + namedMapProviderCache, + tileBackend, + } = collaborators; + + this.namedMapController = new NamedMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ); + + this.tileTemplateController = new TileTemplateController( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authBackend, + userLimitsBackend + ); + + this.adminTemplateController = new AdminTemplateController( + authBackend, + templateMaps, + userLimitsBackend + ); + } + + register (apiRouter, templatePaths) { + const templateRouter = router({ mergeParams: true }); + + this.namedMapController.register(templateRouter); + this.tileTemplateController.register(templateRouter); + this.adminTemplateController.register(templateRouter); + + templatePaths.forEach(path => apiRouter.use(path, templateRouter)); + } +}; diff --git a/lib/cartodb/api/template/tile-template-controller.js b/lib/cartodb/api/template/tile-template-controller.js new file mode 100644 index 00000000..71a45e6e --- /dev/null +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -0,0 +1,95 @@ +const coordinates = require('../middlewares/coordinates'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const namedMapProvider = require('../middlewares/named-map-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const vectorError = require('../middlewares/vector-error'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +module.exports = class TileTemplateController { + constructor ( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authBackend, + userLimitsBackend + ) { + this.namedMapProviderCache = namedMapProviderCache; + this.tileBackend = tileBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } + + register (templateRouter) { + templateRouter.get('/:template_id/:layer/:z/:x/:y.(:format)', this.middlewares()); + } + + middlewares () { + return [ + coordinates(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), + cleanUpQueryParams(), + namedMapProvider({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'NAMED_MAP_TILE' + }), + getTile({ + tileBackend: this.tileBackend, + label: 'NAMED_MAP_TILE' + }), + setContentTypeHeader(), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + vectorError() + ]; + } +}; + +function getTile ({ tileBackend, label }) { + return function getTileMiddleware (req, res, next) { + const { mapConfigProvider } = res.locals; + const { layer, z, x, y, format } = req.params; + const params = { layer, z, x, y, format }; + + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => { + req.profiler.add(stats); + req.profiler.done('render-' + format); + + if (err) { + err.label = label; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.statusCode = 200; + res.body = tile; + + next(); + }); + }; +} + +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/backends/auth.js similarity index 93% rename from lib/cartodb/api/auth_api.js rename to lib/cartodb/backends/auth.js index e2b3fe46..879a1a18 100644 --- a/lib/cartodb/api/auth_api.js +++ b/lib/cartodb/backends/auth.js @@ -5,16 +5,16 @@ * @param {MapStore} mapStore * @param {TemplateMaps} templateMaps * @constructor - * @type {AuthApi} + * @type {AuthBackend} */ -function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) { +function AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps) { this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.mapStore = mapStore; this.templateMaps = templateMaps; } -module.exports = AuthApi; +module.exports = AuthBackend; // Check if the user is authorized by a signer // @@ -23,7 +23,7 @@ module.exports = AuthApi; // null if the request is not signed by anyone // or will be a string cartodb username otherwise. // -AuthApi.prototype.authorizedBySigner = function(req, res, callback) { +AuthBackend.prototype.authorizedBySigner = function(req, res, callback) { if ( ! res.locals.token || ! res.locals.signer ) { return callback(null, false); // no signer requested } @@ -58,7 +58,7 @@ function isValidApiKey(apikey) { // @param callback function(err, authorized) // NOTE: authorized is expected to be 0 or 1 (integer) // -AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) { +AuthBackend.prototype.authorizedByAPIKey = function(user, res, callback) { const apikeyToken = res.locals.api_key; const basicAuthUsername = res.locals.basicAuthUsername; @@ -121,7 +121,7 @@ function usernameMatches (basicAuthUsername, requestUsername) { * @param res - standard res object. Contains the auth parameters in locals * @param callback function(err, allowed) is access allowed not? */ -AuthApi.prototype.authorize = function(req, res, callback) { +AuthBackend.prototype.authorize = function(req, res, callback) { var user = res.locals.user; this.authorizedByAPIKey(user, res, (err, isAuthorizedByApikey) => { diff --git a/lib/cartodb/api/filter_stats_api.js b/lib/cartodb/backends/filter-stats.js similarity index 89% rename from lib/cartodb/api/filter_stats_api.js rename to lib/cartodb/backends/filter-stats.js index 31e90a25..ba427405 100644 --- a/lib/cartodb/api/filter_stats_api.js +++ b/lib/cartodb/backends/filter-stats.js @@ -2,11 +2,11 @@ var _ = require('underscore'); var step = require('step'); var AnalysisFilter = require('../models/filter/analysis'); -function FilterStatsApi(pgQueryRunner) { +function FilterStatsBackends(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = FilterStatsApi; +module.exports = FilterStatsBackends; function getEstimatedRows(pgQueryRunner, username, query, callback) { pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) { @@ -23,7 +23,7 @@ function getEstimatedRows(pgQueryRunner, username, query, callback) { }); } -FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { +FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { var stats = {}; var self = this; step( diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index c060f964..d8320334 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,4 +1,27 @@ var queryUtils = require('../../utils/query-utils'); +const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); +var SubstitutionTokens = require('../../utils/substitution-tokens'); + +// Instantiate a query with tokens for a given zoom level +function queryForZoom(sql, zoom, singleTile=false) { + const tileRes = 256; + const wmSize = 6378137.0*2*Math.PI; + const nTiles = Math.pow(2, zoom); + const tileSize = wmSize / nTiles; + const resolution = tileSize / tileRes; + const scaleDenominator = resolution / 0.00028; + const x0 = -wmSize/2, y0 = -wmSize/2; + let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`; + if (singleTile) { + bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`; + } + return SubstitutionTokens.replace(sql, { + bbox: bbox, + scale_denominator: scaleDenominator, + pixel_width: resolution, + pixel_height: resolution + }); +} function MapnikLayerStats () { this._types = { @@ -11,17 +34,260 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; +function queryPromise(dbConnection, query) { + return new Promise((resolve, reject) => { + dbConnection.query(query, (err, res) => err ? reject(err) : resolve(res)); + }); + } + +function columnAggregations(field) { + if (field.type === 'number') { + return ['min', 'max', 'avg', 'sum']; + } + if (field.type === 'date') { // TODO other types too? + return ['min', 'max']; + } + return []; +} + +function _getSQL(ctx, query, type='pre', zoom=0) { + let sql; + if (type === 'pre') { + sql = ctx.preQuery; + } + else { + sql = ctx.aggrQuery; + } + sql = queryForZoom(sql, zoom || 0); + return query(sql); +} + +function _estimatedFeatureCount(ctx) { + return queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryRowEstimation)) + .then(res => ({ estimatedFeatureCount: res.rows[0].rows })) + .catch(() => ({ estimatedFeatureCount: -1 })); +} + +function _featureCount(ctx) { + if (ctx.metaOptions.featureCount) { + // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query + return queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryActualRowCount)) + .then(res => ({ featureCount: res.rows[0].rows })); + } + return Promise.resolve(); +} + +function _aggrFeatureCount(ctx) { + if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) { + // We expect as zoom level as the value of aggrFeatureCount + // TODO: it'd be nice to admit an array of zoom levels to + // return metadata for multiple levels. + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, queryUtils.getQueryActualRowCount, 'post', ctx.metaOptions.aggrFeatureCount) + ).then(res => ({ aggrFeatureCount: res.rows[0].rows })); + } + return Promise.resolve(); +} + +function _geometryType(ctx) { + if (ctx.metaOptions.geometryType) { + const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); + return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn))) + .then(res => ({ geometryType: res.rows[0].geom_type })); + } + return Promise.resolve(); +} + +function _columns(ctx) { + if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { + // note: post-aggregation columns are in layer.options.columns when aggregation is present + return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0))) + .then(res => formatResultFields(ctx.dbConnection, res.fields)); + } + return Promise.resolve(); +} + +// combine a list of results merging the properties of all the objects +// undefined results are admitted and ignored +function mergeResults(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + if (a === undefined) { + return b; + } + if (b === undefined) { + return a; + } + return Object.assign({}, a, b); + }); + } +} + +// deeper (1 level) combination of a list of objects: +// mergeColumns([{ col1: { a: 1 }, col2: { a: 2 } }, { col1: { b: 3 } }]) => { col1: { a: 1, b: 3 }, col2: { a: 2 } } +function mergeColumns(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + let c = Object.assign({}, b || {}, a || {}); + Object.keys(c).forEach(key => { + if (b.hasOwnProperty(key)) { + c[key] = Object.assign(c[key], b[key]); + } + }); + return c; + }); + } +} + +const SAMPLE_SEED = 0.5; +const DEFAULT_SAMPLE_ROWS = 100; + +function _sample(ctx, numRows) { + if (ctx.metaOptions.sample) { + const sampleProb = Math.min(ctx.metaOptions.sample.num_rows / numRows, 1); + // We'll use a safety limit just in case numRows is a bad estimate + const requestedRows = ctx.metaOptions.sample.num_rows || DEFAULT_SAMPLE_ROWS; + const limit = Math.ceil(requestedRows * 1.5); + let columns = ctx.metaOptions.sample.include_columns; + return queryPromise(ctx.dbConnection, _getSQL( + ctx, + sql => queryUtils.getQuerySample(sql, sampleProb, limit, SAMPLE_SEED, columns) + )).then(res => ({ sample: res.rows })); + } + return Promise.resolve(); +} + +function _columnStats(ctx, columns) { + if (!columns) { + return Promise.resolve(); + } + if (ctx.metaOptions.columnStats) { + let queries = []; + let aggr = []; + queries.push(new Promise(resolve => resolve(columns))); // add columns as first result + Object.keys(columns).forEach(name => { + aggr = aggr.concat( + columnAggregations(columns[name]) + .map(fn => `${fn}(${name}) AS ${name}_${fn}`) + ); + if (columns[name].type === 'string') { + const topN = ctx.metaOptions.columnStats.topCategories || 1024; + const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ? + ctx.metaOptions.columnStats.includeNulls : + true; + + // TODO: ctx.metaOptions.columnStats.maxCategories + // => use PG stats to dismiss columns with more distinct values + queries.push( + queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls)) + ).then(res => ({ [name]: { categories: res.rows } })) + ); + } + }); + queries.push( + queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`) + ).then(res => { + let stats = {}; + Object.keys(columns).forEach(name => { + stats[name] = {}; + columnAggregations(columns[name]).forEach(fn => { + stats[name][fn] = res.rows[0][`${name}_${fn}`]; + }); + }); + return stats; + }) + ); + return Promise.all(queries).then(results => ({ columns: mergeColumns(results) })); + } + return Promise.resolve({ columns }); +} + +// This is adapted from SQL API: +function fieldType(cname) { + let tname; + switch (true) { + case /bool/.test(cname): + tname = 'boolean'; + break; + case /int|float|numeric/.test(cname): + tname = 'number'; + break; + case /text|char|unknown/.test(cname): + tname = 'string'; + break; + case /date|time/.test(cname): + tname = 'date'; + break; + default: + tname = cname; + } + if ( tname && cname.match(/^_/) ) { + tname += '[]'; + } + return tname; +} + +// columns are returned as an object { columnName1: { type1: ...}, ..} +// for consistency with SQL API +function formatResultFields(dbConnection, fields = []) { + let nfields = {}; + for (let field of fields) { + const cname = dbConnection.typeName(field.dataTypeID); + let tname; + if ( ! cname ) { + tname = 'unknown(' + field.dataTypeID + ')'; + } else { + tname = fieldType(cname); + } + nfields[field.name] = { type: tname }; + } + return nfields; +} + MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); - // This query would gather stats for postgresql table if not exists - dbConnection.query(queryRowCountSql, function (err, res) { - if (err) { - return callback(null, {estimatedFeatureCount: -1}); - } else { - // We decided that the relation is 1 row == 1 feature - return callback(null, {estimatedFeatureCount: res.rows[0].rows}); - } + let aggrQuery = layer.options.sql; + let preQuery = layer.options.sql_raw || aggrQuery; + + let ctx = { + dbConnection, + preQuery, + aggrQuery, + metaOptions: layer.options.metadata || {} + }; + + // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere + // we would set queries.results.estimatedFeatureCount and queries.results.geometryType + // (if metaOptions.geometryType) from it. + + // TODO: compute _sample with _featureCount when available + // TODO: add support for sample.exclude option by, in that case, forcing the columns query and + // passing the results to the sample query function. + + Promise.all([ + _estimatedFeatureCount(ctx).then( + ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount) + .then(sampleResults => mergeResults([sampleResults, { estimatedFeatureCount }])) + ), + _featureCount(ctx), + _aggrFeatureCount(ctx), + _geometryType(ctx), + _columns(ctx).then(columns => _columnStats(ctx, columns)) + ]).then(results => { + callback(null, mergeResults(results)); + }).catch(error => { + callback(error); }); }; diff --git a/lib/cartodb/api/overviews_metadata_api.js b/lib/cartodb/backends/overviews-metadata.js similarity index 86% rename from lib/cartodb/api/overviews_metadata_api.js rename to lib/cartodb/backends/overviews-metadata.js index 55b4966f..31045534 100644 --- a/lib/cartodb/api/overviews_metadata_api.js +++ b/lib/cartodb/backends/overviews-metadata.js @@ -1,10 +1,10 @@ var SubstitutionTokens = require('../utils/substitution-tokens'); -function OverviewsMetadataApi(pgQueryRunner) { +function OverviewsMetadataBackend(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = OverviewsMetadataApi; +module.exports = OverviewsMetadataBackend; function prepareSql(sql) { return sql && SubstitutionTokens.replace(sql, { @@ -15,7 +15,7 @@ function prepareSql(sql) { }); } -OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) { +OverviewsMetadataBackend.prototype.getOverviewsMetadata = function (username, sql, callback) { // FIXME: Currently using internal function _cdb_schema_name // CDB_Overviews should provide the schema information directly. var query = 'SELECT *, _cdb_schema_name(base_table)' + diff --git a/lib/cartodb/api/tables_extent_api.js b/lib/cartodb/backends/tables-extent.js similarity index 90% rename from lib/cartodb/api/tables_extent_api.js rename to lib/cartodb/backends/tables-extent.js index 7b534a3f..5e78a6b1 100644 --- a/lib/cartodb/api/tables_extent_api.js +++ b/lib/cartodb/backends/tables-extent.js @@ -1,8 +1,8 @@ -function TablesExtentApi(pgQueryRunner) { +function TablesExtentBackend(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = TablesExtentApi; +module.exports = TablesExtentBackend; /** * Given a username and a list of tables it will return the estimated extent in SRID 4326 for all the tables based on @@ -13,7 +13,7 @@ module.exports = TablesExtentApi; * `table_name` format as valid input * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north` */ -TablesExtentApi.prototype.getBounds = function (username, tables, callback) { +TablesExtentBackend.prototype.getBounds = function (username, tables, callback) { var estimatedExtentSQLs = tables.map(function(table) { return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')"; }); diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/backends/user-limits.js similarity index 81% rename from lib/cartodb/api/user_limits_api.js rename to lib/cartodb/backends/user-limits.js index caf9f151..519f1cd1 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/backends/user-limits.js @@ -5,9 +5,9 @@ var step = require('step'); * @param metadataBackend * @param options * @constructor - * @type {UserLimitsApi} + * @type {UserLimitsBackend} */ -function UserLimitsApi(metadataBackend, options) { +function UserLimitsBackend(metadataBackend, options) { this.metadataBackend = metadataBackend; this.options = options || {}; this.options.limits = this.options.limits || {}; @@ -15,9 +15,9 @@ function UserLimitsApi(metadataBackend, options) { this.preprareRateLimit(); } -module.exports = UserLimitsApi; +module.exports = UserLimitsBackend; -UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) { +UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callback) { var self = this; var limits = { @@ -40,7 +40,7 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) }); }; -UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { +UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { var self = this; step( @@ -80,12 +80,12 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call ); }; -UserLimitsApi.prototype.preprareRateLimit = function () { +UserLimitsBackend.prototype.preprareRateLimit = function () { if (this.options.limits.rateLimitsEnabled) { this.metadataBackend.loadRateLimitsScript(); } }; -UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) { +UserLimitsBackend.prototype.getRateLimit = function (user, endpointGroup, callback) { this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback); }; diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index e0850822..b64decb1 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -10,14 +10,14 @@ function NamedMapProviderCache( templateMaps, pgConnection, metadataBackend, - userLimitsApi, + userLimitsBackend, mapConfigAdapter, affectedTablesCache ) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.mapConfigAdapter = mapConfigAdapter; this.affectedTablesCache = affectedTablesCache; @@ -36,7 +36,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.templateMaps, this.pgConnection, this.metadataBackend, - this.userLimitsApi, + this.userLimitsBackend, this.mapConfigAdapter, this.affectedTablesCache, user, diff --git a/lib/cartodb/controllers/index.js b/lib/cartodb/controllers/index.js deleted file mode 100644 index 429a4b87..00000000 --- a/lib/cartodb/controllers/index.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - Analyses: require('./analyses'), - Layergroup: require('./layergroup'), - Map: require('./map'), - NamedMaps: require('./named_maps'), - NamedMapsAdmin: require('./named_maps_admin'), - ServerInfo: require('./server_info') -}; diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/controllers/layergroup/analysis.js deleted file mode 100644 index 3022bce3..00000000 --- a/lib/cartodb/controllers/layergroup/analysis.js +++ /dev/null @@ -1,75 +0,0 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const sendResponse = require('../../middleware/send-response'); -const dbParamsFromResLocals = require('../../utils/database-params'); - -module.exports = class AnalysisController { - constructor ( - analysisStatusBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ) { - this.analysisStatusBackend = analysisStatusBackend; - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; - this.surrogateKeysCache = surrogateKeysCache; - } - - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - - app.get( - `${mapConfigBasePath}/:token/analysis/node/:nodeId`, - cors(), - user(), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), - cleanUpQueryParams(), - analysisNodeStatus(this.analysisStatusBackend), - sendResponse() - ); - - } -}; - -function analysisNodeStatus (analysisStatusBackend) { - return function analysisNodeStatusMiddleware(req, res, next) { - const { nodeId } = req.params; - const dbParams = dbParamsFromResLocals(res.locals); - - analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET NODE STATUS'; - return next(err); - } - - res.set({ - 'Cache-Control': 'public,max-age=5', - 'Last-Modified': new Date().toUTCString() - }); - - res.body = nodeStatus; - - next(); - }); - }; -} diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/controllers/layergroup/dataview.js deleted file mode 100644 index bca27f1e..00000000 --- a/lib/cartodb/controllers/layergroup/dataview.js +++ /dev/null @@ -1,199 +0,0 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); - -const ALLOWED_DATAVIEW_QUERY_PARAMS = [ - 'filters', // json - 'own_filter', // 0, 1 - 'no_filters', // 0, 1 - 'bbox', // w,s,e,n - 'start', // number - 'end', // number - 'column_type', // string - 'bins', // number - 'aggregation', //string - 'offset', // number - 'q', // widgets search - 'categories', // number -]; - -module.exports = class DataviewController { - constructor ( - dataviewBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ) { - this.dataviewBackend = dataviewBackend; - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; - this.surrogateKeysCache = surrogateKeysCache; - } - - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - - // Undocumented/non-supported API endpoint methods. - // Use at your own peril. - - app.get( - `${mapConfigBasePath}/:token/dataview/:dataviewName`, - cors(), - user(), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), - cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getDataview(this.dataviewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); - - app.get( - `${mapConfigBasePath}/:token/:layer/widget/:dataviewName`, - cors(), - user(), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), - cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getDataview(this.dataviewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); - - app.get( - `${mapConfigBasePath}/:token/dataview/:dataviewName/search`, - cors(), - user(), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - dataviewSearch(this.dataviewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); - - app.get( - `${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`, - cors(), - user(), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - dataviewSearch(this.dataviewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() - ); - } -}; - -function getDataview (dataviewBackend) { - return function getDataviewMiddleware (req, res, next) { - const { user, mapConfigProvider } = res.locals; - const { dataviewName } = req.params; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - - const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query); - - dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET DATAVIEW'; - return next(err); - } - - res.body = dataview; - - next(); - }); - }; -} - -function dataviewSearch (dataviewBackend) { - return function dataviewSearchMiddleware (req, res, next) { - const { user, mapConfigProvider } = res.locals; - const { dataviewName } = req.params; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - - const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); - - dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET DATAVIEW SEARCH'; - return next(err); - } - - res.body = searchResult; - - next(); - }); - }; -} diff --git a/lib/cartodb/controllers/layergroup/index.js b/lib/cartodb/controllers/layergroup/index.js deleted file mode 100644 index e4c1770c..00000000 --- a/lib/cartodb/controllers/layergroup/index.js +++ /dev/null @@ -1,114 +0,0 @@ -const DataviewBackend = require('../../backends/dataview'); -const AnalysisStatusBackend = require('../../backends/analysis-status'); - -const TileController = require('./tile'); -const AttributesController = require('./attributes'); -const StaticController = require('./static'); -const DataviewController = require('./dataview'); -const AnalysisController = require('./analysis'); - -/** - * @param {prepareContext} prepareContext - * @param {PgConnection} pgConnection - * @param {MapStore} mapStore - * @param {TileBackend} tileBackend - * @param {PreviewBackend} previewBackend - * @param {AttributesBackend} attributesBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {AnalysisBackend} analysisBackend - * @constructor - */ -function LayergroupController( - pgConnection, - mapStore, - tileBackend, - previewBackend, - attributesBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - analysisBackend, - authApi -) { - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.tileBackend = tileBackend; - this.previewBackend = previewBackend; - this.attributesBackend = attributesBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - - this.dataviewBackend = new DataviewBackend(analysisBackend); - this.analysisStatusBackend = new AnalysisStatusBackend(); - this.authApi = authApi; -} - -module.exports = LayergroupController; - -LayergroupController.prototype.register = function(app) { - - const tileController = new TileController( - this.tileBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - tileController.register(app); - - const attributesController = new AttributesController( - this.attributesBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - attributesController.register(app); - - const staticController = new StaticController( - this.previewBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - staticController.register(app); - - const dataviewController = new DataviewController( - this.dataviewBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - dataviewController.register(app); - - const analysisController = new AnalysisController( - this.analysisStatusBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - analysisController.register(app); - - -}; diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js deleted file mode 100644 index 544c12b2..00000000 --- a/lib/cartodb/controllers/layergroup/tile.js +++ /dev/null @@ -1,234 +0,0 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); -const layergroupToken = require('../../middleware/layergroup-token'); -const coordinates = require('../../middleware/coordinates'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); -const vectorError = require('../../middleware/vector-error'); - -const SUPPORTED_FORMATS = { - grid_json: true, - json_torque: true, - torque_json: true, - png: true, - png32: true, - mvt: true -}; - -module.exports = class TileController { - constructor ( - tileBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ) { - this.tileBackend = tileBackend; - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; - this.surrogateKeysCache = surrogateKeysCache; - } - - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - - app.get( - `${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, - cors(), - user(), - layergroupToken(), - coordinates(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - cleanUpQueryParams(), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getTile(this.tileBackend, 'map_tile'), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - incrementSuccessMetrics(global.statsClient), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError(), - sendResponse() - ); - - app.get( - `${mapConfigBasePath}/:token/:z/:x/:y.:format`, - cors(), - user(), - layergroupToken(), - coordinates(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - cleanUpQueryParams(), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getTile(this.tileBackend, 'map_tile'), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - incrementSuccessMetrics(global.statsClient), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError(), - sendResponse() - ); - - app.get( - `${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, - distinguishLayergroupFromStaticRoute(), - cors(), - user(), - layergroupToken(), - coordinates(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - cleanUpQueryParams(), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getTile(this.tileBackend, 'maplayer_tile'), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - incrementSuccessMetrics(global.statsClient), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError(), - sendResponse() - ); - } -}; - -function distinguishLayergroupFromStaticRoute () { - return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) { - if (req.params.token === 'static') { - return next('route'); - } - - next(); - }; -} - -function parseFormat (format = '') { - const prettyFormat = format.replace('.', '_'); - return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; -} - -function getStatusCode(tile, format){ - return tile.length === 0 && format === 'mvt' ? 204 : 200; -} - -function getTile (tileBackend, profileLabel = 'tile') { - return function getTileMiddleware (req, res, next) { - req.profiler.start(`windshaft.${profileLabel}`); - - const { mapConfigProvider } = res.locals; - const { token } = res.locals; - const { layer, z, x, y, format } = req.params; - - const params = { token, layer, z, x, y, format }; - - tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { - req.profiler.add(stats); - - if (err) { - return next(err); - } - - if (headers) { - res.set(headers); - } - - const formatStat = parseFormat(req.params.format); - - res.statusCode = getStatusCode(tile, formatStat); - res.body = tile; - - next(); - }); - }; -} - -function incrementSuccessMetrics (statsClient) { - return function incrementSuccessMetricsMiddleware (req, res, next) { - const formatStat = parseFormat(req.params.format); - - statsClient.increment('windshaft.tiles.success'); - statsClient.increment(`windshaft.tiles.${formatStat}.success`); - - next(); - }; -} - -function incrementErrorMetrics (statsClient) { - return function incrementErrorMetricsMiddleware (err, req, res, next) { - const formatStat = parseFormat(req.params.format); - - statsClient.increment('windshaft.tiles.error'); - statsClient.increment(`windshaft.tiles.${formatStat}.error`); - - next(err); - }; -} - -function tileError () { - return function tileErrorMiddleware (err, req, res, next) { - if (err.message === 'Tile does not exist' && req.params.format === 'mvt') { - res.statusCode = 204; - return next(); - } - - // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 - let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); - - // Rewrite mapnik parsing errors to start with layer number - const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); - - if (matches) { - errMsg = `style${matches[2]}: ${matches[1]}`; - } - - err.message = errMsg; - err.label = 'TILE RENDER'; - - next(err); - }; -} diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js deleted file mode 100644 index 004dc0c1..00000000 --- a/lib/cartodb/controllers/map.js +++ /dev/null @@ -1,592 +0,0 @@ -const _ = require('underscore'); -const windshaft = require('windshaft'); -const MapConfig = windshaft.model.MapConfig; -const Datasource = windshaft.model.Datasource; -const ResourceLocator = require('../models/resource-locator'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const credentials = require('../middleware/credentials'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const authorize = require('../middleware/authorize'); -const cacheControlHeader = require('../middleware/cache-control-header'); -const cacheChannelHeader = require('../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../middleware/last-modified-header'); -const sendResponse = require('../middleware/send-response'); -const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); -const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); -const LayergroupMetadata = require('../utils/layergroup-metadata'); -const rateLimit = require('../middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; - -/** - * @param {AuthApi} authApi - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @param {MapBackend} mapBackend - * @param metadataBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {MapConfigAdapter} mapConfigAdapter - * @param {StatsBackend} statsBackend - * @constructor - */ -function MapController ( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTables, - mapConfigAdapter, - statsBackend, - authApi -) { - this.pgConnection = pgConnection; - this.templateMaps = templateMaps; - this.mapBackend = mapBackend; - this.metadataBackend = metadataBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; - - this.mapConfigAdapter = mapConfigAdapter; - const resourceLocator = new ResourceLocator(global.environment); - this.layergroupMetadata = new LayergroupMetadata(resourceLocator); - - this.statsBackend = statsBackend; - this.authApi = authApi; -} - -module.exports = MapController; - -MapController.prototype.register = function(app) { - const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app; - - app.get( - `${mapConfigBasePath}`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); - - app.post( - `${mapConfigBasePath}`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); - - const useTemplate = true; - - app.get( - `${templateBasePath}/:template_id/jsonp`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) - ); - - app.post( - `${templateBasePath}/:template_id`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) - ); - - app.options(`${mapConfigBasePath}`, cors('Content-Type')); -}; - -MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { - const isTemplateInstantiation = useTemplate; - const useTemplateHash = useTemplate; - const includeQuery = !useTemplate; - const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP'; - const addContext = !useTemplate; - - return [ - cors(), - user(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, endpointGroup), - cleanUpQueryParams(['aggregation']), - initProfiler(isTemplateInstantiation), - checkJsonContentType(), - this.getCreateMapMiddlewares(useTemplate), - incrementMapViewCount(this.metadataBackend), - augmentLayergroupData(), - cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader({ now: true }), - setLastUpdatedTimeToLayergroup(), - setLayerStats(this.pgConnection, this.statsBackend), - setLayergroupIdHeader(this.templateMaps ,useTemplateHash), - setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), - setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), - setTurboCartoMetadataToLayergroup(this.layergroupMetadata), - setAggregationMetadataToLayergroup(this.layergroupMetadata), - setTilejsonMetadataToLayergroup(this.layergroupMetadata), - sendResponse(), - augmentError({ label, addContext }) - ]; -}; - -MapController.prototype.getCreateMapMiddlewares = function (useTemplate) { - if (useTemplate) { - return [ - checkInstantiteLayergroup(), - getTemplate( - this.templateMaps, - this.pgConnection, - this.metadataBackend, - this.userLimitsApi, - this.mapConfigAdapter, - this.layergroupAffectedTables - ), - instantiateLayergroup( - this.mapBackend, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTables - ) - ]; - } - - return [ - checkCreateLayergroup(), - prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup ( - this.mapBackend, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTables - ) - ]; -}; - -function initProfiler (isTemplateInstantiation) { - const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; - - return function initProfilerMiddleware (req, res, next) { - req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); - req.profiler.done(`${operation}.initProfilerMiddleware`); - next(); - }; -} - -function checkJsonContentType () { - return function checkJsonContentTypeMiddleware(req, res, next) { - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('POST data must be of type application/json')); - } - - req.profiler.done('checkJsonContentTypeMiddleware'); - - next(); - }; -} - -function checkInstantiteLayergroup () { - return function checkInstantiteLayergroupMiddleware(req, res, next) { - if (req.method === 'GET') { - const { callback, config } = req.query; - - if (callback === undefined || callback.length === 0) { - return next(new Error('callback parameter should be present and be a function name')); - } - - if (config) { - try { - req.body = JSON.parse(config); - } catch(e) { - return next(new Error('Invalid config parameter, should be a valid JSON')); - } - } - } - - req.profiler.done('checkInstantiteLayergroup'); - - return next(); - }; -} - -function checkCreateLayergroup () { - return function checkCreateLayergroupMiddleware (req, res, next) { - if (req.method === 'GET') { - const { config } = req.query; - - if (!config) { - return next(new Error('layergroup GET needs a "config" parameter')); - } - - try { - req.body = JSON.parse(config); - } catch (err) { - return next(err); - } - } - - req.profiler.done('checkCreateLayergroup'); - return next(); - }; -} - -function getTemplate ( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - affectedTablesCache -) { - return function getTemplateMiddleware (req, res, next) { - const templateParams = req.body; - const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const { template_id } = req.params; - const { auth_token } = req.query; - - const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); - - const mapConfigProvider = new NamedMapMapConfigProvider( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - affectedTablesCache, - user, - template_id, - templateParams, - auth_token, - params - ); - - mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { - req.profiler.done('named.getMapConfig'); - if (err) { - return next(err); - } - - res.locals.mapConfig = mapConfig; - res.locals.rendererParams = rendererParams; - res.locals.mapConfigProvider = mapConfigProvider; - - next(); - }); - }; -} - -function prepareAdapterMapConfig (mapConfigAdapter) { - return function prepareAdapterMapConfigMiddleware(req, res, next) { - const requestMapConfig = req.body; - - const { user, api_key } = res.locals; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); - - const context = { - analysisConfiguration: { - user, - db: { - host: dbhost, - port: dbport, - dbname: dbname, - user: dbuser, - pass: dbpassword - }, - batch: { - username: user, - apiKey: api_key - } - } - }; - - mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => { - req.profiler.done('anonymous.getMapConfig'); - if (err) { - return next(err); - } - - req.body = requestMapConfig; - res.locals.context = context; - - next(); - }); - }; -} - -function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { - return function createLayergroupMiddleware (req, res, next) { - const requestMapConfig = req.body; - - const { context } = res.locals; - const { user, cache_buster, api_key } = res.locals; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - - const params = { - cache_buster, api_key, - dbuser, dbname, dbpassword, dbhost, dbport - }; - - const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapConfig = new MapConfig(requestMapConfig, datasource); - - const mapConfigProvider = new CreateLayergroupMapConfigProvider( - mapConfig, - user, - userLimitsApi, - pgConnection, - affectedTablesCache, - params - ); - - res.locals.mapConfig = mapConfig; - res.locals.analysesResults = context.analysesResults; - - const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport }; - - mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => { - req.profiler.done('createLayergroup'); - if (err) { - return next(err); - } - - res.body = layergroup; - res.locals.mapConfigProvider = mapConfigProvider; - - next(); - }); - }; -} - -function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { - return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapConfig, rendererParams } = res.locals; - const mapConfigProvider = new CreateLayergroupMapConfigProvider( - mapConfig, - user, - userLimitsApi, - pgConnection, - affectedTablesCache, - rendererParams - ); - - mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { - req.profiler.done('createLayergroup'); - if (err) { - return next(err); - } - - res.body = layergroup; - - const { mapConfigProvider } = res.locals; - - res.locals.analysesResults = mapConfigProvider.analysesResults; - res.locals.template = mapConfigProvider.template; - res.locals.context = mapConfigProvider.context; - - next(); - }); - }; -} - -function incrementMapViewCount (metadataBackend) { - return function incrementMapViewCountMiddleware(req, res, next) { - const { mapConfig, user } = res.locals; - - // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { - req.profiler.done('incMapviewCount'); - - if (err) { - global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); - } - - next(); - }); - }; -} - -function augmentLayergroupData () { - return function augmentLayergroupDataMiddleware (req, res, next) { - const layergroup = res.body; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - - next(); - }; -} - -function setLastUpdatedTimeToLayergroup () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { mapConfigProvider, analysesResults } = res.locals; - const layergroup = res.body; - - mapConfigProvider.createAffectedTables((err, affectedTables) => { - if (err) { - return next(err); - } - - if (!affectedTables) { - return next(); - } - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }); - }; -} - -function getLastUpdatedTime(analysesResults, lastUpdateTime) { - if (!Array.isArray(analysesResults)) { - return lastUpdateTime; - } - return analysesResults.reduce(function(lastUpdateTime, analysis) { - return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { - var nodeUpdatedAtDate = node.getUpdatedAt(); - var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; - return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; - }, lastUpdateTime); - }, lastUpdateTime); -} - -function setLayerStats (pgConnection, statsBackend) { - return function setLayerStatsMiddleware(req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - statsBackend.getStats(mapConfig, connection, function(err, layersStats) { - if (err) { - return next(err); - } - - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } - - next(); - }); - }); - }; -} - -function setLayergroupIdHeader (templateMaps, useTemplateHash) { - return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { user, template } = res.locals; - const layergroup = res.body; - - if (useTemplateHash) { - var templateHash = templateMaps.fingerPrint(template).substring(0, 8); - layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; - } - - res.set('X-Layergroup-Id', layergroup.layergroupid); - - next(); - }; -} - -function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { - return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); - - next(); - }; -} - -function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { - return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { user, analysesResults = [] } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); - - next(); - }; -} - -function setTurboCartoMetadataToLayergroup (layergroupMetadata) { - return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setAggregationMetadataToLayergroup (layergroupMetadata) { - return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setTilejsonMetadataToLayergroup (layergroupMetadata) { - return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); - - next(); - }; -} - -function augmentError (options) { - const { addContext = false, label = 'MAPS CONTROLLER' } = options; - - return function augmentErrorMiddleware (err, req, res, next) { - req.profiler.done('error'); - const { mapConfig } = res.locals; - - if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; - } - - err.label = label; - - next(err); - }; -} - -function populateError(err, mapConfig) { - var error = new Error(err.message); - error.http_status = err.http_status; - - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - return error; -} diff --git a/lib/cartodb/middleware/cors.js b/lib/cartodb/middleware/cors.js deleted file mode 100644 index 65b7cf4f..00000000 --- a/lib/cartodb/middleware/cors.js +++ /dev/null @@ -1,14 +0,0 @@ -module.exports = function cors (extraHeaders) { - return function corsMiddleware (req, res, next) { - let baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token"; - - if(extraHeaders) { - baseHeaders += ", " + extraHeaders; - } - - res.set("Access-Control-Allow-Origin", "*"); - res.set("Access-Control-Allow-Headers", baseHeaders); - - next(); - }; -}; diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 8c1ada5d..80e021b1 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -52,6 +52,11 @@ module.exports = class BaseDataview { if (err) { return callback(err); } + + if (!result || !result.rows || !result.rows.length) { + return callback(new Error('The column type could not be determined')); + } + const pgType = result.rows[0].pg_typeof; callback(null, getPGTypeName(pgType)); }, readOnlyTransaction); diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 67c5eac4..9752ab57 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -101,6 +101,12 @@ module.exports = class AggregationMapConfigAdapter { aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); } + if (!layer.options.sql_raw) { + // if sql_wrap is present, the original query should already be + // in sql_raw (with sql being the wrapped query); + // otherwise we keep the now the original query in sql_raw + layer.options.sql_raw = layer.options.sql; + } layer.options.sql = aggregationSql; mapConfig.getLayerAggregationColumns(index, (err, columns) => { @@ -159,6 +165,7 @@ module.exports = class AggregationMapConfigAdapter { } _getAggregationMetadata (mapConfig, layer, adapted) { + // also: pre-aggr query, columns, ... if (!adapted) { return { png: false, mvt: false }; } diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js index 72c29593..a104c592 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js @@ -2,9 +2,9 @@ var step = require('step'); var queue = require('queue-async'); var _ = require('underscore'); -function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) { - this.overviewsMetadataApi = overviewsMetadataApi; - this.filterStatsApi = filterStatsApi; +function MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend) { + this.overviewsMetadataBackend = overviewsMetadataBackend; + this.filterStatsBackend = filterStatsBackend; } module.exports = MapConfigOverviewsAdapter; @@ -25,7 +25,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) { return done(null, layer); } - self.overviewsMetadataApi.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){ + self.overviewsMetadataBackend.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){ if (err) { done(err, layer); } else { @@ -53,7 +53,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf function collectStatsData(err, filters, unfiltered_query) { var next_step = this; if ( filters ) { - self.filterStatsApi.getFilterStats( + self.filterStatsBackend.getFilterStats( user, unfiltered_query, filters, function(err, stats) { diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index 2bd30152..31c4f7c4 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -117,7 +117,9 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer, var layerSql = layer.options.sql; var layerRawSql = layer.options.sql_raw; - if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) { + if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql && layer.options.sql_wrap) { + // For wrapped queries we'll derive the tokens from the data extent + // instead of the whole Earth/root tile. var self = this; var tokensQuery = tokensQueryTpl({_sql: layerRawSql}); return pg.query(tokensQuery, function(err, resultSet) { diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 1f19d395..bc36d3ae 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -7,16 +7,23 @@ const QueryTables = require('cartodb-query-tables'); /** * @param {MapConfig} mapConfig * @param {String} user - * @param {UserLimitsApi} userLimitsApi + * @param {UserLimitsBackend} userLimitsBackend * @param {Object} params * @constructor * @type {CreateLayergroupMapConfigProvider} */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) { +function CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsBackend, + pgConnection, + affectedTablesCache, + params +) { this.mapConfig = mapConfig; this.user = user; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.pgConnection = pgConnection; this.affectedTablesCache = affectedTablesCache; this.params = params; @@ -36,7 +43,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); + self.userLimitsBackend.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index d9f9da83..1fd9dadf 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -7,15 +7,15 @@ const QueryTables = require('cartodb-query-tables'); /** * @param {MapStore} mapStore * @param {String} user - * @param {UserLimitsApi} userLimitsApi + * @param {UserLimitsBackend} userLimitsBackend * @param {Object} params * @constructor * @type {MapStoreMapConfigProvider} */ -function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) { +function MapStoreMapConfigProvider(mapStore, user, userLimitsBackend, pgConnection, affectedTablesCache, params) { this.mapStore = mapStore; this.user = user; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.pgConnection = pgConnection; this.affectedTablesCache = affectedTablesCache; this.token = params.token; @@ -38,7 +38,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); + self.userLimitsBackend.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 065d20f4..2d9a88c2 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -15,7 +15,7 @@ function NamedMapMapConfigProvider( templateMaps, pgConnection, metadataBackend, - userLimitsApi, + userLimitsBackend, mapConfigAdapter, affectedTablesCache, owner, @@ -27,7 +27,7 @@ function NamedMapMapConfigProvider( this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.mapConfigAdapter = mapConfigAdapter; this.owner = owner; @@ -125,7 +125,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { function prepareContextLimits(err, _mapConfig) { assert.ifError(err); mapConfig = _mapConfig; - self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this); + self.userLimitsBackend.getRenderLimits(self.owner, self.params.api_key, this); }, function cacheAndReturnMapConfig(err, renderLimits) { self.err = err; diff --git a/lib/cartodb/controllers/server_info.js b/lib/cartodb/server-info-controller.js similarity index 80% rename from lib/cartodb/controllers/server_info.js rename to lib/cartodb/server-info-controller.js index c70de9fa..17e0649c 100644 --- a/lib/cartodb/controllers/server_info.js +++ b/lib/cartodb/server-info-controller.js @@ -1,4 +1,4 @@ -var HealthCheck = require('../monitoring/health_check'); +var HealthCheck = require('./monitoring/health_check'); var WELCOME_MSG = "This is the CartoDB Maps API, " + "see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html"; @@ -12,10 +12,10 @@ function ServerInfoController(versions) { module.exports = ServerInfoController; -ServerInfoController.prototype.register = function(app) { - app.get('/health', this.health.bind(this)); - app.get('/', this.welcome.bind(this)); - app.get('/version', this.version.bind(this)); +ServerInfoController.prototype.register = function(monitorRouter) { + monitorRouter.get('/health', this.health.bind(this)); + monitorRouter.get('/', this.welcome.bind(this)); + monitorRouter.get('/version', this.version.bind(this)); }; ServerInfoController.prototype.welcome = function(req, res) { diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 8535783d..ea51f2db 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -1,273 +1,39 @@ -var express = require('express'); -var bodyParser = require('body-parser'); -var RedisPool = require('redis-mpool'); -var cartodbRedis = require('cartodb-redis'); -var _ = require('underscore'); +const _ = require('underscore'); +const express = require('express'); +const windshaft = require('windshaft'); +const { mapnik } = windshaft; -var controller = require('./controllers'); +const jsonReplacer = require('./utils/json-replacer'); -var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); -var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); -var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); -var FastlyCacheBackend = require('./cache/backend/fastly'); +const ApiRouter = require('./api/api-router'); +const ServerInfoController = require('./server-info-controller'); -var StatsClient = require('./stats/client'); -const stats = require('./middleware/stats'); +const StatsClient = require('./stats/client'); -var RendererStatsReporter = require('./stats/reporter/renderer'); +module.exports = function createServer (serverOptions) { + validateOptions(serverOptions); -var windshaft = require('windshaft'); -var mapnik = windshaft.mapnik; - -var TemplateMaps = require('./backends/template_maps.js'); -var OverviewsMetadataApi = require('./api/overviews_metadata_api'); -var FilterStatsApi = require('./api/filter_stats_api'); -var UserLimitsApi = require('./api/user_limits_api'); -var AuthApi = require('./api/auth_api'); -var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); -var NamedMapProviderCache = require('./cache/named_map_provider_cache'); -var PgQueryRunner = require('./backends/pg_query_runner'); -var PgConnection = require('./backends/pg_connection'); - -var AnalysisBackend = require('./backends/analysis'); - -var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; -var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); - -var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); -var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter'); -var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter'); -var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter'); -var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); -var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); -var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter'); -var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter'); -var MapConfigAdapter = require('./models/mapconfig/adapter'); - -var StatsBackend = require('./backends/stats'); - -const lzmaMiddleware = require('./middleware/lzma'); -const errorMiddleware = require('./middleware/error-middleware'); - -module.exports = function(serverOptions) { // Make stats client globally accessible global.statsClient = StatsClient.getInstance(serverOptions.statsd); - var redisPool = new RedisPool(_.defaults(global.environment.redis, { - name: 'windshaft-server', - unwatchOnRelease: false, - noReadyCheck: true - })); - - redisPool.on('status', function(status) { - var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.'; - global.statsClient.gauge(keyPrefix + 'count', status.count); - global.statsClient.gauge(keyPrefix + 'unused', status.unused); - global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); - }); - - var metadataBackend = cartodbRedis({pool: redisPool}); - var pgConnection = new PgConnection(metadataBackend); - var pgQueryRunner = new PgQueryRunner(pgConnection); - var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); - var filterStatsApi = new FilterStatsApi(pgQueryRunner); - var userLimitsApi = new UserLimitsApi(metadataBackend, { - limits: { - cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, - render: serverOptions.renderer.mapnik.limits.render || 0, - rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled - } - }); - - var templateMaps = new TemplateMaps(redisPool, { - max_user_templates: global.environment.maxUserTemplates - }); - - var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions)); - - function invalidateNamedMap (owner, templateName) { - var startTime = Date.now(); - surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { - var logMessage = JSON.stringify({ - username: owner, - type: 'named_map_invalidation', - elapsed: Date.now() - startTime, - error: !!err ? JSON.stringify(err.message) : undefined - }); - if (err) { - global.logger.warn(logMessage); - } else { - global.logger.info(logMessage); - } - }); - } - ['update', 'delete'].forEach(function(eventType) { - templateMaps.on(eventType, invalidateNamedMap); - }); - serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions); - validateOptions(serverOptions); - bootstrapFonts(serverOptions); - // initialize express server - var app = bootstrap(serverOptions); - // Extend windshaft with all the elements of the options object - _.extend(app, serverOptions); + const app = express(); - var mapStore = new windshaft.storage.MapStore({ - pool: redisPool, - expire_time: serverOptions.grainstore.default_layergroup_ttl - }); + app.enable('jsonp callback'); + app.disable('x-powered-by'); + app.disable('etag'); + app.set('json replacer', jsonReplacer()); - var onTileErrorStrategy; - if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { - onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { + const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment }); + apiRouter.register(app); - function isRenderTimeoutError (err) { - return err.message === 'Render timed out'; - } + const versions = getAndValidateVersions(serverOptions); - function isDatasourceTimeoutError (err) { - return err.message && err.message.match(/canceling statement due to statement timeout/i); - } - - function isTimeoutError (err) { - return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); - } - - function isRasterFormat (format) { - return format === 'png' || format === 'jpg'; - } - - if (isTimeoutError(err) && isRasterFormat(format)) { - return callback(null, timeoutErrorTile, { - 'Content-Type': 'image/png', - }, {}); - } else { - return callback(err, tile, headers, stats); - } - }; - } - - var rendererFactory = new windshaft.renderer.Factory({ - onTileErrorStrategy: onTileErrorStrategy, - mapnik: { - redisPool: redisPool, - grainstore: serverOptions.grainstore, - mapnik: serverOptions.renderer.mapnik - }, - http: serverOptions.renderer.http, - mvt: serverOptions.renderer.mvt - }); - - // initialize render cache - var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, { - ttl: 60000, // 60 seconds TTL by default - statsInterval: 60000 // reports stats every milliseconds defined here - }); - var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); - var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); - rendererStatsReporter.start(); - - var attributesBackend = new windshaft.backend.Attributes(); - var previewBackend = new windshaft.backend.Preview(rendererCache); - var tileBackend = new windshaft.backend.Tile(rendererCache); - var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); - var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); - - var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); - - var statsBackend = new StatsBackend(); - - var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); - app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - - var mapConfigAdapter = new MapConfigAdapter( - new MapConfigNamedLayersAdapter(templateMaps, pgConnection), - new MapConfigBufferSizeAdapter(), - new SqlWrapMapConfigAdapter(), - new DataviewsWidgetsAdapter(), - new AnalysisMapConfigAdapter(analysisBackend), - new AggregationMapConfigAdapter(pgConnection), - new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi), - new TurboCartoAdapter() - ); - - var namedMapProviderCache = new NamedMapProviderCache( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - layergroupAffectedTablesCache - ); - - ['update', 'delete'].forEach(function(eventType) { - templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); - }); - - var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); - - var TablesExtentApi = require('./api/tables_extent_api'); - var tablesExtentApi = new TablesExtentApi(pgQueryRunner); - - var versions = getAndValidateVersions(serverOptions); - - /******************************************************************************************************************* - * Routing - ******************************************************************************************************************/ - - new controller.Layergroup( - pgConnection, - mapStore, - tileBackend, - previewBackend, - attributesBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - analysisBackend, - authApi - ).register(app); - - new controller.Map( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - mapConfigAdapter, - statsBackend, - authApi - ).register(app); - - new controller.NamedMaps( - namedMapProviderCache, - tileBackend, - previewBackend, - surrogateKeysCache, - tablesExtentApi, - metadataBackend, - pgConnection, - authApi, - userLimitsApi - ).register(app); - - new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app); - - new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app); - - new controller.ServerInfo(versions).register(app); - - /******************************************************************************************************************* - * END Routing - ******************************************************************************************************************/ - - app.use(errorMiddleware()); + const serverInfoController = new ServerInfoController(versions); + serverInfoController.register(app); return app; }; @@ -278,6 +44,22 @@ function validateOptions(opts) { } } +function bootstrapFonts(opts) { + // Set carto renderer configuration for MMLStore + opts.grainstore.carto_env = opts.grainstore.carto_env || {}; + var cenv = opts.grainstore.carto_env; + cenv.validation_data = cenv.validation_data || {}; + if ( ! cenv.validation_data.fonts ) { + mapnik.register_system_fonts(); + mapnik.register_default_fonts(); + cenv.validation_data.fonts = _.keys(mapnik.fontFiles()); + } +} + +function mapnikVersion(opts) { + return opts.grainstore.mapnik_version || mapnik.versions.mapnik; +} + function getAndValidateVersions(options) { // jshint undef:false var warn = console.warn.bind(console); @@ -309,127 +91,10 @@ function getAndValidateVersions(options) { }); // Be nice and warn if configured mapnik version is != installed mapnik version - if (mapnik.versions.mapnik !== options.grainstore.mapnik_version) { - warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' + + if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) { + warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' + ' != configured mapnik version (' + options.grainstore.mapnik_version + ')'); } return installedDependenciesVersions; } - -function bootstrapFonts(opts) { - // Set carto renderer configuration for MMLStore - opts.grainstore.carto_env = opts.grainstore.carto_env || {}; - var cenv = opts.grainstore.carto_env; - cenv.validation_data = cenv.validation_data || {}; - if ( ! cenv.validation_data.fonts ) { - mapnik.register_system_fonts(); - mapnik.register_default_fonts(); - cenv.validation_data.fonts = _.keys(mapnik.fontFiles()); - } -} - -function bootstrap(opts) { - var app; - if (_.isObject(opts.https)) { - // use https if possible - app = express.createServer(opts.https); - } else { - // fall back to http by default - app = express(); - } - app.enable('jsonp callback'); - app.disable('x-powered-by'); - app.disable('etag'); - - // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 - // See: http://expressjs.com/en/4x/api.html#app.set - app.set('json replacer', function (key, value) { - if (value !== value) { - return 'NaN'; - } - - if (value === Infinity) { - return 'Infinity'; - } - - if (value === -Infinity) { - return '-Infinity'; - } - - return value; - }); - - app.use(bodyParser.json()); - - app.use(function bootstrap$prepareRequestResponse(req, res, next) { - if (global.environment && global.environment.api_hostname) { - res.set('X-Served-By-Host', global.environment.api_hostname); - } - - next(); - }); - - app.use(stats({ - enabled: opts.useProfiler, - statsClient: global.statsClient - })); - - app.use(lzmaMiddleware()); - - // temporary measure until we upgrade to newer version expressjs so we can check err.status - app.use(function(err, req, res, next) { - if (err) { - if (err.name === 'SyntaxError') { - res.status(400).json({ errors: [err.name + ': ' + err.message] }); - } else { - next(err); - } - } else { - next(); - } - }); - - setupLogger(app, opts); - - return app; -} - -function setupLogger(app, opts) { - if (global.log4js && opts.log_format) { - var loggerOpts = { - // Allowing for unbuffered logging is mainly - // used to avoid hanging during unit testing. - // TODO: provide an explicit teardown function instead, - // releasing any event handler or timer set by - // this component. - buffer: !opts.unbuffered_logging, - // optional log format - format: opts.log_format - }; - app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'}))); - } -} - -function surrogateKeysCacheBackends(serverOptions) { - var cacheBackends = []; - - if (serverOptions.varnish_purge_enabled) { - cacheBackends.push( - new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) - ); - } - - if (serverOptions.fastly && - !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { - cacheBackends.push( - new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) - ); - } - - return cacheBackends; -} - -function mapnikVersion(opts) { - return opts.grainstore.mapnik_version || mapnik.versions.mapnik; -} diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 1891e208..dff1280e 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -16,6 +16,15 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, { snapToGrid: false, clipByBox2d: false, metrics: false, + postgis: { + simplify_geometries: false, + extent: '-20037508.3,-20037508.3,20037508.3,20037508.3', + row_limit: 65535, + persist_connection: false, + use_overviews: true, + max_size: 500, + twkb_encoding: true + }, limits: {} }, http: {} @@ -49,12 +58,14 @@ module.exports = { port: global.environment.port, host: global.environment.host }, + // FIXME: Remove it. This is no longer needed, paths are defined in routers // This is for inline maps and table maps base_url: global.environment.base_url_legacy || '/tiles/:table', /// @deprecated with Windshaft-0.17.0 ///base_url_notable: '/tiles', + // FIXME: Remove it. This is no longer needed, paths are defined in routers // This is for Detached maps // // "maps" is the official, while @@ -62,14 +73,61 @@ module.exports = { // base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)', + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)', + // Base URLs for the APIs + // + // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + routes: global.environment.routes || { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + }, + grainstore: { map: { // TODO: allow to specify in configuration srid: 3857 }, - datasource: global.environment.postgres, + datasource: rendererConfig.mapnik.postgis || global.environment.postgres || {}, cachedir: global.environment.millstone.cache_basedir, use_workers: rendererConfig.mapnik.useCartocssWorkers || false, mapnik_version: global.environment.mapnik_version, @@ -82,18 +140,9 @@ module.exports = { statsInterval: rendererConfig.statsInterval }, renderer: { - mvt: rendererConfig.mvt, - mapnik: _.defaults(rendererConfig.mapnik, { - geojson: { - dbPoolParams: { - size: 16, - idleTimeout: 3000, - reapInterval: 1000 - }, - clipByBox2d: false - } - }), - torque: rendererConfig.torque, + mvt: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.mvt), + mapnik: rendererConfig.mapnik, + torque: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.torque), http: rendererConfig.http }, diff --git a/lib/cartodb/utils/json-replacer.js b/lib/cartodb/utils/json-replacer.js new file mode 100644 index 00000000..cccac1f6 --- /dev/null +++ b/lib/cartodb/utils/json-replacer.js @@ -0,0 +1,19 @@ +module.exports = function jsonReplacerFactory () { + // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 + // See: http://expressjs.com/en/4x/api.html#app.set + return function jsonReplacer (key, value) { + if (value !== value) { + return 'NaN'; + } + + if (value === Infinity) { + return 'Infinity'; + } + + if (value === -Infinity) { + return '-Infinity'; + } + + return value; + }; +}; diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index c1b56276..867db0cd 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -21,10 +21,15 @@ module.exports.extractTableNames = function extractTableNames(query) { ].join(''); }; +module.exports.getQueryActualRowCount = function (query) { + return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`; +}; + function getQueryRowEstimation(query) { return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows'; } -module.exports.getQueryRowCount = getQueryRowEstimation; + +module.exports.getQueryRowEstimation = getQueryRowEstimation; module.exports.getAggregationMetadata = ctx => ` WITH @@ -71,3 +76,96 @@ module.exports.countNaNs = function countNaNs(ctx) { `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` }`; }; + +module.exports.getQueryTopCategories = function(query, column, topN, includeNulls=false) { + const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`; + return ` + SELECT ${column} AS category, COUNT(*) AS frequency + FROM (${query}) AS __cdb_query + ${where} + GROUP BY ${column} ORDER BY 2 DESC + LIMIT ${topN} + `; +}; + +function columnSelector(columns) { + if (!columns) { + return '*'; + } + if (typeof columns === 'string') { + return columns; + } + if (Array.isArray(columns)) { + return columns.map(name => `"${name}"`).join(', '); + } + throw new TypeError(`Bad argument type for columns: ${typeof columns}`); +} + +module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5, columns = null) { + const singleTable = simpleQueryTable(query); + if (singleTable) { + return getTableSample(singleTable.table, columns || singleTable.columns, sampleProb, limit, randomSeed); + } + const limitClause = limit ? `LIMIT ${limit}` : ''; + return ` + WITH __cdb_rndseed AS ( + SELECT setseed(${randomSeed}) + ) + SELECT ${columnSelector(columns)} + FROM (${query}) AS __cdb_query + WHERE random() < ${sampleProb} + ${limitClause} + `; +}; + +function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) { + const limitClause = limit ? `LIMIT ${limit}` : ''; + sampleProb *= 100; + randomSeed *= Math.pow(2, 31) -1; + return ` + SELECT ${columnSelector(columns)} + FROM ${table} + TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause} + `; +} + +function simpleQueryTable(sql) { + const basicQuery = + /\s*SELECT\s+([\*a-z0-9_,\s]+?)\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; + const unwrappedQuery = new RegExp("^"+basicQuery.source+"$", 'i'); + // queries for named maps are wrapped like this: + var wrappedQuery = new RegExp( + "^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" + + basicQuery.source + + "\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$", + 'i' + ); + let match = sql.match(unwrappedQuery); + if (!match) { + match = sql.match(wrappedQuery); + } + if (match) { + const columns = match[1]; + const schema = match[3]; + const table = match[4]; + return { table: schema ? `${schema}.${table}` : table, columns }; + } + return false; +} + +module.exports.getQueryGeometryType = function(query, geometryColumn) { + return ` + SELECT ST_GeometryType(${geometryColumn}) AS geom_type + FROM (${query}) AS __cdb_query + WHERE ${geometryColumn} IS NOT NULL + LIMIT 1 + `; +}; + +module.exports.getQueryLimited = function(query, limit=0) { + return ` + SELECT * + FROM (${query}) AS __cdb_query + LIMIT ${limit} + `; +}; diff --git a/package.json b/package.json index ebdf0f1c..25167026 100644 --- a/package.json +++ b/package.json @@ -24,41 +24,41 @@ "Simon Martin " ], "dependencies": { - "basic-auth": "^2.0.0", - "body-parser": "^1.18.2", - "camshaft": "0.61.8", - "cartodb-psql": "0.10.2", + "basic-auth": "2.0.0", + "body-parser": "1.18.3", + "camshaft": "0.61.10", + "cartodb-psql": "0.11.0", "cartodb-query-tables": "0.3.0", "cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#remove-auth-fallback", - "debug": "^3.1.0", - "dot": "~1.0.2", - "express": "~4.16.0", - "fastly-purge": "~1.0.1", - "glob": "^7.1.2", + "debug": "3.1.0", + "dot": "1.1.2", + "express": "4.16.3", + "fastly-purge": "1.0.1", + "glob": "7.1.2", "log4js": "cartodb/log4js-node#cdb", - "lru-cache": "2.6.5", - "lzma": "~2.3.2", - "node-statsd": "~0.0.7", - "on-headers": "^1.0.1", - "queue-async": "~1.0.7", + "lru-cache": "4.1.3", + "lzma": "2.3.2", + "node-statsd": "0.1.1", + "on-headers": "1.0.1", + "queue-async": "1.1.0", "redis-mpool": "0.5.0", - "request": "2.85.0", - "semver": "~5.3.0", - "step": "~0.0.6", - "step-profiler": "~0.3.0", + "request": "2.87.0", + "semver": "5.5.0", + "step": "1.0.0", + "step-profiler": "0.3.0", "turbo-carto": "0.20.2", - "underscore": "~1.6.0", - "windshaft": "4.7.1", - "yargs": "~5.0.0" + "underscore": "1.6.0", + "windshaft": "4.8.1", + "yargs": "11.1.0" }, "devDependencies": { - "istanbul": "~0.4.3", - "jshint": "~2.9.4", - "mocha": "~3.4.1", - "moment": "~2.18.1", - "nock": "~2.11.0", + "istanbul": "0.4.5", + "jshint": "2.9.5", + "mocha": "3.5.3", + "moment": "2.22.1", + "nock": "9.2.6", "redis": "2.8.0", - "strftime": "~0.8.2" + "strftime": "0.10.0" }, "scripts": { "lint": "jshint lib test", diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 6658eb81..96401f15 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -100,6 +100,18 @@ describe('aggregation', function () { from generate_series(-3, 3) x `; + const POINTS_SQL_PAIRS = ` + -- Generate pairs of near points + select + x + 4 as cartodb_id, + st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0), 4326) as the_geom, + st_transform( + st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0),4326), + 3857) as the_geom_webmercator, + x as value + from generate_series(-6, 6) x + `; + function createVectorMapConfig (layers = [ { type: 'cartodb', @@ -130,6 +142,7 @@ describe('aggregation', function () { before(function () { serverOptions.renderer.mvt.usePostGIS = usePostGIS; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; }); after(function (){ @@ -138,6 +151,7 @@ describe('aggregation', function () { afterEach(function (done) { this.testClient.drain(done); + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); it('should return a layergroup indicating the mapconfig was aggregated', function (done) { @@ -2173,6 +2187,168 @@ describe('aggregation', function () { }); }); + ['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`default pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.ok(body.metadata.layers[0].meta.stats.estimatedFeatureCount > 0); + + done(); + }); + }); + + it(`on demand post-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + aggrFeatureCount: 10 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 13); + + done(); + }); + }); + + it(`post-aggregation count adapts to zoom level with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + aggrFeatureCount: 0 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 9); + + done(); + }); + }); + + it(`on-demand pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + featureCount: true + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.featureCount, 13); + + done(); + }); + }); + }); }); }); diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js index 42fb19d0..98316d2d 100644 --- a/test/acceptance/analysis/named-maps.js +++ b/test/acceptance/analysis/named-maps.js @@ -261,6 +261,41 @@ describe('named-maps analysis', function() { ); }); + it('should return and an error requesting unsupported image format', function(done) { + assert.response( + server, + { + url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.gif', + method: 'GET', + encoding: 'binary', + headers: { + host: username + } + }, + { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + assert.ifError(err); + assert.deepEqual( + JSON.parse(res.body), + { + errors:['Unsupported image format \"gif\"'], + errors_with_context:[{ + type: 'unknown', + message: 'Unsupported image format \"gif\"' + }] + } + ); + done(); + + } + ); + }); + }); describe('auto-instantiation', function() { diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 204fcac7..9330b36f 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -491,4 +491,4 @@ describe('Count aggregation', function () { this.testClient.drain(done); }); }); -}); \ No newline at end of file +}); diff --git a/test/acceptance/dataviews/error-cases.js b/test/acceptance/dataviews/error-cases.js index 3e6cc2a8..78acba5d 100644 --- a/test/acceptance/dataviews/error-cases.js +++ b/test/acceptance/dataviews/error-cases.js @@ -3,85 +3,211 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); -describe('histogram-dataview', function() { +describe('dataview error cases', function() { + + describe('generic errors', function() { - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); - var ERROR_RESPONSE = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - function createMapConfig(dataviews) { - return { - version: '1.5.0', - layers: [ - { - "type": "cartodb", - "options": { - "source": { - "id": "HEAD" - }, - "cartocss": "#points { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0" - } - } - ], - dataviews: dataviews, - analyses: [ - { - "id": "HEAD", - "type": "source", - "params": { - "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x" - } - } - ] + var ERROR_RESPONSE = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } }; - } - - it('should fail when invalid dataviews object is provided, string case', function(done) { - var mapConfig = createMapConfig("wadus-string"); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { - assert.ok(!err, err); - - assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]); - - done(); + + function createMapConfig(dataviews) { + return { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + dataviews: dataviews, + analyses: [ + { + "id": "HEAD", + "type": "source", + "params": { + "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x" + } + } + ] + }; + } + + it('should fail when invalid dataviews object is provided, string case', function(done) { + var mapConfig = createMapConfig("wadus-string"); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { + assert.ok(!err, err); + + assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]); + + done(); + }); + }); + + it('should fail when invalid dataviews object is provided, array case', function(done) { + var mapConfig = createMapConfig([]); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { + assert.ok(!err, err); + + assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]); + + done(); + }); + }); + + it('should work with empty but valid objects', function(done) { + var mapConfig = createMapConfig({}); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err, err); + + assert.ok(layergroup); + assert.ok(layergroup.layergroupid); + + done(); + }); }); }); - it('should fail when invalid dataviews object is provided, array case', function(done) { - var mapConfig = createMapConfig([]); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { - assert.ok(!err, err); + describe('pg_typeof', function() { - assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]); - - done(); + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } }); - }); - it('should work with empty but valid objects', function(done) { - var mapConfig = createMapConfig({}); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err, err); + function createMapConfig(query) { + query = query || 'select * from populated_places_simple_reduced'; - assert.ok(layergroup); - assert.ok(layergroup.layergroupid); + return { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: query, + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', + cartocss_version: '2.0.1', + source: { id: "a0" } + } + } + ], + analyses: [{ + id: "a0", + type: "source", + params: { + query: query + } + }], + dataviews: { + aggregation_count_dataview: { + type: "aggregation", + source: { id: "a0" }, + options: { + column: "adm0name", + aggregation: "count", + aggregationColumn: "adm0name" + } + } + } + }; + } - done(); + it('should work without filters', function(done) { + this.testClient = new TestClient(createMapConfig()); + this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err) { + assert.ifError(err); + done(); + }); }); + + it('should work with filters', function(done) { + var params = { + filters: { + dataviews: {aggregation_count_dataview: {accept: ['Canada']}} + } + }; + + this.testClient = new TestClient(createMapConfig()); + this.testClient.getDataview('aggregation_count_dataview', params, function(err) { + assert.ifError(err); + done(); + }); + }); + + it('should return an error if the column used by dataview does not exist', function(done) { + const query = 'select cartodb_id, the_geom, the_geom_webmercator from populated_places_simple_reduced'; + + const params = { + response: { + status: 404, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + const expectedResponseBody = { + errors:['column "adm0name" does not exist'], + errors_with_context:[{ + type:'unknown', + message:'column "adm0name" does not exist' + }] + }; + + this.testClient = new TestClient(createMapConfig(query)); + this.testClient.getDataview('aggregation_count_dataview', params, function(err, result) { + assert.ifError(err); + assert.deepEqual(result, expectedResponseBody); + done(); + }); + }); + + it('should not fail if query does not return rows', function(done) { + const query = 'select * from populated_places_simple_reduced limit 0'; + + const expectedResponseBody = { + aggregation: 'count', + count: 0, + nulls: 0, + nans: 0, + infinities: 0, + min: 0, + max: 0, + categoriesCount: 0, + categories: [], + type: 'aggregation' + }; + + this.testClient = new TestClient(createMapConfig(query)); + this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { + assert.ifError(err); + assert.deepEqual(result, expectedResponseBody); + done(); + }); + }); + }); }); diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index b95c9a74..309b96d8 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -234,7 +234,7 @@ describe('tests from old api translated to multilayer', function() { var parsed = JSON.parse(res.body); assert.ok(parsed.errors); assert.equal(parsed.errors.length, 1); - assert.ok(parsed.errors[0].match(/^Unexpected token W/)); + assert.ok(parsed.errors[0].match(/Unexpected token W/)); done(); } @@ -373,6 +373,7 @@ describe('tests from old api translated to multilayer', function() { }; // reset internal cacheChannel cache + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); assert.response(server, diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index 9844c667..69c4e56c 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -282,4 +282,57 @@ describe('named maps static view', function() { }); }); + it('should return an error requesting unsupported image format', function (done) { + var view = { + zoom: 4, + center: { + lng: 40, + lat: 20 + } + }; + + templateMaps.addTemplate(username, createTemplate(view), function (err) { + if (err) { + return done(err); + } + + var url = `/api/v1/map/static/named/${templateName}/640/480.gif`; + + + var requestOptions = { + url: url, + method: 'GET', + headers: { + host: username + }, + encoding: 'binary' + }; + + var expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + // this could be removed once named maps are invalidated, otherwise you hits the cache + var server = new CartodbWindshaft(serverOptions); + + assert.response(server, requestOptions, expectedResponse, function (res, err) { + assert.ifError(err); + assert.deepEqual( + JSON.parse(res.body), + { + errors:['Unsupported image format \"gif\"'], + errors_with_context:[{ + type: 'unknown', + message: 'Unsupported image format \"gif\"' + }] + } + ); + done(); + }); + }); + }); + }); diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index 4dc089c9..b83a9720 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -51,19 +51,19 @@ describe('server_gettile', function() { var mapConfig = testClient.defaultTableMapConfig('test_table'); testClient.withLayergroup(mapConfig, function (err, requestTile, finish) { requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); lastXwc = xwc; requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); assert.ok(xwc >= lastXwc); requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(!xwc); finish(done); @@ -103,7 +103,7 @@ describe('server_gettile', function() { testClient.withLayergroup(mapConfig, validateLayergroup, function(err, requestTile, finish) { requestTile(tileUrl, function(err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(!xwc); requestTile(tileUrl, function (err, res) { @@ -111,7 +111,7 @@ describe('server_gettile', function() { res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time" ); - assert.ok(res.headers['x-windshaft-cache'] >= 0); + assert.ok(parseInt(res.headers['x-windshaft-cache'], 10) >= 0); assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) { diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 8b8ec339..243875a0 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -7,8 +7,35 @@ var overviewsQueryRewriter = new OverviewsQueryRewriter({ }); module.exports = _.extend({}, serverOptions, { + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url: '/database/:dbname/table/:table', + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_mapconfig: '/database/:dbname/layergroup', + + routes: { + v0: { + paths: [ + '/tiles', + '/database/:dbname' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + }, + + grainstore: { datasource: { geometry_field: 'the_geom', diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js index eb9f8954..925d5028 100644 --- a/test/acceptance/rate-limit.test.js +++ b/test/acceptance/rate-limit.test.js @@ -5,11 +5,11 @@ const redis = require('redis'); const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const TestClient = require('../support/test-client'); -const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api'); -const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit'); +const UserLimitsBackend = require('../../lib/cartodb/backends/user-limits'); +const rateLimitMiddleware = require('../../lib/cartodb/api/middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; -let userLimitsApi; +let userLimitsApi; let rateLimit; let redisClient; let testClient; @@ -112,11 +112,11 @@ function assertGetLayergroupRequest (status, limit, remaining, reset, retry, don 'Carto-Rate-Limit-Reset': reset } }; - + if(retry) { response.headers['Retry-After'] = retry; } - + testClient.getLayergroup({ response }, err => { assert.ifError(err); if (done) { @@ -133,11 +133,11 @@ function assertRateLimitRequest (status, limit, remaining, reset, retry, done) { "Carto-Rate-Limit-Remaining": remaining, "Carto-Rate-Limit-Reset": reset }; - + if(retry) { expectedHeaders['Retry-After'] = retry; } - + assert.deepEqual(res.headers, expectedHeaders); if(status === 200) { @@ -160,7 +160,7 @@ describe('rate limit', function() { before(function() { global.environment.enabledFeatures.rateLimitsEnabled = true; global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; - + redisClient = redis.createClient(global.environment.redis.port); testClient = new TestClient(createMapConfig(), 1234); }); @@ -183,7 +183,7 @@ describe('rate limit', function() { done(); }); }); - }); + }); it('should not be rate limited', function (done) { const count = 1; @@ -218,7 +218,7 @@ describe('rate limit middleware', function () { const redisPool = new RedisPool(global.environment.redis); const metadataBackend = cartodbRedis({ pool: redisPool }); - userLimitsApi = new UserLimitsApi(metadataBackend, { + userLimitsApi = new UserLimitsBackend(metadataBackend, { limits: { rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled } @@ -257,9 +257,9 @@ describe('rate limit middleware', function () { it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) { userLimitsApi.metadataBackend.redisCmd( - 8, - 'SCRIPT', - ['FLUSH'], + 8, + 'SCRIPT', + ['FLUSH'], function () { assertRateLimitRequest(200, 1, 0, 1); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); @@ -277,7 +277,7 @@ describe('rate limit and vector tiles', function () { before(function(done) { global.environment.enabledFeatures.rateLimitsEnabled = true; global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true; - + redisClient = redis.createClient(global.environment.redis.port); const count = 1; const period = 1; @@ -287,9 +287,9 @@ describe('rate limit and vector tiles', function () { testClient = new TestClient(createMapConfig(), 1234); testClient.getLayergroup({status: 200}, (err, res) => { assert.ifError(err); - + layergroupid = res.layergroupid; - + done(); }); }); @@ -336,12 +336,12 @@ describe('rate limit and vector tiles', function () { testClient.getTile(0, 0, 0, tileParams(204, '1', '0', '1'), (err) => { assert.ifError(err); - + testClient.getTile( - 0, - 0, - 0, - tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'), + 0, + 0, + 0, + tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'), (err, res, tile) => { assert.ifError(err); @@ -350,11 +350,11 @@ describe('rate limit and vector tiles', function () { assert.equal(tileJSON.length, 2); assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); - + done(); } - ); + ); }); - + }); }); diff --git a/test/acceptance/regressions.js b/test/acceptance/regressions.js index 698d1401..e49065d3 100644 --- a/test/acceptance/regressions.js +++ b/test/acceptance/regressions.js @@ -1,7 +1,10 @@ require('../support/test_helper'); var assert = require('../support/assert'); +const helper = require('../support/test_helper'); var TestClient = require('../support/test-client'); const LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); +const CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); +const serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); describe('regressions', function() { @@ -38,6 +41,49 @@ describe('regressions', function() { }); }); + // See: https://github.com/CartoDB/Windshaft-cartodb/pull/956 + it('"/user/localhost/api/v1/map" should create an anonymous map', function (done) { + const server = new CartodbWindshaft(serverOptions); + const layergroup = { + version: '1.7.0', + layers: [ + { + type: 'mapnik', + options: { + sql: TestClient.SQL.ONE_POINT, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + + const keysToDelete = {}; + + assert.response(server, + { + url: '/user/localhost/api/v1/map', + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + function(res, err) { + if (err) { + return done(err); + } + + const body = JSON.parse(res.body); + assert.ok(body.layergroupid); + + keysToDelete['map_cfg|' + LayergroupToken.parse(body.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + helper.deleteRedisKeys(keysToDelete, done); + } + ); + }); + describe('map instantiation', function () { const apikeyToken = 'regular1'; const mapConfig = { diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index cdac7bb1..4e8413ae 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -49,7 +49,7 @@ describe('Create mapnik layergroup', function() { sql: [ 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address', ' from test_table t1, test_table_2 t2', - ' where t1.cartodb_id = t2.cartodb_id;' + ' where t1.cartodb_id = t2.cartodb_id' ].join(''), cartocss_version: cartocssVersion, cartocss: cartocss @@ -74,6 +74,26 @@ describe('Create mapnik layergroup', function() { } }; + var mapnikLayerNullCats = { + type: 'mapnik', + options: { + sql: ` + WITH geom AS ( + SELECT + 'SRID=4326;POINT(0 0)'::geometry AS the_geom, + 'SRID=3857;POINT(0 0)'::geometry AS the_geom_webmercator + ) + SELECT 1 AS cartodb_id, 'A' As cat, geom.* FROM geom + UNION + SELECT 2 AS cartodb_id, 'B' As cat, geom.* FROM geom + UNION + SELECT 2 AS cartodb_id, NULL::text As cat, geom.* FROM geom + `, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + function mapnikBasicLayerId(index) { return 'layer' + index; } @@ -90,7 +110,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); @@ -107,7 +127,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -127,7 +147,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -147,7 +167,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); @@ -164,7 +184,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -183,7 +203,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); @@ -204,7 +224,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); @@ -224,7 +244,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function (err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); @@ -245,7 +265,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); @@ -265,7 +285,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); @@ -273,7 +293,288 @@ describe('Create mapnik layergroup', function() { assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats')); assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1)); assert.equal(layergroup.metadata.layers[2].type, 'http'); + global.environment.enabledFeatures.layerStats = true; testClient.drain(done); }); }); + + function layerWithMetadata(layer, metadata) { + return Object.assign(layer, { + options: Object.assign(layer.options, { metadata }) + }); + } + + it('should provide columns as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columns: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { type: 'number' }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { type: 'string' } + }; + assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns); + testClient.drain(done); + }); + }); + + // metadata categories are ordered only partially by descending frequency; + // this orders them completely to avoid ambiguities when comparing + function withSortedCategories(columns) { + function catOrder(a, b) { + if (a.frequency !== b.frequency) { + return b.frequency - a.frequency; + } + if (a.category < b.category) { + return -1; + } + if (a.category > b.category) { + return +1; + } + return 0; + } + let sorted = {}; + Object.keys(columns).forEach(name => { + let data = columns[name]; + if (data.hasOwnProperty('categories')) { + data = Object.assign(data, { categories: data.categories.sort(catOrder)}); + } + sorted[name] = data; + }); + return sorted; + } + + it('should provide column stats as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columnStats: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { + type: 'number', + avg: 3, + max: 5, + min: 1, + sum: 15 + }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { + type: 'string', + categories: [ + { + category: "Calle de la Palma 72, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle de Pérez Galdós 9, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle Divino Pastor 12, Madrid, Spain", + frequency: 1 + }, + { + category: "Manuel Fernández y González 8, Madrid, Spain", + frequency: 1 + }, + { + category: "Plaza Conde de Toreno 2, Madrid, Spain", + frequency: 1 + } + ] + } + }; + assert.deepEqual( + withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns), + withSortedCategories(expectedColumns) + ); + testClient.drain(done); + }); + }); + + it('should limit the number of categories as requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columnStats: { topCategories: 2 } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.address.categories.length, 2); + testClient.drain(done); + }); + }); + + it('should include null categories if requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayerNullCats, { + columnStats: { includeNulls: true } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.cat.categories.length, 3); + testClient.drain(done); + }); + }); + + it('should not include null categories if not requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayerNullCats, { + columnStats: { includeNulls: false } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.cat.categories.length, 2); + testClient.drain(done); + }); + }); + + it('should provide row count as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + testClient.drain(done); + }); + }); + + it('should provide geometry type as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + testClient.drain(done); + }); + }); + + it('should provide a sample as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + sample: { num_rows: 3 } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); + const expectedCols = [ 'cartodb_id', 'address', 'the_geom', 'the_geom_webmercator' ].sort(); + assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols); + testClient.drain(done); + }); + }); + + it('can specify sample columns', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + sample: { + num_rows: 3, + include_columns: [ 'cartodb_id', 'address', 'the_geom' ] + } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); + const expectedCols = [ 'cartodb_id', 'address', 'the_geom' ].sort(); + assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols); + testClient.drain(done); + }); + }); + + + it('should only provide requested optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true, + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.sample, undefined); + assert.equal(layergroup.metadata.layers[0].meta.stats.columns, undefined); + testClient.drain(done); + }); + }); + }); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 810e1c99..faea283e 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -29,6 +29,7 @@ describe('template_api', function() { before(function () { server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); }); @@ -1060,6 +1061,7 @@ describe('template_api', function() { assert.ok(cc); assert.equal(cc, expectedCC); // hack simulating restart... + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache var get_request = { url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1', diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 52ead026..6e5423e1 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -2,6 +2,7 @@ require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; @@ -438,6 +439,77 @@ describe('user database timeout limit', function () { }); }); + + if (process.env.POSTGIS_VERSION === '2.4') { + describe('fetching vector tiles via PostGIS renderer', function() { + const usePostGIS = true; + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, done); + }); + + it('"mvt" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'mvt', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/x-protobuf' + } + }, + cacheBuster: true + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + + done(); + }); + }); + }); + }); + } }); }); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 93770080..79822c7b 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -139,7 +139,7 @@ describe('user render timeout limit', function () { }); it('layergroup creation works but tile request fails due to render timeout', function (done) { - this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { + this.testClient.getTile(0, 0, 0, { cacheBuster: true }, (err, res, tile) => { assert.ifError(err); assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { @@ -180,7 +180,8 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { @@ -201,56 +202,52 @@ describe('user render timeout limit', function () { }); }); - if (process.env.POSTGIS_VERSION === '2.4') { - describe('vector (PostGIS)', vector(true)); - } - - describe('vector (mapnik)', vector(false)); - function vector(usePostGIS) { + describe('vector tile via mapnik renderer', function () { + const usePostGIS = false; const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; - return function () { - beforeEach(function (done) { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserDatabaseTimeoutLimit(50, done); - }); - afterEach(function (done) { - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { - if (err) { - return done(err); + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + response: { + status: 429, + headers: { + 'Content-Type': 'application/x-protobuf' } - this.testClient.drain(done); - }); + }, + cacheBuster: true + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + + done(); }); - - it('layergroup creation works but vector tile request fails due to render timeout', function (done) { - const params = { - format: 'mvt', - response: { - status: 429, - headers: { - 'Content-Type': 'application/x-protobuf' - } - } - }; - - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { - assert.ifError(err); - - var tileJSON = tile.toJSON(); - assert.equal(Array.isArray(tileJSON), true); - assert.equal(tileJSON.length, 2); - assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); - assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); - - done(); - }); - }); - }; - } + }); + }); describe('interativity', function () { beforeEach(function (done) { @@ -277,7 +274,8 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { @@ -365,7 +363,7 @@ describe('user render timeout limit', function () { }); }); - it('layergroup creation works and render tile fails', function (done) { + it('layergroup creation works and render static center tile fails', function (done) { const params = { zoom: 0, lat: 0, @@ -378,7 +376,8 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getStaticCenter(params, function (err, res, timeoutError) { diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js index 87a87935..d5414a7e 100644 --- a/test/integration/mapconfig_overviews_adapter.js +++ b/test/integration/mapconfig_overviews_adapter.js @@ -5,18 +5,18 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); -var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api'); +var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata'); +var FilterStatsBackend = require('../../lib/cartodb/backends/filter-stats'); var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter'); var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); -var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); -var filterStatsApi = new FilterStatsApi(pgQueryRunner); +var overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); +var filterStatsBackend = new FilterStatsBackend(pgQueryRunner); -var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi); +var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend); describe('MapConfigOverviewsAdapter', function() { diff --git a/test/integration/overviews-metadata-api.js b/test/integration/overviews-metadata-api.js index 4d015870..ce82a1d9 100644 --- a/test/integration/overviews-metadata-api.js +++ b/test/integration/overviews-metadata-api.js @@ -7,24 +7,24 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); +var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata'); -describe('OverviewsMetadataApi', function() { +describe('OverviewsMetadataBackend', function() { - var overviewsMetadataApi; + var overviewsMetadataBackend; before(function() { var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); - overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); }); it('should return an empty relation for tables that have no overviews', function(done) { var query = 'select * from test_table'; - overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) { assert.ok(!err, err); assert.deepEqual(result, {}); @@ -35,7 +35,7 @@ describe('OverviewsMetadataApi', function() { it('should return overviews metadata', function(done) { var query = 'select * from test_table_overviews'; - overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) { assert.ok(!err, err); assert.deepEqual(result, { diff --git a/test/support/test-client.js b/test/support/test-client.js index 5e08394a..2a5840f5 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -125,6 +125,14 @@ function resErr2errRes(callback) { }; } +function layergroupidTemplate (layergroupId, params) { + const { token, signer, cacheBuster } = LayergroupToken.parse(layergroupId); + + // {user}@{token}:{cache_buster} + // {token}:{cache_buster} + return `${signer ? signer + '@' : ''}${token}:${params.cacheBuster ? Date.now() : cacheBuster }`; +} + TestClient.prototype.getWidget = function(widgetName, params, callback) { var self = this; @@ -726,7 +734,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = '/api/v1/map/' + layergroupId + '/'; + url = `/api/v1/map/${layergroupidTemplate(layergroupId, params)}/`; var layers = params.layers; @@ -769,7 +777,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }, params.response); - var isPng = format.match(/png$/); if (isPng) { @@ -954,7 +961,9 @@ TestClient.prototype.getStaticCenter = function (params, callback) { self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = `/api/v1/map/static/center/${layergroupId}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`; + const layergroupid = layergroupidTemplate(layergroupId, params); + + url = `/api/v1/map/static/center/${layergroupid}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`; if (self.apiKey) { url += '?' + qs.stringify({api_key: self.apiKey}); diff --git a/test/unit/cartodb/error-middleware.test.js b/test/unit/cartodb/error-middleware.test.js index 861bec54..88b1ed6e 100644 --- a/test/unit/cartodb/error-middleware.test.js +++ b/test/unit/cartodb/error-middleware.test.js @@ -1,7 +1,7 @@ require('../../support/test_helper.js'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware'); describe('error-middleware', function() { @@ -28,7 +28,7 @@ describe('error-middleware', function() { error.subtype = 'test subtype'; const errors = [error, error]; - + const req = {}; const res = { headers: {}, @@ -43,7 +43,7 @@ describe('error-middleware', function() { send () {} }; - const errorHeader = { + const errorHeader = { mainError: { statusCode: 400, message: error.message, @@ -133,7 +133,7 @@ describe('error-middleware', function() { error.subtype = badString; const errors = [error, error]; - + const req = {}; const res = { headers: {}, @@ -148,7 +148,7 @@ describe('error-middleware', function() { send () {} }; - const errorHeader = { + const errorHeader = { mainError: { statusCode: 400, message: escapedString, diff --git a/test/unit/cartodb/error_messages.test.js b/test/unit/cartodb/error_messages.test.js index bfe0b03a..ecc6fe3c 100644 --- a/test/unit/cartodb/error_messages.test.js +++ b/test/unit/cartodb/error_messages.test.js @@ -2,7 +2,7 @@ require('../../support/test_helper'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware'); describe('error messages clean up', function() { diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js index 3ad81962..9701d7ca 100644 --- a/test/unit/cartodb/lzmaMiddleware.test.js +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -1,7 +1,7 @@ var assert = require('assert'); var testHelper = require('../../support/test_helper'); -var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma'); +var lzmaMiddleware = require('../../../lib/cartodb/api/middlewares/lzma'); describe('lzma-middleware', function() { diff --git a/test/unit/cartodb/middlewares/coordinates.test.js b/test/unit/cartodb/middlewares/coordinates.test.js index 11cd2626..5b60aa82 100644 --- a/test/unit/cartodb/middlewares/coordinates.test.js +++ b/test/unit/cartodb/middlewares/coordinates.test.js @@ -1,5 +1,5 @@ const assert = require('assert'); -const coordinates = require('../../../../lib/cartodb/middleware/coordinates'); +const coordinates = require('../../../../lib/cartodb/api/middlewares/coordinates'); describe('coordinates middleware', function () { it('should return error: invalid zoom paramenter (-1)', function (done) { diff --git a/test/unit/cartodb/ported/windshaft_server.test.js b/test/unit/cartodb/ported/windshaft_server.test.js index e6f07ea7..2bc1423c 100644 --- a/test/unit/cartodb/ported/windshaft_server.test.js +++ b/test/unit/cartodb/ported/windshaft_server.test.js @@ -1,6 +1,5 @@ require('../../../support/test_helper.js'); -var _ = require('underscore'); var assert = require('assert'); var cartodbServer = require('../../../../lib/cartodb/server'); var serverOptions = require('../../../../lib/cartodb/server_options'); @@ -30,15 +29,7 @@ describe('windshaft', function() { function(){ var ws = cartodbServer({unbuffered_logging:true}); ws.listen(); - }, /Cannot read property 'mapnik' of undefined/ + }, /Must initialise server with/ ); }); - - it('options are set on main windshaft object', function(){ - var ws = cartodbServer(serverOptions); - assert.ok(_.isObject(ws.bind)); - assert.ok(_.isObject(ws.grainstore)); - assert.equal(ws.base_url, '/tiles/:table'); - }); - }); diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index ff311cc6..02b63dc7 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -4,13 +4,13 @@ var _ = require('underscore'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../../lib/cartodb/backends/pg_connection'); -var AuthApi = require('../../../lib/cartodb/api/auth_api'); +var AuthBackend = require('../../../lib/cartodb/backends/auth'); var TemplateMaps = require('../../../lib/cartodb/backends/template_maps'); -const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/clean-up-query-params'); -const authorizeMiddleware = require('../../../lib/cartodb/middleware/authorize'); -const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/db-conn-setup'); -const credentialsMiddleware = require('../../../lib/cartodb/middleware/credentials'); +const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/api/middlewares/clean-up-query-params'); +const authorizeMiddleware = require('../../../lib/cartodb/api/middlewares/authorize'); +const dbConnSetupMiddleware = require('../../../lib/cartodb/api/middlewares/db-conn-setup'); +const credentialsMiddleware = require('../../../lib/cartodb/api/middlewares/credentials'); var windshaft = require('windshaft'); @@ -31,10 +31,10 @@ describe('prepare-context', function() { var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var templateMaps = new TemplateMaps(redisPool); - var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); + var authBackend = new AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps); cleanUpQueryParams = cleanUpQueryParamsMiddleware(); - authorize = authorizeMiddleware(authApi); + authorize = authorizeMiddleware(authBackend); dbConnSetup = dbConnSetupMiddleware(pgConnection); setCredentials = credentialsMiddleware(); }); diff --git a/yarn.lock b/yarn.lock index 357b8175..4f98787a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,20 +2,20 @@ # yarn lockfile v1 -"@carto/mapnik@3.6.2-carto.8": - version "3.6.2-carto.8" - resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.8.tgz#70448689d9b14d644bebd079f5714871c458a46d" +"@carto/mapnik@3.6.2-carto.10": + version "3.6.2-carto.10" + resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.10.tgz#a97c951dcdac09d0eb35b3ea71e5eeaa206c1af6" dependencies: mapnik-vector-tile cartodb/mapnik-vector-tile#v1.6.1-carto.1 nan "2.10.0" - node-pre-gyp "0.7.0" + node-pre-gyp "0.10.0" protozero "1.5.1" -"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb7": - version "2.5.1-cdb7" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/ec881cb9ac52113f895f23857430e2d434bb99a6" +"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb9": + version "2.5.1-cdb9" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/5129e43223cb55daed31373c7a36c98eb6178fc1" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.10" "@mapbox/sphericalmercator" "~1.0.1" mapnik-pool "~0.1.3" @@ -23,11 +23,11 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -"abaculus@github:cartodb/abaculus#2.0.3-cdb8": - version "2.0.3-cdb8" - resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/31c03f2442943d4c47740fa154cda753b5cccd8a" +"abaculus@github:cartodb/abaculus#2.0.3-cdb10": + version "2.0.3-cdb10" + resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/90d537028bb8af8a35e7a40c46493066dd8a76b3" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.10" d3-queue "^2.0.2" sphericalmercator "1.0.x" @@ -35,11 +35,11 @@ abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" -accepts@~1.3.4: - version "1.3.4" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f" +accepts@~1.3.5: + version "1.3.5" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.5.tgz#eb777df6011723a3b14e8a72c0805c8e86746bd2" dependencies: - mime-types "~2.1.16" + mime-types "~2.1.18" negotiator "0.6.1" ajv@^4.9.1: @@ -74,6 +74,10 @@ ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" +ansi-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" + ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" @@ -143,7 +147,7 @@ balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" -basic-auth@^2.0.0: +basic-auth@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.0.tgz#015db3f353e02e56377755f962742e8981e7bbba" dependencies: @@ -165,7 +169,7 @@ block-stream@*: dependencies: inherits "~2.0.0" -body-parser@1.18.2, body-parser@^1.18.2: +body-parser@1.18.2: version "1.18.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.2.tgz#87678a19d84b47d859b83199bd59bce222b10454" dependencies: @@ -180,6 +184,21 @@ body-parser@1.18.2, body-parser@^1.18.2: raw-body "2.3.2" type-is "~1.6.15" +body-parser@1.18.3: + version "1.18.3" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.3.tgz#5b292198ffdd553b3a0f20ded0592b956955c8b4" + dependencies: + bytes "3.0.0" + content-type "~1.0.4" + debug "2.6.9" + depd "~1.1.2" + http-errors "~1.6.3" + iconv-lite "0.4.23" + on-finished "~2.3.0" + qs "6.5.2" + raw-body "2.3.3" + type-is "~1.6.16" + boom@2.x.x: version "2.10.1" resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f" @@ -238,13 +257,17 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.61.8: - version "0.61.8" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.8.tgz#75669c6c14791a93433e79a8892298e88cb0fce2" +camelcase@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd" + +camshaft@0.61.10: + version "0.61.10" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.10.tgz#9055bbc577dbd87d38a0e712bf6157afd7704dca" dependencies: async "^1.5.2" bunyan "1.8.1" - cartodb-psql "^0.10.1" + cartodb-psql "0.11.0" debug "^3.1.0" dot "^1.0.3" request "2.85.0" @@ -267,13 +290,13 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" +carto@cartodb/carto#master: + version "0.15.1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/31abb8bee02df605521247b0223d508320f7d4c8" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" - underscore "~1.6.0" + underscore "1.8.3" "carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" @@ -289,12 +312,12 @@ cartocolor@4.0.0: dependencies: colorbrewer "1.0.0" -cartodb-psql@0.10.2, cartodb-psql@^0.10.1: - version "0.10.2" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.2.tgz#8c505066e4a635cfa0ee4c603769c83f6e2187dd" +cartodb-psql@0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.11.0.tgz#6b4eae0876ee56944a61fe5f4acc6a8b0b11233f" dependencies: debug "^3.1.0" - pg cartodb/node-postgres#6.1.6-cdb1 + pg CartoDB/node-postgres#6.4.2-cdb1 underscore "~1.6.0" cartodb-query-tables@0.3.0: @@ -320,13 +343,16 @@ center-align@^0.1.1: align-text "^0.1.3" lazy-cache "^1.0.3" -"chai@>=1.9.2 <4.0.0": - version "3.5.0" - resolved "https://registry.yarnpkg.com/chai/-/chai-3.5.0.tgz#4d02637b067fe958bdbfdd3a40ec56fef7373247" +chai@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chai/-/chai-4.1.2.tgz#0f64584ba642f0f2ace2806279f4f06ca23ad73c" dependencies: assertion-error "^1.0.1" - deep-eql "^0.1.3" - type-detect "^1.0.0" + check-error "^1.0.1" + deep-eql "^3.0.0" + get-func-name "^2.0.0" + pathval "^1.0.0" + type-detect "^4.0.0" chalk@^1.1.3: version "1.1.3" @@ -338,6 +364,14 @@ chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" +check-error@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" + +chownr@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181" + chroma-js@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.1.1.tgz#9bb9434959336ece75700aaadfeedc71806d8c05" @@ -365,6 +399,14 @@ cliui@^3.2.0: strip-ansi "^3.0.1" wrap-ansi "^2.0.0" +cliui@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-4.1.0.tgz#348422dbe82d800b3022eef4f6ac10bf2e4d1b49" + dependencies: + string-width "^2.1.1" + strip-ansi "^4.0.0" + wrap-ansi "^2.0.0" + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -423,6 +465,14 @@ core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" +cross-spawn@^5.0.1: + version "5.1.0" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-5.1.0.tgz#e8bd0efee58fcff6f8f94510a0a554bbfa235449" + dependencies: + lru-cache "^4.0.1" + shebang-command "^1.2.0" + which "^1.2.9" + cryptiles@2.x.x: version "2.0.5" resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8" @@ -439,6 +489,10 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" +d3@3.5.17: + version "3.5.17" + resolved "https://registry.yarnpkg.com/d3/-/d3-3.5.17.tgz#bc46748004378b21a360c9fc7cf5231790762fb8" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -449,25 +503,25 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b" +debug@2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: - ms "0.7.2" + ms "0.7.1" -debug@2.6.9, debug@^2.2.0: +debug@2.6.8: + version "2.6.8" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" + dependencies: + ms "2.0.0" + +debug@2.6.9, debug@^2.1.2, debug@^2.2.0: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: ms "2.0.0" -debug@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" - dependencies: - ms "2.0.0" - -debug@^3.1.0, debug@~3.1.0: +debug@3.1.0, debug@^3.1.0, debug@~3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: @@ -477,11 +531,11 @@ decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" -deep-eql@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2" +deep-eql@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df" dependencies: - type-detect "0.1.1" + type-detect "^4.0.0" deep-equal@^1.0.0: version "1.0.1" @@ -507,7 +561,7 @@ depd@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" -depd@~1.1.1: +depd@~1.1.1, depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -551,7 +605,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@1.1.2, dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -575,7 +633,7 @@ ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" -encodeurl@~1.0.1: +encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -636,15 +694,27 @@ etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" +execa@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777" + dependencies: + cross-spawn "^5.0.1" + get-stream "^3.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + exit@0.1.2, exit@0.1.x: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" -express@~4.16.0: - version "4.16.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c" +express@4.16.3: + version "4.16.3" + resolved "https://registry.yarnpkg.com/express/-/express-4.16.3.tgz#6af8a502350db3246ecc4becf6b5a34d22f7ed53" dependencies: - accepts "~1.3.4" + accepts "~1.3.5" array-flatten "1.1.1" body-parser "1.18.2" content-disposition "0.5.2" @@ -652,26 +722,26 @@ express@~4.16.0: cookie "0.3.1" cookie-signature "1.0.6" debug "2.6.9" - depd "~1.1.1" - encodeurl "~1.0.1" + depd "~1.1.2" + encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "1.1.0" + finalhandler "1.1.1" fresh "0.5.2" merge-descriptors "1.0.1" methods "~1.1.2" on-finished "~2.3.0" parseurl "~1.3.2" path-to-regexp "0.1.7" - proxy-addr "~2.0.2" + proxy-addr "~2.0.3" qs "6.5.1" range-parser "~1.2.0" safe-buffer "5.1.1" - send "0.16.1" - serve-static "1.13.1" + send "0.16.2" + serve-static "1.13.2" setprototypeof "1.1.0" - statuses "~1.3.1" - type-is "~1.6.15" + statuses "~1.4.0" + type-is "~1.6.16" utils-merge "1.0.1" vary "~1.1.2" @@ -695,22 +765,22 @@ fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" -fastly-purge@~1.0.1: +fastly-purge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/fastly-purge/-/fastly-purge-1.0.1.tgz#3bdfe9ea1d0fbf2a65712f2f5fe2eca63fcb5960" dependencies: request "^2.55.0" -finalhandler@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.0.tgz#ce0b6855b45853e791b2fcc680046d88253dd7f5" +finalhandler@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.1.tgz#eebf4ed840079c83f4249038c9d703008301b105" dependencies: debug "2.6.9" - encodeurl "~1.0.1" + encodeurl "~1.0.2" escape-html "~1.0.3" on-finished "~2.3.0" parseurl "~1.3.2" - statuses "~1.3.1" + statuses "~1.4.0" unpipe "~1.0.0" find-up@^1.0.0: @@ -720,6 +790,12 @@ find-up@^1.0.0: path-exists "^2.0.0" pinkie-promise "^2.0.0" +find-up@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" + dependencies: + locate-path "^2.0.0" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -748,6 +824,12 @@ fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" +fs-minipass@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.5.tgz#06c277218454ec288df77ada54a03b8702aacb9d" + dependencies: + minipass "^2.2.1" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -805,6 +887,14 @@ get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" +get-func-name@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41" + +get-stream@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -822,6 +912,17 @@ glob@7.1.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@7.1.2, glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + glob@^5.0.15: version "5.0.15" resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" @@ -842,17 +943,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1, glob@^7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -943,6 +1033,10 @@ hawk@~6.0.2: hoek "4.x.x" sntp "2.x.x" +he@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" + hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -981,6 +1075,15 @@ http-errors@1.6.2, http-errors@~1.6.2: setprototypeof "1.0.3" statuses ">= 1.3.1 < 2" +http-errors@1.6.3, http-errors@~1.6.3: + version "1.6.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.3.tgz#8b55680bb4be283a0b5bf4ea2e38580be1d9320d" + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.0" + statuses ">= 1.4.0 < 2" + http-signature@~1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf" @@ -1005,6 +1108,18 @@ iconv-lite@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" +iconv-lite@0.4.23, iconv-lite@^0.4.4: + version "0.4.23" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" + dependencies: + safer-buffer ">= 2.1.2 < 3" + +ignore-walk@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.1.tgz#a83e62e7d272ac0e3b551aaa82831a19b69f82f8" + dependencies: + minimatch "^3.0.4" + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -1024,9 +1139,9 @@ invert-kv@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" -ipaddr.js@1.5.2: - version "1.5.2" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.5.2.tgz#d4b505bde9946987ccf0fc58d9010ff9607e3fa0" +ipaddr.js@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.6.0.tgz#e3fa357b773da619f26e95f049d055c72796f86b" is-arrayish@^0.2.1: version "0.2.1" @@ -1048,6 +1163,14 @@ is-fullwidth-code-point@^1.0.0: dependencies: number-is-nan "^1.0.0" +is-fullwidth-code-point@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" + +is-stream@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" + is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -1072,7 +1195,7 @@ isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" -istanbul@~0.4.3: +istanbul@0.4.5: version "0.4.5" resolved "https://registry.yarnpkg.com/istanbul/-/istanbul-0.4.5.tgz#65c7d73d4c4da84d4f3ac310b918fb0b8033733b" dependencies: @@ -1110,7 +1233,7 @@ jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" -jshint@~2.9.4: +jshint@2.9.5: version "2.9.5" resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.5.tgz#1e7252915ce681b40827ee14248c46d34e9aa62c" dependencies: @@ -1137,7 +1260,7 @@ json-stable-stringify@^1.0.1: dependencies: jsonify "~0.0.0" -json-stringify-safe@~5.0.1: +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" @@ -1191,6 +1314,13 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" +locate-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" + dependencies: + p-locate "^2.0.0" + path-exists "^3.0.0" + lodash._baseassign@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz#8c38a099500f215ad09e59f1722fd0c52bfe0a4e" @@ -1214,7 +1344,7 @@ lodash._isiterateecall@^3.0.0: version "3.0.9" resolved "https://registry.yarnpkg.com/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz#5203ad7ba425fae842460e696db9cf3e6aac057c" -lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.1.0, lodash.assign@^4.2.0: +lodash.assign@^4.0.3, lodash.assign@^4.0.6: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" @@ -1242,14 +1372,14 @@ lodash.keys@^3.0.0: lodash.isarguments "^3.0.0" lodash.isarray "^3.0.0" -lodash@2.4.1: - version "2.4.1" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.1.tgz#5b7723034dda4d262e5a46fb2c58d7cc22f71420" - lodash@3.7.x: version "3.7.0" resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.7.0.tgz#3678bd8ab995057c07ade836ed2ef087da811d45" +lodash@^4.17.5: + version "4.17.10" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7" + lodash@^4.5.1: version "4.17.5" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511" @@ -1267,11 +1397,14 @@ longest@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" -lru-cache@2.6.5: - version "2.6.5" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.6.5.tgz#e56d6354148ede8d7707b58d143220fd08df0fd5" +lru-cache@4.1.3, lru-cache@^4.0.1: + version "4.1.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c" + dependencies: + pseudomap "^1.0.2" + yallist "^2.1.2" -lzma@~2.3.2: +lzma@2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/lzma/-/lzma-2.3.2.tgz#3783b24858b9c0e747a0df3cbf1fb5fcaa92c441" @@ -1292,7 +1425,7 @@ mapnik-reference@~8.5.3: dependencies: semver "^5.1.0" -"mapnik-vector-tile@github:cartodb/mapnik-vector-tile#v1.6.1-carto.1": +mapnik-vector-tile@cartodb/mapnik-vector-tile#v1.6.1-carto.1: version "1.6.1-carto.1" resolved "https://codeload.github.com/cartodb/mapnik-vector-tile/tar.gz/0111f7117946179d62ec7a6eba2f4e9fb355d05e" @@ -1300,6 +1433,12 @@ media-typer@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" +mem@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/mem/-/mem-1.1.0.tgz#5edd52b485ca1d900fe64895505399a0dfa45f76" + dependencies: + mimic-fn "^1.0.0" + merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -1327,12 +1466,22 @@ mime-db@~1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17, mime-types@~2.1.7: +mime-db@~1.33.0: + version "1.33.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.33.0.tgz#a3492050a5cb9b63450541e39d9788d2272783db" + +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.7: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: mime-db "~1.30.0" +mime-types@~2.1.18: + version "2.1.18" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.18.tgz#6f323f60a83d11146f831ff11fd66e2fe5503bb8" + dependencies: + mime-db "~1.33.0" + mime@1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" @@ -1345,6 +1494,10 @@ mime@~1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" +mimic-fn@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" + "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -1363,35 +1516,49 @@ minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" +minipass@^2.2.1, minipass@^2.2.4: + version "2.3.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.3.0.tgz#2e11b1c46df7fe7f1afbe9a490280add21ffe384" + dependencies: + safe-buffer "^5.1.1" + yallist "^3.0.0" + +minizlib@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.1.0.tgz#11e13658ce46bc3a70a267aac58359d1e0c29ceb" + dependencies: + minipass "^2.2.1" + mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: minimist "0.0.8" -mocha@~3.4.1: - version "3.4.2" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" +mocha@3.5.3: + version "3.5.3" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.5.3.tgz#1e0480fe36d2da5858d1eb6acc38418b26eaa20d" dependencies: browser-stdout "1.3.0" commander "2.9.0" - debug "2.6.0" + debug "2.6.8" diff "3.2.0" escape-string-regexp "1.0.5" glob "7.1.1" growl "1.9.2" + he "1.1.1" json3 "3.3.2" lodash.create "3.1.1" mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6, moment@~2.18.1: - version "2.18.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" +moment@2.22.1, moment@^2.10.6: + version "2.22.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.22.1.tgz#529a2e9bf973f259c9643d237fda84de3a26e8ad" -ms@0.7.2: - version "0.7.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" +ms@0.7.1: + version "0.7.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" ms@2.0.0: version "2.0.0" @@ -1405,11 +1572,11 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@2.10.0: +nan@2.10.0, nan@^2.0.8: version "2.10.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0: +nan@^2.3.4, nan@^2.4.0: version "2.8.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" @@ -1429,35 +1596,46 @@ ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" +needle@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/needle/-/needle-2.2.1.tgz#b5e325bd3aae8c2678902fa296f729455d1d3a7d" + dependencies: + debug "^2.1.2" + iconv-lite "^0.4.4" + sax "^1.2.4" + negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" -nock@~2.11.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/nock/-/nock-2.11.0.tgz#2554cc09f0c3077100aae185b88d030bad950fcd" +nock@9.2.6: + version "9.2.6" + resolved "https://registry.yarnpkg.com/nock/-/nock-9.2.6.tgz#496ddb2c32e6d0848cda1b3ea51fbe054d4454d3" dependencies: - chai ">=1.9.2 <4.0.0" - debug "^1.0.4" + chai "^4.1.2" + debug "^3.1.0" deep-equal "^1.0.0" - lodash "2.4.1" + json-stringify-safe "^5.0.1" + lodash "^4.17.5" mkdirp "^0.5.0" - propagate "0.3.x" + propagate "^1.0.0" + qs "^6.5.1" + semver "^5.5.0" -node-pre-gyp@0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.7.0.tgz#55aeffbaed93b50d0a4657d469198cd80ac9df36" +node-pre-gyp@0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.10.0.tgz#6e4ef5bb5c5203c6552448828c852c40111aac46" dependencies: detect-libc "^1.0.2" mkdirp "^0.5.1" + needle "^2.2.0" nopt "^4.0.1" + npm-packlist "^1.1.6" npmlog "^4.0.2" rc "^1.1.7" - request "2.83.0" rimraf "^2.6.1" semver "^5.3.0" - tar "^2.2.1" - tar-pack "^3.4.0" + tar "^4" node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: version "0.6.39" @@ -1475,9 +1653,9 @@ node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: tar "^2.2.1" tar-pack "^3.4.0" -node-statsd@~0.0.7: - version "0.0.7" - resolved "https://registry.yarnpkg.com/node-statsd/-/node-statsd-0.0.7.tgz#96d4bbd21dff2d798b3374a3e9329cfecce3afa8" +node-statsd@0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/node-statsd/-/node-statsd-0.1.1.tgz#27a59348763d0af7a037ac2a031fef3f051013d3" nopt@3.x: version "3.0.6" @@ -1501,6 +1679,23 @@ normalize-package-data@^2.3.2: semver "2 || 3 || 4 || 5" validate-npm-package-license "^3.0.1" +npm-bundled@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.3.tgz#7e71703d973af3370a9591bafe3a63aca0be2308" + +npm-packlist@^1.1.6: + version "1.1.10" + resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.1.10.tgz#1039db9e985727e464df066f4cf0ab6ef85c398a" + dependencies: + ignore-walk "^3.0.1" + npm-bundled "^1.0.1" + +npm-run-path@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f" + dependencies: + path-key "^2.0.0" + npmlog@^4.0.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" @@ -1532,7 +1727,7 @@ on-finished@~2.3.0: dependencies: ee-first "1.1.1" -on-headers@^1.0.1: +on-headers@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.1.tgz#928f5d0f470d49342651ea6794b0857c100693f7" @@ -1570,6 +1765,14 @@ os-locale@^1.4.0: dependencies: lcid "^1.0.0" +os-locale@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-2.1.0.tgz#42bc2900a6b5b8bd17376c8e882b65afccf24bf2" + dependencies: + execa "^0.7.0" + lcid "^1.0.0" + mem "^1.1.0" + os-tmpdir@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" @@ -1581,9 +1784,29 @@ osenv@^0.1.4: os-homedir "^1.0.0" os-tmpdir "^1.0.0" -packet-reader@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.2.0.tgz#819df4d010b82d5ea5671f8a1a3acf039bcd7700" +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + +p-limit@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.2.0.tgz#0e92b6bedcb59f022c13d0f1949dc82d15909f1c" + dependencies: + p-try "^1.0.0" + +p-locate@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" + dependencies: + p-limit "^1.1.0" + +p-try@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" + +packet-reader@0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.3.1.tgz#cd62e60af8d7fea8a705ec4ff990871c46871f27" parse-json@^2.2.0: version "2.2.0" @@ -1601,10 +1824,18 @@ path-exists@^2.0.0: dependencies: pinkie-promise "^2.0.0" +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" +path-key@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -1617,6 +1848,10 @@ path-type@^1.0.0: pify "^2.0.0" pinkie-promise "^2.0.0" +pathval@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.0.tgz#b942e6d4bde653005ef6b71361def8727d0645e0" + performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" @@ -1650,20 +1885,20 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: - version "6.1.6" - resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" +"pg@github:CartoDB/node-postgres#6.4.2-cdb1": + version "6.4.2" + resolved "https://codeload.github.com/CartoDB/node-postgres/tar.gz/449fac1d6da711ffcc6694ae3c89f85244f48bdc" dependencies: buffer-writer "1.0.1" js-string-escape "1.0.1" - packet-reader "0.2.0" + packet-reader "0.3.1" pg-connection-string "0.1.3" pg-pool "1.*" pg-types "1.*" - pgpass "1.x" + pgpass "1.*" semver "4.3.2" -pgpass@1.x: +pgpass@1.*: version "1.0.2" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" dependencies: @@ -1750,20 +1985,24 @@ progress-stream@~0.5.x: speedometer "~0.1.2" through2 "~0.2.3" -propagate@0.3.x: - version "0.3.1" - resolved "https://registry.yarnpkg.com/propagate/-/propagate-0.3.1.tgz#e3a84404a7ece820dd6bbea9f6d924e3135ae09c" +propagate@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-1.0.0.tgz#00c2daeedda20e87e3782b344adba1cddd6ad709" protozero@1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.5.1.tgz#5a27df6fb6e1ed743f510812ae76c082f5b16638" -proxy-addr@~2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.2.tgz#6571504f47bb988ec8180253f85dd7e14952bdec" +proxy-addr@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.3.tgz#355f262505a621646b3130a728eb647e22055341" dependencies: forwarded "~0.1.2" - ipaddr.js "1.5.2" + ipaddr.js "1.6.0" + +pseudomap@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" punycode@^1.4.1: version "1.4.1" @@ -1773,10 +2012,18 @@ qs@6.5.1, qs@~6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" +qs@6.5.2, qs@^6.5.1: + version "6.5.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" + qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" +queue-async@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.1.0.tgz#d5f88e32fd01fee4f6d946d21d17abd16ac65f35" + queue-async@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" @@ -1794,6 +2041,15 @@ raw-body@2.3.2: iconv-lite "0.4.19" unpipe "1.0.0" +raw-body@2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.3.tgz#1b324ece6b5706e153855bc1148c65bb7f6ea0c3" + dependencies: + bytes "3.0.0" + http-errors "1.6.3" + iconv-lite "0.4.23" + unpipe "1.0.0" + rc@^1.1.7: version "1.2.5" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.5.tgz#275cd687f6e3b36cc756baa26dfee80a790301fd" @@ -1904,9 +2160,9 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@2.83.0, request@2.x, request@^2.55.0: - version "2.83.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" +request@2.85.0: + version "2.85.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -1931,9 +2187,34 @@ request@2.83.0, request@2.x, request@^2.55.0: tunnel-agent "^0.6.0" uuid "^3.1.0" -request@2.85.0: - version "2.85.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" +request@2.87.0: + version "2.87.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.87.0.tgz#32f00235cd08d482b4d0d68db93a829c0ed5756e" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + tough-cookie "~2.3.3" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + +request@2.x, request@^2.55.0: + version "2.83.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -1992,11 +2273,23 @@ safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, s version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" -safe-json-stringify@~1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" +safe-buffer@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: +safe-json-stringify@~1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz#bd2b6dad1ebafab3c24672a395527f01804b7e19" + +"safer-buffer@>= 2.1.2 < 3": + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + +sax@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -2004,6 +2297,10 @@ semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" +semver@5.5.0, semver@^5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" + semver@~4.3.3: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" @@ -2012,14 +2309,14 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -send@0.16.1: - version "0.16.1" - resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" +send@0.16.2: + version "0.16.2" + resolved "https://registry.yarnpkg.com/send/-/send-0.16.2.tgz#6ecca1e0f8c156d141597559848df64730a6bbc1" dependencies: debug "2.6.9" - depd "~1.1.1" + depd "~1.1.2" destroy "~1.0.4" - encodeurl "~1.0.1" + encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" @@ -2028,16 +2325,16 @@ send@0.16.1: ms "2.0.0" on-finished "~2.3.0" range-parser "~1.2.0" - statuses "~1.3.1" + statuses "~1.4.0" -serve-static@1.13.1: - version "1.13.1" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.1.tgz#4c57d53404a761d8f2e7c1e8a18a47dbf278a719" +serve-static@1.13.2: + version "1.13.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.2.tgz#095e8472fd5b46237db50ce486a43f4b86c6cec1" dependencies: - encodeurl "~1.0.1" + encodeurl "~1.0.2" escape-html "~1.0.3" parseurl "~1.3.2" - send "0.16.1" + send "0.16.2" set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" @@ -2051,6 +2348,16 @@ setprototypeof@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" +shebang-command@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" + dependencies: + shebang-regex "^1.0.0" + +shebang-regex@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" + shelljs@0.3.x: version "0.3.0" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" @@ -2059,6 +2366,10 @@ signal-exit@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" +simple-statistics@^0.9.0: + version "0.9.2" + resolved "https://registry.yarnpkg.com/simple-statistics/-/simple-statistics-0.9.2.tgz#3e35cb10308fc76963aa4ee7252e6a6ea10d28e4" + single-line-log@~0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/single-line-log/-/single-line-log-0.3.1.tgz#a7ad6507f218ce5dfe16c4bf2d659246419e7a06" @@ -2109,7 +2420,11 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" @@ -2150,23 +2465,35 @@ sshpk@^1.7.0: jsbn "~0.1.0" tweetnacl "~0.14.0" -"statuses@>= 1.3.1 < 2", statuses@~1.3.1: +"statuses@>= 1.3.1 < 2": version "1.3.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" -step-profiler@~0.3.0: +"statuses@>= 1.4.0 < 2": + version "1.5.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" + +statuses@~1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087" + +step-profiler@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/step-profiler/-/step-profiler-0.3.0.tgz#841368ce44f2330c862edd5ec60e7fbec148f0b3" dependencies: debug "^2.2.0" -step@~0.0.5, step@~0.0.6: +step@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" + +step@~0.0.5: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" -strftime@~0.8.2: - version "0.8.4" - resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" +strftime@0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.10.0.tgz#b3f0fa419295202a5a289f6d6be9f4909a617193" string-width@^1.0.1, string-width@^1.0.2: version "1.0.2" @@ -2176,6 +2503,13 @@ string-width@^1.0.1, string-width@^1.0.2: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" +string-width@^2.0.0, string-width@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" + dependencies: + is-fullwidth-code-point "^2.0.0" + strip-ansi "^4.0.0" + string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" @@ -2196,12 +2530,22 @@ strip-ansi@^3.0.0, strip-ansi@^3.0.1: dependencies: ansi-regex "^2.0.0" +strip-ansi@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" + dependencies: + ansi-regex "^3.0.0" + strip-bom@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" dependencies: is-utf8 "^0.2.0" +strip-eof@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" + strip-json-comments@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-1.0.4.tgz#1e15fbcac97d3ee99bf2d73b4c656b082bbafb91" @@ -2247,6 +2591,18 @@ tar@^2.2.1: fstream "^1.0.2" inherits "2" +tar@^4: + version "4.4.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.2.tgz#60685211ba46b38847b1ae7ee1a24d744a2cd462" + dependencies: + chownr "^1.0.1" + fs-minipass "^1.2.5" + minipass "^2.2.4" + minizlib "^1.1.0" + mkdirp "^0.5.0" + safe-buffer "^5.1.2" + yallist "^3.0.2" + through2@~0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/through2/-/through2-0.2.3.tgz#eb3284da4ea311b6cc8ace3653748a52abf25a3f" @@ -2258,30 +2614,33 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb12": - version "0.6.18-cdb12" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/e3c0d80e604ca4a5dfad648ee6f6fb355d415a88" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb14": + version "0.6.18-cdb14" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/6d06f728833d3e34d1adcd05567b3f4379f547bb" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.10" generic-pool "~2.4.0" mime "~1.6.0" sphericalmercator "~1.0.4" step "~0.0.5" -tilelive@5.12.2: - version "5.12.2" - resolved "https://registry.yarnpkg.com/tilelive/-/tilelive-5.12.2.tgz#9ede3aefded4b04e0be32e0dfc6a1ce01247eef7" +tilelive@5.12.3: + version "5.12.3" + resolved "https://registry.yarnpkg.com/tilelive/-/tilelive-5.12.3.tgz#9c8c770e1194aa8d353d9a1d03147404307cdebd" dependencies: minimist "~0.2.0" progress-stream "~0.5.x" queue-async "~1.0.7" sphericalmercator "~1.0.1" -torque.js@~2.11.0: - version "2.11.5" - resolved "https://registry.yarnpkg.com/torque.js/-/torque.js-2.11.5.tgz#8fe8dfb53fb76497a3c427fcb7ba18e427c1ef6b" +torque.js@2.16.2: + version "2.16.2" + resolved "https://registry.yarnpkg.com/torque.js/-/torque.js-2.16.2.tgz#fadeaeb2adce161eca03ea2fe16ef0e80aa02967" dependencies: - carto CartoDB/carto#0.15.1-cdb1 + carto cartodb/carto#master + d3 "3.5.17" + turbo-carto "0.19.0" + turf-jenks "~1.0.1" tough-cookie@~2.3.0, tough-cookie@~2.3.3: version "2.3.3" @@ -2295,6 +2654,17 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +turbo-carto@0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.0.tgz#83fb1932acd42acb426312eef216b5f6ac34708e" + dependencies: + cartocolor "4.0.0" + colorbrewer "1.0.0" + debug "2.2.0" + es6-promise "3.1.2" + postcss "5.0.19" + postcss-value-parser "3.3.0" + turbo-carto@0.20.2: version "0.20.2" resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.2.tgz#2b737597a65c2918432f70ea414f12fbec2b6a6f" @@ -2306,6 +2676,12 @@ turbo-carto@0.20.2: postcss "5.0.19" postcss-value-parser "3.3.0" +turf-jenks@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/turf-jenks/-/turf-jenks-1.0.1.tgz#5c72aea13a716eff2f0023a72cfa549a00202d7f" + dependencies: + simple-statistics "^0.9.0" + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" @@ -2316,13 +2692,9 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" -type-detect@0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-0.1.1.tgz#0ba5ec2a885640e470ea4e8505971900dac58822" - -type-detect@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" +type-detect@^4.0.0: + version "4.0.8" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" type-is@~1.6.15: version "1.6.15" @@ -2331,6 +2703,13 @@ type-is@~1.6.15: media-typer "0.3.0" mime-types "~2.1.15" +type-is@~1.6.16: + version "1.6.16" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.16.tgz#f89ce341541c672b25ee7ae3c73dee3b2be50194" + dependencies: + media-typer "0.3.0" + mime-types "~2.1.18" + uglify-js@^2.6: version "2.8.29" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" @@ -2348,6 +2727,10 @@ uid-number@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" +underscore@1.6.0, underscore@~1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" + underscore@1.8.2: version "1.8.2" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.2.tgz#64df2eb590899de950782f3735190ba42ebf311d" @@ -2356,10 +2739,6 @@ underscore@1.8.3: version "1.8.3" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" -underscore@~1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" - unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -2399,7 +2778,11 @@ which-module@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" -which@^1.1.1: +which-module@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" + +which@^1.1.1, which@^1.2.9: version "1.3.0" resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" dependencies: @@ -2419,29 +2802,29 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.7.1: - version "4.7.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.1.tgz#2b92753b2f6e97b239e15e1576ec312cc7dfeb13" +windshaft@4.8.1: + version "4.8.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.8.1.tgz#7c1ef21e4f885643f6347bc5871274c533fce206" dependencies: - "@carto/mapnik" "3.6.2-carto.8" - "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb7 - abaculus cartodb/abaculus#2.0.3-cdb8 + "@carto/mapnik" "3.6.2-carto.10" + "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb9 + abaculus cartodb/abaculus#2.0.3-cdb10 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "^0.10.1" - debug "^3.1.0" - dot "~1.0.2" + cartodb-psql "0.11.0" + debug "3.1.0" + dot "1.1.2" grainstore "1.9.0" - queue-async "~1.0.7" - redis-mpool "^0.5.0" - request "2.85.0" - semver "~5.0.3" - sphericalmercator "1.0.4" - step "~0.0.6" - tilelive "5.12.2" - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb12 - torque.js "~2.11.0" - underscore "~1.6.0" + queue-async "1.1.0" + redis-mpool "0.5.0" + request "2.87.0" + semver "5.5.0" + sphericalmercator "1.0.5" + step "1.0.0" + tilelive "5.12.3" + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb14 + torque.js "2.16.2" + underscore "1.6.0" wordwrap@0.0.2: version "0.0.2" @@ -2480,6 +2863,14 @@ y18n@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" +yallist@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" + +yallist@^3.0.0, yallist@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" + yargs-parser@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4" @@ -2487,12 +2878,28 @@ yargs-parser@^2.4.1: camelcase "^3.0.0" lodash.assign "^4.0.6" -yargs-parser@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-3.2.0.tgz#5081355d19d9d0c8c5d81ada908cb4e6d186664f" +yargs-parser@^9.0.2: + version "9.0.2" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-9.0.2.tgz#9ccf6a43460fe4ed40a9bb68f48d43b8a68cc077" dependencies: - camelcase "^3.0.0" - lodash.assign "^4.1.0" + camelcase "^4.1.0" + +yargs@11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-11.1.0.tgz#90b869934ed6e871115ea2ff58b03f4724ed2d77" + dependencies: + cliui "^4.0.0" + decamelize "^1.1.1" + find-up "^2.1.0" + get-caller-file "^1.0.1" + os-locale "^2.0.0" + require-directory "^2.1.1" + require-main-filename "^1.0.1" + set-blocking "^2.0.0" + string-width "^2.0.0" + which-module "^2.0.0" + y18n "^3.2.1" + yargs-parser "^9.0.2" yargs@^4.2.0: version "4.8.1" @@ -2522,25 +2929,6 @@ yargs@~3.10.0: decamelize "^1.0.0" window-size "0.1.0" -yargs@~5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-5.0.0.tgz#3355144977d05757dbb86d6e38ec056123b3a66e" - dependencies: - cliui "^3.2.0" - decamelize "^1.1.1" - get-caller-file "^1.0.1" - lodash.assign "^4.2.0" - os-locale "^1.4.0" - read-pkg-up "^1.0.1" - require-directory "^2.1.1" - require-main-filename "^1.0.1" - set-blocking "^2.0.0" - string-width "^1.0.2" - which-module "^1.0.0" - window-size "^0.2.0" - y18n "^3.2.1" - yargs-parser "^3.2.0" - zipfile@~0.5.5: version "0.5.11" resolved "https://registry.yarnpkg.com/zipfile/-/zipfile-0.5.11.tgz#a46c1b76706138fb9bcf83890193caec679f685e"