diff --git a/.travis.yml b/.travis.yml index fefef779..fc228792 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,8 @@ -sudo: required -dist: trusty - -services: - - docker - -before_install: - - docker pull cartoimages/windshaft-testing - -script: - - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh - -language: generic - +jobs: + include: + - sudo: required + services: + - docker + language: generic + before_install: docker pull carto/nodejs-xenial-pg101:latest + script: npm run docker-test -- 10.15.1 # Node.js version diff --git a/HOWTO_RELEASE b/HOWTO_RELEASE index a2927b60..0a398d01 100644 --- a/HOWTO_RELEASE +++ b/HOWTO_RELEASE @@ -1,8 +1,8 @@ 1. Test (make clean all check), fix if broken before proceeding -2. Ensure proper version in package.json +2. Ensure proper version in package.json and package-lock.json 3. Ensure NEWS section exists for the new version, review it, add release date -4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}` -5. Commit package.json, yarn.lock, NEWS +4. If there are modified dependencies in package.json, update them with `npm upgrade {{package_name}}@{{version}}` +5. Commit package.json, package-lock.json, NEWS 6. git tag -a Major.Minor.Patch # use NEWS section as content 7. Stub NEWS/package for next version diff --git a/INSTALL.md b/INSTALL.md index 45654b0b..f5102f3b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,35 +1,23 @@ -# Installing Windshaft-CartoDB # +# Installing Windshaft-CartoDB -## Requirements ## -Make sure that you have the requirements needed. These are +## Requirements -- Core - - Node.js >=6.9.x - - yarn >=0.27.5 <1.0.0 - - PostgreSQL >8.3.x, PostGIS >1.5.x - - Redis >2.4.0 (http://www.redis.io) - - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik). - - Windshaft: check [Windshaft dependencies and installation notes](https://github.com/CartoDB/Windshaft#dependencies) - - libcairo2-dev, libpango1.0-dev, libjpeg8-dev and libgif-dev for server side canvas support +Make sure that you have the requirements needed. These are: -- For cache control (optional) - - CartoDB 0.9.5+ (for `CDB_QueryTables`) - - Varnish (http://www.varnish-cache.org) +- Node 10.x +- npm 6.x +- PostgreSQL >= 10.0 +- PostGIS >= 2.4 +- CARTO Postgres Extension >= 0.24.1 +- Redis >= 4 +- libcairo2-dev, libpango1.0-dev, libjpeg8-dev and libgif-dev for server side canvas support +- C++11 (to build internal dependencies if needed) -On Ubuntu 14.04 the dependencies can be installed with +### Optional -```shell -sudo apt-get update -sudo apt-get install -y make g++ pkg-config git-core \ - libgif-dev libjpeg-dev libcairo2-dev \ - libhiredis-dev redis-server \ - nodejs nodejs-legacy npm \ - postgresql-9.3-postgis-2.1 postgresql-plpython-9.3 postgresql-server-dev-9.3 -``` +- Varnish (http://www.varnish-cache.org) -On Ubuntu 12.04 the [cartodb/cairo PPA](https://launchpad.net/~cartodb/+archive/ubuntu/cairo) may be useful. - -## PostGIS setup ## +## PostGIS setup A `template_postgis` database is expected. One can be set up with @@ -38,16 +26,16 @@ createdb --owner postgres --template template0 template_postgis psql -d template_postgis -c 'CREATE EXTENSION postgis;' ``` -## Build/install ## +## Build/install To fetch and build all node-based dependencies, run: -``` -yarn +```shell +npm install ``` -Note that the ```yarn``` step will populate the node_modules/ +Note that the ```npm``` step will populate the node_modules/ directory with modules, some of which being compiled on demand. If you happen to have startup errors you may need to force rebuilding those modules. At any time just wipe out the node_modules/ directory and run -```yarn``` again. +```npm``` again. diff --git a/NEWS.md b/NEWS.md index 64305a89..1e592442 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,191 @@ # Changelog -## 5.3.2 -Released yyyy-mm-dd - - Upgrades Windshaft to 4.5.3 +## 7.0.1 +Released 2019-mm-dd + + +## 7.0.0 +Released 2019-02-22 + +Breaking changes: +- Drop support for Node.js 6 +- Drop support for npm 3 +- Stop supporting `yarn.lock` +- Drop support for Postgres 9.5 +- Drop support for PosGIS 2.2 +- Drop support for Redis 3 + +Announcements: +- In configuration, set `clipByBox2d` to true by default +- Update docs: compatible Node.js and npm versions +- Report fine-grained Garbage Collector stats +- Adding Authorization to Access-Control-Allow-Headers (https://github.com/CartoDB/CartoDB-SQL-API/issues/534) +- Update deps: + - windshaft@4.13.1: Upgrade tilelive-mapnik to version 0.6.18-cdb18 + - camshaft@0.63.4: Improve error message for exceeded batch SQL API payload size: add suggestions about what the user can do about it. +- Update dev deps: + - jshint@2.9.7 + - mocha@5.2.0 +- Be able to customize max waiting workers parameter +- Handle 'max waitingClients count exceeded' error as "429, You are over platform's limits" + +## 6.5.1 +Released 2018-12-26 + +Bug Fixes: +- Update carto-package.json + +## 6.5.0 +Released 2018-12-26 + +New features +- Suport Node.js 10 +- Configure travis to run docker tests against Node.js 6 & 10 versions +- Aggregation time dimensions +- Update sample configurations to use PostGIS to generate MVT's by default (as in production) +- Upgrades Windshaft to [4.12.1](https://github.com/CartoDB/Windshaft/blob/4.12.1/NEWS.md#version-4121) + - `pg-mvt`: Use `query-rewriter` to compose the query to render a MVT tile. If not defined, it will use a Default Query Rewriter. + - `pg-mvt`: Fix bug while building query and there is no columns defined for the layer. + - `pg-mvt`: Accept trailing semicolon in input queries. + - `Renderer Cache Entry`: Do not throw errors for integrity checks. + - Fix bug when releasing the renderer cache entry in some scenarios. + - Upgrade grainstore to [1.10.0](https://github.com/CartoDB/grainstore/releases/tag/1.10.0) +- Upgrade cartodb-redis to [2.1.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/2.1.0) +- Upgrade cartodb-query-tables to [0.4.0](https://github.com/CartoDB/node-cartodb-query-tables/releases/tag/0.4.0) +- Upgrade cartodb-psql to [0.13.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.13.1) +- Upgrade turbo-carto to [0.21.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.21.0) +- Upgrade camshaft to [0.63.1](https://github.com/CartoDB/camshaft/releases/tag/0.63.1) +- Upgrade redis-mpool to [0.7.0](https://github.com/CartoDB/node-redis-mpool/releases/tag/0.7.0) + +Bug Fixes: +- Prevent from uncaught exception: Range filter Error from camshaft when getting analysis query. +- Make all modules to use strict mode semantics. + +## 6.4.0 +Released 2018-09-24 + +- Upgrades Camshaft to [0.62.3](https://github.com/CartoDB/camshaft/releases/tag/0.61.11): + - Build query from node's cache to compute output columns when building analysis + - Adds metadata columns for street level geocoding +- Remove use of `step` module to handle asynchronous code, now it's defined as development dependency. +- Bug Fixes: (#1020) + - Fix bug in date-wrapper regarding columns with spaces + - Fix bug in aggregation-query regarding columns with spaces +- Upgrades Windshaft to [4.10.0](https://github.com/CartoDB/Windshaft/blob/4.10.0/NEWS.md#version-4100) + - `pg-mvt`: + - Now matches the behaviour of the `mapnik` renderer for MVTs. + - Removed undocummented filtering by `layer.options.columns`. + - Implement timeout in getTile. + - Several bugfixes. + - Dependency updates: Fixed a bug in Mapnik MVT renderer and cleanup in `tilelive-mapnik`. + - [MapConfig 1.8.0 released](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.8.0.md) with new options for MVTs: + - Add **`vector_extent`** option in MapConfig to setup the layer extent. + - Add **`vector_simplify_extent`** option in MapConfig to configure the simplification process. + - Remove use of `step` module to handle asynchronous code, now it's defined as development dependency. + +## 6.3.0 +Released 2018-07-26 + +- Upgrades Camshaft to [0.62.1](https://github.com/CartoDB/camshaft/releases/tag/0.62.1): + - Support for batch street-level geocoding. [0.62.1](https://github.com/CartoDB/camshaft/releases/tag/0.62.1) + +## 6.2.0 +Released 2018-07-20 + +Notice: +- This release changes the way that authentication works internally. You'll need to run `bundle exec rake carto:api_key:create_default` in your development environment to keep working. + +New features: +- CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5 +- Upgrades Windshaft to [4.8.3](https://github.com/CartoDB/Windshaft/blob/4.8.3/NEWS.md#version-483) which includes: + - Update internal deps. + - A fix in mapnik-vector-tile to avoid grouping together properties with the same value but a different type. + - Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible). + - MVT: Disable simplify_distance to avoid multiple simplifications. + - Fix a bug with zero length lines not being rendered when using the marker symbolizer. + - Reduce size of npm package + - Omit attributes validation in layers with aggregation to avoid potentially long instantiation times +- Upgrades Camshaft to [0.61.11](https://github.com/CartoDB/camshaft/releases/tag/0.61.11): + - Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses. [0.61.10](https://github.com/CartoDB/camshaft/releases/tag/0.61.10) + - Quote name columns when performing trade area analysis to avoid Syntax Errors. [0.61.11](https://github.com/CartoDB/camshaft/releases/tag/0.61.11) +- Update other deps: + - body-parser: 1.18.3 + - cartodb-psql: 0.11.0 + - cartodb-redis: 2.0.1 + - dot: 1.1.2 + - express: 4.16.3 + - lru-cache: 4.1.3 + - node-statsd: 0.1.1, + - queue-async: 1.1.0 + - request: 2.87.0 + - semver: 5.5.0 + - step: 1.0.0 + - turbo-carto: 0.20.4 + - yargs: 11.1.0 +- Update devel deps: + - istanbul: 0.4.5 + - jshint: 2.9.5 + - mocha: 3.5.3 + - moment: 2.22.1 + - nock: 9.2.6 + - strftime: 0.10.0 +- Optional instantiation metadata stats (https://github.com/CartoDB/Windshaft-cartodb/pull/952) +- Experimental dates_as_numbers support +- Tiles base urls with api key + +Bug Fixes: +- Validates tile coordinates (z/x/y) from request params to be a valid integer value. +- Static maps fails for unsupported formats +- Handling errors extracting the column type on dataviews +- Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens +- Fix numeric histogram bounds when `start` and `end` are specified (#991) +- Static maps filters correctly if `layer` option is passed in the url. +- Aggregation doesn't return out-of-tile, partially aggregated clusters +- Aggregation was not accurate for high zoom, far away from the origin tiles + +Announcements: + * Improve error message when the DB query is over the user's limits + +## 6.1.0 +Released 2018-04-16 + +New features: +- Aggreation filters +- Upgrades Windshaft to 4.7.0, which includes @carto/mapnik v3.6.2-carto.7 with improvements to metrics and markers caching. It also adds an option to disable the markers symbolizer caches in mapnik. + +Bug Fixes: +- Non-default aggregation selected the wrong columns (e.g. for vector tiles) +- Aggregation dimensions with alias where broken +- cartodb_id was not unique accross aggregated vector tiles + +## 6.0.0 +Released 2018-03-19 +Backward incompatible changes: + - Needs Redis v4 + +New features: + - Upgrades camshaft to 0.61.8 + - Upgrades cartodb-redis to 1.0.0 + - Rate limit feature (disabled by default) + - Fixes for tests with PG11 + +## 5.4.0 +Released 2018-03-15 + - Upgrades Windshaft to 4.5.7 ([Mapnik top metrics](https://github.com/CartoDB/Windshaft/pull/597), [AttributesBackend allows multiple features if all the attributes are the same](https://github.com/CartoDB/Windshaft/pull/602)) + - Implemented middleware to authorize users via new Api Key system + - Keep the old authorization system as fallback + - Aggregation widget: Remove NULL categories in 'count' aggregations too + - Update request to 2.85.0 + - Update camshaft to 0.61.4 (Fixes for AOI and Merge analyses) + - Update windshaft to 4.6.0, which in turn updates @carto/mapnik to 3.6.2-carto.4 and related dependencies. It brings in a cache for rasterized symbols. See https://github.com/CartoDB/node-mapnik/blob/v3.6.2-carto/CHANGELOG.carto.md#362-carto4 + - PostGIS: Variables in postgis SQL queries must now additionally be wrapped in `!` (refs [#29](https://github.com/CartoDB/mapnik/issues/29), [mapnik/#3618](https://github.com/mapnik/mapnik/pull/3618)): +```sql +-- Before +SELECT ... WHERE trait = @variable + +-- Now +SELECT ... WHERE trait = !@variable! +``` ## 5.3.1 Released 2018-02-13 diff --git a/README.md b/README.md index bdc20f57..79a17cbd 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,10 @@ Upgrading Checkout your commit/branch. If you need to reinstall dependencies (you can check [NEWS](NEWS.md)) do the following: +```sh +$ rm -rf node_modules +$ npm install ``` -rm -rf node_modules; yarn -``` - -Run ---- ``` node app.js @@ -71,12 +69,12 @@ See [CONTRIBUTING.md](CONTRIBUTING.md). ### Developing with a custom windshaft version If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is -to use `yarn link`. You can read more about it at [yarn-link: Symlink a package folder](https://yarnpkg.com/en/docs/cli/link). +to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link.html). **Quick start**: ```shell -~/windshaft-directory $ yarn -~/windshaft-directory $ yarn link -~/windshaft-cartodb-directory $ yarn link windshaft +~/windshaft-directory $ npm install +~/windshaft-directory $ npm link +~/windshaft-cartodb-directory $ npm link windshaft ``` diff --git a/app.js b/app.js index 1dd465f0..f4de15fe 100755 --- a/app.js +++ b/app.js @@ -1,3 +1,5 @@ +'use strict'; + var http = require('http'); var https = require('https'); var path = require('path'); @@ -100,8 +102,6 @@ if ( global.environment.log_filename ) { global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); -global.environment.api_hostname = require('os').hostname().split('.')[0]; - // Include cartodb_windshaft only _after_ the "global" variable is set // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 var cartodbWindshaft = require('./lib/cartodb/server'); @@ -128,6 +128,40 @@ listener.on('listening', function() { ); }); +function getCPUUsage (oldUsage) { + let usage; + + if (oldUsage && oldUsage._start) { + usage = Object.assign({}, process.cpuUsage(oldUsage._start.cpuUsage)); + usage.time = Date.now() - oldUsage._start.time; + } else { + usage = Object.assign({}, process.cpuUsage()); + usage.time = process.uptime() * 1000; // s to ms + } + + usage.percent = (usage.system + usage.user) / (usage.time * 10); + + Object.defineProperty(usage, '_start', { + value: { + cpuUsage: process.cpuUsage(), + time: Date.now() + } + }); + + return usage; +} + +let previousCPUUsage = getCPUUsage(); +setInterval(function cpuUsageMetrics () { + const CPUUsage = getCPUUsage(previousCPUUsage); + + Object.keys(CPUUsage).forEach(property => { + global.statsClient.gauge(`windshaft.cpu.${property}`, CPUUsage[property]); + }); + + previousCPUUsage = CPUUsage; +}, 5000); + setInterval(function() { var memoryUsage = process.memoryUsage(); Object.keys(memoryUsage).forEach(function(k) { @@ -154,9 +188,46 @@ if (global.gc) { if (gcInterval > 0) { setInterval(function gcForcedCycle() { - var start = Date.now(); global.gc(); - global.statsClient.timing('windshaft.gc', Date.now() - start); }, gcInterval); } } + +const gcStats = require('gc-stats')(); + +gcStats.on('stats', function ({ pauseMS, gctype }) { + global.statsClient.timing('windshaft.gc', pauseMS); + global.statsClient.timing(`windshaft.gctype.${getGCTypeValue(gctype)}`, pauseMS); +}); + +function getGCTypeValue (type) { + // 1: Scavenge (minor GC) + // 2: Mark/Sweep/Compact (major GC) + // 4: Incremental marking + // 8: Weak/Phantom callback processing + // 15: All + let value; + + switch (type) { + case 1: + value = 'Scavenge'; + break; + case 2: + value = 'MarkSweepCompact'; + break; + case 4: + value = 'IncrementalMarking'; + break; + case 8: + value = 'ProcessWeakCallbacks'; + break; + case 15: + value = 'All'; + break; + default: + value = 'Unkown'; + break; + } + + return value; +} diff --git a/carto-package.json b/carto-package.json new file mode 100644 index 00000000..e65b0f8f --- /dev/null +++ b/carto-package.json @@ -0,0 +1,17 @@ +{ + "name": "carto_windshaft", + "current_version": { + "requires": { + "node": "^10.15.1", + "npm": "^6.4.1", + "mapnik": "==3.0.15.9", + "crankshaft": "~0.8.1" + }, + "works_with": { + "redis": ">=4.0.0", + "postgresql": ">=10.0.0", + "postgis": ">=2.4.4.5", + "carto_postgresql_ext": ">=0.24.1" + } + } +} diff --git a/config/environments/development.js.example b/config/environments/development.js.example index df2273c7..ed0aaf58 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -15,26 +15,59 @@ var config = { // Base URLs for the APIs // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', @@ -54,12 +87,12 @@ var config = { // idle socket timeout, in milliseconds ,socket_timeout: 600000 ,enable_cors: true - ,cache_enabled: false + ,cache_enabled: true ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' // If log_filename is given logs will be written // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - ,log_filename: undefined + ,log_filename: 'logs/node-windshaft.log' // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>' @@ -67,39 +100,25 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS - type: "postgis", user: "publicuser", password: "public", host: '127.0.0.1', port: 5432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - /* experimental - geometry_field: "the_geom", - extent: "-180,-90,180,90", - srid: 4326, - */ - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' ,statsd: { host: 'localhost', port: 8125, - prefix: 'dev.', + prefix: 'dev.', // could be hostname, better not containing dots cacheDns: true // support all allowed node-statsd options } @@ -108,18 +127,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { - //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, - //PostGIS 2.4 is required for this to work - //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + //If enabled, MVTs will be generated with PostGIS directly + //If disabled, MVTs will be generated with Mapnik MVT + usePostGIS: true }, mapnik: { // The size of the pool of internal mapnik backend @@ -128,6 +138,10 @@ var config = { // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, + // The maximum number of waiting clients of the pool of internal mapnik backend + // This maximum number is per mapnik renderer created in Windshaft's RendererFactory + poolMaxWaitingClients: 64, + // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML. // This will prevent blocking the main thread. useCartocssWorkers: false, @@ -171,7 +185,31 @@ var config = { // SQL queries will be wrapped with ST_ClipByBox2D // Returning the portion of a geometry falling within a rectangle // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + clipByBox2d: true, + + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500, + twkb_encoding: true + }, limits: { // Time in milliseconds a render request can take before it fails, some notes: @@ -186,31 +224,17 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, // Require metrics to the renderer - metrics: false + metrics: false, + + // Options for markers attributes, ellipses and images caches + markers_symbolizer_caches: { + disabled: false + } }, http: { timeout: 2000, // the timeout in ms for a http tile request @@ -226,16 +250,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { @@ -256,7 +271,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: '/tmp/analysis.log' + filename: 'logs/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. @@ -326,6 +341,12 @@ var config = { // X-Tiler-Profile header containing elapsed timing for various // steps taken for producing the response. ,useProfiler:true + ,serverMetadata: { + cdn_url: { + http: undefined, + https: undefined + } + } // Settings for the health check available at /health ,health: { enabled: false, @@ -343,7 +364,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 9f7436df..bc34db81 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -15,26 +15,59 @@ var config = { // Base URLs for the APIs // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map', @@ -67,26 +100,18 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - max_size: 500 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -102,18 +127,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { - //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, - //PostGIS 2.4 is required for this to work - //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + //If enabled, MVTs will be generated with PostGIS directly + //If disabled, MVTs will be generated with Mapnik MVT + usePostGIS: true }, mapnik: { // The size of the pool of internal mapnik backend @@ -122,6 +138,10 @@ var config = { // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, + // The maximum number of waiting clients of the pool of internal mapnik backend + // This maximum number is per mapnik renderer created in Windshaft's RendererFactory + poolMaxWaitingClients: 64, + // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML. // This will prevent blocking the main thread. useCartocssWorkers: false, @@ -165,7 +185,31 @@ var config = { // SQL queries will be wrapped with ST_ClipByBox2D // Returning the portion of a geometry falling within a rectangle // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + clipByBox2d: true, + + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500, + twkb_encoding: true + }, limits: { // Time in milliseconds a render request can take before it fails, some notes: @@ -180,33 +224,17 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, // Require metrics to the renderer - metrics: false + metrics: false, + // Options for markers attributes, ellipses and images caches + markers_symbolizer_caches: { + disabled: false + } }, http: { timeout: 2000, // the timeout in ms for a http tile request @@ -222,16 +250,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { @@ -252,7 +271,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: 'logs/analysis.log' + filename: 'logs/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. @@ -345,7 +364,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: false + layerStats: false, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 66c5a88c..41f8e281 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -15,29 +15,62 @@ var config = { // Base URLs for the APIs // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/maps/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "/api/v1/maps" is the the new API, - // "/tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { - http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map', https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' } @@ -55,7 +88,7 @@ var config = { ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: true - ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type] (:res[X-Tiler-Errors])' + ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' // If log_filename is given logs will be written // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal @@ -67,33 +100,25 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' ,statsd: { host: 'localhost', port: 8125, - prefix: 'stage.:host.', + prefix: 'stage.:host.', // could be hostname, better not containing dots cacheDns: true // support all allowed node-statsd options } @@ -102,18 +127,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { - //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, - //PostGIS 2.4 is required for this to work - //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + //If enabled, MVTs will be generated with PostGIS directly + //If disabled, MVTs will be generated with Mapnik MVT + usePostGIS: true }, mapnik: { // The size of the pool of internal mapnik backend @@ -122,6 +138,10 @@ var config = { // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, + // The maximum number of waiting clients of the pool of internal mapnik backend + // This maximum number is per mapnik renderer created in Windshaft's RendererFactory + poolMaxWaitingClients: 64, + // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML. // This will prevent blocking the main thread. useCartocssWorkers: false, @@ -165,7 +185,31 @@ var config = { // SQL queries will be wrapped with ST_ClipByBox2D // Returning the portion of a geometry falling within a rectangle // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + clipByBox2d: true, + + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500, + twkb_encoding: true + }, limits: { // Time in milliseconds a render request can take before it fails, some notes: @@ -180,33 +224,17 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, // Require metrics to the renderer - metrics: false + metrics: false, + // Options for markers attributes, ellipses and images caches + markers_symbolizer_caches: { + disabled: false + } }, http: { timeout: 2000, // the timeout in ms for a http tile request @@ -222,16 +250,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { @@ -252,7 +271,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: 'logs/analysis.log' + filename: 'logs/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. @@ -330,7 +349,7 @@ var config = { } // Settings for the health check available at /health ,health: { - enabled: false, + enabled: true, username: 'localhost', z: 0, x: 0, @@ -345,7 +364,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 0c98c4b6..05c5e94c 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -16,28 +16,62 @@ var config = { // Base URLs for the APIs // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { - http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map' + http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + https: 'https://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map' } // Maximum number of connections for one process @@ -54,11 +88,11 @@ var config = { ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: false - ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' + ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' // If log_filename is given logs will be written // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - //,log_filename: 'logs/node-windshaft.log' + ,log_filename: '/tmp/node-windshaft.log' // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>' @@ -66,33 +100,25 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: 'test_windshaft_cartodb_user_<%= user_id %>_pass' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "test_windshaft_publicuser", password: "public", host: '127.0.0.1', port: 5432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } - ,mapnik_version: '' + ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' ,statsd: { host: 'localhost', port: 8125, - prefix: 'test.:host.', + prefix: 'test.:host.', // could be hostname, better not containing dots cacheDns: true // support all allowed node-statsd options } @@ -101,18 +127,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { - //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, - //PostGIS 2.4 is required for this to work - //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + //If enabled, MVTs will be generated with PostGIS directly + //If disabled, MVTs will be generated with Mapnik MVT + usePostGIS: true }, mapnik: { // The size of the pool of internal mapnik backend @@ -121,6 +138,10 @@ var config = { // Important: check the configuration of uv_threadpool_size to use suitable value poolSize: 8, + // The maximum number of waiting clients of the pool of internal mapnik backend + // This maximum number is per mapnik renderer created in Windshaft's RendererFactory + poolMaxWaitingClients: 64, + // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML. // This will prevent blocking the main thread. useCartocssWorkers: false, @@ -164,7 +185,31 @@ var config = { // SQL queries will be wrapped with ST_ClipByBox2D // Returning the portion of a geometry falling within a rectangle // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + clipByBox2d: true, + + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500, + twkb_encoding: false + }, limits: { // Time in milliseconds a render request can take before it fails, some notes: @@ -179,32 +224,17 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer - 'cache-features': true, + 'cache-features': false, // Require metrics to the renderer - metrics: false + metrics: false, + + // Options for markers attributes, ellipses and images caches + markers_symbolizer_caches: { + disabled: false + } }, http: { timeout: 2000, // the timeout in ms for a http tile request @@ -222,16 +252,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { @@ -252,7 +273,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: 'node-windshaft.log' + filename: '/tmp/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. @@ -322,6 +343,12 @@ var config = { // X-Tiler-Profile header containing elapsed timing for various // steps taken for producing the response. ,useProfiler:true + ,serverMetadata: { + cdn_url: { + http: undefined, + https: undefined + } + } // Settings for the health check available at /health ,health: { enabled: false, @@ -339,7 +366,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/docker-bash.sh b/docker-bash.sh new file mode 100755 index 00000000..5666ea1d --- /dev/null +++ b/docker-bash.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo "*********************" +echo "To install Node.js, run:" +echo "/src/nodejs-install.sh" +echo "Use NODEJS_VERSION env var to select the Node.js version" +echo " " +echo "To start postgres, run:" +echo "/etc/init.d/postgresql start" +echo "*********************" +echo " " + +docker run -it -v `pwd`:/srv carto/nodejs-xenial-pg101:latest bash diff --git a/docker-test.sh b/docker-test.sh old mode 100644 new mode 100755 index 8cc8c093..44821126 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,11 +1,4 @@ -export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres +#!/bin/bash -npm install -g yarn@0.27.5 -yarn - - /etc/init.d/postgresql start - -createdb template_postgis && createuser publicuser -psql -c "CREATE EXTENSION postgis" template_postgis - -POSTGIS_VERSION=2.4 npm test \ No newline at end of file +docker run -e "NODEJS_VERSION=${1}" -v `pwd`:/srv carto/nodejs-xenial-pg101:latest bash run_tests_docker.sh && \ + docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v diff --git a/docker/Dockerfile-nodejs-xenial-pg101:latest b/docker/Dockerfile-nodejs-xenial-pg101:latest new file mode 100644 index 00000000..f61e60dc --- /dev/null +++ b/docker/Dockerfile-nodejs-xenial-pg101:latest @@ -0,0 +1,92 @@ +FROM ubuntu:xenial + +# Use UTF8 to avoid encoding problems with pgsql +ENV LANG C.UTF-8 +ENV NPROCS 1 +ENV JOBS 1 +ENV CXX g++-4.9 +ENV PGUSER postgres + +# Add external repos +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + curl \ + software-properties-common \ + locales \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && add-apt-repository -y ppa:cartodb/postgresql-10 \ + && add-apt-repository -y ppa:cartodb/gis \ + && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash \ + && . ~/.nvm/nvm.sh \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 + +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + g++-4.9 \ + gcc-4.9 \ + git \ + libcairo2-dev \ + libgdal-dev \ + libgdal1i \ + libgdal20 \ + libgeos-dev \ + libgif-dev \ + libjpeg8-dev \ + libjson-c-dev \ + libpango1.0-dev \ + libpixman-1-dev \ + libproj-dev \ + libprotobuf-c-dev \ + libxml2-dev \ + gdal-bin \ + make \ + nodejs \ + protobuf-c-compiler \ + pkg-config \ + wget \ + zip \ + postgresql-10 \ + postgresql-10-plproxy \ + postgis=2.4.4.6+carto-1 \ + postgresql-10-postgis-2.4=2.4.4.6+carto-1 \ + postgresql-10-postgis-2.4-scripts=2.4.4.6+carto-1 \ + postgresql-10-postgis-scripts=2.4.4.6+carto-1 \ + postgresql-client-10 \ + postgresql-client-common \ + postgresql-common \ + postgresql-contrib \ + postgresql-plpython-10 \ + postgresql-server-dev-10 \ + && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \ + && tar xvzf redis-4.0.8.tar.gz \ + && cd redis-4.0.8 \ + && make \ + && make install \ + && cd .. \ + && rm redis-4.0.8.tar.gz \ + && rm -R redis-4.0.8 \ + && apt-get purge -y wget protobuf-c-compiler \ + && apt-get autoremove -y + +# Configure PostgreSQL +RUN set -ex \ + && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \ + && echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && /etc/init.d/postgresql start \ + && createdb template_postgis \ + && createuser publicuser \ + && psql -c "CREATE EXTENSION postgis" template_postgis \ + && /etc/init.d/postgresql stop + +WORKDIR /srv +EXPOSE 5858 + +COPY ./scripts/nodejs-install.sh /src/nodejs-install.sh +RUN chmod 777 /src/nodejs-install.sh +CMD /src/nodejs-install.sh + diff --git a/docker/Dockerfile-nodejs10-xenial-pg101:postgis-2.4.4.5 b/docker/Dockerfile-nodejs10-xenial-pg101:postgis-2.4.4.5 new file mode 100644 index 00000000..f0c3f738 --- /dev/null +++ b/docker/Dockerfile-nodejs10-xenial-pg101:postgis-2.4.4.5 @@ -0,0 +1,88 @@ +FROM ubuntu:xenial + +# Use UTF8 to avoid encoding problems with pgsql +ENV LANG C.UTF-8 +ENV NPROCS 1 +ENV JOBS 1 +ENV CXX g++-4.9 +ENV PGUSER postgres + +# Add external repos +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + curl \ + software-properties-common \ + locales \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && add-apt-repository -y ppa:cartodb/postgresql-10 \ + && add-apt-repository -y ppa:cartodb/gis \ + && curl -sL https://deb.nodesource.com/setup_10.x | bash \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 + +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + g++-4.9 \ + gcc-4.9 \ + git \ + libcairo2-dev \ + libgdal-dev \ + libgdal1i \ + libgdal20 \ + libgeos-dev \ + libgif-dev \ + libjpeg8-dev \ + libjson-c-dev \ + libpango1.0-dev \ + libpixman-1-dev \ + libproj-dev \ + libprotobuf-c-dev \ + libxml2-dev \ + gdal-bin \ + make \ + nodejs \ + protobuf-c-compiler \ + pkg-config \ + wget \ + zip \ + postgresql-10 \ + postgresql-10-plproxy \ + postgis=2.4.4.5+carto-1 \ + postgresql-10-postgis-2.4=2.4.4.5+carto-1 \ + postgresql-10-postgis-2.4-scripts=2.4.4.5+carto-1 \ + postgresql-10-postgis-scripts=2.4.4.5+carto-1 \ + postgresql-client-10 \ + postgresql-client-common \ + postgresql-common \ + postgresql-contrib \ + postgresql-plpython-10 \ + postgresql-server-dev-10 \ + && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \ + && tar xvzf redis-4.0.8.tar.gz \ + && cd redis-4.0.8 \ + && make \ + && make install \ + && cd .. \ + && rm redis-4.0.8.tar.gz \ + && rm -R redis-4.0.8 \ + && apt-get purge -y wget protobuf-c-compiler \ + && apt-get autoremove -y + +# Configure PostgreSQL +RUN set -ex \ + && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \ + && echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && /etc/init.d/postgresql start \ + && createdb template_postgis \ + && createuser publicuser \ + && psql -c "CREATE EXTENSION postgis" template_postgis \ + && /etc/init.d/postgresql stop + +WORKDIR /srv +EXPOSE 5858 + +CMD /etc/init.d/postgresql start diff --git a/docker/Dockerfile-nodejs6-xenial-pg101:latest b/docker/Dockerfile-nodejs6-xenial-pg101:latest new file mode 100644 index 00000000..d1d70589 --- /dev/null +++ b/docker/Dockerfile-nodejs6-xenial-pg101:latest @@ -0,0 +1,89 @@ +FROM ubuntu:xenial + +# Use UTF8 to avoid encoding problems with pgsql +ENV LANG C.UTF-8 +ENV NPROCS 1 +ENV JOBS 1 +ENV CXX g++-4.9 +ENV PGUSER postgres + +# Add external repos +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + curl \ + software-properties-common \ + locales \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && add-apt-repository -y ppa:cartodb/postgresql-10 \ + && add-apt-repository -y ppa:cartodb/gis \ + && curl -sL https://deb.nodesource.com/setup_6.x | bash \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 + +# Install dependencies and PostGIS 2.4 from sources +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + g++-4.9 \ + gcc-4.9 \ + git \ + libcairo2-dev \ + libgdal-dev \ + libgdal1i \ + libgdal20 \ + libgeos-dev \ + libgif-dev \ + libjpeg8-dev \ + libjson-c-dev \ + libpango1.0-dev \ + libpixman-1-dev \ + libproj-dev \ + libprotobuf-c-dev \ + libxml2-dev \ + gdal-bin \ + make \ + nodejs \ + protobuf-c-compiler \ + pkg-config \ + wget \ + zip \ + postgresql-10 \ + postgresql-10-plproxy \ + postgresql-10-postgis-2.4 \ + postgresql-10-postgis-2.4-scripts \ + postgresql-10-postgis-scripts \ + postgresql-client-10 \ + postgresql-client-common \ + postgresql-common \ + postgresql-contrib \ + postgresql-plpython-10 \ + postgresql-server-dev-10 \ + postgis \ + && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \ + && tar xvzf redis-4.0.8.tar.gz \ + && cd redis-4.0.8 \ + && make \ + && make install \ + && cd .. \ + && rm redis-4.0.8.tar.gz \ + && rm -R redis-4.0.8 \ + && apt-get purge -y wget protobuf-c-compiler \ + && apt-get autoremove -y + +# Configure PostgreSQL +RUN set -ex \ + && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \ + && echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && /etc/init.d/postgresql start \ + && createdb template_postgis \ + && createuser publicuser \ + && psql -c "CREATE EXTENSION postgis" template_postgis \ + && /etc/init.d/postgresql stop + +WORKDIR /srv +EXPOSE 5858 + +CMD /etc/init.d/postgresql start diff --git a/docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4 b/docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4 new file mode 100644 index 00000000..d1d70589 --- /dev/null +++ b/docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4 @@ -0,0 +1,89 @@ +FROM ubuntu:xenial + +# Use UTF8 to avoid encoding problems with pgsql +ENV LANG C.UTF-8 +ENV NPROCS 1 +ENV JOBS 1 +ENV CXX g++-4.9 +ENV PGUSER postgres + +# Add external repos +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + curl \ + software-properties-common \ + locales \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && add-apt-repository -y ppa:cartodb/postgresql-10 \ + && add-apt-repository -y ppa:cartodb/gis \ + && curl -sL https://deb.nodesource.com/setup_6.x | bash \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 + +# Install dependencies and PostGIS 2.4 from sources +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + g++-4.9 \ + gcc-4.9 \ + git \ + libcairo2-dev \ + libgdal-dev \ + libgdal1i \ + libgdal20 \ + libgeos-dev \ + libgif-dev \ + libjpeg8-dev \ + libjson-c-dev \ + libpango1.0-dev \ + libpixman-1-dev \ + libproj-dev \ + libprotobuf-c-dev \ + libxml2-dev \ + gdal-bin \ + make \ + nodejs \ + protobuf-c-compiler \ + pkg-config \ + wget \ + zip \ + postgresql-10 \ + postgresql-10-plproxy \ + postgresql-10-postgis-2.4 \ + postgresql-10-postgis-2.4-scripts \ + postgresql-10-postgis-scripts \ + postgresql-client-10 \ + postgresql-client-common \ + postgresql-common \ + postgresql-contrib \ + postgresql-plpython-10 \ + postgresql-server-dev-10 \ + postgis \ + && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \ + && tar xvzf redis-4.0.8.tar.gz \ + && cd redis-4.0.8 \ + && make \ + && make install \ + && cd .. \ + && rm redis-4.0.8.tar.gz \ + && rm -R redis-4.0.8 \ + && apt-get purge -y wget protobuf-c-compiler \ + && apt-get autoremove -y + +# Configure PostgreSQL +RUN set -ex \ + && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \ + && echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && /etc/init.d/postgresql start \ + && createdb template_postgis \ + && createuser publicuser \ + && psql -c "CREATE EXTENSION postgis" template_postgis \ + && /etc/init.d/postgresql stop + +WORKDIR /srv +EXPOSE 5858 + +CMD /etc/init.d/postgresql start diff --git a/docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4.4.5 b/docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4.4.5 new file mode 100644 index 00000000..c4d7b091 --- /dev/null +++ b/docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4.4.5 @@ -0,0 +1,88 @@ +FROM ubuntu:xenial + +# Use UTF8 to avoid encoding problems with pgsql +ENV LANG C.UTF-8 +ENV NPROCS 1 +ENV JOBS 1 +ENV CXX g++-4.9 +ENV PGUSER postgres + +# Add external repos +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + curl \ + software-properties-common \ + locales \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && add-apt-repository -y ppa:cartodb/postgresql-10 \ + && add-apt-repository -y ppa:cartodb/gis \ + && curl -sL https://deb.nodesource.com/setup_6.x | bash \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 + +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + g++-4.9 \ + gcc-4.9 \ + git \ + libcairo2-dev \ + libgdal-dev \ + libgdal1i \ + libgdal20 \ + libgeos-dev \ + libgif-dev \ + libjpeg8-dev \ + libjson-c-dev \ + libpango1.0-dev \ + libpixman-1-dev \ + libproj-dev \ + libprotobuf-c-dev \ + libxml2-dev \ + gdal-bin \ + make \ + nodejs \ + protobuf-c-compiler \ + pkg-config \ + wget \ + zip \ + postgresql-10 \ + postgresql-10-plproxy \ + postgis=2.4.4.5+carto-1 \ + postgresql-10-postgis-2.4=2.4.4.5+carto-1 \ + postgresql-10-postgis-2.4-scripts=2.4.4.5+carto-1 \ + postgresql-10-postgis-scripts=2.4.4.5+carto-1 \ + postgresql-client-10 \ + postgresql-client-common \ + postgresql-common \ + postgresql-contrib \ + postgresql-plpython-10 \ + postgresql-server-dev-10 \ + && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \ + && tar xvzf redis-4.0.8.tar.gz \ + && cd redis-4.0.8 \ + && make \ + && make install \ + && cd .. \ + && rm redis-4.0.8.tar.gz \ + && rm -R redis-4.0.8 \ + && apt-get purge -y wget protobuf-c-compiler \ + && apt-get autoremove -y + +# Configure PostgreSQL +RUN set -ex \ + && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \ + && echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \ + && /etc/init.d/postgresql start \ + && createdb template_postgis \ + && createuser publicuser \ + && psql -c "CREATE EXTENSION postgis" template_postgis \ + && /etc/init.d/postgresql stop + +WORKDIR /srv +EXPOSE 5858 + +CMD /etc/init.d/postgresql start diff --git a/docker/reference.md b/docker/reference.md new file mode 100644 index 00000000..31dd26cb --- /dev/null +++ b/docker/reference.md @@ -0,0 +1,23 @@ +After running the tests with docker, you will need Docker installed and the docker image downloaded. + +## Install docker +`sudo apt install docker.io && sudo usermod -aG docker $(whoami)` + +## Download image +`docker pull carto/IMAGE` + +## Carto account +https://hub.docker.com/r/carto/ + +## Update image +- Edit the docker image file with your desired changes +- Build image: + - `docker build -t carto/IMAGE -f docker/DOCKER_FILE docker/` + +- Upload to docker hub: + - Login into docker hub: + - `docker login` + - Create tag: + - `docker tag carto/IMAGE carto/IMAGE` + - Upload: + - `docker push carto/IMAGE` diff --git a/docker/scripts/nodejs-install.sh b/docker/scripts/nodejs-install.sh new file mode 100644 index 00000000..f86775a7 --- /dev/null +++ b/docker/scripts/nodejs-install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + +if [ -z $NODEJS_VERSION ]; then + NODEJS_VERSION="10" + NODEJS_VERSION_OPTIONS="--lts" +fi + +nvm install $NODEJS_VERSION $NODEJS_VERSION_OPTIONS +nvm alias default $NODEJS_VERSION +nvm use default diff --git a/docs/MapConfig-Aggregation-extension.md b/docs/MapConfig-Aggregation-extension.md index 850b179a..bf2172f2 100644 --- a/docs/MapConfig-Aggregation-extension.md +++ b/docs/MapConfig-Aggregation-extension.md @@ -29,7 +29,7 @@ The value of this attribute can be `false` to explicitly disable aggregation for // object, defines the columns of the aggregated datasets. Each property corresponds to a columns name and // should contain an object with two properties: "aggregate_function" (one of "sum", "max", "min", "avg", "mode" or "count"), // and "aggregated_column" (the name of a column of the original layer query or "*") - // A column defined as `"_cdb_features_count": {"aggregate_function": "count", aggregated_column: "*"}` + // A column defined as `"_cdb_feature_count": {"aggregate_function": "count", aggregated_column: "*"}` // is always generated in addition to the defined columns. // The column names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used // for aggregated columns, as they correspond to columns always present in the result. diff --git a/docs/aggregation.md b/docs/aggregation.md index 2fb361c3..506e7c77 100644 --- a/docs/aggregation.md +++ b/docs/aggregation.md @@ -10,7 +10,7 @@ Aggregation is available only for point geometries. During aggregation the point When no placement or columns are specified a special default aggregation is performed. -This special mode performs only spatial aggregation (using a grid defined by the requested tile and the resolution, parameter, as all the other cases), and returns a _random_ record from each group (grid cell) with all its columns and an additional `_cdb_features_count` with the number of features in the group. +This special mode performs only spatial aggregation (using a grid defined by the requested tile and the resolution, parameter, as all the other cases), and returns a _random_ record from each group (grid cell) with all its columns and an additional `_cdb_feature_count` with the number of features in the group. Regarding the randomness of the sample: currently we use the row with the minimum `cartodb_id` value in each group. @@ -18,7 +18,7 @@ The rationale behind having this special aggregation with all the original colum ### User defined aggregations -When either a explicit placement or columns are requested we no longer use the special, query; we use one determined by the placement (which will default to "centroid"), and it will have as columns only the aggregated columns specified, in addition to `_cdb_features_count`, which is always present. +When either a explicit placement or columns are requested we no longer use the special, query; we use one determined by the placement (which will default to "centroid"), and it will have as columns only the aggregated columns specified, in addition to `_cdb_feature_count`, which is always present. We might decide in the future to allow sampling column values for any of the different placement modes. @@ -134,6 +134,10 @@ of the original dataset applying three different aggregate functions. > Note that you can use the original column names as names of the result, but all the result column names must be unique. In particular, the names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used for aggregated columns, as they correspond to columns always present in the result. +#### Limitations: +* The iso text format does not admit `starting` or `count` parameters +* Cyclic units (day of the week, etc.) don't admit `count` or `starting` either. + ### `resolution` Defines the cell-size of the spatial aggregation grid. This is equivalent to the [CartoCSS `-torque-resolution`](https://carto.com/docs/carto-engine/cartocss/properties-for-torque/#-torque-resolution-float) property of Torque maps. @@ -185,3 +189,80 @@ This is the minimum number of (estimated) rows in the dataset (query results) fo ] } ``` + +### `filters` + +Aggregated data can be filtered by imposing filtering conditions on the aggregated columns. + +Each condition is represented by one or more parameters: + +* `{ "equal": V }` selects an specific value of the aggregated column. +* `{ "not_equal": V }` selects values different from the one specified. +* `{ "in": [v1, v2, v3] }` selects any value from a list. +* `{ "not_in": [v1, v2, v3] }` selects any value not in a list. +* `{ "less_than": v }` selects values strictly less than the one given. +* `{ "less_than_or_equal_to": v }` selects values less than or equal to the one given. +* `{ "greater_than": v }` selects values strictly greater than the one given. +* `{ "greater_than_or_equal_to": v }` selects values greater than or equal to the one given. + +One of the *less* conditions can be combined with one of the *greater* conditions to select a range of values, for example: +* `{ "greater_than": v1, "less_than": v2 }` +* `{ "greater_than_or_equal_to": v1, "less_than": v2 }` +* `{ "greater_than": v1, "less_than_or_equal_to": v2 }` +* `{ "greater_than_or_equal_to": v1, "less_than_or_equal_to": v2 }` + +For a given column, multiple conditions can be passed in an array; the conditions will logically ORed (any of the conditions have to be verifid for the value to be selected): + +* `"myvalue": [ { "equal": 10 }, { "less_than": 0 }]` will select values of the column `myvalue` which are equal to 10 **or** less than 0. + +In addition, the filters applied to different columns are logically combined with AND (all the conditions have to be satisfied for an element to be selected); for example with the following `filters` parameter we'll select aggregated records which have a `total_value` > 100 **and** a category equal to "a". + +```json +{ + "total_value": { "greater_than": 100 }, + "category": { "equal": "a" } +} +``` + +Note that the filtered columns have to be defined with the `columns` parameter, except for `_cdb_feature_count`, which is always implicitly defined and can be filtered too. + +#### Example + +```json +{ + "version": "1.7.0", + "extent": [-20037508.5, -20037508.5, 20037508.5, 20037508.5], + "srid": 3857, + "maxzoom": 18, + "minzoom": 3, + "layers": [ + { + "type": "mapnik", + "options": { + "sql": "select * from table", + "cartocss": "#table { marker-width: [total]; marker-fill: ramp(value, (red, green, blue), jenks); }", + "cartocss_version": "2.3.0", + "aggregation": { + "placement": "centroid", + "columns": { + "total_value": { + "aggregate_function": "sum", + "aggregated_column": "value" + }, + "category": { + "aggregate_function": "mode", + "aggregated_column": "category" + } + }, + "filters" : { + "total_value": { "greater_than": 100 }, + "category": { "equal": "a" } + }, + "resolution": 2, + "threshold": 500000 + } + } + } + ] +} +``` diff --git a/docs/static_maps_api.md b/docs/static_maps_api.md index 34e4e907..554f892d 100644 --- a/docs/static_maps_api.md +++ b/docs/static_maps_api.md @@ -11,7 +11,7 @@ Begin by instantiating either a Named or Anonymous Map using the `layergroupid t #### Definition ```bash -GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format} +GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format}{{?}extra_options} ``` #### Params @@ -58,6 +58,9 @@ Note: you can see this endpoint as GET /api/v1/map/static/bbox/{token}/{west},{south},{east},{north}/{width}/{height}.{format}` ``` +#### Extra options + * Layer: List of layers to be shown in the image (by default `all`), for example `?layer=0,1`. + ### Named Map #### Definition diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js new file mode 100644 index 00000000..fafc90b3 --- /dev/null +++ b/lib/cartodb/api/api-router.js @@ -0,0 +1,323 @@ +'use strict'; + +const { Router: router } = require('express'); + +const RedisPool = require('redis-mpool'); +const cartodbRedis = require('cartodb-redis'); + +const windshaft = require('windshaft'); + +const PgConnection = require('../backends/pg_connection'); +const AnalysisBackend = require('../backends/analysis'); +const AnalysisStatusBackend = require('../backends/analysis-status'); +const DataviewBackend = require('../backends/dataview'); +const TemplateMaps = require('../backends/template_maps.js'); +const PgQueryRunner = require('../backends/pg_query_runner'); +const StatsBackend = require('../backends/stats'); +const AuthBackend = require('../backends/auth'); + +const UserLimitsBackend = require('../backends/user-limits'); +const OverviewsMetadataBackend = require('../backends/overviews-metadata'); +const FilterStatsApi = require('../backends/filter-stats'); +const TablesExtentBackend = require('../backends/tables-extent'); + +const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables'); +const SurrogateKeysCache = require('../cache/surrogate_keys_cache'); +const VarnishHttpCacheBackend = require('../cache/backend/varnish_http'); +const FastlyCacheBackend = require('../cache/backend/fastly'); +const NamedMapProviderCache = require('../cache/named_map_provider_cache'); +const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); + +const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); +const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter'); +const MapConfigBufferSizeAdapter = require('../models/mapconfig/adapter/mapconfig-buffer-size-adapter'); +const AnalysisMapConfigAdapter = require('../models/mapconfig/adapter/analysis-mapconfig-adapter'); +const MapConfigOverviewsAdapter = require('../models/mapconfig/adapter/mapconfig-overviews-adapter'); +const TurboCartoAdapter = require('../models/mapconfig/adapter/turbo-carto-adapter'); +const DataviewsWidgetsAdapter = require('../models/mapconfig/adapter/dataviews-widgets-adapter'); +const AggregationMapConfigAdapter = require('../models/mapconfig/adapter/aggregation-mapconfig-adapter'); +const MapConfigAdapter = require('../models/mapconfig/adapter'); +const VectorMapConfigAdapter = require('../models/mapconfig/adapter/vector-mapconfig-adapter'); + +const ResourceLocator = require('../models/resource-locator'); +const LayergroupMetadata = require('../utils/layergroup-metadata'); +const RendererStatsReporter = require('../stats/reporter/renderer'); + +const initializeStatusCode = require('./middlewares/initialize-status-code'); +const logger = require('./middlewares/logger'); +const bodyParser = require('body-parser'); +const servedByHostHeader = require('./middlewares/served-by-host-header'); +const stats = require('./middlewares/stats'); +const lzmaMiddleware = require('./middlewares/lzma'); +const cors = require('./middlewares/cors'); +const user = require('./middlewares/user'); +const sendResponse = require('./middlewares/send-response'); +const syntaxError = require('./middlewares/syntax-error'); +const errorMiddleware = require('./middlewares/error-middleware'); + +const MapRouter = require('./map/map-router'); +const TemplateRouter = require('./template/template-router'); + +module.exports = class ApiRouter { + constructor ({ serverOptions, environmentOptions }) { + this.serverOptions = serverOptions; + + const redisOptions = Object.assign({ + name: 'windshaft-server', + unwatchOnRelease: false, + noReadyCheck: true + }, environmentOptions.redis); + + const redisPool = new RedisPool(redisOptions); + + redisPool.on('status', function (status) { + var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.'; + global.statsClient.gauge(keyPrefix + 'count', status.count); + global.statsClient.gauge(keyPrefix + 'unused', status.unused); + global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); + }); + + const metadataBackend = cartodbRedis({ pool: redisPool }); + const pgConnection = new PgConnection(metadataBackend); + + const mapStore = new windshaft.storage.MapStore({ + pool: redisPool, + expire_time: serverOptions.grainstore.default_layergroup_ttl + }); + + const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); + + const rendererCacheOpts = Object.assign({ + ttl: 60000, // 60 seconds TTL by default + statsInterval: 60000 // reports stats every milliseconds defined here + }, serverOptions.renderCache || {}); + + const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); + const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); + rendererStatsReporter.start(); + + const tileBackend = new windshaft.backend.Tile(rendererCache); + const attributesBackend = new windshaft.backend.Attributes(); + const previewBackend = new windshaft.backend.Preview(rendererCache); + const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); + const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + + const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions); + const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends); + const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache }); + + const analysisStatusBackend = new AnalysisStatusBackend(); + const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); + const dataviewBackend = new DataviewBackend(analysisBackend); + const statsBackend = new StatsBackend(); + + const userLimitsBackend = new UserLimitsBackend(metadataBackend, { + limits: { + cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, + render: serverOptions.renderer.mapnik.limits.render || 0, + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled + } + }); + const authBackend = new AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps); + + const layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); + + if (process.env.NODE_ENV === 'test') { + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + } + + const pgQueryRunner = new PgQueryRunner(pgConnection); + const overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); + + const filterStatsBackend = new FilterStatsApi(pgQueryRunner); + const tablesExtentBackend = new TablesExtentBackend(pgQueryRunner); + + const mapConfigAdapter = new MapConfigAdapter( + new MapConfigNamedLayersAdapter(templateMaps, pgConnection), + new MapConfigBufferSizeAdapter(), + new SqlWrapMapConfigAdapter(), + new DataviewsWidgetsAdapter(), + new AnalysisMapConfigAdapter(analysisBackend), + new VectorMapConfigAdapter(pgConnection), + new AggregationMapConfigAdapter(pgConnection), + new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend), + new TurboCartoAdapter() + ); + + const resourceLocator = new ResourceLocator(global.environment); + const layergroupMetadata = new LayergroupMetadata(resourceLocator); + + const namedMapProviderCache = new NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + layergroupAffectedTablesCache + ); + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); + }); + + const collaborators = { + analysisStatusBackend, + attributesBackend, + dataviewBackend, + previewBackend, + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache, + templateMaps, + mapBackend, + metadataBackend, + mapConfigAdapter, + statsBackend, + layergroupMetadata, + namedMapProviderCache, + tablesExtentBackend + }; + + this.mapRouter = new MapRouter({ collaborators }); + this.templateRouter = new TemplateRouter({ collaborators }); + } + + register (app) { + // FIXME: we need a better way to reset cache while running tests + if (process.env.NODE_ENV === 'test') { + app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; + } + + Object.keys(this.serverOptions.routes).forEach(apiVersion => { + const routes = this.serverOptions.routes[apiVersion]; + + const apiRouter = router({ mergeParams: true }); + + apiRouter.use(logger(this.serverOptions)); + apiRouter.use(initializeStatusCode()); + apiRouter.use(bodyParser.json()); + apiRouter.use(servedByHostHeader()); + apiRouter.use(stats({ + enabled: this.serverOptions.useProfiler, + statsClient: global.statsClient + })); + apiRouter.use(lzmaMiddleware()); + apiRouter.use(cors()); + apiRouter.use(user()); + + this.templateRouter.register(apiRouter, routes.template.paths); + this.mapRouter.register(apiRouter, routes.map.paths); + + apiRouter.use(sendResponse()); + apiRouter.use(syntaxError()); + apiRouter.use(errorMiddleware()); + + const apiPaths = routes.paths; + + apiPaths.forEach(path => app.use(path, apiRouter)); + }); + } +}; + + +function createTemplateMaps ({ redisPool, surrogateKeysCache }) { + const templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates + }); + + function invalidateNamedMap (owner, templateName) { + var startTime = Date.now(); + surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { + var logMessage = JSON.stringify({ + username: owner, + type: 'named_map_invalidation', + elapsed: Date.now() - startTime, + error: !!err ? JSON.stringify(err.message) : undefined + }); + if (err) { + global.logger.warn(logMessage); + } else { + global.logger.info(logMessage); + } + }); + } + + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, invalidateNamedMap); + }); + + return templateMaps; +} + +function createSurrogateKeysCacheBackends(serverOptions) { + var cacheBackends = []; + + if (serverOptions.varnish_purge_enabled) { + cacheBackends.push( + new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) + ); + } + + if (serverOptions.fastly && + !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { + cacheBackends.push( + new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) + ); + } + + return cacheBackends; +} + +const timeoutErrorTilePath = __dirname + '/../../../assets/render-timeout-fallback.png'; +const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); + +function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) { + var onTileErrorStrategy; + if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) { + onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { + + function isRenderTimeoutError (err) { + return err.message === 'Render timed out'; + } + + function isDatasourceTimeoutError (err) { + return err.message && err.message.match(/canceling statement due to statement timeout/i); + } + + function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); + } + + function isRasterFormat (format) { + return format === 'png' || format === 'jpg'; + } + + if (isTimeoutError(err) && isRasterFormat(format)) { + return callback(null, timeoutErrorTile, { + 'Content-Type': 'image/png', + }, {}); + } else { + return callback(err, tile, headers, stats); + } + }; + } + + const rendererFactory = new windshaft.renderer.Factory({ + onTileErrorStrategy: onTileErrorStrategy, + mapnik: { + redisPool: redisPool, + grainstore: serverOptions.grainstore, + mapnik: serverOptions.renderer.mapnik + }, + http: serverOptions.renderer.http, + mvt: serverOptions.renderer.mvt, + torque: serverOptions.renderer.torque + }); + + return rendererFactory; +} diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js deleted file mode 100644 index 562614c3..00000000 --- a/lib/cartodb/api/auth_api.js +++ /dev/null @@ -1,138 +0,0 @@ -var assert = require('assert'); -var step = require('step'); - -/** - * - * @param {PgConnection} pgConnection - * @param metadataBackend - * @param {MapStore} mapStore - * @param {TemplateMaps} templateMaps - * @constructor - * @type {AuthApi} - */ -function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) { - this.pgConnection = pgConnection; - this.metadataBackend = metadataBackend; - this.mapStore = mapStore; - this.templateMaps = templateMaps; -} - -module.exports = AuthApi; - -// Check if the user is authorized by a signer -// -// @param res express response object -// @param callback function(err, signed_by) signed_by will be -// null if the request is not signed by anyone -// or will be a string cartodb username otherwise. -// -AuthApi.prototype.authorizedBySigner = function(res, callback) { - if ( ! res.locals.token || ! res.locals.signer ) { - return callback(null, false); // no signer requested - } - - var self = this; - - var layergroup_id = res.locals.token; - var auth_token = res.locals.auth_token; - - this.mapStore.load(layergroup_id, function(err, mapConfig) { - if (err) { - return callback(err); - } - - var authorized = self.templateMaps.isAuthorized(mapConfig.obj().template, auth_token); - - return callback(null, authorized); - }); -}; - -// Check if a request is authorized by api_key -// -// @param user -// @param req express request object -// @param callback function(err, authorized) -// NOTE: authorized is expected to be 0 or 1 (integer) -// -AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) { - var givenKey = req.query.api_key || req.query.map_key; - if ( ! givenKey && req.body ) { - // check also in request body - givenKey = req.body.api_key || req.body.map_key; - } - if ( ! givenKey ) { - return callback(null, 0); // no api key, no authorization... - } - - var self = this; - - step( - function () { - self.metadataBackend.getUserMapKey(user, this); - }, - function checkApiKey(err, val){ - assert.ifError(err); - return val && givenKey === val; - }, - function finish(err, authorized) { - callback(err, authorized); - } - ); -}; - -/** - * Check access authorization - * - * @param req - standard req object. Importantly contains table and host information - * @param res - standard res object. Contains the auth parameters in locals - * @param callback function(err, allowed) is access allowed not? - */ -AuthApi.prototype.authorize = function(req, res, callback) { - var self = this; - var user = res.locals.user; - - step( - function () { - self.authorizedByAPIKey(user, req, this); - }, - function checkApiKey(err, authorized){ - req.profiler.done('authorizedByAPIKey'); - assert.ifError(err); - - // if not authorized by api_key, continue - if (!authorized) { - // not authorized by api_key, check if authorized by signer - return self.authorizedBySigner(res, this); - } - - // authorized by api key, login as the given username and stop - self.pgConnection.setDBAuth(user, res.locals, function(err) { - callback(err, true); // authorized (or error) - }); - }, - function checkSignAuthorized(err, authorized) { - if (err) { - return callback(err); - } - - if ( ! authorized ) { - // request not authorized by signer. - - // if no signer name was given, let dbparams and - // PostgreSQL do the rest. - // - if ( ! res.locals.signer ) { - return callback(null, true); // authorized so far - } - - // if signer name was given, return no authorization - return callback(null, false); - } - - self.pgConnection.setDBAuth(user, res.locals, function(err) { - req.profiler.done('setDBAuth'); - callback(err, true); // authorized (or error) - }); - } - ); -}; diff --git a/lib/cartodb/api/filter_stats_api.js b/lib/cartodb/api/filter_stats_api.js deleted file mode 100644 index 31e90a25..00000000 --- a/lib/cartodb/api/filter_stats_api.js +++ /dev/null @@ -1,59 +0,0 @@ -var _ = require('underscore'); -var step = require('step'); -var AnalysisFilter = require('../models/filter/analysis'); - -function FilterStatsApi(pgQueryRunner) { - this.pgQueryRunner = pgQueryRunner; -} - -module.exports = FilterStatsApi; - -function getEstimatedRows(pgQueryRunner, username, query, callback) { - pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) { - if (err){ - callback(err); - return; - } - var rows; - if ( result_rows[0] && result_rows[0]['QUERY PLAN'] && - result_rows[0]['QUERY PLAN'][0] && result_rows[0]['QUERY PLAN'][0].Plan ) { - rows = result_rows[0]['QUERY PLAN'][0].Plan['Plan Rows']; - } - return callback(null, rows); - }); -} - -FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { - var stats = {}; - var self = this; - step( - function getUnfilteredRows() { - getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this); - }, - function receiveUnfilteredRows(err, rows) { - if (err){ - callback(err); - return; - } - stats.unfiltered_rows = rows; - this(null, rows); - }, - function getFilteredRows() { - if ( filters && !_.isEmpty(filters)) { - var analysisFilter = new AnalysisFilter(filters); - var query = analysisFilter.sql(unfiltered_query); - getEstimatedRows(self.pgQueryRunner, username, query, this); - } else { - this(null, null); - } - }, - function receiveFilteredRows(err, rows) { - if (err){ - callback(err); - return; - } - stats.filtered_rows = rows; - callback(null, stats); - } - ); -}; diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/api/map/analyses-catalog-controller.js similarity index 57% rename from lib/cartodb/controllers/analyses.js rename to lib/cartodb/api/map/analyses-catalog-controller.js index fb75c633..f7f66346 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -1,37 +1,54 @@ -var PSQL = require('cartodb-psql'); -var cors = require('../middleware/cors'); -var userMiddleware = require('../middleware/user'); +'use strict'; -function AnalysesController(prepareContext) { - this.prepareContext = prepareContext; -} +const PSQL = require('cartodb-psql'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const authorize = require('../middlewares/authorize'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheControlHeader = require('../middlewares/cache-control-header'); +const dbParamsFromResLocals = require('../../utils/database-params'); -module.exports = AnalysesController; +module.exports = class AnalysesController { + constructor (pgConnection, authBackend, userLimitsBackend) { + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } -AnalysesController.prototype.register = function (app) { - app.get( - `${app.base_url_mapconfig}/analyses/catalog`, - cors(), - userMiddleware, - this.prepareContext, - this.createPGClient(), - this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), - this.getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), - this.prepareResponse(), - this.setCacheControlHeader(), - this.sendResponse(), - this.unathorizedError() - ); + register (mapRouter) { + mapRouter.get('/analyses/catalog', this.middlewares()); + } + + middlewares () { + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), + cleanUpQueryParams(), + createPGClient(), + getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), + getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), + prepareResponse(), + cacheControlHeader({ ttl: 10, revalidate: true }), + unauthorizedError() + ]; + } }; -AnalysesController.prototype.createPGClient = function () { +function createPGClient () { return function createPGClientMiddleware (req, res, next) { - res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); + const dbParams = dbParamsFromResLocals(res.locals); + + res.locals.pg = new PSQL(dbParams); + next(); }; -}; +} -AnalysesController.prototype.getDataFromQuery = function ({ queryTemplate, key }) { +function getDataFromQuery({ queryTemplate, key }) { const readOnlyTransactionOn = true; return function getCatalogMiddleware(req, res, next) { @@ -48,9 +65,9 @@ AnalysesController.prototype.getDataFromQuery = function ({ queryTemplate, key } next(); }, readOnlyTransactionOn); }; -}; +} -AnalysesController.prototype.prepareResponse = function () { +function prepareResponse () { return function prepareResponseMiddleware (req, res, next) { const { catalog, tables } = res.locals; @@ -87,32 +104,14 @@ AnalysesController.prototype.prepareResponse = function () { return -1; }); + res.statusCode = 200; res.body = { catalog: analysisCatalog }; next(); }; -}; +} -AnalysesController.prototype.setCacheControlHeader = function () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - next(); - }; -}; - -AnalysesController.prototype.sendResponse = function() { - return function sendResponseMiddleware (req, res) { - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - }; -}; - -AnalysesController.prototype.unathorizedError = function () { +function unauthorizedError () { return function unathorizedErrorMiddleware(err, req, res, next) { if (err.message.match(/permission\sdenied/)) { err = new Error('Unauthorized'); @@ -121,7 +120,7 @@ AnalysesController.prototype.unathorizedError = function () { next(err); }; -}; +} const catalogQueryTpl = ctx => ` SELECT analysis_def->>'type' as type, * FROM cdb_analysis_catalog WHERE username = '${ctx._username}' @@ -145,23 +144,3 @@ var tablesQueryTpl = ctx => ` FROM analysis_tables ORDER BY size DESC `; - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js new file mode 100644 index 00000000..0375d9ed --- /dev/null +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -0,0 +1,61 @@ +'use strict'; + +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const dbParamsFromResLocals = require('../../utils/database-params'); + +module.exports = class AnalysisLayergroupController { + constructor (analysisStatusBackend, pgConnection, userLimitsBackend, authBackend) { + this.analysisStatusBackend = analysisStatusBackend; + this.pgConnection = pgConnection; + this.userLimitsBackend = userLimitsBackend; + this.authBackend = authBackend; + } + + register (mapRouter) { + mapRouter.get('/:token/analysis/node/:nodeId', this.middlewares()); + } + + middlewares () { + return [ + layergroupToken(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), + cleanUpQueryParams(), + analysisNodeStatus(this.analysisStatusBackend) + ]; + } +}; + +function analysisNodeStatus (analysisStatusBackend) { + return function analysisNodeStatusMiddleware(req, res, next) { + const { nodeId } = req.params; + const dbParams = dbParamsFromResLocals(res.locals); + + analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET NODE STATUS'; + return next(err); + } + + res.set({ + 'Cache-Control': 'public,max-age=5', + 'Last-Modified': new Date().toUTCString() + }); + + res.statusCode = 200; + res.body = nodeStatus; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/anonymous-map-controller.js b/lib/cartodb/api/map/anonymous-map-controller.js new file mode 100644 index 00000000..ef47f5b6 --- /dev/null +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -0,0 +1,224 @@ +'use strict'; + +const windshaft = require('windshaft'); +const MapConfig = windshaft.model.MapConfig; +const Datasource = windshaft.model.Datasource; +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const initProfiler = require('../middlewares/init-profiler'); +const checkJsonContentType = require('../middlewares/check-json-content-type'); +const incrementMapViewCount = require('../middlewares/increment-map-view-count'); +const augmentLayergroupData = require('../middlewares/augment-layergroup-data'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup'); +const layerStats = require('../middlewares/layer-stats'); +const layergroupIdHeader = require('../middlewares/layergroup-id-header'); +const layergroupMetadata = require('../middlewares/layergroup-metadata'); +const mapError = require('../middlewares/map-error'); +const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +module.exports = class AnonymousMapController { + /** + * @param {AuthBackend} authBackend + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @param {MapBackend} mapBackend + * @param metadataBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsBackend} userLimitsBackend + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigAdapter} mapConfigAdapter + * @param {StatsBackend} statsBackend + * @constructor + */ + constructor ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTables = layergroupAffectedTables; + this.mapConfigAdapter = mapConfigAdapter; + this.statsBackend = statsBackend; + this.authBackend = authBackend; + this.layergroupMetadata = layergroupMetadata; + } + + register (mapRouter) { + mapRouter.options('/'); + mapRouter.get('/', this.middlewares()); + mapRouter.post('/', this.middlewares()); + } + + middlewares () { + const isTemplateInstantiation = false; + const useTemplateHash = false; + const includeQuery = true; + const label = 'ANONYMOUS LAYERGROUP'; + const addContext = true; + + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), + cleanUpQueryParams(['aggregation']), + initProfiler(isTemplateInstantiation), + checkJsonContentType(), + checkCreateLayergroup(), + prepareAdapterMapConfig(this.mapConfigAdapter), + createLayergroup ( + this.mapBackend, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTables + ), + incrementMapViewCount(this.metadataBackend), + augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + lastUpdatedTimeLayergroup(), + layerStats(this.pgConnection, this.statsBackend), + layergroupIdHeader(this.templateMaps, useTemplateHash), + layergroupMetadata(this.layergroupMetadata, includeQuery), + mapError({ label, addContext }) + ]; + } +}; + +function checkCreateLayergroup () { + return function checkCreateLayergroupMiddleware (req, res, next) { + if (req.method === 'GET') { + const { config } = req.query; + + if (!config) { + return next(new Error('layergroup GET needs a "config" parameter')); + } + + try { + req.body = JSON.parse(config); + } catch (err) { + return next(err); + } + } + + req.profiler.done('checkCreateLayergroup'); + return next(); + }; +} + +function prepareAdapterMapConfig (mapConfigAdapter) { + return function prepareAdapterMapConfigMiddleware(req, res, next) { + const requestMapConfig = req.body; + + const { user, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + const context = { + analysisConfiguration: { + user, + db: { + host: dbhost, + port: dbport, + dbname: dbname, + user: dbuser, + pass: dbpassword + }, + batch: { + username: user, + apiKey: api_key + } + } + }; + + mapConfigAdapter.getMapConfig(user, + requestMapConfig, + params, + context, + (err, requestMapConfig, stats = { overviewsAddedToMapconfig : false }) => { + req.profiler.done('anonymous.getMapConfig'); + + stats.mapType = 'anonymous'; + req.profiler.add(stats); + + + if (err) { + return next(err); + } + + req.body = requestMapConfig; + res.locals.context = context; + + next(); + }); + }; +} + +function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) { + return function createLayergroupMiddleware (req, res, next) { + const requestMapConfig = req.body; + + const { context } = res.locals; + const { user, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = { + cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport + }; + + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapConfig = new MapConfig(requestMapConfig, datasource); + + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsBackend, + pgConnection, + affectedTablesCache, + params + ); + + res.locals.mapConfig = mapConfig; + res.locals.analysesResults = context.analysesResults; + + const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport }; + + mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup, stats = {}) => { + req.profiler.add(stats); + + if (err) { + return next(err); + } + + res.statusCode = 200; + res.body = layergroup; + res.locals.mapConfigProvider = mapConfigProvider; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/attributes-layergroup-controller.js b/lib/cartodb/api/map/attributes-layergroup-controller.js new file mode 100644 index 00000000..9ab394d7 --- /dev/null +++ b/lib/cartodb/api/map/attributes-layergroup-controller.js @@ -0,0 +1,91 @@ +'use strict'; + +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); + +module.exports = class AttributesLayergroupController { + constructor ( + attributesBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ) { + this.attributesBackend = attributesBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authBackend = authBackend; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (mapRouter) { + mapRouter.get('/:token/:layer/attributes/:fid', this.middlewares()); + } + + middlewares () { + return [ + layergroupToken(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getFeatureAttributes(this.attributesBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } +}; + +function getFeatureAttributes (attributesBackend) { + return function getFeatureAttributesMiddleware (req, res, next) { + req.profiler.start('windshaft.maplayer_attribute'); + + const { mapConfigProvider } = res.locals; + const { token } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { layer, fid } = req.params; + + const params = { + token, + dbuser, dbname, dbpassword, dbhost, dbport, + layer, fid + }; + + attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET ATTRIBUTES'; + return next(err); + } + + res.statusCode = 200; + res.body = tile; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js new file mode 100644 index 00000000..d82556b8 --- /dev/null +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -0,0 +1,144 @@ +'use strict'; + +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); + +const ALLOWED_DATAVIEW_QUERY_PARAMS = [ + 'filters', // json + 'own_filter', // 0, 1 + 'no_filters', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'offset', // number + 'q', // widgets search + 'categories', // number +]; + +module.exports = class DataviewLayergroupController { + constructor ( + dataviewBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ) { + this.dataviewBackend = dataviewBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authBackend = authBackend; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (mapRouter) { + // Undocumented/non-supported API endpoint methods. + // Use at your own peril. + + mapRouter.get('/:token/dataview/:dataviewName', this.middlewares({ + action: 'get', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW + })); + + mapRouter.get('/:token/:layer/widget/:dataviewName', this.middlewares({ + action: 'get', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW + })); + + mapRouter.get('/:token/dataview/:dataviewName/search', this.middlewares({ + action: 'search', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH + })); + + mapRouter.get('/:token/:layer/widget/:dataviewName/search', this.middlewares({ + action: 'search', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH + })); + } + + middlewares ({ action, rateLimitGroup }) { + return [ + layergroupToken(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, rateLimitGroup), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + action === 'search' ? dataviewSearch(this.dataviewBackend) : getDataview(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } +}; + +function getDataview (dataviewBackend) { + return function getDataviewMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW'; + return next(err); + } + + res.statusCode = 200; + res.body = dataview; + + next(); + }); + }; +} + +function dataviewSearch (dataviewBackend) { + return function dataviewSearchMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW SEARCH'; + return next(err); + } + + res.statusCode = 200; + res.body = searchResult; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js new file mode 100644 index 00000000..dac55670 --- /dev/null +++ b/lib/cartodb/api/map/map-router.js @@ -0,0 +1,131 @@ +'use strict'; + +const { Router: router } = require('express'); + +const AnalysisLayergroupController = require('./analysis-layergroup-controller'); +const AttributesLayergroupController = require('./attributes-layergroup-controller'); +const DataviewLayergroupController = require('./dataview-layergroup-controller'); +const PreviewLayergroupController = require('./preview-layergroup-controller'); +const TileLayergroupController = require('./tile-layergroup-controller'); +const AnonymousMapController = require('./anonymous-map-controller'); +const PreviewTemplateController = require('./preview-template-controller'); +const AnalysesCatalogController = require('./analyses-catalog-controller'); + +module.exports = class MapRouter { + constructor ({ collaborators }) { + const { + analysisStatusBackend, + attributesBackend, + dataviewBackend, + previewBackend, + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache, + templateMaps, + mapBackend, + metadataBackend, + mapConfigAdapter, + statsBackend, + layergroupMetadata, + namedMapProviderCache, + tablesExtentBackend + } = collaborators; + + this.analysisLayergroupController = new AnalysisLayergroupController( + analysisStatusBackend, + pgConnection, + userLimitsBackend, + authBackend + ); + + this.attributesLayergroupController = new AttributesLayergroupController( + attributesBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.dataviewLayergroupController = new DataviewLayergroupController( + dataviewBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.previewLayergroupController = new PreviewLayergroupController( + previewBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.tileLayergroupController = new TileLayergroupController( + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); + + this.anonymousMapController = new AnonymousMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ); + + this.previewTemplateController = new PreviewTemplateController( + namedMapProviderCache, + previewBackend, + surrogateKeysCache, + tablesExtentBackend, + metadataBackend, + pgConnection, + authBackend, + userLimitsBackend + ); + + this.analysesController = new AnalysesCatalogController( + pgConnection, + authBackend, + userLimitsBackend + ); + } + + register (apiRouter, mapPaths) { + const mapRouter = router({ mergeParams: true }); + + this.analysisLayergroupController.register(mapRouter); + this.attributesLayergroupController.register(mapRouter); + this.dataviewLayergroupController.register(mapRouter); + this.previewLayergroupController.register(mapRouter); + this.tileLayergroupController.register(mapRouter); + this.anonymousMapController.register(mapRouter); + this.previewTemplateController.register(mapRouter); + this.analysesController.register(mapRouter); + + mapPaths.forEach(path => apiRouter.use(path, mapRouter)); + } +}; diff --git a/lib/cartodb/api/map/preview-layergroup-controller.js b/lib/cartodb/api/map/preview-layergroup-controller.js new file mode 100644 index 00000000..bf516f5e --- /dev/null +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -0,0 +1,158 @@ +'use strict'; + +const layergroupToken = require('../middlewares/layergroup-token'); +const coordinates = require('../middlewares/coordinates'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const noop = require('../middlewares/noop'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const checkStaticImageFormat = require('../middlewares/check-static-image-format'); + +module.exports = class PreviewLayergroupController { + constructor ( + previewBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ) { + this.previewBackend = previewBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authBackend = authBackend; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (mapRouter) { + mapRouter.get('/static/center/:token/:z/:lat/:lng/:width/:height.:format', this.middlewares({ + validateZoom: true, + previewType: 'centered' + })); + + mapRouter.get('/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', this.middlewares({ + validateZoom: false, + previewType: 'bbox' + })); + } + + middlewares ({ validateZoom, previewType }) { + const forcedFormat = 'png'; + + let getPreviewImage; + + if (previewType === 'centered') { + getPreviewImage = getPreviewImageByCenter; + } + + if (previewType === 'bbox') { + getPreviewImage = getPreviewImageByBoundingBox; + } + + return [ + layergroupToken(), + validateZoom ? coordinates({ z: true, x: false, y: false }) : noop(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + cleanUpQueryParams(['layer']), + checkStaticImageFormat(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), + getPreviewImage(this.previewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } +}; + +function getPreviewImageByCenter (previewBackend) { + return function getPreviewImageByCenterMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const zoom = +req.params.z; + const center = { + lng: +req.params.lng, + lat: +req.params.lat + }; + + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.statusCode = 200; + res.body = image; + + next(); + }); + }; +} + +function getPreviewImageByBoundingBox (previewBackend) { + return function getPreviewImageByBoundingBoxMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const bounds = { + west: +req.params.west, + north: +req.params.north, + east: +req.params.east, + south: +req.params.south + }; + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.statusCode = 200; + res.body = image; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js new file mode 100644 index 00000000..eea382de --- /dev/null +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -0,0 +1,369 @@ +'use strict'; + +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const namedMapProvider = require('../middlewares/named-map-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const checkStaticImageFormat = require('../middlewares/check-static-image-format'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +const DEFAULT_ZOOM_CENTER = { + zoom: 1, + center: { + lng: 0, + lat: 0 + } +}; + +function numMapper(n) { + return +n; +} + +module.exports = class PreviewTemplateController { + constructor ( + namedMapProviderCache, + previewBackend, + surrogateKeysCache, + tablesExtentBackend, + metadataBackend, + pgConnection, + authBackend, + userLimitsBackend + ) { + this.namedMapProviderCache = namedMapProviderCache; + this.previewBackend = previewBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.tablesExtentBackend = tablesExtentBackend; + this.metadataBackend = metadataBackend; + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } + + register (mapRouter) { + mapRouter.get('/static/named/:template_id/:width/:height.:format', this.middlewares()); + } + + middlewares () { + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), + cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), + checkStaticImageFormat(), + namedMapProvider({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'STATIC_VIZ_MAP', forcedFormat: 'png' + }), + getTemplate({ label: 'STATIC_VIZ_MAP' }), + prepareLayerFilterFromPreviewLayers({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'STATIC_VIZ_MAP' + }), + getStaticImageOptions({ tablesExtentBackend: this.tablesExtentBackend }), + getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), + setContentTypeHeader(), + incrementMapViews({ metadataBackend: this.metadataBackend }), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } +}; + +function getTemplate ({ label }) { + return function getTemplateMiddleware (req, res, next) { + const { mapConfigProvider } = res.locals; + + mapConfigProvider.getTemplate((err, template) => { + if (err) { + err.label = label; + return next(err); + } + + res.locals.template = template; + + next(); + }); + }; +} + +function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) { + return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) { + const { template } = res.locals; + const { config, auth_token } = req.query; + + if (!template || !template.view || !template.view.preview_layers) { + return next(); + } + + var previewLayers = template.view.preview_layers; + var layerVisibilityFilter = []; + + template.layergroup.layers.forEach((layer, index) => { + if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) { + layerVisibilityFilter.push(''+index); + } + }); + + if (!layerVisibilityFilter.length) { + return next(); + } + + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, format } = req.params; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, format + }; + + // overwrites 'all' default filter + params.layer = layerVisibilityFilter.join(','); + + // recreates the provider + namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, provider) => { + if (err) { + err.label = label; + return next(err); + } + + res.locals.mapConfigProvider = provider; + + next(); + }); + }; +} + +function getStaticImageOptions ({ tablesExtentBackend }) { + return function getStaticImageOptionsMiddleware(req, res, next) { + const { user, mapConfigProvider, template } = res.locals; + const { zoom, lon, lat, bbox } = req.query; + const params = { zoom, lon, lat, bbox }; + + const imageOpts = getImageOptions(params, template); + + if (imageOpts) { + res.locals.imageOpts = imageOpts; + return next(); + } + + res.locals.imageOpts = DEFAULT_ZOOM_CENTER; + + mapConfigProvider.createAffectedTables((err, affectedTables) => { + if (err) { + return next(); + } + + var tables = affectedTables.tables || []; + + if (tables.length === 0) { + return next(); + } + + tablesExtentBackend.getBounds(user, tables, (err, bounds) => { + if (err) { + return next(); + } + + res.locals.imageOpts = bounds; + + return next(); + }); + }); + }; +} + +function getImageOptions (params, template) { + const { zoom, lon, lat, bbox } = params; + + let imageOpts = getImageOptionsFromCoordinates(zoom, lon, lat); + if (imageOpts) { + return imageOpts; + } + + imageOpts = getImageOptionsFromBoundingBox(bbox); + if (imageOpts) { + return imageOpts; + } + + imageOpts = getImageOptionsFromTemplate(template, zoom); + if (imageOpts) { + return imageOpts; + } +} + +function getImageOptionsFromCoordinates (zoom, lon, lat) { + if ([zoom, lon, lat].map(numMapper).every(Number.isFinite)) { + return { + zoom: zoom, + center: { + lng: lon, + lat: lat + } + }; + } +} + + +function getImageOptionsFromTemplate (template, zoom) { + if (template.view) { + var zoomCenter = templateZoomCenter(template.view); + if (zoomCenter) { + if (Number.isFinite(+zoom)) { + zoomCenter.zoom = +zoom; + } + + return zoomCenter; + } + + var bounds = templateBounds(template.view); + if (bounds) { + return bounds; + } + } +} + +function getImageOptionsFromBoundingBox (bbox = '') { + var _bbox = bbox.split(',').map(numMapper); + + if (_bbox.length === 4 && _bbox.every(Number.isFinite)) { + return { + bounds: { + west: _bbox[0], + south: _bbox[1], + east: _bbox[2], + north: _bbox[3] + } + }; + } +} + +function getImage({ previewBackend, label }) { + return function getImageMiddleware (req, res, next) { + const { imageOpts, mapConfigProvider } = res.locals; + const { zoom, center, bounds } = imageOpts; + + let { width, height } = req.params; + + width = +width; + height = +height; + + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + + if (zoom !== undefined && center) { + return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center, + (err, image, headers, stats) => { + req.profiler.add(stats); + + if (err) { + err.label = label; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.statusCode = 200; + res.body = image; + + next(); + }); + } + + previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => { + req.profiler.add(stats); + req.profiler.done('render-' + format); + + if (err) { + err.label = label; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.statusCode = 200; + res.body = image; + + next(); + }); + }; +} + +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} + +function incrementMapViewsError (ctx) { + return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`; +} + +function incrementMapViews ({ metadataBackend }) { + return function incrementMapViewsMiddleware(req, res, next) { + const { user, mapConfigProvider } = res.locals; + + mapConfigProvider.getMapConfig((err, mapConfig) => { + if (err) { + global.logger.log(incrementMapViewsError({ user, err })); + return next(); + } + + const statTag = mapConfig.obj().stat_tag; + + metadataBackend.incMapviewCount(user, statTag, (err) => { + if (err) { + global.logger.log(incrementMapViewsError({ user, err })); + } + + next(); + }); + }); + }; +} + +function templateZoomCenter(view) { + if (view.zoom !== undefined && view.center) { + return { + zoom: view.zoom, + center: view.center + }; + } + return false; +} + +function templateBounds(view) { + if (view.bounds) { + var hasAllBounds = ['west', 'south', 'east', 'north'].every(prop => Number.isFinite(view.bounds[prop])); + + if (hasAllBounds) { + return { + bounds: { + west: view.bounds.west, + south: view.bounds.south, + east: view.bounds.east, + north: view.bounds.north + } + }; + } else { + return false; + } + } + return false; +} diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js new file mode 100644 index 00000000..422fa8b6 --- /dev/null +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -0,0 +1,168 @@ +'use strict'; + +const layergroupToken = require('../middlewares/layergroup-token'); +const coordinates = require('../middlewares/coordinates'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const vectorError = require('../middlewares/vector-error'); + +const SUPPORTED_FORMATS = { + grid_json: true, + json_torque: true, + torque_json: true, + png: true, + png32: true, + mvt: true +}; + +module.exports = class TileLayergroupController { + constructor ( + tileBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ) { + this.tileBackend = tileBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authBackend = authBackend; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (mapRouter) { + // REGEXP: doesn't match with `val` + const not = (val) => `(?!${val})([^\/]+?)`; + + // Sadly the path that matches 1 also matches with 2 so we need to tell to express + // that performs only the middlewares of the first path that matches + // for that we use one array to group all paths. + mapRouter.get([ + `/:token/:z/:x/:y@:scale_factor?x.:format`, // 1 + `/:token/:z/:x/:y.:format`, // 2 + `/:token${not('static')}/:layer/:z/:x/:y.(:format)` + ], this.middlewares()); + } + + middlewares () { + return [ + layergroupToken(), + coordinates(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError() + ]; + } +}; + +function parseFormat (format = '') { + const prettyFormat = format.replace('.', '_'); + return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; +} + +function getStatusCode(tile, format){ + return tile.length === 0 && format === 'mvt' ? 204 : 200; +} + +function getTile (tileBackend) { + return function getTileMiddleware (req, res, next) { + req.profiler.start(`windshaft.${req.params.layer ? 'maplayer_tile' : 'map_tile'}`); + + const { mapConfigProvider } = res.locals; + const { token } = res.locals; + const { layer, z, x, y, format } = req.params; + + const params = { token, layer, z, x, y, format }; + + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { + req.profiler.add(stats); + + if (err) { + return next(err); + } + + if (headers) { + res.set(headers); + } + + const formatStat = parseFormat(req.params.format); + + res.statusCode = getStatusCode(tile, formatStat); + res.body = tile; + + next(); + }); + }; +} + +function incrementSuccessMetrics (statsClient) { + return function incrementSuccessMetricsMiddleware (req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.success'); + statsClient.increment(`windshaft.tiles.${formatStat}.success`); + + next(); + }; +} + +function incrementErrorMetrics (statsClient) { + return function incrementErrorMetricsMiddleware (err, req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.error'); + statsClient.increment(`windshaft.tiles.${formatStat}.error`); + + next(err); + }; +} + +function tileError () { + return function tileErrorMiddleware (err, req, res, next) { + + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + + // Rewrite mapnik parsing errors to start with layer number + const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); + + if (matches) { + errMsg = `style${matches[2]}: ${matches[1]}`; + } + + err.message = errMsg; + err.label = 'TILE RENDER'; + + next(err); + }; +} diff --git a/lib/cartodb/api/middlewares/augment-layergroup-data.js b/lib/cartodb/api/middlewares/augment-layergroup-data.js new file mode 100644 index 00000000..d58e2fb2 --- /dev/null +++ b/lib/cartodb/api/middlewares/augment-layergroup-data.js @@ -0,0 +1,16 @@ +'use strict'; + +const _ = require('underscore'); + +module.exports = function augmentLayergroupData () { + return function augmentLayergroupDataMiddleware (req, res, next) { + const layergroup = res.body; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + _.extend(layergroup, global.environment.serverMetadata); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/context/authorize.js b/lib/cartodb/api/middlewares/authorize.js similarity index 64% rename from lib/cartodb/middleware/context/authorize.js rename to lib/cartodb/api/middlewares/authorize.js index a42b5407..619f1f74 100644 --- a/lib/cartodb/middleware/context/authorize.js +++ b/lib/cartodb/api/middlewares/authorize.js @@ -1,9 +1,10 @@ -module.exports = function authorizeMiddleware (authApi) { - return function (req, res, next) { - req.profiler.done('req2params.setup'); +'use strict'; - authApi.authorize(req, res, (err, authorized) => { +module.exports = function authorize (authBackend) { + return function authorizeMiddleware (req, res, next) { + authBackend.authorize(req, res, (err, authorized) => { req.profiler.done('authorize'); + if (err) { return next(err); } diff --git a/lib/cartodb/api/middlewares/cache-channel-header.js b/lib/cartodb/api/middlewares/cache-channel-header.js new file mode 100644 index 00000000..03b3d2e1 --- /dev/null +++ b/lib/cartodb/api/middlewares/cache-channel-header.js @@ -0,0 +1,26 @@ +'use strict'; + +module.exports = function setCacheChannelHeader () { + return function setCacheChannelHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider } = res.locals; + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Cache Channel Header:', err); + return next(); + } + + if (!affectedTables) { + return next(); + } + + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/cache-control-header.js b/lib/cartodb/api/middlewares/cache-control-header.js new file mode 100644 index 00000000..d72ee0c3 --- /dev/null +++ b/lib/cartodb/api/middlewares/cache-control-header.js @@ -0,0 +1,21 @@ +'use strict'; + +const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365; + +module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) { + return function setCacheControlHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const directives = [ 'public', `max-age=${ttl}` ]; + + if (revalidate) { + directives.push('must-revalidate'); + } + + res.set('Cache-Control', directives.join(',')); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/check-json-content-type.js b/lib/cartodb/api/middlewares/check-json-content-type.js new file mode 100644 index 00000000..6f02c750 --- /dev/null +++ b/lib/cartodb/api/middlewares/check-json-content-type.js @@ -0,0 +1,13 @@ +'use strict'; + +module.exports = function checkJsonContentType () { + return function checkJsonContentTypeMiddleware(req, res, next) { + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('POST data must be of type application/json')); + } + + req.profiler.done('checkJsonContentTypeMiddleware'); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/check-static-image-format.js b/lib/cartodb/api/middlewares/check-static-image-format.js new file mode 100644 index 00000000..411240db --- /dev/null +++ b/lib/cartodb/api/middlewares/check-static-image-format.js @@ -0,0 +1,13 @@ +'use strict'; + +const VALID_IMAGE_FORMATS = ['png', 'jpg']; + +module.exports = function checkStaticImageFormat () { + return function checkStaticImageFormatMiddleware (req, res, next) { + if(!VALID_IMAGE_FORMATS.includes(req.params.format)) { + return next(new Error(`Unsupported image format "${req.params.format}"`)); + } + + next(); + }; +}; diff --git a/lib/cartodb/middleware/context/clean-up-query-params.js b/lib/cartodb/api/middlewares/clean-up-query-params.js similarity index 51% rename from lib/cartodb/middleware/context/clean-up-query-params.js rename to lib/cartodb/api/middlewares/clean-up-query-params.js index 280fe986..898b5358 100644 --- a/lib/cartodb/middleware/context/clean-up-query-params.js +++ b/lib/cartodb/api/middlewares/clean-up-query-params.js @@ -1,3 +1,5 @@ +'use strict'; + const _ = require('underscore'); // Whitelist query parameters and attach format @@ -14,19 +16,16 @@ const REQUEST_QUERY_PARAMS_WHITELIST = [ 'filters' // json ]; -module.exports = function cleanUpQueryParamsMiddleware () { - return function cleanUpQueryParams (req, res, next) { - var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST; +module.exports = function cleanUpQueryParamsMiddleware (customQueryParams = []) { + if (!Array.isArray(customQueryParams)) { + throw new Error('customQueryParams must receive an Array of params'); + } - if (Array.isArray(res.locals.allowedQueryParams)) { - allowedQueryParams = allowedQueryParams.concat(res.locals.allowedQueryParams); - } + return function cleanUpQueryParams (req, res, next) { + const allowedQueryParams = [...REQUEST_QUERY_PARAMS_WHITELIST, ...customQueryParams]; req.query = _.pick(req.query, allowedQueryParams); - // bring all query values onto res.locals object - _.extend(res.locals, req.query); - next(); }; }; diff --git a/lib/cartodb/api/middlewares/coordinates.js b/lib/cartodb/api/middlewares/coordinates.js new file mode 100644 index 00000000..5c48439c --- /dev/null +++ b/lib/cartodb/api/middlewares/coordinates.js @@ -0,0 +1,43 @@ +'use strict'; + +const positiveIntegerNumberRegExp = /^\d+$/; +const integerNumberRegExp = /^-?\d+$/; +const invalidZoomMessage = function (zoom) { + return `Invalid zoom value (${zoom}). It should be an integer number greather than or equal to 0`; +}; +const invalidCoordXMessage = function (x) { + return `Invalid coodinate 'x' value (${x}). It should be an integer number`; +}; +const invalidCoordYMessage = function (y) { + return `Invalid coodinate 'y' value (${y}). It should be an integer number greather than or equal to 0`; +}; + +module.exports = function coordinates (validate = { z: true, x: true, y: true }) { + return function coordinatesMiddleware (req, res, next) { + const { z, x, y } = req.params; + + if (validate.z && !positiveIntegerNumberRegExp.test(z)) { + const err = new Error(invalidZoomMessage(z)); + err.http_status = 400; + + return next(err); + } + + // Negative values for x param are valid. The x param is wrapped + if (validate.x && !integerNumberRegExp.test(x)) { + const err = new Error(invalidCoordXMessage(x)); + err.http_status = 400; + + return next(err); + } + + if (validate.y && !positiveIntegerNumberRegExp.test(y)) { + const err = new Error(invalidCoordYMessage(y)); + err.http_status = 400; + + return next(err); + } + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/cors.js b/lib/cartodb/api/middlewares/cors.js new file mode 100644 index 00000000..87e41dc7 --- /dev/null +++ b/lib/cartodb/api/middlewares/cors.js @@ -0,0 +1,21 @@ +'use strict'; + +module.exports = function cors () { + return function corsMiddleware (req, res, next) { + const headers = [ + 'X-Requested-With', + 'X-Prototype-Version', + 'X-CSRF-Token', + 'Authorization' + ]; + + if (req.method === 'OPTIONS') { + headers.push('Content-Type'); + } + + res.set("Access-Control-Allow-Origin", "*"); + res.set("Access-Control-Allow-Headers", headers.join(', ')); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/credentials.js b/lib/cartodb/api/middlewares/credentials.js new file mode 100644 index 00000000..71316f4b --- /dev/null +++ b/lib/cartodb/api/middlewares/credentials.js @@ -0,0 +1,87 @@ +'use strict'; + +const basicAuth = require('basic-auth'); + +module.exports = function credentials () { + return function credentialsMiddleware(req, res, next) { + const apikeyCredentials = getApikeyCredentialsFromRequest(req); + + res.locals.api_key = apikeyCredentials.token; + res.locals.basicAuthUsername = apikeyCredentials.username; + res.set('vary', 'Authorization'); //Honor Authorization header when caching. + + return next(); + }; +}; + +function getApikeyCredentialsFromRequest(req) { + let apikeyCredentials = { + token: null, + username: null, + }; + + for (let getter of apikeyGetters) { + apikeyCredentials = getter(req); + if (apikeyTokenFound(apikeyCredentials)) { + break; + } + } + + return apikeyCredentials; +} + +const apikeyGetters = [ + getApikeyTokenFromHeaderAuthorization, + getApikeyTokenFromRequestQueryString, + getApikeyTokenFromRequestBody, +]; + +function getApikeyTokenFromHeaderAuthorization(req) { + const credentials = basicAuth(req); + + if (credentials) { + return { + username: credentials.username, + token: credentials.pass + }; + } else { + return { + username: null, + token: null, + }; + } +} + +function getApikeyTokenFromRequestQueryString(req) { + let token = null; + + if (req.query && req.query.api_key) { + token = req.query.api_key; + } else if (req.query && req.query.map_key) { + token = req.query.map_key; + } + + return { + username: null, + token: token, + }; +} + +function getApikeyTokenFromRequestBody(req) { + let token = null; + + if (req.body && req.body.api_key) { + token = req.body.api_key; + } else if (req.body && req.body.map_key) { + token = req.body.map_key; + } + + return { + username: null, + token: token, + }; +} + +function apikeyTokenFound(apikey) { + return !!apikey && !!apikey.token; +} diff --git a/lib/cartodb/middleware/context/db-conn-setup.js b/lib/cartodb/api/middlewares/db-conn-setup.js similarity index 64% rename from lib/cartodb/middleware/context/db-conn-setup.js rename to lib/cartodb/api/middlewares/db-conn-setup.js index dec48f6d..0ab4d105 100644 --- a/lib/cartodb/middleware/context/db-conn-setup.js +++ b/lib/cartodb/api/middlewares/db-conn-setup.js @@ -1,31 +1,32 @@ +'use strict'; + const _ = require('underscore'); -module.exports = function dbConnSetupMiddleware(pgConnection) { - return function dbConnSetup(req, res, next) { - const user = res.locals.user; +module.exports = function dbConnSetup (pgConnection) { + return function dbConnSetupMiddleware (req, res, next) { + const { user } = res.locals; + pgConnection.setDBConn(user, res.locals, (err) => { + req.profiler.done('dbConnSetup'); + if (err) { if (err.message && -1 !== err.message.indexOf('name not found')) { err.http_status = 404; } - req.profiler.done('req2params'); + return next(err); } - // Add default database connection parameters - // if none given _.defaults(res.locals, { dbuser: global.environment.postgres.user, dbpassword: global.environment.postgres.password, dbhost: global.environment.postgres.host, dbport: global.environment.postgres.port }); - + res.set('X-Served-By-DB-Host', res.locals.dbhost); - req.profiler.done('req2params'); - - next(null); + next(); }); }; }; diff --git a/lib/cartodb/middleware/error-middleware.js b/lib/cartodb/api/middlewares/error-middleware.js similarity index 79% rename from lib/cartodb/middleware/error-middleware.js rename to lib/cartodb/api/middlewares/error-middleware.js index 85f38936..688b861a 100644 --- a/lib/cartodb/middleware/error-middleware.js +++ b/lib/cartodb/api/middlewares/error-middleware.js @@ -1,3 +1,5 @@ +'use strict'; + const _ = require('underscore'); const debug = require('debug')('windshaft:cartodb:error-middleware'); @@ -7,7 +9,7 @@ module.exports = function errorMiddleware (/* options */) { // jshint maxcomplexity:9 var allErrors = Array.isArray(err) ? err : [err]; - allErrors = populateTimeoutErrors(allErrors); + allErrors = populateLimitErrors(allErrors); const label = err.label || 'UNKNOWN'; err = allErrors[0] || new Error(label); @@ -15,10 +17,6 @@ module.exports = function errorMiddleware (/* options */) { var statusCode = findStatusCode(err); - if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') { - statusCode = 204; - } - setErrorHeader(allErrors, statusCode, res); debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack); @@ -50,26 +48,53 @@ function isDatasourceTimeoutError (err) { return err.message && err.message.match(/canceling statement due to statement timeout/i); } -function isTimeoutError (err) { - return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); +function isTimeoutError (errorTypes) { + return errorTypes.renderTimeoutError || errorTypes.datasourceTimeoutError; } -function populateTimeoutErrors (errors) { +function getErrorTypes(error) { + return { + renderTimeoutError: isRenderTimeoutError(error), + datasourceTimeoutError: isDatasourceTimeoutError(error), + }; +} + +function isMaxWaitingClientsError (err) { + return err.message === 'max waitingClients count exceeded'; +} + +function populateLimitErrors (errors) { return errors.map(function (error) { - if (isRenderTimeoutError(error)) { - error.subtype = 'render'; + if (isMaxWaitingClientsError(error)) { + error.message = 'You are over platform\'s limits: Max render capacity exceeded.' + + ' Contact CARTO support for more details.'; + error.type = 'limit'; + error.subtype = 'render-capacity'; + error.http_status = 429; + + return error; } - if (isDatasourceTimeoutError(error)) { - error.subtype = 'datasource'; - } + const errorTypes = getErrorTypes(error); - if (isTimeoutError(error)) { + if (isTimeoutError(errorTypes)) { error.message = 'You are over platform\'s limits. Please contact us to know more details'; error.type = 'limit'; error.http_status = 429; } + if (errorTypes.datasourceTimeoutError) { + error.subtype = 'datasource'; + error.message = 'You are over platform\'s limits: SQL query timeout error.' + + ' Refactor your query before running again or contact CARTO support for more details.'; + } + + if (errorTypes.renderTimeoutError) { + error.subtype = 'render'; + error.message = 'You are over platform\'s limits: Render timeout error.' + + ' Contact CARTO support for more details.'; + } + return error; }); } @@ -186,15 +211,15 @@ function setErrorHeader(errors, statusCode, res) { subtype: error.subtype }; }); - + res.set('X-Tiler-Errors', stringifyForLogs(errorsLog)); } /** - * Remove problematic nested characters + * Remove problematic nested characters * from object for logs RegEx - * - * @param {Object} object + * + * @param {Object} object */ function stringifyForLogs(object) { Object.keys(object).map(key => { diff --git a/lib/cartodb/api/middlewares/increment-map-view-count.js b/lib/cartodb/api/middlewares/increment-map-view-count.js new file mode 100644 index 00000000..6afb8c76 --- /dev/null +++ b/lib/cartodb/api/middlewares/increment-map-view-count.js @@ -0,0 +1,18 @@ +'use strict'; + +module.exports = function incrementMapViewCount (metadataBackend) { + return function incrementMapViewCountMiddleware(req, res, next) { + const { mapConfig, user } = res.locals; + + // Error won't blow up, just be logged. + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { + req.profiler.done('incMapviewCount'); + + if (err) { + global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); + } + + next(); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/init-profiler.js b/lib/cartodb/api/middlewares/init-profiler.js new file mode 100644 index 00000000..86916822 --- /dev/null +++ b/lib/cartodb/api/middlewares/init-profiler.js @@ -0,0 +1,11 @@ +'use strict'; + +module.exports = function initProfiler (isTemplateInstantiation) { + const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; + + return function initProfilerMiddleware (req, res, next) { + req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); + req.profiler.done(`${operation}.initProfilerMiddleware`); + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/initialize-status-code.js b/lib/cartodb/api/middlewares/initialize-status-code.js new file mode 100644 index 00000000..db35bb51 --- /dev/null +++ b/lib/cartodb/api/middlewares/initialize-status-code.js @@ -0,0 +1,11 @@ +'use strict'; + +module.exports = function initializeStatusCode () { + return function initializeStatusCodeMiddleware (req, res, next) { + if (req.method !== 'OPTIONS') { + res.statusCode = 404; + } + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/last-modified-header.js b/lib/cartodb/api/middlewares/last-modified-header.js new file mode 100644 index 00000000..3f6c6e6a --- /dev/null +++ b/lib/cartodb/api/middlewares/last-modified-header.js @@ -0,0 +1,40 @@ +'use strict'; + +module.exports = function setLastModifiedHeader () { + return function setLastModifiedHeaderMiddleware(req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider, cache_buster } = res.locals; + + if (cache_buster) { + const cacheBuster = parseInt(cache_buster, 10); + const lastModifiedDate = Number.isFinite(cacheBuster) ? new Date(cacheBuster) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Last Modified Header:', err); + return next(); + } + + if (!affectedTables) { + res.set('Last-Modified', new Date().toUTCString()); + + return next(); + } + + const lastUpdatedAt = affectedTables.getLastUpdatedAt(); + const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? new Date(lastUpdatedAt) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/last-updated-time-layergroup.js b/lib/cartodb/api/middlewares/last-updated-time-layergroup.js new file mode 100644 index 00000000..d0520316 --- /dev/null +++ b/lib/cartodb/api/middlewares/last-updated-time-layergroup.js @@ -0,0 +1,41 @@ +'use strict'; + +module.exports = function setLastUpdatedTimeToLayergroup () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { mapConfigProvider, analysesResults } = res.locals; + const layergroup = res.body; + + mapConfigProvider.createAffectedTables((err, affectedTables) => { + if (err) { + return next(err); + } + + if (!affectedTables) { + return next(); + } + + var lastUpdateTime = affectedTables.getLastUpdatedAt(); + + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + next(); + }); + }; +}; + +function getLastUpdatedTime(analysesResults, lastUpdateTime) { + if (!Array.isArray(analysesResults)) { + return lastUpdateTime; + } + return analysesResults.reduce(function(lastUpdateTime, analysis) { + return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { + var nodeUpdatedAtDate = node.getUpdatedAt(); + var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; + return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; + }, lastUpdateTime); + }, lastUpdateTime); +} diff --git a/lib/cartodb/api/middlewares/layer-stats.js b/lib/cartodb/api/middlewares/layer-stats.js new file mode 100644 index 00000000..2df472f7 --- /dev/null +++ b/lib/cartodb/api/middlewares/layer-stats.js @@ -0,0 +1,28 @@ +'use strict'; + +module.exports = function setLayerStats (pgConnection, statsBackend) { + return function setLayerStatsMiddleware(req, res, next) { + const { user, mapConfig } = res.locals; + const layergroup = res.body; + + pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); + } + + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { + if (err) { + return next(err); + } + + if (layersStats.length > 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + + next(); + }); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/layergroup-id-header.js b/lib/cartodb/api/middlewares/layergroup-id-header.js new file mode 100644 index 00000000..94470972 --- /dev/null +++ b/lib/cartodb/api/middlewares/layergroup-id-header.js @@ -0,0 +1,17 @@ +'use strict'; + +module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash) { + return function setLayergroupIdHeaderMiddleware (req, res, next) { + const { user, template } = res.locals; + const layergroup = res.body; + + if (useTemplateHash) { + var templateHash = templateMaps.fingerPrint(template).substring(0, 8); + layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + } + + res.set('X-Layergroup-Id', layergroup.layergroupid); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/layergroup-metadata.js b/lib/cartodb/api/middlewares/layergroup-metadata.js new file mode 100644 index 00000000..ae1fddba --- /dev/null +++ b/lib/cartodb/api/middlewares/layergroup-metadata.js @@ -0,0 +1,17 @@ +'use strict'; + +module.exports = function setMetadataToLayergroup (layergroupMetadata, includeQuery) { + return function setMetadataToLayergroupMiddleware (req, res, next) { + const { user, mapConfig, analysesResults = [], context, api_key: userApiKey } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); + layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); + layergroupMetadata.addDateWrappingMetadata (layergroup, mapConfig.obj()); + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig, userApiKey); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/layergroup-token.js b/lib/cartodb/api/middlewares/layergroup-token.js new file mode 100644 index 00000000..fcc6959b --- /dev/null +++ b/lib/cartodb/api/middlewares/layergroup-token.js @@ -0,0 +1,30 @@ +'use strict'; + +const LayergroupToken = require('../../models/layergroup-token'); +const authErrorMessageTemplate = function (signer, user) { + return `Cannot use map signature of user "${signer}" on db of user "${user}"`; +}; + +module.exports = function layergroupToken () { + return function layergroupTokenMiddleware (req, res, next) { + const user = res.locals.user; + const layergroupToken = LayergroupToken.parse(req.params.token); + + res.locals.token = layergroupToken.token; + res.locals.cache_buster = layergroupToken.cacheBuster; + + if (layergroupToken.signer) { + res.locals.signer = layergroupToken.signer; + + if (res.locals.signer !== user) { + const err = new Error(authErrorMessageTemplate(res.locals.signer, user)); + err.type = 'auth'; + err.http_status = (req.query && req.query.callback) ? 200: 403; + + return next(err); + } + } + + return next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/logger.js b/lib/cartodb/api/middlewares/logger.js new file mode 100644 index 00000000..da198d64 --- /dev/null +++ b/lib/cartodb/api/middlewares/logger.js @@ -0,0 +1,24 @@ +'use strict'; + +module.exports = function logger (options) { + if (!global.log4js || !options.log_format) { + return function dummyLoggerMiddleware (req, res, next) { + next(); + }; + } + + const opts = { + level: 'info', + // Allowing for unbuffered logging is mainly + // used to avoid hanging during unit testing. + // TODO: provide an explicit teardown function instead, + // releasing any event handler or timer set by + // this component. + buffer: !options.unbuffered_logging, + // optional log format + format: options.log_format + }; + const logger = global.log4js.getLogger(); + + return global.log4js.connectLogger(logger, opts); +}; diff --git a/lib/cartodb/api/middlewares/lzma.js b/lib/cartodb/api/middlewares/lzma.js new file mode 100644 index 00000000..dee7ee61 --- /dev/null +++ b/lib/cartodb/api/middlewares/lzma.js @@ -0,0 +1,35 @@ +'use strict'; + +const LZMA = require('lzma').LZMA; + +module.exports = function lzma () { + const lzmaWorker = new LZMA(); + + return function lzmaMiddleware (req, res, next) { + if (!req.query.hasOwnProperty('lzma')) { + return next(); + } + + // Decode (from base64) + var lzma = new Buffer(req.query.lzma, 'base64') + .toString('binary') + .split('') + .map(function(c) { + return c.charCodeAt(0) - 128; + }); + + // Decompress + lzmaWorker.decompress(lzma, function(result) { + try { + delete req.query.lzma; + Object.assign(req.query, JSON.parse(result)); + + req.profiler.done('lzma'); + + next(); + } catch (err) { + next(new Error('Error parsing lzma as JSON: ' + err)); + } + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/map-error.js b/lib/cartodb/api/middlewares/map-error.js new file mode 100644 index 00000000..7c0ec278 --- /dev/null +++ b/lib/cartodb/api/middlewares/map-error.js @@ -0,0 +1,37 @@ +'use strict'; + +module.exports = function mapError (options) { + const { addContext = false, label = 'MAPS CONTROLLER' } = options; + + return function mapErrorMiddleware (err, req, res, next) { + req.profiler.done('error'); + const { mapConfig } = res.locals; + + if (addContext) { + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; + } + + err.label = label; + + next(err); + }; +}; + +function populateError(err, mapConfig) { + var error = new Error(err.message); + error.http_status = err.http_status; + + if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { + error.http_status = 400; + } + + error.type = 'layer'; + error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; + error.layer = { + id: mapConfig.getLayerId(err.layerIndex), + index: err.layerIndex, + type: mapConfig.layerType(err.layerIndex) + }; + + return error; +} diff --git a/lib/cartodb/api/middlewares/map-store-map-config-provider.js b/lib/cartodb/api/middlewares/map-store-map-config-provider.js new file mode 100644 index 00000000..37c624c5 --- /dev/null +++ b/lib/cartodb/api/middlewares/map-store-map-config-provider.js @@ -0,0 +1,40 @@ +'use strict'; + +const MapStoreMapConfigProvider = require('../../models/mapconfig/provider/map-store-provider'); + +module.exports = function createMapStoreMapConfigProvider ( + mapStore, + userLimitsBackend, + pgConnection, + affectedTablesCache, + forcedFormat = null +) { + return function createMapStoreMapConfigProviderMiddleware (req, res, next) { + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { layer: layerFromParams, z, x, y, scale_factor, format } = req.params; + const { layer: layerFromQuery } = req.query; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + layer: (layerFromQuery || layerFromParams), z, x, y, scale_factor, format + }; + + if (forcedFormat) { + params.format = forcedFormat; + params.layer = params.layer || 'all'; + } + + res.locals.mapConfigProvider = new MapStoreMapConfigProvider( + mapStore, + user, + userLimitsBackend, + pgConnection, + affectedTablesCache, + params + ); + + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/named-map-provider.js b/lib/cartodb/api/middlewares/named-map-provider.js new file mode 100644 index 00000000..9134dd29 --- /dev/null +++ b/lib/cartodb/api/middlewares/named-map-provider.js @@ -0,0 +1,34 @@ +'use strict'; + +module.exports = function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) { + return function getNamedMapProviderMiddleware (req, res, next) { + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, layer: layerFromParams, z, x, y, format } = req.params; + const { layer: layerFromQuery } = req.query; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format + }; + + if (forcedFormat) { + params.format = forcedFormat; + params.layer = params.layer || 'all'; + } + + const { config, auth_token } = req.query; + + namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { + if (err) { + err.label = label; + return next(err); + } + + res.locals.mapConfigProvider = namedMapProvider; + + next(); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/noop.js b/lib/cartodb/api/middlewares/noop.js new file mode 100644 index 00000000..9cfa684b --- /dev/null +++ b/lib/cartodb/api/middlewares/noop.js @@ -0,0 +1,7 @@ +'use strict'; + +module.exports = function noop () { + return function noopMiddleware (req, res, next) { + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/rate-limit.js b/lib/cartodb/api/middlewares/rate-limit.js new file mode 100644 index 00000000..14452254 --- /dev/null +++ b/lib/cartodb/api/middlewares/rate-limit.js @@ -0,0 +1,72 @@ +'use strict'; + +const RATE_LIMIT_ENDPOINTS_GROUPS = { + ANONYMOUS: 'anonymous', + STATIC: 'static', + STATIC_NAMED: 'static_named', + DATAVIEW: 'dataview', + DATAVIEW_SEARCH: 'dataview_search', + ANALYSIS: 'analysis', + ANALYSIS_CATALOG: 'analysis_catalog', + TILE: 'tile', + ATTRIBUTES: 'attributes', + NAMED_LIST: 'named_list', + NAMED_CREATE: 'named_create', + NAMED_GET: 'named_get', + NAMED: 'named', + NAMED_UPDATE: 'named_update', + NAMED_DELETE: 'named_delete', + NAMED_TILES: 'named_tiles' +}; + +function rateLimit(userLimitsBackend, endpointGroup = null) { + if (!isRateLimitEnabled(endpointGroup)) { + return function rateLimitDisabledMiddleware(req, res, next) { next(); }; + } + + return function rateLimitMiddleware(req, res, next) { + userLimitsBackend.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) { + if (err) { + return next(err); + } + + if (!userRateLimit) { + return next(); + } + + const [isBlocked, limit, remaining, retry, reset] = userRateLimit; + + res.set({ + 'Carto-Rate-Limit-Limit': limit, + 'Carto-Rate-Limit-Remaining': remaining, + 'Carto-Rate-Limit-Reset': reset + }); + + if (isBlocked) { + // retry is floor rounded in seconds by redis-cell + res.set('Retry-After', retry + 1); + + let rateLimitError = new Error( + 'You are over platform\'s limits: too many requests.' + + ' Please contact us to know more details' + ); + rateLimitError.http_status = 429; + rateLimitError.type = 'limit'; + rateLimitError.subtype = 'rate-limit'; + return next(rateLimitError); + } + + return next(); + }); + }; +} + + +function isRateLimitEnabled(endpointGroup) { + return global.environment.enabledFeatures.rateLimitsEnabled && + endpointGroup && + global.environment.enabledFeatures.rateLimitsByEndpoint[endpointGroup]; +} + +module.exports = rateLimit; +module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS; diff --git a/lib/cartodb/api/middlewares/send-response.js b/lib/cartodb/api/middlewares/send-response.js new file mode 100644 index 00000000..3e858aff --- /dev/null +++ b/lib/cartodb/api/middlewares/send-response.js @@ -0,0 +1,19 @@ +'use strict'; + +module.exports = function sendResponse () { + return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); + + res.status(res.statusCode); + + if (Buffer.isBuffer(res.body)) { + return res.send(res.body); + } + + if (req.query.callback) { + return res.jsonp(res.body); + } + + res.json(res.body); + }; +}; diff --git a/lib/cartodb/api/middlewares/served-by-host-header.js b/lib/cartodb/api/middlewares/served-by-host-header.js new file mode 100644 index 00000000..297cc945 --- /dev/null +++ b/lib/cartodb/api/middlewares/served-by-host-header.js @@ -0,0 +1,13 @@ +'use strict'; + +const os = require('os'); + +module.exports = function servedByHostHeader () { + const hostname = os.hostname().split('.')[0]; + + return function servedByHostHeaderMiddleware (req, res, next) { + res.set('X-Served-By-Host', hostname); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/api/middlewares/stats.js similarity index 80% rename from lib/cartodb/middleware/stats.js rename to lib/cartodb/api/middlewares/stats.js index 489ee645..51dbdf53 100644 --- a/lib/cartodb/middleware/stats.js +++ b/lib/cartodb/api/middlewares/stats.js @@ -1,11 +1,13 @@ -const Profiler = require('../stats/profiler_proxy'); +'use strict'; + +const Profiler = require('../../stats/profiler_proxy'); const debug = require('debug')('windshaft:cartodb:stats'); const onHeaders = require('on-headers'); -module.exports = function statsMiddleware(options) { +module.exports = function stats (options) { const { enabled = true, statsClient } = options; - return function stats(req, res, next) { + return function statsMiddleware (req, res, next) { req.profiler = new Profiler({ statsd_client: statsClient, profile: enabled diff --git a/lib/cartodb/api/middlewares/surrogate-key-header.js b/lib/cartodb/api/middlewares/surrogate-key-header.js new file mode 100644 index 00000000..f616b009 --- /dev/null +++ b/lib/cartodb/api/middlewares/surrogate-key-header.js @@ -0,0 +1,33 @@ +'use strict'; + +const NamedMapsCacheEntry = require('../../cache/model/named_maps_entry'); +const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); + +module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { user, mapConfigProvider } = res.locals; + + if (mapConfigProvider instanceof NamedMapMapConfigProvider) { + surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName())); + } + + if (req.method !== 'GET') { + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Surrogate Key Header:', err); + return next(); + } + + if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) { + return next(); + } + + surrogateKeysCache.tag(res, affectedTables); + + next(); + }); + }; +}; diff --git a/lib/cartodb/api/middlewares/syntax-error.js b/lib/cartodb/api/middlewares/syntax-error.js new file mode 100644 index 00000000..c293cef3 --- /dev/null +++ b/lib/cartodb/api/middlewares/syntax-error.js @@ -0,0 +1,12 @@ +'use strict'; + +module.exports = function syntaxError () { + return function syntaxErrorMiddleware (err, req, res, next) { + if (err.name === 'SyntaxError') { + err.http_status = 400; + err.message = `${err.name}: ${err.message}`; + } + + next(err); + }; +}; diff --git a/lib/cartodb/api/middlewares/user.js b/lib/cartodb/api/middlewares/user.js new file mode 100644 index 00000000..4dda75e7 --- /dev/null +++ b/lib/cartodb/api/middlewares/user.js @@ -0,0 +1,13 @@ +'use strict'; + +const CdbRequest = require('../../models/cdb_request'); + +module.exports = function user () { + const cdbRequest = new CdbRequest(); + + return function userMiddleware(req, res, next) { + res.locals.user = cdbRequest.userByReq(req); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/api/middlewares/vector-error.js similarity index 77% rename from lib/cartodb/middleware/vector-error.js rename to lib/cartodb/api/middlewares/vector-error.js index f42f1c87..4de1ebce 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/api/middlewares/vector-error.js @@ -1,12 +1,13 @@ -const fs = require('fs'); +'use strict'; -const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt'); +const fs = require('fs'); +const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../../assets/render-timeout-fallback.mvt'); module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { if(req.params.format === 'mvt') { - if (isTimeoutError(err)) { + if (isTimeoutError(err) || isRateLimitError(err)) { res.set('Content-Type', 'application/x-protobuf'); return res.status(429).send(timeoutErrorVectorTile); } @@ -28,3 +29,7 @@ function isDatasourceTimeoutError (err) { function isTimeoutError (err) { return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); } + +function isRateLimitError (err) { + return err.type === 'limit' && err.subtype === 'rate-limit'; +} diff --git a/lib/cartodb/api/template/admin-template-controller.js b/lib/cartodb/api/template/admin-template-controller.js new file mode 100644 index 00000000..c8d2ba97 --- /dev/null +++ b/lib/cartodb/api/template/admin-template-controller.js @@ -0,0 +1,233 @@ +'use strict'; + +const { templateName } = require('../../backends/template_maps'); +const credentials = require('../middlewares/credentials'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +module.exports = class AdminTemplateController { + /** + * @param {AuthBackend} authBackend + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @constructor + */ + constructor (authBackend, templateMaps, userLimitsBackend) { + this.authBackend = authBackend; + this.templateMaps = templateMaps; + this.userLimitsBackend = userLimitsBackend; + } + + register (templateRouter) { + templateRouter.options(`/:template_id`); + + templateRouter.post('/', this.middlewares({ + action: 'create', + label: 'POST TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE + })); + + templateRouter.put('/:template_id', this.middlewares({ + action: 'update', + label: 'PUT TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE + })); + + templateRouter.get('/:template_id', this.middlewares({ + action: 'get', + label: 'GET TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET + })); + + templateRouter.delete('/:template_id', this.middlewares({ + action: 'delete', + label: 'DELETE TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE + })); + + templateRouter.get('/', this.middlewares({ + action: 'list', + label: 'GET TEMPLATE LIST', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST + })); + } + + middlewares ({ action, label, rateLimitGroup }) { + let template; + + if (action === 'create') { + template = createTemplate; + } + + if (action === 'update') { + template = updateTemplate; + } + + if (action === 'get') { + template = retrieveTemplate; + } + + if (action === 'delete') { + template = destroyTemplate; + } + + if (action === 'list') { + template = listTemplates; + } + + return [ + credentials(), + authorizedByAPIKey({ authBackend: this.authBackend, action, label }), + rateLimit(this.userLimitsBackend, rateLimitGroup), + checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), + template({ templateMaps: this.templateMaps }) + ]; + } +}; + +function checkContentType ({ label }) { + return function checkContentTypeMiddleware (req, res, next) { + if ((req.method === 'POST' || req.method === 'PUT') && !req.is('application/json')) { + const error = new Error(`${req.method} template data must be of type application/json`); + error.label = label; + return next(error); + } + + next(); + }; +} + +function authorizedByAPIKey ({ authBackend, action, label }) { + return function authorizedByAPIKeyMiddleware (req, res, next) { + const { user } = res.locals; + + authBackend.authorizedByAPIKey(user, res, (err, authenticated, apikey) => { + if (err) { + return next(err); + } + + if (!authenticated) { + const error = new Error(`Only authenticated users can ${action} templated maps`); + error.http_status = 403; + error.label = label; + return next(error); + } + + if (apikey.type !== 'master') { + const error = new Error('Forbidden'); + error.type = 'auth'; + error.subtype = 'api-key-does-not-grant-access'; + error.http_status = 403; + + return next(error); + } + + next(); + }); + }; +} + +function createTemplate ({ templateMaps }) { + return function createTemplateMiddleware (req, res, next) { + const { user } = res.locals; + const template = req.body; + + templateMaps.addTemplate(user, template, (err, templateId) => { + if (err) { + return next(err); + } + + res.statusCode = 200; + res.body = { template_id: templateId }; + + next(); + }); + }; +} + +function updateTemplate ({ templateMaps }) { + return function updateTemplateMiddleware (req, res, next) { + const { user } = res.locals; + const template = req.body; + const templateId = templateName(req.params.template_id); + + templateMaps.updTemplate(user, templateId, template, (err) => { + if (err) { + return next(err); + } + + res.statusCode = 200; + res.body = { template_id: templateId }; + + next(); + }); + }; +} + +function retrieveTemplate ({ templateMaps }) { + return function retrieveTemplateMiddleware (req, res, next) { + req.profiler.start('windshaft-cartodb.get_template'); + + const { user } = res.locals; + const templateId = templateName(req.params.template_id); + + templateMaps.getTemplate(user, templateId, (err, template) => { + if (err) { + return next(err); + } + + if (!template) { + const error = new Error(`Cannot find template '${templateId}' of user '${user}'`); + error.http_status = 404; + return next(error); + } + // auth_id was added by ourselves, + // so we remove it before returning to the user + delete template.auth_id; + + res.statusCode = 200; + res.body = { template }; + + next(); + }); + }; +} + +function destroyTemplate ({ templateMaps }) { + return function destroyTemplateMiddleware (req, res, next) { + req.profiler.start('windshaft-cartodb.delete_template'); + + const { user } = res.locals; + const templateId = templateName(req.params.template_id); + + templateMaps.delTemplate(user, templateId, (err/* , tpl_val */) => { + if (err) { + return next(err); + } + + res.statusCode = 204; + res.body = ''; + + next(); + }); + }; +} + +function listTemplates ({ templateMaps }) { + return function listTemplatesMiddleware (req, res, next) { + req.profiler.start('windshaft-cartodb.get_template_list'); + + const { user } = res.locals; + + templateMaps.listTemplates(user, (err, templateIds) => { + if (err) { + return next(err); + } + + res.statusCode = 200; + res.body = { template_ids: templateIds }; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js new file mode 100644 index 00000000..b28143dd --- /dev/null +++ b/lib/cartodb/api/template/named-template-controller.js @@ -0,0 +1,220 @@ +'use strict'; + +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const initProfiler = require('../middlewares/init-profiler'); +const checkJsonContentType = require('../middlewares/check-json-content-type'); +const incrementMapViewCount = require('../middlewares/increment-map-view-count'); +const augmentLayergroupData = require('../middlewares/augment-layergroup-data'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup'); +const layerStats = require('../middlewares/layer-stats'); +const layergroupIdHeader = require('../middlewares/layergroup-id-header'); +const layergroupMetadata = require('../middlewares/layergroup-metadata'); +const mapError = require('../middlewares/map-error'); +const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); +const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +module.exports = class NamedMapController { + /** + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @param {MapBackend} mapBackend + * @param metadataBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsBackend} userLimitsBackend + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigAdapter} mapConfigAdapter + * @param {StatsBackend} statsBackend + * @param {AuthBackend} authBackend + * @param layergroupMetadata + * @constructor + */ + constructor ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTables = layergroupAffectedTables; + this.mapConfigAdapter = mapConfigAdapter; + this.statsBackend = statsBackend; + this.authBackend = authBackend; + this.layergroupMetadata = layergroupMetadata; + } + + register (templateRouter) { + templateRouter.get('/:template_id/jsonp', this.middlewares()); + templateRouter.post('/:template_id', this.middlewares()); + } + + middlewares () { + const isTemplateInstantiation = true; + const useTemplateHash = true; + const includeQuery = false; + const label = 'NAMED MAP LAYERGROUP'; + const addContext = false; + + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), + cleanUpQueryParams(['aggregation']), + initProfiler(isTemplateInstantiation), + checkJsonContentType(), + checkInstantiteLayergroup(), + getTemplate( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsBackend, + this.mapConfigAdapter, + this.layergroupAffectedTables + ), + instantiateLayergroup( + this.mapBackend, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTables + ), + incrementMapViewCount(this.metadataBackend), + augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + lastUpdatedTimeLayergroup(), + layerStats(this.pgConnection, this.statsBackend), + layergroupIdHeader(this.templateMaps ,useTemplateHash), + layergroupMetadata(this.layergroupMetadata, includeQuery), + mapError({ label, addContext }) + ]; + } +}; + +function checkInstantiteLayergroup () { + return function checkInstantiteLayergroupMiddleware(req, res, next) { + if (req.method === 'GET') { + const { callback, config } = req.query; + + if (callback === undefined || callback.length === 0) { + return next(new Error('callback parameter should be present and be a function name')); + } + + if (config) { + try { + req.body = JSON.parse(config); + } catch(e) { + return next(new Error('Invalid config parameter, should be a valid JSON')); + } + } + } + + req.profiler.done('checkInstantiteLayergroup'); + + return next(); + }; +} + +function getTemplate ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + affectedTablesCache +) { + return function getTemplateMiddleware (req, res, next) { + const templateParams = req.body; + const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id } = req.params; + const { auth_token } = req.query; + + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + const mapConfigProvider = new NamedMapMapConfigProvider( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + affectedTablesCache, + user, + template_id, + templateParams, + auth_token, + params + ); + + mapConfigProvider.getMapConfig((err, mapConfig, rendererParams, context, stats = {}) => { + req.profiler.done('named.getMapConfig'); + + stats.mapType = 'named'; + req.profiler.add(stats); + + if (err) { + return next(err); + } + + res.locals.mapConfig = mapConfig; + res.locals.rendererParams = rendererParams; + res.locals.mapConfigProvider = mapConfigProvider; + + next(); + }); + }; +} + +function instantiateLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) { + return function instantiateLayergroupMiddleware (req, res, next) { + const { user, mapConfig, rendererParams } = res.locals; + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsBackend, + pgConnection, + affectedTablesCache, + rendererParams + ); + + mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup, stats = {}) => { + req.profiler.add(stats); + + if (err) { + return next(err); + } + + res.statusCode = 200; + res.body = layergroup; + + const { mapConfigProvider } = res.locals; + + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js new file mode 100644 index 00000000..0299679d --- /dev/null +++ b/lib/cartodb/api/template/template-router.js @@ -0,0 +1,66 @@ +'use strict'; + +const { Router: router } = require('express'); + +const NamedMapController = require('./named-template-controller'); +const AdminTemplateController = require('./admin-template-controller'); +const TileTemplateController = require('./tile-template-controller'); + +module.exports = class TemplateRouter { + constructor ({ collaborators }) { + const { + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata, + namedMapProviderCache, + tileBackend, + } = collaborators; + + this.namedMapController = new NamedMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ); + + this.tileTemplateController = new TileTemplateController( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authBackend, + userLimitsBackend + ); + + this.adminTemplateController = new AdminTemplateController( + authBackend, + templateMaps, + userLimitsBackend + ); + } + + register (apiRouter, templatePaths) { + const templateRouter = router({ mergeParams: true }); + + this.namedMapController.register(templateRouter); + this.tileTemplateController.register(templateRouter); + this.adminTemplateController.register(templateRouter); + + templatePaths.forEach(path => apiRouter.use(path, templateRouter)); + } +}; diff --git a/lib/cartodb/api/template/tile-template-controller.js b/lib/cartodb/api/template/tile-template-controller.js new file mode 100644 index 00000000..daf751e6 --- /dev/null +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -0,0 +1,97 @@ +'use strict'; + +const coordinates = require('../middlewares/coordinates'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const namedMapProvider = require('../middlewares/named-map-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const vectorError = require('../middlewares/vector-error'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +module.exports = class TileTemplateController { + constructor ( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authBackend, + userLimitsBackend + ) { + this.namedMapProviderCache = namedMapProviderCache; + this.tileBackend = tileBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } + + register (templateRouter) { + templateRouter.get('/:template_id/:layer/:z/:x/:y.(:format)', this.middlewares()); + } + + middlewares () { + return [ + coordinates(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), + cleanUpQueryParams(), + namedMapProvider({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'NAMED_MAP_TILE' + }), + getTile({ + tileBackend: this.tileBackend, + label: 'NAMED_MAP_TILE' + }), + setContentTypeHeader(), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + vectorError() + ]; + } +}; + +function getTile ({ tileBackend, label }) { + return function getTileMiddleware (req, res, next) { + const { mapConfigProvider } = res.locals; + const { layer, z, x, y, format } = req.params; + const params = { layer, z, x, y, format }; + + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => { + req.profiler.add(stats); + req.profiler.done('render-' + format); + + if (err) { + err.label = label; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.statusCode = 200; + res.body = tile; + + next(); + }); + }; +} + +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js deleted file mode 100644 index 0f4f0f4c..00000000 --- a/lib/cartodb/api/user_limits_api.js +++ /dev/null @@ -1,79 +0,0 @@ -var step = require('step'); - -/** - * - * @param metadataBackend - * @param options - * @constructor - * @type {UserLimitsApi} - */ -function UserLimitsApi(metadataBackend, options) { - this.metadataBackend = metadataBackend; - this.options = options || {}; - this.options.limits = this.options.limits || {}; -} - -module.exports = UserLimitsApi; - -UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) { - var self = this; - - var limits = { - cacheOnTimeout: self.options.limits.cacheOnTimeout || false, - render: self.options.limits.render || 0 - }; - - self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { - if (err) { - return callback(err); - } - - if (timeoutRenderLimit && timeoutRenderLimit.render) { - if (Number.isFinite(timeoutRenderLimit.render)) { - limits.render = timeoutRenderLimit.render; - } - } - - return callback(null, limits); - }); -}; - -UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { - var self = this; - - step( - function isAuthorized() { - var next = this; - - if (!apiKey) { - return next(null, false); - } - - self.metadataBackend.getUserMapKey(username, function (err, userApiKey) { - if (err) { - return next(err); - } - - return next(null, userApiKey === apiKey); - }); - }, - function getUserTimeoutRenderLimits(err, authorized) { - var next = this; - - if (err) { - return next(err); - } - - self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { - if (err) { - return next(err); - } - - next(null, { - render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic - }); - }); - }, - callback - ); -}; diff --git a/lib/cartodb/backends/analysis-status.js b/lib/cartodb/backends/analysis-status.js index 97f851d2..1c636f4e 100644 --- a/lib/cartodb/backends/analysis-status.js +++ b/lib/cartodb/backends/analysis-status.js @@ -1,3 +1,5 @@ +'use strict'; + var PSQL = require('cartodb-psql'); function AnalysisStatusBackend() { @@ -5,16 +7,14 @@ function AnalysisStatusBackend() { module.exports = AnalysisStatusBackend; - -AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { - var nodeId = params.nodeId; - +AnalysisStatusBackend.prototype.getNodeStatus = function (nodeId, dbParams, callback) { var statusQuery = [ 'SELECT node_id, status, updated_at, last_error_message as error_message', 'FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\'' ].join(' '); - var pg = new PSQL(dbParamsFromReqParams(params)); + var pg = new PSQL(dbParams); + pg.query(statusQuery, function(err, result) { if (err) { return callback(err, result); @@ -36,23 +36,3 @@ AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { return callback(null, statusResponse); }, true); // use read-only transaction }; - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/backends/auth.js b/lib/cartodb/backends/auth.js new file mode 100644 index 00000000..0a17e4a2 --- /dev/null +++ b/lib/cartodb/backends/auth.js @@ -0,0 +1,180 @@ +'use strict'; + +/** + * + * @param {PgConnection} pgConnection + * @param metadataBackend + * @param {MapStore} mapStore + * @param {TemplateMaps} templateMaps + * @constructor + * @type {AuthBackend} + */ +function AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps) { + this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; + this.mapStore = mapStore; + this.templateMaps = templateMaps; +} + +module.exports = AuthBackend; + +// Check if the user is authorized by a signer +// +// @param res express response object +// @param callback function(err, signed_by) signed_by will be +// null if the request is not signed by anyone +// or will be a string cartodb username otherwise. +// +AuthBackend.prototype.authorizedBySigner = function(req, res, callback) { + if ( ! res.locals.token || ! res.locals.signer ) { + return callback(null, false); // no signer requested + } + + var self = this; + + var layergroup_id = res.locals.token; + var auth_token = req.query.auth_token; + + this.mapStore.load(layergroup_id, function(err, mapConfig) { + if (err) { + return callback(err); + } + + var authorized = self.templateMaps.isAuthorized(mapConfig.obj().template, auth_token); + + return callback(null, authorized); + }); +}; + +function isValidApiKey(apikey) { + return apikey.type && + apikey.user && + apikey.databasePassword && + apikey.databaseRole; +} + +// Check if a request is authorized by api_key +// +// @param user +// @param res express response object +// @param callback function(err, authorized) +// NOTE: authorized is expected to be 0 or 1 (integer) +// +AuthBackend.prototype.authorizedByAPIKey = function(user, res, callback) { + const apikeyToken = res.locals.api_key; + const basicAuthUsername = res.locals.basicAuthUsername; + + if ( ! apikeyToken ) { + return callback(null, false); // no api key, no authorization... + } + + this.metadataBackend.getApikey(user, apikeyToken, (err, apikey) => { + if (err) { + if (isNameNotFoundError(err)) { + err.http_status = 404; + } + + return callback(err); + } + + if ( !isValidApiKey(apikey)) { + const error = new Error('Unauthorized'); + error.type = 'auth'; + error.subtype = 'api-key-not-found'; + error.http_status = 401; + + return callback(error); + } + + if (!usernameMatches(basicAuthUsername, res.locals.user)) { + const error = new Error('Forbidden'); + error.type = 'auth'; + error.subtype = 'api-key-username-mismatch'; + error.http_status = 403; + + return callback(error); + } + + if (!apikey.grantsMaps) { + const error = new Error('Forbidden'); + error.type = 'auth'; + error.subtype = 'api-key-does-not-grant-access'; + error.http_status = 403; + + return callback(error); + } + + return callback(null, true, apikey); + }); +}; + +function isNameNotFoundError (err) { + return err.message && -1 !== err.message.indexOf('name not found'); +} + +function usernameMatches (basicAuthUsername, requestUsername) { + return !(basicAuthUsername && (basicAuthUsername !== requestUsername)); +} + +/** + * Check access authorization + * + * @param req - standard req object. Importantly contains table and host information + * @param res - standard res object. Contains the auth parameters in locals + * @param callback function(err, allowed) is access allowed not? + */ +AuthBackend.prototype.authorize = function(req, res, callback) { + var user = res.locals.user; + + this.authorizedByAPIKey(user, res, (err, isAuthorizedByApikey) => { + if (err) { + return callback(err); + } + + if (isAuthorizedByApikey) { + return this.pgConnection.setDBAuth(user, res.locals, 'regular', function (err) { + req.profiler.done('setDBAuth'); + + if (err) { + return callback(err); + } + + callback(null, true); + }); + } + + this.authorizedBySigner(req, res, (err, isAuthorizedBySigner) => { + if (err) { + return callback(err); + } + + if (isAuthorizedBySigner) { + return this.pgConnection.setDBAuth(user, res.locals, 'master', function (err) { + req.profiler.done('setDBAuth'); + + if (err) { + return callback(err); + } + + callback(null, true); + }); + } + + // if no signer name was given, use default api key + if (!res.locals.signer) { + return this.pgConnection.setDBAuth(user, res.locals, 'default', function (err) { + req.profiler.done('setDBAuth'); + + if (err) { + return callback(err); + } + + callback(null, true); + }); + } + + // if signer name was given, return no authorization + return callback(null, false); + }); + }); +}; diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index c110f328..acdf24a7 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -1,13 +1,11 @@ -var assert = require('assert'); +'use strict'; var _ = require('underscore'); var PSQL = require('cartodb-psql'); -var step = require('step'); - var BBoxFilter = require('../models/filter/bbox'); - var DataviewFactory = require('../models/dataview/factory'); var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory'); +const dbParamsFromReqParams = require('../utils/database-params'); var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' @@ -23,53 +21,76 @@ function DataviewBackend(analysisBackend) { module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { + const dataviewName = params.dataviewName; - var dataviewName = params.dataviewName; - step( - function getMapConfig() { - mapConfigProvider.getMapConfig(this); - }, - function runDataviewQuery(err, mapConfig) { - assert.ifError(err); + mapConfigProvider.getMapConfig(function (err, mapConfig) { + if (err) { + return callback(err); + } - var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); - if (!dataviewDefinition) { - throw new Error("Dataview '" + dataviewName + "' does not exists"); - } + var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!dataviewDefinition) { + const error = new Error(`Dataview '${dataviewName}' does not exist`); + error.type = 'dataview'; + error.http_status = 400; + return callback(error); + } + if (!validFilterParams(params)) { + const error = new Error('Both own_filter and no_filters cannot be sent in the same request'); + error.type = 'dataview'; + error.http_status = 400; + return callback(error); + } + + var pg; + var overrideParams; + var dataview; + + try { + pg = new PSQL(dbParamsFromReqParams(params)); + var query = getQueryWithFilters(dataviewDefinition, params); + var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); + var dataviewFactory = DataviewFactoryWithOverviews.getFactory(overviewsQueryRewriter, queryRewriteData, { + bbox: params.bbox + }); + dataview = dataviewFactory.getDataview(query, dataviewDefinition); var ownFilter = +params.own_filter; - var noFilters = +params.no_filters; - if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) { - err = new Error(); - err.message = 'Both own_filter and no_filters cannot be sent in the same request'; - err.type = 'dataview'; - err.http_status = 400; + overrideParams = getOverrideParams(params, !!ownFilter); + } catch (error) { + return callback(error); + } + + dataview.getResult(pg, overrideParams, function (err, dataviewResult, stats = {}) { + if (err) { return callback(err); } - var pg = new PSQL(dbParamsFromReqParams(params)); - - var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); - if (params.bbox) { - var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); - query = bboxFilter.sql(query); - } - - var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); - - var dataviewFactory = DataviewFactoryWithOverviews.getFactory( - overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } - ); - - var dataview = dataviewFactory.getDataview(query, dataviewDefinition); - dataview.getResult(pg, getOverrideParams(params, !!ownFilter), this); - }, - function returnCallback(err, result) { - return callback(err, result); - } - ); + return callback(null, dataviewResult, stats); + }); + }); }; +function validFilterParams (params) { + var ownFilter = +params.own_filter; + var noFilters = +params.no_filters; + + return !(Number.isFinite(ownFilter) && Number.isFinite(noFilters)); +} + +function getQueryWithFilters (dataviewDefinition, params) { + var ownFilter = +params.own_filter; + var noFilters = +params.no_filters; + var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); + + if (params.bbox) { + var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + + return query; +} + function getDataviewQuery(dataviewDefinition, ownFilter, noFilters) { if (noFilters) { return dataviewDefinition.sql.no_filters; @@ -131,62 +152,57 @@ function getOverrideParams(params, ownFilter) { } DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewName, params, callback) { - step( - function getMapConfig() { - mapConfigProvider.getMapConfig(this); - }, - function runDataviewSearchQuery(err, mapConfig) { - assert.ifError(err); - - var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); - if (!dataviewDefinition) { - throw new Error("Dataview '" + dataviewName + "' does not exists"); - } - - var pg = new PSQL(dbParamsFromReqParams(params)); - - var ownFilter = +params.own_filter; - ownFilter = !!ownFilter; - - var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; - - if (params.bbox) { - var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); - query = bboxFilter.sql(query); - } - - var userQuery = params.q; - - var dataview = DataviewFactory.getDataview(query, dataviewDefinition); - dataview.search(pg, userQuery, this); - }, - function returnCallback(err, result) { - return callback(err, result); + mapConfigProvider.getMapConfig(function (err, mapConfig) { + if (err) { + return callback(err); } - ); + + var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!dataviewDefinition) { + const error = new Error(`Dataview '${dataviewName}' does not exist`); + error.type = 'dataview'; + error.http_status = 400; + return callback(error); + } + + var pg; + var query; + var dataview; + var userQuery = params.q; + + try { + pg = new PSQL(dbParamsFromReqParams(params)); + query = getQueryWithOwnFilters(dataviewDefinition, params); + dataview = DataviewFactory.getDataview(query, dataviewDefinition); + } catch (error) { + return callback(error); + } + + dataview.search(pg, userQuery, function (err, result) { + if (err) { + return callback(err); + } + + return callback(null, result); + }); + }); }; +function getQueryWithOwnFilters (dataviewDefinition, params) { + var ownFilter = +params.own_filter; + ownFilter = !!ownFilter; + + var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; + + if (params.bbox) { + var bboxFilter = new BBoxFilter({ column: 'the_geom', srid: 4326 }, { bbox: params.bbox }); + query = bboxFilter.sql(query); + } + + return query; +} + function getDataviewDefinition(mapConfig, dataviewName) { var dataviews = mapConfig.dataviews || {}; return dataviews[dataviewName]; } - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js new file mode 100644 index 00000000..c5fb48bb --- /dev/null +++ b/lib/cartodb/backends/filter-stats.js @@ -0,0 +1,53 @@ +'use strict'; + +var _ = require('underscore'); +var AnalysisFilter = require('../models/filter/analysis'); + +function FilterStatsBackends(pgQueryRunner) { + this.pgQueryRunner = pgQueryRunner; +} + +module.exports = FilterStatsBackends; + +function getEstimatedRows(pgQueryRunner, username, query, callback) { + pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) { + if (err){ + callback(err); + return; + } + var rows; + if ( result_rows[0] && result_rows[0]['QUERY PLAN'] && + result_rows[0]['QUERY PLAN'][0] && result_rows[0]['QUERY PLAN'][0].Plan ) { + rows = result_rows[0]['QUERY PLAN'][0].Plan['Plan Rows']; + } + return callback(null, rows); + }); +} + +FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { + var stats = {}; + + getEstimatedRows(this.pgQueryRunner, username, unfiltered_query, (err, rows) => { + if (err){ + return callback(err); + } + + stats.unfiltered_rows = rows; + + if (!filters || _.isEmpty(filters)) { + return callback(null, stats); + } + + var analysisFilter = new AnalysisFilter(filters); + var query = analysisFilter.sql(unfiltered_query); + + getEstimatedRows(this.pgQueryRunner, username, query, (err, rows) => { + if (err){ + return callback(err); + } + + stats.filtered_rows = rows; + return callback(null, stats); + }); + }); +}; diff --git a/lib/cartodb/backends/layer-stats/empty-layer-stats.js b/lib/cartodb/backends/layer-stats/empty-layer-stats.js index 0760c0b6..72c76e84 100644 --- a/lib/cartodb/backends/layer-stats/empty-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/empty-layer-stats.js @@ -1,3 +1,5 @@ +'use strict'; + function EmptyLayerStats(types) { this._types = types || {}; } diff --git a/lib/cartodb/backends/layer-stats/factory.js b/lib/cartodb/backends/layer-stats/factory.js index 8aacdb5a..19eb0701 100644 --- a/lib/cartodb/backends/layer-stats/factory.js +++ b/lib/cartodb/backends/layer-stats/factory.js @@ -1,3 +1,5 @@ +'use strict'; + var LayerStats = require('./layer-stats'); var EmptyLayerStats = require('./empty-layer-stats'); var MapnikLayerStats = require('./mapnik-layer-stats'); diff --git a/lib/cartodb/backends/layer-stats/layer-stats.js b/lib/cartodb/backends/layer-stats/layer-stats.js index 2464fb22..38e9b461 100644 --- a/lib/cartodb/backends/layer-stats/layer-stats.js +++ b/lib/cartodb/backends/layer-stats/layer-stats.js @@ -1,3 +1,5 @@ +'use strict'; + var queue = require('queue-async'); function LayerStats(layerStatsIterator) { diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index c060f964..a8a181cc 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,4 +1,8 @@ -var queryUtils = require('../../utils/query-utils'); +'use strict'; + +const queryUtils = require('../../utils/query-utils'); +const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); +const aggregationQuery = require('../../models/aggregation/aggregation-query'); function MapnikLayerStats () { this._types = { @@ -11,17 +15,297 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; +function columnAggregations(field) { + if (field.type === 'number') { + return ['min', 'max', 'avg', 'sum']; + } + if (field.type === 'date') { // TODO other types too? + return ['min', 'max']; + } + if (field.type === 'timeDimension') { + return ['min', 'max']; + } + return []; +} + +function _getSQL(ctx, query, type='pre', zoom=0) { + let sql; + if (type === 'pre') { + sql = ctx.preQuery; + } + else { + sql = ctx.aggrQuery; + } + sql = queryUtils.subsituteTokensForZoom(sql, zoom || 0); + return query(sql); +} + +function _estimatedFeatureCount(ctx) { + return queryUtils.queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryRowEstimation)) + .then(res => ({ estimatedFeatureCount: res.rows[0].rows })) + .catch(() => ({ estimatedFeatureCount: -1 })); +} + +function _featureCount(ctx) { + if (ctx.metaOptions.featureCount) { + // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query + return queryUtils.queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryActualRowCount)) + .then(res => ({ featureCount: res.rows[0].rows })); + } + return Promise.resolve(); +} + +function _aggrFeatureCount(ctx) { + if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) { + // We expect as zoom level as the value of aggrFeatureCount + // TODO: it'd be nice to admit an array of zoom levels to + // return metadata for multiple levels. + return queryUtils.queryPromise( + ctx.dbConnection, + _getSQL(ctx, queryUtils.getQueryActualRowCount, 'post', ctx.metaOptions.aggrFeatureCount) + ).then(res => ({ aggrFeatureCount: res.rows[0].rows })); + } + return Promise.resolve(); +} + +function _geometryType(ctx) { + if (ctx.metaOptions.geometryType) { + const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); + const sqlQuery = _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)); + return queryUtils.queryPromise(ctx.dbConnection, sqlQuery) + .then(res => ({ geometryType: (res.rows[0] || {}).geom_type })); + } + return Promise.resolve(); +} + +function _columns(ctx) { + if (ctx.metaOptions.columns || ctx.metaOptions.columnStats || ctx.metaOptions.dimensions) { + // note: post-aggregation columns are in layer.options.columns when aggregation is present + return queryUtils.queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0))) + .then(res => formatResultFields(ctx.dbConnection, res.fields)); + } + return Promise.resolve(); +} + +// combine a list of results merging the properties of all the objects +// undefined results are admitted and ignored +function mergeResults(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + if (a === undefined) { + return b; + } + if (b === undefined) { + return a; + } + return Object.assign({}, a, b); + }); + } +} + +// deeper (1 level) combination of a list of objects: +// mergeColumns([{ col1: { a: 1 }, col2: { a: 2 } }, { col1: { b: 3 } }]) => { col1: { a: 1, b: 3 }, col2: { a: 2 } } +function mergeColumns(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + let c = Object.assign({}, b || {}, a || {}); + Object.keys(c).forEach(key => { + if (b.hasOwnProperty(key)) { + c[key] = Object.assign(c[key], b[key]); + } + }); + return c; + }); + } +} + +const SAMPLE_SEED = 0.5; +const DEFAULT_SAMPLE_ROWS = 100; + +function _sample(ctx, numRows) { + if (ctx.metaOptions.sample) { + const sampleProb = Math.min(ctx.metaOptions.sample.num_rows / numRows, 1); + // We'll use a safety limit just in case numRows is a bad estimate + const requestedRows = ctx.metaOptions.sample.num_rows || DEFAULT_SAMPLE_ROWS; + const limit = Math.ceil(requestedRows * 1.5); + let columns = ctx.metaOptions.sample.include_columns; + return queryUtils.queryPromise(ctx.dbConnection, _getSQL( + ctx, + sql => queryUtils.getQuerySample(sql, sampleProb, limit, SAMPLE_SEED, columns) + )).then(res => ({ sample: res.rows })); + } + return Promise.resolve(); +} + +function _columnsMetadataRequired(options) { + // We need determine the columns of a query + // if either column stats or dimension stats are required, + // since we'll ultimately use the same query to fetch both + return options.columnStats || options.dimensions; +} + +function _columnStats(ctx, columns, dimensions) { + if (!columns) { + return Promise.resolve(); + } + if (_columnsMetadataRequired(ctx.metaOptions)) { + let queries = []; + let aggr = []; + if (ctx.metaOptions.columnStats) { + queries.push(new Promise(resolve => resolve({ columns }))); // add columns as first result + Object.keys(columns).forEach(name => { + aggr = aggr.concat( + columnAggregations(columns[name]) + .map(fn => `${fn}("${name}") AS "${name}_${fn}"`) + ); + if (columns[name].type === 'string') { + const topN = ctx.metaOptions.columnStats.topCategories || 1024; + const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ? + ctx.metaOptions.columnStats.includeNulls : + true; + + // TODO: ctx.metaOptions.columnStats.maxCategories + // => use PG stats to dismiss columns with more distinct values + queries.push( + queryUtils.queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls)) + ).then(res => ({ columns: { [name]: { categories: res.rows } } })) + ); + } + }); + } + const dimensionsStats = {}; + let dimensionsInfo = {}; + if (ctx.metaOptions.dimensions && dimensions) { + dimensionsInfo = aggregationQuery.infoForOptions({ dimensions }); + Object.keys(dimensionsInfo).forEach(dimName => { + const info = dimensionsInfo[dimName]; + if (info.type === 'timeDimension') { + dimensionsStats[dimName] = { + params: info.params + }; + aggr = aggr.concat( + columnAggregations(info).map(fn => `${fn}(${info.sql}) AS "${dimName}_${fn}"`) + ); + } + }); + } + queries.push( + queryUtils.queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`) + ).then(res => { + let stats = { columns: {}, dimensions: {} }; + Object.keys(columns).forEach(name => { + stats.columns[name] = {}; + columnAggregations(columns[name]).forEach(fn => { + stats.columns[name][fn] = res.rows[0][`${name}_${fn}`]; + }); + }); + Object.keys(dimensionsInfo).forEach(name => { + stats.dimensions[name] = stats.dimensions[name] || Object.assign({}, dimensionsStats[name]); + let type = null; + columnAggregations(dimensionsInfo[name]).forEach(fn => { + type = type || + fieldTypeSafe(ctx.dbConnection, res.fields.find(f => f.name === `${name}_${fn}`)); + stats.dimensions[name][fn] = res.rows[0][`${name}_${fn}`]; + }); + stats.dimensions[name].type = type; + }); + return stats; + }) + ); + return Promise.all(queries).then(results => ({ + columns: mergeColumns(results.map(r => r.columns)), + dimensions: mergeColumns(results.map( r => r.dimensions)) + })); + } + return Promise.resolve({ columns }); +} + +// This is adapted from SQL API: +function fieldType(cname) { + let tname; + switch (true) { + case /bool/.test(cname): + tname = 'boolean'; + break; + case /int|float|numeric/.test(cname): + tname = 'number'; + break; + case /text|char|unknown/.test(cname): + tname = 'string'; + break; + case /date|time/.test(cname): + tname = 'date'; + break; + default: + tname = cname; + } + if ( tname && cname.match(/^_/) ) { + tname += '[]'; + } + return tname; +} + +function fieldTypeSafe(dbConnection, field) { + const cname = dbConnection.typeName(field.dataTypeID); + return cname ? fieldType(cname) : `unknown(${field.dataTypeID})`; +} + +// columns are returned as an object { columnName1: { type1: ...}, ..} +// for consistency with SQL API +function formatResultFields(dbConnection, fields = []) { + let nfields = {}; + for (let field of fields) { + nfields[field.name] = { type: fieldTypeSafe(dbConnection, field) }; + } + return nfields; +} + MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); - // This query would gather stats for postgresql table if not exists - dbConnection.query(queryRowCountSql, function (err, res) { - if (err) { - return callback(null, {estimatedFeatureCount: -1}); - } else { - // We decided that the relation is 1 row == 1 feature - return callback(null, {estimatedFeatureCount: res.rows[0].rows}); - } + let aggrQuery = layer.options.sql; + let preQuery = layer.options.sql_raw || aggrQuery; + + let ctx = { + dbConnection, + preQuery, + aggrQuery, + metaOptions: layer.options.metadata || {}, + }; + + // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere + // we would set queries.results.estimatedFeatureCount and queries.results.geometryType + // (if metaOptions.geometryType) from it. + + // TODO: compute _sample with _featureCount when available + // TODO: add support for sample.exclude option by, in that case, forcing the columns query and + // passing the results to the sample query function. + + const dimensions = (layer.options.aggregation || {}).dimensions; + + Promise.all([ + _estimatedFeatureCount(ctx).then( + ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount) + .then(sampleResults => mergeResults([sampleResults, { estimatedFeatureCount }])) + ), + _featureCount(ctx), + _aggrFeatureCount(ctx), + _geometryType(ctx), + _columns(ctx).then(columns => _columnStats(ctx, columns, dimensions)) + ]).then(results => { + results = mergeResults(results); + callback(null, results); + }).catch(error => { + callback(error); }); }; diff --git a/lib/cartodb/backends/layer-stats/torque-layer-stats.js b/lib/cartodb/backends/layer-stats/torque-layer-stats.js index 00b4def2..af198b1e 100644 --- a/lib/cartodb/backends/layer-stats/torque-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/torque-layer-stats.js @@ -1,3 +1,5 @@ +'use strict'; + function TorqueLayerStats() { this._types = { torque: true diff --git a/lib/cartodb/api/overviews_metadata_api.js b/lib/cartodb/backends/overviews-metadata.js similarity index 55% rename from lib/cartodb/api/overviews_metadata_api.js rename to lib/cartodb/backends/overviews-metadata.js index 55b4966f..f066c4ef 100644 --- a/lib/cartodb/api/overviews_metadata_api.js +++ b/lib/cartodb/backends/overviews-metadata.js @@ -1,25 +1,22 @@ -var SubstitutionTokens = require('../utils/substitution-tokens'); +'use strict'; -function OverviewsMetadataApi(pgQueryRunner) { +const queryUtils = require('../utils/query-utils'); + +function OverviewsMetadataBackend(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = OverviewsMetadataApi; +module.exports = OverviewsMetadataBackend; -function prepareSql(sql) { - return sql && SubstitutionTokens.replace(sql, { - bbox: 'ST_MakeEnvelope(0,0,0,0)', - scale_denominator: '0', - pixel_width: '1', - pixel_height: '1' - }); -} - -OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) { +OverviewsMetadataBackend.prototype.getOverviewsMetadata = function (username, sql, callback) { // FIXME: Currently using internal function _cdb_schema_name // CDB_Overviews should provide the schema information directly. - var query = 'SELECT *, _cdb_schema_name(base_table)' + - ' FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))'; + const query = ` + SELECT *, _cdb_schema_name(base_table) + FROM CDB_Overviews( + CDB_QueryTablesText($windshaft$${queryUtils.substituteDummyTokens(sql)}$windshaft$) + ); + `; this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) { if (err){ callback(err); diff --git a/lib/cartodb/backends/pg_connection.js b/lib/cartodb/backends/pg_connection.js index d98bb703..9c7ed4c3 100644 --- a/lib/cartodb/backends/pg_connection.js +++ b/lib/cartodb/backends/pg_connection.js @@ -1,7 +1,8 @@ -var assert = require('assert'); -var step = require('step'); +'use strict'; + var PSQL = require('cartodb-psql'); var _ = require('underscore'); +const debug = require('debug')('cachechan'); function PgConnection(metadataBackend) { this.metadataBackend = metadataBackend; @@ -20,45 +21,59 @@ module.exports = PgConnection; // // @param callback function(err) // -PgConnection.prototype.setDBAuth = function(username, params, callback) { - var self = this; - - var user_params = {}; - var auth_user = global.environment.postgres_auth_user; - var auth_pass = global.environment.postgres_auth_pass; - step( - function getId() { - self.metadataBackend.getUserId(username, this); - }, - function(err, user_id) { - assert.ifError(err); - user_params.user_id = user_id; - var dbuser = _.template(auth_user, user_params); - _.extend(params, {dbuser:dbuser}); - - // skip looking up user_password if postgres_auth_pass - // doesn't contain the "user_password" label - if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) { - return null; +PgConnection.prototype.setDBAuth = function(username, params, apikeyType, callback) { + if (apikeyType === 'master') { + this.metadataBackend.getMasterApikey(username, (err, apikey) => { + if (err) { + if (isNameNotFoundError(err)) { + err.http_status = 404; + } + return callback(err); } - self.metadataBackend.getUserDBPass(username, this); - }, - function(err, user_password) { - assert.ifError(err); - user_params.user_password = user_password; - if ( auth_pass ) { - var dbpass = _.template(auth_pass, user_params); - _.extend(params, {dbpassword:dbpass}); + params.dbuser = apikey.databaseRole; + params.dbpassword = apikey.databasePassword; + + return callback(); + }); + } else if (apikeyType === 'regular') { //Actually it can be any type of api key + this.metadataBackend.getApikey(username, params.api_key, (err, apikey) => { + if (err) { + if (isNameNotFoundError(err)) { + err.http_status = 404; + } + return callback(err); } - return true; - }, - function finish(err) { - callback(err); - } - ); + + params.dbuser = apikey.databaseRole; + params.dbpassword = apikey.databasePassword; + + return callback(); + }); + } else if (apikeyType === 'default') { + this.metadataBackend.getApikey(username, 'default_public', (err, apikey) => { + if (err) { + if (isNameNotFoundError(err)) { + err.http_status = 404; + } + return callback(err); + } + + params.dbuser = apikey.databaseRole; + params.dbpassword = apikey.databasePassword; + + return callback(); + }); + } else { + return callback(new Error(`Invalid Apikey type: ${apikeyType}, valid ones: master, regular, default`)); + } }; +function isNameNotFoundError (err) { + return err.message && -1 !== err.message.indexOf('name not found'); +} + + // Set db connection parameters to those for the given username // // @param dbowner cartodb username of database owner, @@ -71,36 +86,30 @@ PgConnection.prototype.setDBAuth = function(username, params, callback) { // @param callback function(err) // PgConnection.prototype.setDBConn = function(dbowner, params, callback) { - var self = this; - // Add default database connection parameters - // if none given _.defaults(params, { - dbuser: global.environment.postgres.user, - dbpassword: global.environment.postgres.password, + // dbuser: global.environment.postgres.user, + // dbpassword: global.environment.postgres.password, dbhost: global.environment.postgres.host, dbport: global.environment.postgres.port }); - step( - function getConnectionParams() { - self.metadataBackend.getUserDBConnectionParams(dbowner, this); - }, - function extendParams(err, dbParams){ - assert.ifError(err); - // we don't want null values or overwrite a non public user - if (params.dbuser !== 'publicuser' || !dbParams.dbuser) { - delete dbParams.dbuser; - } - if ( dbParams ) { - _.extend(params, dbParams); - } - return null; - }, - function finish(err) { - callback(err); - } - ); -}; + this.metadataBackend.getUserDBConnectionParams(dbowner, (err, dbParams) => { + if (err) { + return callback(err); + } + + // we don’t want null values or overwrite a non public user + if (params.dbuser !== 'publicuser' || !dbParams.dbuser) { + delete dbParams.dbuser; + } + + if (dbParams) { + _.extend(params, dbParams); + } + + callback(); + }); +}; /** * Returns a `cartodb-psql` object for a given username. @@ -109,28 +118,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) { */ PgConnection.prototype.getConnection = function(username, callback) { - var self = this; + debug("getConn1"); - var params = {}; - - require('debug')('cachechan')("getConn1"); - step( - function setAuth() { - self.setDBAuth(username, params, this); - }, - function setConn(err) { - assert.ifError(err); - self.setDBConn(username, params, this); - }, - function openConnection(err) { - assert.ifError(err); - return callback(err, new PSQL({ - user: params.dbuser, - pass: params.dbpass, - host: params.dbhost, - port: params.dbport, - dbname: params.dbname - })); + this.getDatabaseParams(username, (err, databaseParams) => { + if (err) { + return callback(err); } - ); + return callback(err, new PSQL({ + user: databaseParams.dbuser, + pass: databaseParams.dbpass, + host: databaseParams.dbhost, + port: databaseParams.dbport, + dbname: databaseParams.dbname + })); + + }); +}; + +PgConnection.prototype.getDatabaseParams = function(username, callback) { + const databaseParams = {}; + + this.setDBAuth(username, databaseParams, 'master', err => { + if (err) { + return callback(err); + } + + this.setDBConn(username, databaseParams, err => { + if (err) { + return callback(err); + } + + callback(null, databaseParams); + }); + }); }; diff --git a/lib/cartodb/backends/pg_query_runner.js b/lib/cartodb/backends/pg_query_runner.js index bf57f166..c96041c4 100644 --- a/lib/cartodb/backends/pg_query_runner.js +++ b/lib/cartodb/backends/pg_query_runner.js @@ -1,6 +1,6 @@ -var assert = require('assert'); +'use strict'; + var PSQL = require('cartodb-psql'); -var step = require('step'); function PgQueryRunner(pgConnection) { this.pgConnection = pgConnection; @@ -16,31 +16,23 @@ module.exports = PgQueryRunner; * @param {Function} callback function({Error}, {Array}) second argument is guaranteed to be an array */ PgQueryRunner.prototype.run = function(username, query, callback) { - var self = this; - var params = {}; - - step( - function setAuth() { - self.pgConnection.setDBAuth(username, params, this); - }, - function setConn(err) { - assert.ifError(err); - self.pgConnection.setDBConn(username, params, this); - }, - function executeQuery(err) { - assert.ifError(err); - var psql = new PSQL({ - user: params.dbuser, - pass: params.dbpass, - host: params.dbhost, - port: params.dbport, - dbname: params.dbname - }); - psql.query(query, function(err, resultSet) { - resultSet = resultSet || {}; - return callback(err, resultSet.rows || []); - }); + this.pgConnection.getDatabaseParams(username, (err, databaseParams) => { + if (err) { + return callback(err); } - ); + + const psql = new PSQL({ + user: databaseParams.dbuser, + pass: databaseParams.dbpass, + host: databaseParams.dbhost, + port: databaseParams.dbport, + dbname: databaseParams.dbname + }); + + psql.query(query, function (err, resultSet) { + resultSet = resultSet || {}; + return callback(err, resultSet.rows || []); + }); + }); }; diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index b0385bac..c3e5303d 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -1,3 +1,5 @@ +'use strict'; + var layerStats = require('./layer-stats/factory'); function StatsBackend() { diff --git a/lib/cartodb/api/tables_extent_api.js b/lib/cartodb/backends/tables-extent.js similarity index 89% rename from lib/cartodb/api/tables_extent_api.js rename to lib/cartodb/backends/tables-extent.js index 7b534a3f..ff41701c 100644 --- a/lib/cartodb/api/tables_extent_api.js +++ b/lib/cartodb/backends/tables-extent.js @@ -1,8 +1,10 @@ -function TablesExtentApi(pgQueryRunner) { +'use strict'; + +function TablesExtentBackend(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = TablesExtentApi; +module.exports = TablesExtentBackend; /** * Given a username and a list of tables it will return the estimated extent in SRID 4326 for all the tables based on @@ -13,7 +15,7 @@ module.exports = TablesExtentApi; * `table_name` format as valid input * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north` */ -TablesExtentApi.prototype.getBounds = function (username, tables, callback) { +TablesExtentBackend.prototype.getBounds = function (username, tables, callback) { var estimatedExtentSQLs = tables.map(function(table) { return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')"; }); diff --git a/lib/cartodb/backends/template_maps.js b/lib/cartodb/backends/template_maps.js index 75a482fe..ea2d93fd 100644 --- a/lib/cartodb/backends/template_maps.js +++ b/lib/cartodb/backends/template_maps.js @@ -1,7 +1,7 @@ -var assert = require('assert'); +'use strict'; + var crypto = require('crypto'); var debug = require('debug')('windshaft:templates'); -var step = require('step'); var _ = require('underscore'); var dot = require('dot'); @@ -69,27 +69,19 @@ TemplateMaps.prototype._userTemplateLimit = function() { * @param callback - function to pass results too. */ TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) { - var redisClient; - var that = this; - var db = that.db_signatures; + this.redis_pool.acquire(this.db_signatures, (err, redisClient) => { + if (err) { + return callback(err); + } - step( - function getRedisClient() { - that.redis_pool.acquire(db, this); - }, - function executeQuery(err, data) { - assert.ifError(err); - redisClient = data; - redisArgs.push(this); - redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs); - }, - function releaseRedisClient(err, data) { - if ( ! _.isUndefined(redisClient) ) { - that.redis_pool.release(db, redisClient); - } - callback(err, data); - } - ); + redisClient[redisFunc.toUpperCase()](...redisArgs, (err, data) => { + this.redis_pool.release(this.db_signatures, redisClient); + if (err) { + return callback(err); + } + return callback(null, data); + }); + }); }; var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i; @@ -184,6 +176,37 @@ function templateDefaults(template) { }); } +/** + * Checks if the if the user reaches the templetes limit + * + * @param userTemplatesKey user templat key in Redis + * @param owner cartodb username of the template owner + * @param callback returns error if the user reaches the limit + */ +TemplateMaps.prototype._checkUserTemplatesLimit = function(userTemplatesKey, owner, callback) { + const limit = this._userTemplateLimit(); + + if(!limit) { + return callback(); + } + + this._redisCmd('HLEN', [userTemplatesKey], (err, numberOfTemplates) => { + if (err) { + return callback(err); + } + + if (numberOfTemplates >= limit) { + const limitReachedError = new Error( + `User '${owner}' reached limit on number of templates (${numberOfTemplates}/${limit})` + ); + limitReachedError.http_status = 409; + return callback(limitReachedError); + } + + return callback(); + }); +}; + //--------------- PUBLIC API ------------------------------------- // Add a template @@ -199,52 +222,41 @@ function templateDefaults(template) { // Return template identifier (only valid for given user) // TemplateMaps.prototype.addTemplate = function(owner, template, callback) { - var self = this; - template = templateDefaults(template); var invalidError = this._checkInvalidTemplate(template); - if ( invalidError ) { + if (invalidError) { return callback(invalidError); } - var templateName = template.name; - var userTemplatesKey = this.key_usr_tpl({ owner:owner }); - var limit = this._userTemplateLimit(); + var userTemplatesKey = this.key_usr_tpl({ owner }); - step( - function checkLimit() { - if ( ! limit ) { - return 0; - } - self._redisCmd('HLEN', [ userTemplatesKey ], this); - }, - function installTemplateIfDoesNotExist(err, numberOfTemplates) { - assert.ifError(err); - if ( limit && numberOfTemplates >= limit ) { - var limitReachedError = new Error("User '" + owner + "' reached limit on number of templates (" + - numberOfTemplates + "/" + limit + ")"); - limitReachedError.http_status = 409; - throw limitReachedError; - } - self._redisCmd('HSETNX', [ userTemplatesKey, templateName, JSON.stringify(template) ], this); - }, - function validateInstallation(err, wasSet) { - assert.ifError(err); - if ( ! wasSet ) { - throw new Error("Template '" + templateName + "' of user '" + owner + "' already exists"); - } - - return true; - }, - function finish(err) { - if (!err) { - self.emit('add', owner, templateName, template); - } - - callback(err, templateName, template); + this._checkUserTemplatesLimit(userTemplatesKey, owner, err => { + if (err) { + return callback(err); } - ); + + let templateString; + try { + templateString = JSON.stringify(template); + } catch (error) { + return callback(error); + } + + this._redisCmd('HSETNX', [userTemplatesKey, template.name, templateString], (err, wasSet) => { + if (err) { + return callback(err); + } + + if (!wasSet) { + var templateExistsError = new Error(`Template '${template.name}' of user '${owner}' already exists`); + return callback(templateExistsError); + } + + this.emit('add', owner, template.name, template); + return callback(null, template.name, template); + }); + }); }; // Delete a template @@ -257,26 +269,18 @@ TemplateMaps.prototype.addTemplate = function(owner, template, callback) { // @param callback function(err) // TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) { - var self = this; - step( - function deleteTemplate() { - self._redisCmd('HDEL', [ self.key_usr_tpl({ owner:owner }), tpl_id ], this); - }, - function handleDeletion(err, deleted) { - assert.ifError(err); - if (!deleted) { - throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); - } - return true; - }, - function finish(err) { - if (!err) { - self.emit('delete', owner, tpl_id); - } - - callback(err); + this._redisCmd('HDEL', [ this.key_usr_tpl({ owner:owner }), tpl_id ], (err, deleted) => { + if (err) { + return callback(err); } - ); + + if (!deleted) { + return callback(new Error(`Template '${tpl_id}' of user '${owner}' does not exist`)); + } + + this.emit('delete', owner, tpl_id); + return callback(); + }); }; // Update a template @@ -296,56 +300,58 @@ TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) { // @param callback function(err) // TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) { - - var self = this; - template = templateDefaults(template); var invalidError = this._checkInvalidTemplate(template); - - if ( invalidError ) { + if (invalidError) { return callback(invalidError); } - var templateName = template.name; - - if ( tpl_id !== templateName ) { - return callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + templateName + "')")); + if (tpl_id !== template.name) { + return callback(new Error(`Cannot update name of a map template ('${tpl_id}' != '${template.name}')`)); } - var userTemplatesKey = this.key_usr_tpl({ owner:owner }); + var userTemplatesKey = this.key_usr_tpl({ owner }); - var previousTemplate = null; + this._redisCmd('HGET', [userTemplatesKey, tpl_id], (err, beforeUpdateTemplate) => { + if (err) { + return callback(err); + } - step( - function getExistingTemplate() { - self._redisCmd('HGET', [ userTemplatesKey, tpl_id ], this); - }, - function updateTemplate(err, _currentTemplate) { - assert.ifError(err); - if (!_currentTemplate) { - throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); + if (!beforeUpdateTemplate) { + return callback(new Error(`Template '${tpl_id}' of user '${owner}' does not exist`)); + } + + let templateString; + try { + templateString = JSON.stringify(template); + } catch (error) { + return callback(error); + } + + this._redisCmd('HSET', [userTemplatesKey, template.name, templateString], (err, didSetNewField) => { + if (err) { + return callback(err); } - previousTemplate = _currentTemplate; - self._redisCmd('HSET', [ userTemplatesKey, templateName, JSON.stringify(template) ], this); - }, - function handleTemplateUpdate(err, didSetNewField) { - assert.ifError(err); + if (didSetNewField) { debug('New template created on update operation'); } - return true; - }, - function finish(err) { - if (!err) { - if (self.fingerPrint(JSON.parse(previousTemplate)) !== self.fingerPrint(template)) { - self.emit('update', owner, templateName, template); - } + + let beforeUpdateTemplateObject; + try { + beforeUpdateTemplateObject = JSON.parse(beforeUpdateTemplate); + } catch (error) { + return callback(error); } - callback(err, template); - } - ); + if (this.fingerPrint(beforeUpdateTemplateObject) !== this.fingerPrint(template)) { + this.emit('update', owner, template.name, template); + } + + return callback(null, template); + }); + }); }; // List user templates @@ -370,19 +376,20 @@ TemplateMaps.prototype.listTemplates = function(owner, callback) { // Return full template definition // TemplateMaps.prototype.getTemplate = function(owner, tpl_id, callback) { - var self = this; - step( - function getTemplate() { - self._redisCmd('HGET', [ self.key_usr_tpl({owner:owner}), tpl_id ], this); - }, - function parseTemplate(err, tpl_val) { - assert.ifError(err); - return JSON.parse(tpl_val); - }, - function finish(err, tpl) { - callback(err, tpl); + this._redisCmd('HGET', [this.key_usr_tpl({owner:owner}), tpl_id], (err, template) => { + if (err) { + return callback(err); } - ); + + let templateObject; + try { + templateObject = JSON.parse(template); + } catch (error) { + return callback(error); + } + + return callback(null, templateObject); + }); }; TemplateMaps.prototype.isAuthorized = function(template, authTokens) { diff --git a/lib/cartodb/backends/user-limits.js b/lib/cartodb/backends/user-limits.js new file mode 100644 index 00000000..6177007a --- /dev/null +++ b/lib/cartodb/backends/user-limits.js @@ -0,0 +1,84 @@ +'use strict'; + +/** + * + * @param metadataBackend + * @param options + * @constructor + * @type {UserLimitsBackend} + */ +function UserLimitsBackend(metadataBackend, options) { + this.metadataBackend = metadataBackend; + this.options = options || {}; + this.options.limits = this.options.limits || {}; + + this.preprareRateLimit(); +} + +module.exports = UserLimitsBackend; + +UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callback) { + var self = this; + + var limits = { + cacheOnTimeout: self.options.limits.cacheOnTimeout || false, + render: self.options.limits.render || 0 + }; + + self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) { + if (err) { + return callback(err); + } + + if (timeoutRenderLimit && timeoutRenderLimit.render) { + if (Number.isFinite(timeoutRenderLimit.render)) { + limits.render = timeoutRenderLimit.render; + } + } + + return callback(null, limits); + }); +}; + +UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { + isAuthorized(this.metadataBackend, username, apiKey, (err, authorized) => { + if (err) { + return callback(err); + } + + this.metadataBackend.getUserTimeoutRenderLimits(username, (err, timeoutRenderLimit) => { + if (err) { + return callback(err); + } + + return callback( + null, + { render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic } + ); + }); + }); +}; + +function isAuthorized(metadataBackend, username, apiKey, callback) { + if (!apiKey) { + return callback(null, false); + } + + metadataBackend.getUserMapKey(username, function (err, userApiKey) { + if (err) { + return callback(err); + } + + return callback(null, userApiKey === apiKey); + }); +} + +UserLimitsBackend.prototype.preprareRateLimit = function () { + if (this.options.limits.rateLimitsEnabled) { + this.metadataBackend.loadRateLimitsScript(); + } +}; + +UserLimitsBackend.prototype.getRateLimit = function (user, endpointGroup, callback) { + this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback); +}; diff --git a/lib/cartodb/cache/backend/fastly.js b/lib/cartodb/cache/backend/fastly.js index 1dad622c..e1ca6b35 100644 --- a/lib/cartodb/cache/backend/fastly.js +++ b/lib/cartodb/cache/backend/fastly.js @@ -1,3 +1,5 @@ +'use strict'; + var FastlyPurge = require('fastly-purge'); function FastlyCacheBackend(apiKey, serviceId) { diff --git a/lib/cartodb/cache/backend/varnish_http.js b/lib/cartodb/cache/backend/varnish_http.js index 760e1f11..35fa5330 100644 --- a/lib/cartodb/cache/backend/varnish_http.js +++ b/lib/cartodb/cache/backend/varnish_http.js @@ -1,3 +1,5 @@ +'use strict'; + var request = require('request'); function VarnishHttpCacheBackend(host, port) { diff --git a/lib/cartodb/cache/layergroup_affected_tables.js b/lib/cartodb/cache/layergroup_affected_tables.js index d33c5a73..fb63eb37 100644 --- a/lib/cartodb/cache/layergroup_affected_tables.js +++ b/lib/cartodb/cache/layergroup_affected_tables.js @@ -1,3 +1,5 @@ +'use strict'; + var LruCache = require('lru-cache'); function LayergroupAffectedTables() { diff --git a/lib/cartodb/cache/model/named_maps_entry.js b/lib/cartodb/cache/model/named_maps_entry.js index aaab7f3a..a4537368 100644 --- a/lib/cartodb/cache/model/named_maps_entry.js +++ b/lib/cartodb/cache/model/named_maps_entry.js @@ -1,3 +1,5 @@ +'use strict'; + var crypto = require('crypto'); function NamedMaps(owner, name) { diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index ebafbbac..d0e34883 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -1,3 +1,5 @@ +'use strict'; + var _ = require('underscore'); var dot = require('dot'); var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); @@ -6,12 +8,20 @@ var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + affectedTablesCache +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.mapConfigAdapter = mapConfigAdapter; + this.affectedTablesCache = affectedTablesCache; this.providerCache = new LruCache({ max: 2000 }); } @@ -28,8 +38,9 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.templateMaps, this.pgConnection, this.metadataBackend, - this.userLimitsApi, + this.userLimitsBackend, this.mapConfigAdapter, + this.affectedTablesCache, user, templateId, config, diff --git a/lib/cartodb/cache/surrogate_keys_cache.js b/lib/cartodb/cache/surrogate_keys_cache.js index 907e6345..bf4bc852 100644 --- a/lib/cartodb/cache/surrogate_keys_cache.js +++ b/lib/cartodb/cache/surrogate_keys_cache.js @@ -1,3 +1,5 @@ +'use strict'; + var queue = require('queue-async'); /** diff --git a/lib/cartodb/controllers/index.js b/lib/cartodb/controllers/index.js deleted file mode 100644 index 429a4b87..00000000 --- a/lib/cartodb/controllers/index.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - Analyses: require('./analyses'), - Layergroup: require('./layergroup'), - Map: require('./map'), - NamedMaps: require('./named_maps'), - NamedMapsAdmin: require('./named_maps_admin'), - ServerInfo: require('./server_info') -}; diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js deleted file mode 100644 index 0276527d..00000000 --- a/lib/cartodb/controllers/layergroup.js +++ /dev/null @@ -1,513 +0,0 @@ -var assert = require('assert'); -var step = require('step'); - -var cors = require('../middleware/cors'); -var userMiddleware = require('../middleware/user'); -var allowQueryParams = require('../middleware/allow-query-params'); -var vectorError = require('../middleware/vector-error'); - -var DataviewBackend = require('../backends/dataview'); -var AnalysisStatusBackend = require('../backends/analysis-status'); - -var MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); - -var QueryTables = require('cartodb-query-tables'); - -/** - * @param {AuthApi} authApi - * @param {PgConnection} pgConnection - * @param {MapStore} mapStore - * @param {TileBackend} tileBackend - * @param {PreviewBackend} previewBackend - * @param {AttributesBackend} attributesBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {AnalysisBackend} analysisBackend - * @constructor - */ -function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) { - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.tileBackend = tileBackend; - this.previewBackend = previewBackend; - this.attributesBackend = attributesBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; - - this.dataviewBackend = new DataviewBackend(analysisBackend); - this.analysisStatusBackend = new AnalysisStatusBackend(); - - this.prepareContext = prepareContext; -} - -module.exports = LayergroupController; - -LayergroupController.prototype.register = function(app) { - app.get( - app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format', - cors(), - userMiddleware, - this.prepareContext, - this.tile.bind(this), - vectorError() - ); - - app.get( - app.base_url_mapconfig + '/:token/:z/:x/:y.:format', - cors(), - userMiddleware, - this.prepareContext, - this.tile.bind(this), - vectorError() - ); - - app.get( - app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)', - cors(), - userMiddleware, - validateLayerRouteMiddleware, - this.prepareContext, - this.layer.bind(this), - vectorError() - ); - - app.get( - app.base_url_mapconfig + '/:token/:layer/attributes/:fid', - cors(), - userMiddleware, - this.prepareContext, - this.attributes.bind(this) - ); - - app.get( - app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format', - cors(), - userMiddleware, - allowQueryParams(['layer']), - this.prepareContext, - this.center.bind(this) - ); - - app.get( - app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', - cors(), - userMiddleware, - allowQueryParams(['layer']), - this.prepareContext, - this.bbox.bind(this) - ); - - // Undocumented/non-supported API endpoint methods. - // Use at your own peril. - - var allowedDataviewQueryParams = [ - 'filters', // json - 'own_filter', // 0, 1 - 'no_filters', // 0, 1 - 'bbox', // w,s,e,n - 'start', // number - 'end', // number - 'column_type', // string - 'bins', // number - 'aggregation', //string - 'offset', // number - 'q', // widgets search - 'categories', // number - ]; - - app.get( - app.base_url_mapconfig + '/:token/dataview/:dataviewName', - cors(), - userMiddleware, - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - this.dataview.bind(this) - ); - - app.get( - app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName', - cors(), - userMiddleware, - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - this.dataview.bind(this) - ); - - app.get( - app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', - cors(), - userMiddleware, - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - this.dataviewSearch.bind(this) - ); - - app.get( - app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search', - cors(), - userMiddleware, - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - this.dataviewSearch.bind(this) - ); - - app.get( - app.base_url_mapconfig + '/:token/analysis/node/:nodeId', - cors(), - userMiddleware, - this.prepareContext, - this.analysisNodeStatus.bind(this) - ); -}; - -LayergroupController.prototype.analysisNodeStatus = function(req, res, next) { - var self = this; - - step( - function retrieveNodeStatus() { - self.analysisStatusBackend.getNodeStatus(res.locals, this); - }, - function finish(err, nodeStatus, stats) { - req.profiler.add(stats || {}); - - if (err) { - err.label = 'GET NODE STATUS'; - next(err); - } else { - self.sendResponse(req, res, nodeStatus, 200, { - 'Cache-Control': 'public,max-age=5', - 'Last-Modified': new Date().toUTCString() - }); - } - } - ); -}; - -LayergroupController.prototype.dataview = function(req, res, next) { - var self = this; - - step( - function retrieveDataview() { - var mapConfigProvider = new MapStoreMapConfigProvider( - self.mapStore, res.locals.user, self.userLimitsApi, res.locals - ); - self.dataviewBackend.getDataview( - mapConfigProvider, - res.locals.user, - res.locals, - this - ); - }, - function finish(err, dataview, stats) { - req.profiler.add(stats || {}); - - if (err) { - err.label = 'GET DATAVIEW'; - next(err); - } else { - self.sendResponse(req, res, dataview, 200); - } - } - ); -}; - -LayergroupController.prototype.dataviewSearch = function(req, res, next) { - var self = this; - - step( - function searchDataview() { - var mapConfigProvider = new MapStoreMapConfigProvider( - self.mapStore, res.locals.user, self.userLimitsApi, res.locals - ); - self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this); - }, - function finish(err, searchResult, stats) { - req.profiler.add(stats || {}); - - if (err) { - err.label = 'GET DATAVIEW SEARCH'; - next(err); - } else { - self.sendResponse(req, res, searchResult, 200); - } - } - ); - -}; - -LayergroupController.prototype.attributes = function(req, res, next) { - var self = this; - - req.profiler.start('windshaft.maplayer_attribute'); - - step( - function retrieveFeatureAttributes() { - var mapConfigProvider = new MapStoreMapConfigProvider( - self.mapStore, res.locals.user, self.userLimitsApi, res.locals - ); - self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this); - }, - function finish(err, tile, stats) { - req.profiler.add(stats || {}); - - if (err) { - err.label = 'GET ATTRIBUTES'; - next(err); - } else { - self.sendResponse(req, res, tile, 200); - } - } - ); - -}; - -// Gets a tile for a given token and set of tile ZXY coords. (OSM style) -LayergroupController.prototype.tile = function(req, res, next) { - req.profiler.start('windshaft.map_tile'); - this.tileOrLayer(req, res, next); -}; - -// Gets a tile for a given token, layer set of tile ZXY coords. (OSM style) -LayergroupController.prototype.layer = function(req, res, next) { - req.profiler.start('windshaft.maplayer_tile'); - this.tileOrLayer(req, res, next); -}; - -LayergroupController.prototype.tileOrLayer = function (req, res, next) { - var self = this; - - step( - function mapController$getTileOrGrid() { - self.tileBackend.getTile( - new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals), - req.params, this - ); - }, - function mapController$finalize(err, tile, headers, stats) { - req.profiler.add(stats); - self.finalizeGetTileOrGrid(err, req, res, tile, headers, next); - } - ); -}; - -function getStatusCode(tile, format){ - return tile.length===0 && format==='mvt'? 204:200; -} - -// This function is meant for being called as the very last -// step by all endpoints serving tiles or grids -LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) { - var supportedFormats = { - grid_json: true, - json_torque: true, - torque_json: true, - png: true, - png32: true, - mvt: true - }; - - var formatStat = 'invalid'; - if (req.params.format) { - var format = req.params.format.replace('.', '_'); - if (supportedFormats[format]) { - formatStat = format; - } - } - - if (err) { - // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 - var errMsg = err.message ? ( '' + err.message ) : ( '' + err ); - - // Rewrite mapnik parsing errors to start with layer number - var matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); - if (matches) { - errMsg = 'style'+matches[2]+': ' + matches[1]; - } - err.message = errMsg; - - err.label = 'TILE RENDER'; - next(err); - - global.statsClient.increment('windshaft.tiles.error'); - global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); - } else { - this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers); - global.statsClient.increment('windshaft.tiles.success'); - global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); - } -}; - -LayergroupController.prototype.bbox = function(req, res, next) { - this.staticMap(req, res, +req.params.width, +req.params.height, { - west: +req.params.west, - north: +req.params.north, - east: +req.params.east, - south: +req.params.south - }, null, next); -}; - -LayergroupController.prototype.center = function(req, res, next) { - this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, { - lng: +req.params.lng, - lat: +req.params.lat - }, next); -}; - -LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) { - var format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - // We force always the tile to be generated using PNG because - // is the only format we support by now - res.locals.format = 'png'; - res.locals.layer = res.locals.layer || 'all'; - - var self = this; - - step( - function getImage() { - if (center) { - self.previewBackend.getImage( - new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals), - format, width, height, zoom, center, this); - } else { - self.previewBackend.getImage( - new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals), - format, width, height, zoom /* bounds */, this); - } - }, - function handleImage(err, image, headers, stats) { - req.profiler.done('render-' + format); - req.profiler.add(stats || {}); - - if (err) { - err.label = 'STATIC_MAP'; - next(err); - } else { - res.set('Content-Type', headers['Content-Type'] || 'image/' + format); - self.sendResponse(req, res, image, 200); - } - } - ); -}; - -LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) { - var self = this; - - req.profiler.done('res'); - - res.set('Cache-Control', 'public,max-age=31536000'); - - // Set Last-Modified header - var lastUpdated; - if (res.locals.cache_buster) { - // Assuming cache_buster is a timestamp - lastUpdated = new Date(parseInt(res.locals.cache_buster)); - } else { - lastUpdated = new Date(); - } - res.set('Last-Modified', lastUpdated.toUTCString()); - - var dbName = res.locals.dbname; - step( - function getAffectedTables() { - self.getAffectedTables(res.locals.user, dbName, res.locals.token, this); - }, - function sendResponse(err, affectedTables) { - req.profiler.done('affectedTables'); - if (err) { - global.logger.warn('ERROR generating cache channel: ' + err); - } - if (!!affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - self.surrogateKeysCache.tag(res, affectedTables); - } - - if (headers) { - res.set(headers); - } - - res.status(status); - - if (!Buffer.isBuffer(body) && typeof body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(body); - } else { - res.json(body); - } - } else { - res.send(body); - } - } - ); -}; - -LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) { - - if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) { - return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId)); - } - - var self = this; - step( - function extractSQL() { - step( - function loadFromStore() { - self.mapStore.load(layergroupId, this); - }, - function getSQL(err, mapConfig) { - assert.ifError(err); - - var queries = []; - mapConfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - queries.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); - - return queries.length ? queries.join(';') : null; - }, - this - ); - }, - function findAffectedTables(err, sql) { - assert.ifError(err); - - if ( ! sql ) { - throw new Error("this request doesn't need an X-Cache-Channel generated"); - } - - step( - function getConnection() { - self.pgConnection.getConnection(user, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function buildCacheChannel(err, tables) { - assert.ifError(err); - self.layergroupAffectedTables.set(dbName, layergroupId, tables); - - return tables; - }, - callback - ); -}; - - -function validateLayerRouteMiddleware(req, res, next) { - if (req.params.token === 'static') { - return next('route'); - } - - next(); -} diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js deleted file mode 100644 index 34660ce0..00000000 --- a/lib/cartodb/controllers/map.js +++ /dev/null @@ -1,729 +0,0 @@ -var _ = require('underscore'); -var windshaft = require('windshaft'); -var QueryTables = require('cartodb-query-tables'); - -var ResourceLocator = require('../models/resource-locator'); - -var cors = require('../middleware/cors'); -var userMiddleware = require('../middleware/user'); -const allowQueryParams = require('../middleware/allow-query-params'); - -var MapConfig = windshaft.model.MapConfig; -var Datasource = windshaft.model.Datasource; - -var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); - -var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); -var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); - -/** - * @param {AuthApi} authApi - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @param {MapBackend} mapBackend - * @param metadataBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {MapConfigAdapter} mapConfigAdapter - * @param {StatsBackend} statsBackend - * @constructor - */ -function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, - statsBackend) { - this.pgConnection = pgConnection; - this.templateMaps = templateMaps; - this.mapBackend = mapBackend; - this.metadataBackend = metadataBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; - - this.mapConfigAdapter = mapConfigAdapter; - this.resourceLocator = new ResourceLocator(global.environment); - - this.statsBackend = statsBackend; - this.prepareContext = prepareContext; -} - -module.exports = MapController; - -MapController.prototype.register = function(app) { - const { base_url_mapconfig, base_url_templated } = app; - const useTemplate = true; - - app.get(base_url_mapconfig, this.composeCreateMapMiddleware()); - app.post(base_url_mapconfig, this.composeCreateMapMiddleware()); - app.get(`${base_url_templated}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate)); - app.post(`${base_url_templated}/:template_id`, this.composeCreateMapMiddleware(useTemplate)); - app.options(app.base_url_mapconfig, cors('Content-Type')); -}; - -MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) { - const isTemplateInstantiation = useTemplate; - const useTemplateHash = useTemplate; - const includeQuery = !useTemplate; - const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP'; - const addContext = !useTemplate; - - return [ - cors(), - userMiddleware, - allowQueryParams(['aggregation']), - this.prepareContext, - this.initProfiler(isTemplateInstantiation), - this.checkJsonContentType(), - useTemplate ? this.checkInstantiteLayergroup() : this.checkCreateLayergroup(), - useTemplate ? this.getTemplate() : this.prepareAdapterMapConfig(), - useTemplate ? this.instantiateLayergroup() : this.createLayergroup(), - this.incrementMapViewCount(), - this.augmentLayergroupData(), - this.getAffectedTables(), - this.setCacheChannel(), - this.setLastModified(), - this.setLastUpdatedTimeToLayergroup(), - this.setCacheControl(), - this.setLayerStats(), - this.setLayergroupIdHeader(useTemplateHash), - this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(), - this.setAnalysesMetadataToLayergroup(includeQuery), - this.setTurboCartoMetadataToLayergroup(), - this.setAggregationMetadataToLayergroup(), - this.setTilejsonMetadataToLayergroup(), - this.setSurrogateKeyHeader(), - this.sendResponse(), - this.augmentError({ label, addContext }) - ]; -}; - -MapController.prototype.initProfiler = function (isTemplateInstantiation) { - const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; - - return function initProfilerMiddleware (req, res, next) { - req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); - req.profiler.done(`${operation}.initProfilerMiddleware`); - next(); - }; -}; - -MapController.prototype.checkJsonContentType = function () { - return function checkJsonContentTypeMiddleware(req, res, next) { - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('POST data must be of type application/json')); - } - - req.profiler.done('checkJsonContentTypeMiddleware'); - - next(); - }; -}; - -MapController.prototype.checkInstantiteLayergroup = function () { - return function checkInstantiteLayergroupMiddleware(req, res, next) { - if (req.method === 'GET') { - const { callback, config } = req.query; - - if (callback === undefined || callback.length === 0) { - return next(new Error('callback parameter should be present and be a function name')); - } - - if (config) { - try { - req.body = JSON.parse(config); - } catch(e) { - return next(new Error('Invalid config parameter, should be a valid JSON')); - } - } - } - - req.profiler.done('checkInstantiteLayergroup'); - - return next(); - }; -}; - -MapController.prototype.checkCreateLayergroup = function () { - return function checkCreateLayergroupMiddleware (req, res, next) { - if (req.method === 'GET') { - const { config } = res.locals; - - if (!config) { - return next(new Error('layergroup GET needs a "config" parameter')); - } - - try { - req.body = JSON.parse(config); - } catch (err) { - return next(err); - } - } - - req.profiler.done('checkCreateLayergroup'); - return next(); - }; -}; - -MapController.prototype.getTemplate = function () { - return function getTemplateMiddleware (req, res, next) { - const templateParams = req.body; - const { user } = res.locals; - - const mapconfigProvider = new NamedMapMapConfigProvider( - this.templateMaps, - this.pgConnection, - this.metadataBackend, - this.userLimitsApi, - this.mapConfigAdapter, - user, - req.params.template_id, - templateParams, - res.locals.auth_token, - res.locals - ); - - mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { - req.profiler.done('named.getMapConfig'); - if (err) { - return next(err); - } - - res.locals.mapconfig = mapconfig; - res.locals.rendererParams = rendererParams; - res.locals.mapconfigProvider = mapconfigProvider; - - next(); - }); - }.bind(this); -}; - -MapController.prototype.prepareAdapterMapConfig = function () { - return function prepareAdapterMapConfigMiddleware(req, res, next) { - const requestMapConfig = req.body; - const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; - - const context = { - analysisConfiguration: { - user, - db: { - host: dbhost, - port: dbport, - dbname: dbname, - user: dbuser, - pass: dbpassword - }, - batch: { - username: user, - apiKey: api_key - } - } - }; - - this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { - req.profiler.done('anonymous.getMapConfig'); - if (err) { - return next(err); - } - - req.body = requestMapConfig; - res.locals.context = context; - - next(); - }); - }.bind(this); -}; - -MapController.prototype.createLayergroup = function () { - return function createLayergroupMiddleware (req, res, next) { - const requestMapConfig = req.body; - const { context, user } = res.locals; - const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapconfig = new MapConfig(requestMapConfig, datasource); - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals); - - res.locals.mapconfig = mapconfig; - res.locals.analysesResults = context.analysesResults; - - this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { - req.profiler.done('createLayergroup'); - if (err) { - return next(err); - } - - res.locals.layergroup = layergroup; - - next(); - }); - }.bind(this); -}; - -MapController.prototype.instantiateLayergroup = function () { - return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapconfig, rendererParams } = res.locals; - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); - - this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { - req.profiler.done('createLayergroup'); - if (err) { - return next(err); - } - - res.locals.layergroup = layergroup; - - const { mapconfigProvider } = res.locals; - - res.locals.analysesResults = mapconfigProvider.analysesResults; - res.locals.template = mapconfigProvider.template; - res.locals.templateName = mapconfigProvider.getTemplateName(); - res.locals.context = mapconfigProvider.context; - - next(); - }); - }.bind(this); -}; - -MapController.prototype.incrementMapViewCount = function () { - return function incrementMapViewCountMiddleware(req, res, next) { - const { mapconfig, user } = res.locals; - - // Error won't blow up, just be logged. - this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { - req.profiler.done('incMapviewCount'); - - if (err) { - global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); - } - - next(); - }); - }.bind(this); -}; - -MapController.prototype.augmentLayergroupData = function () { - return function augmentLayergroupDataMiddleware (req, res, next) { - const { layergroup } = res.locals; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - - next(); - }; -}; - -function getTemplateUrl(url) { - return url.https || url.http; -} - -function getTilejson(tiles, grids) { - const tilejson = { - tilejson: '2.2.0', - tiles: tiles.https || tiles.http - }; - - if (grids) { - tilejson.grids = grids.https || grids.http; - } - - return tilejson; -} - -MapController.prototype.setTilejsonMetadataToLayergroup = function () { - return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; - - const isVectorOnlyMapConfig = mapconfig.isVectorOnlyMapConfig(); - let hasMapnikLayers = false; - layergroup.metadata.layers.forEach((layerMetadata, index) => { - const layerId = mapconfig.getLayerId(index); - const rasterResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.png`; - if (mapconfig.layerType(index) === 'mapnik') { - hasMapnikLayers = true; - const vectorResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.mvt`; - const layerTilejson = { - vector: getTilejson(this.resourceLocator.getTileUrls(user, vectorResource)) - }; - if (!isVectorOnlyMapConfig) { - let grids = null; - const layer = mapconfig.getLayer(index); - if (layer.options.interactivity) { - const gridResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.grid.json`; - grids = this.resourceLocator.getTileUrls(user, gridResource); - } - layerTilejson.raster = getTilejson( - this.resourceLocator.getTileUrls(user, rasterResource), - grids - ); - } - layerMetadata.tilejson = layerTilejson; - } else { - layerMetadata.tilejson = { - raster: getTilejson(this.resourceLocator.getTileUrls(user, rasterResource)) - }; - } - }); - - const tilejson = {}; - const url = {}; - - if (hasMapnikLayers) { - const vectorResource = `${layergroup.layergroupid}/{z}/{x}/{y}.mvt`; - tilejson.vector = getTilejson( - this.resourceLocator.getTileUrls(user, vectorResource) - ); - url.vector = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, vectorResource)); - - if (!isVectorOnlyMapConfig) { - const rasterResource = `${layergroup.layergroupid}/{z}/{x}/{y}.png`; - tilejson.raster = getTilejson( - this.resourceLocator.getTileUrls(user, rasterResource) - ); - url.raster = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, rasterResource)); - } - } - - layergroup.metadata.tilejson = tilejson; - layergroup.metadata.url = url; - - next(); - }.bind(this); -}; - -MapController.prototype.getAffectedTables = function () { - return function getAffectedTablesMiddleware (req, res, next) { - const { dbname, layergroup, user, mapconfig } = res.locals; - - this.pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - const sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); - - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - return next(err); - } - - // feed affected tables cache so it can be reused from, for instance, layergroup controller - this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); - - res.locals.affectedTables = affectedTables; - - next(); - }); - }); - }.bind(this); -}; - -MapController.prototype.setCacheChannel = function () { - return function setCacheChannelMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (req.method === 'GET') { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -}; - -MapController.prototype.setLastModified = function () { - return function setLastModifiedMiddleware (req, res, next) { - if (req.method === 'GET') { - res.set('Last-Modified', (new Date()).toUTCString()); - } - - next(); - }; -}; - -MapController.prototype.setLastUpdatedTimeToLayergroup = function () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { affectedTables, layergroup, analysesResults } = res.locals; - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }; -}; - -function getLastUpdatedTime(analysesResults, lastUpdateTime) { - if (!Array.isArray(analysesResults)) { - return lastUpdateTime; - } - return analysesResults.reduce(function(lastUpdateTime, analysis) { - return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { - var nodeUpdatedAtDate = node.getUpdatedAt(); - var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; - return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; - }, lastUpdateTime); - }, lastUpdateTime); -} - -MapController.prototype.setCacheControl = function () { - return function setCacheControlMiddleware (req, res, next) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - } - - next(); - }; -}; - -MapController.prototype.setLayerStats = function () { - return function setLayerStatsMiddleware(req, res, next) { - const { user, mapconfig, layergroup } = res.locals; - - this.pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) { - if (err) { - return next(err); - } - - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } - - next(); - }); - }); - }.bind(this); -}; - -MapController.prototype.setLayergroupIdHeader = function (useTemplateHash) { - return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { layergroup, user, template } = res.locals; - - if (useTemplateHash) { - var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8); - layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; - } - - res.set('X-Layergroup-Id', layergroup.layergroupid); - - next(); - }.bind(this); -}; - -MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function () { - return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; - - this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); - - next(); - }.bind(this); -}; - -// TODO this should take into account several URL patterns -MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) { - this.addDataviewsUrls(username, layergroup, mapConfig); - this.addWidgetsUrl(username, layergroup, mapConfig); -}; - -MapController.prototype.addDataviewsUrls = function(username, layergroup, mapConfig) { - layergroup.metadata.dataviews = layergroup.metadata.dataviews || {}; - var dataviews = mapConfig.dataviews || {}; - - Object.keys(dataviews).forEach(function(dataviewName) { - var resource = layergroup.layergroupid + '/dataview/' + dataviewName; - layergroup.metadata.dataviews[dataviewName] = { - url: this.resourceLocator.getUrls(username, resource) - }; - }.bind(this)); -}; - -MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig) { - if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { - layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { - var mapConfigLayer = mapConfig.layers[layerIndex]; - if (mapConfigLayer.options && mapConfigLayer.options.widgets) { - layer.widgets = layer.widgets || {}; - Object.keys(mapConfigLayer.options.widgets).forEach(function(widgetName) { - var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName; - layer.widgets[widgetName] = { - type: mapConfigLayer.options.widgets[widgetName].type, - url: this.resourceLocator.getUrls(username, resource) - }; - }.bind(this)); - } - return layer; - }.bind(this)); - } -}; - -MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) { - return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, user, analysesResults = [] } = res.locals; - - this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); - - next(); - }.bind(this); -}; - -MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) { - includeQuery = includeQuery || false; - analysesResults = analysesResults || []; - layergroup.metadata.analyses = []; - - analysesResults.forEach(function(analysis) { - var nodes = analysis.getNodes(); - layergroup.metadata.analyses.push({ - nodes: nodes.reduce(function(nodesIdMap, node) { - if (node.params.id) { - var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); - var nodeRepr = { - status: node.getStatus(), - url: this.resourceLocator.getUrls(username, nodeResource) - }; - if (includeQuery) { - nodeRepr.query = node.getQuery(); - } - if (node.getStatus() === 'failed') { - nodeRepr.error_message = node.getErrorMessage(); - } - nodesIdMap[node.params.id] = nodeRepr; - } - - return nodesIdMap; - }.bind(this), {}) - }); - }.bind(this)); -}; - -MapController.prototype.setTurboCartoMetadataToLayergroup = function () { - return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; - - addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context); - - next(); - }; -}; - -function addTurboCartoContextMetadata(layergroup, mapConfig, context) { - if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { - layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { - if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { - layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; - } - return layer; - }); - } -} - -// TODO: see how evolve this function, it's a good candidate to be refactored -MapController.prototype.setAggregationMetadataToLayergroup = function () { - return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; - - addAggregationContextMetadata(layergroup, mapconfig.obj(), context); - - next(); - }; -}; - -function addAggregationContextMetadata(layergroup, mapConfig, context) { - if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { - layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { - if (context.aggregation && Array.isArray(context.aggregation.layers)) { - layer.meta.aggregation = context.aggregation.layers[layerIndex]; - } - return layer; - }); - } -} - -MapController.prototype.setSurrogateKeyHeader = function () { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { affectedTables, user, templateName } = res.locals; - - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - this.surrogateKeysCache.tag(res, affectedTables); - } - - if (templateName) { - this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } - - next(); - }.bind(this); -}; - -MapController.prototype.sendResponse = function () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } - }; -}; - -MapController.prototype.augmentError = function (options) { - const { addContext = false, label = 'MAPS CONTROLLER' } = options; - - return function augmentErrorMiddleware (err, req, res, next) { - req.profiler.done('error'); - const { mapconfig } = res.locals; - - if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; - } - - err.label = label; - - next(err); - }; -}; - -function populateError(err, mapConfig) { - var error = new Error(err.message); - error.http_status = err.http_status; - - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - return error; -} diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js deleted file mode 100644 index 92ba4ea6..00000000 --- a/lib/cartodb/controllers/named_maps.js +++ /dev/null @@ -1,490 +0,0 @@ -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -const cors = require('../middleware/cors'); -const userMiddleware = require('../middleware/user'); -const allowQueryParams = require('../middleware/allow-query-params'); -const vectorError = require('../middleware/vector-error'); - -const DEFAULT_ZOOM_CENTER = { - zoom: 1, - center: { - lng: 0, - lat: 0 - } -}; - -function numMapper(n) { - return +n; -} - -function getRequestParams(locals) { - const params = Object.assign({}, locals); - - delete params.template; - delete params.affectedTablesAndLastUpdate; - delete params.namedMapProvider; - delete params.allowedQueryParams; - - return params; -} - -function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend, - surrogateKeysCache, tablesExtentApi, metadataBackend) { - this.namedMapProviderCache = namedMapProviderCache; - this.tileBackend = tileBackend; - this.previewBackend = previewBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.tablesExtentApi = tablesExtentApi; - this.metadataBackend = metadataBackend; - this.prepareContext = prepareContext; -} - -module.exports = NamedMapsController; - -NamedMapsController.prototype.register = function(app) { - app.get( - app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)', - cors(), - userMiddleware, - this.prepareContext, - this.getNamedMapProvider('NAMED_MAP_TILE'), - this.getAffectedTables(), - this.getTile('NAMED_MAP_TILE'), - this.setSurrogateKey(), - this.setCacheChannelHeader(), - this.setLastModifiedHeader(), - this.setCacheControlHeader(), - this.setContentTypeHeader(), - this.respond(), - vectorError() - ); - - app.get( - app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', - cors(), - userMiddleware, - allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), - this.prepareContext, - this.getNamedMapProvider('STATIC_VIZ_MAP'), - this.getAffectedTables(), - this.getTemplate('STATIC_VIZ_MAP'), - this.prepareLayerFilterFromPreviewLayers('STATIC_VIZ_MAP'), - this.getStaticImageOptions(), - this.getImage('STATIC_VIZ_MAP'), - this.incrementMapViews(), - this.setSurrogateKey(), - this.setCacheChannelHeader(), - this.setLastModifiedHeader(), - this.setCacheControlHeader(), - this.setContentTypeHeader(), - this.respond() - ); -}; - -NamedMapsController.prototype.getNamedMapProvider = function (label) { - return function getNamedMapProviderMiddleware (req, res, next) { - const { user } = res.locals; - const { config, auth_token } = req.query; - const { template_id } = req.params; - - // We force always the tile to be generated using PNG because - // is the only format we support by now - res.locals.format = 'png'; - res.locals.layer = res.locals.layer || 'all'; - - const params = getRequestParams(res.locals); - - this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { - if (err) { - err.label = label; - return next(err); - } - - res.locals.namedMapProvider = namedMapProvider; - - next(); - }); - }.bind(this); -}; - -NamedMapsController.prototype.getAffectedTables = function () { - return function getAffectedTables (req, res, next) { - const { namedMapProvider } = res.locals; - - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { - req.profiler.done('affectedTables'); - - if (err) { - return next(err); - } - - res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate; - - next(); - }); - }.bind(this); -}; - -NamedMapsController.prototype.getTemplate = function (label) { - return function getTemplateMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; - - namedMapProvider.getTemplate((err, template) => { - if (err) { - err.label = label; - return next(err); - } - - res.locals.template = template; - - next(); - }); - }; -}; - -NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (label) { - return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) { - const { user, template } = res.locals; - const { template_id } = req.params; - const { config, auth_token } = req.query; - - if (!template || !template.view || !template.view.preview_layers) { - return next(); - } - - var previewLayers = template.view.preview_layers; - var layerVisibilityFilter = []; - - template.layergroup.layers.forEach((layer, index) => { - if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) { - layerVisibilityFilter.push(''+index); - } - }); - - if (!layerVisibilityFilter.length) { - return next(); - } - - const params = getRequestParams(res.locals); - - // overwrites 'all' default filter - params.layer = layerVisibilityFilter.join(','); - - // recreates the provider - this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, provider) => { - if (err) { - err.label = label; - return next(err); - } - - res.locals.namedMapProvider = provider; - - next(); - }); - }.bind(this); -}; - -NamedMapsController.prototype.getTile = function (label) { - return function getTileMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; - - this.tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => { - req.profiler.add(stats); - - if (err) { - err.label = label; - return next(err); - } - - res.locals.body = tile; - res.locals.headers = headers; - res.locals.stats = stats; - - next(); - }); - }.bind(this); -}; - -NamedMapsController.prototype.getStaticImageOptions = function () { - return function getStaticImageOptionsMiddleware(req, res, next) { - const { user, namedMapProvider, template } = res.locals; - - const imageOpts = getImageOptions(res.locals, template); - - if (imageOpts) { - res.locals.imageOpts = imageOpts; - return next(); - } - - res.locals.imageOpts = DEFAULT_ZOOM_CENTER; - - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { - if (err) { - return next(); - } - - var affectedTables = affectedTablesAndLastUpdate.tables || []; - - if (affectedTables.length === 0) { - return next(); - } - - this.tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => { - if (err) { - return next(); - } - - res.locals.imageOpts = bounds; - - return next(); - }); - }); - }.bind(this); -}; - -function getImageOptions (params, template) { - const { zoom, lon, lat, bbox } = params; - - let imageOpts = getImageOptionsFromCoordinates(zoom, lon, lat); - if (imageOpts) { - return imageOpts; - } - - imageOpts = getImageOptionsFromBoundingBox(bbox); - if (imageOpts) { - return imageOpts; - } - - imageOpts = getImageOptionsFromTemplate(template, zoom); - if (imageOpts) { - return imageOpts; - } -} - -function getImageOptionsFromCoordinates (zoom, lon, lat) { - if ([zoom, lon, lat].map(numMapper).every(Number.isFinite)) { - return { - zoom: zoom, - center: { - lng: lon, - lat: lat - } - }; - } -} - - -function getImageOptionsFromTemplate (template, zoom) { - if (template.view) { - var zoomCenter = templateZoomCenter(template.view); - if (zoomCenter) { - if (Number.isFinite(+zoom)) { - zoomCenter.zoom = +zoom; - } - - return zoomCenter; - } - - var bounds = templateBounds(template.view); - if (bounds) { - return bounds; - } - } -} - -function getImageOptionsFromBoundingBox (bbox = '') { - var _bbox = bbox.split(',').map(numMapper); - - if (_bbox.length === 4 && _bbox.every(Number.isFinite)) { - return { - bounds: { - west: _bbox[0], - south: _bbox[1], - east: _bbox[2], - north: _bbox[3] - } - }; - } -} - -NamedMapsController.prototype.getImage = function (label) { - return function getImageMiddleware (req, res, next) { - const { imageOpts, namedMapProvider } = res.locals; - const { zoom, center, bounds } = imageOpts; - - let { width, height } = req.params; - - width = +width; - height = +height; - - const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - - if (zoom !== undefined && center) { - return this.previewBackend.getImage(namedMapProvider, format, width, height, zoom, center, - (err, image, headers, stats) => { - if (err) { - err.label = label; - return next(err); - } - - res.locals.body = image; - res.locals.headers = headers; - res.locals.stats = stats; - - next(); - }); - } - - this.previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => { - if (err) { - err.label = label; - return next(err); - } - - res.locals.body = image; - res.locals.headers = headers; - res.locals.stats = stats; - - next(); - }); - }.bind(this); -}; - -function incrementMapViewsError (ctx) { - return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`; -} - -NamedMapsController.prototype.incrementMapViews = function () { - return function incrementMapViewsMiddleware(req, res, next) { - const { user, namedMapProvider } = res.locals; - - namedMapProvider.getMapConfig((err, mapConfig) => { - if (err) { - global.logger.log(incrementMapViewsError({ user, err })); - return next(); - } - - const statTag = mapConfig.obj().stat_tag; - - this.metadataBackend.incMapviewCount(user, statTag, (err) => { - if (err) { - global.logger.log(incrementMapViewsError({ user, err })); - } - - next(); - }); - }); - }.bind(this); -}; - -function templateZoomCenter(view) { - if (view.zoom !== undefined && view.center) { - return { - zoom: view.zoom, - center: view.center - }; - } - return false; -} - -function templateBounds(view) { - if (view.bounds) { - var hasAllBounds = ['west', 'south', 'east', 'north'].every(prop => Number.isFinite(view.bounds[prop])); - - if (hasAllBounds) { - return { - bounds: { - west: view.bounds.west, - south: view.bounds.south, - east: view.bounds.east, - north: view.bounds.north - } - }; - } else { - return false; - } - } - return false; -} - -NamedMapsController.prototype.setCacheChannelHeader = function () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel()); - } - - next(); - }; -}; - -NamedMapsController.prototype.setSurrogateKey = function () { - return function setSurrogateKeyMiddleware(req, res, next) { - const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals; - - this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName())); - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - if (affectedTablesAndLastUpdate.tables.length > 0) { - this.surrogateKeysCache.tag(res, affectedTablesAndLastUpdate); - } - } - - next(); - }.bind(this); -}; - -NamedMapsController.prototype.setLastModifiedHeader = function () { - return function setLastModifiedHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - var lastModifiedDate; - if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) { - lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt()); - } else { - lastModifiedDate = new Date(); - } - - res.set('Last-Modified', lastModifiedDate.toUTCString()); - } - - next(); - }; - }; - -NamedMapsController.prototype.setCacheControlHeader = function () { - return function setCacheControlHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - // we increase cache control as we can invalidate it - res.set('Cache-Control', 'public,max-age=31536000'); - } - - next(); - }; - }; - -NamedMapsController.prototype.setContentTypeHeader = function () { - return function setContentTypeHeaderMiddleware(req, res, next) { - const { headers = {} } = res.locals; - - res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png'); - - next(); - }; -}; - -NamedMapsController.prototype.respond = function () { - return function respondMiddleware (req, res) { - const { body, stats = {}, format } = res.locals; - - req.profiler.done('render-' + format); - req.profiler.add(stats); - - res.status(200); - res.send(body); - }; -}; diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js deleted file mode 100644 index a4d05467..00000000 --- a/lib/cartodb/controllers/named_maps_admin.js +++ /dev/null @@ -1,204 +0,0 @@ -const { templateName } = require('../backends/template_maps'); -const cors = require('../middleware/cors'); -const userMiddleware = require('../middleware/user'); - -/** - * @param {AuthApi} authApi - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @constructor - */ -function NamedMapsAdminController(authApi, templateMaps) { - this.authApi = authApi; - this.templateMaps = templateMaps; -} - -module.exports = NamedMapsAdminController; - -NamedMapsAdminController.prototype.register = function (app) { - const { base_url_templated } = app; - - app.post( - `${base_url_templated}/`, - cors(), - userMiddleware, - this.checkContentType('POST', 'POST TEMPLATE'), - this.authorizedByAPIKey('create', 'POST TEMPLATE'), - this.create() - ); - - app.put( - `${base_url_templated}/:template_id`, - cors(), - userMiddleware, - this.checkContentType('PUT', 'PUT TEMPLATE'), - this.authorizedByAPIKey('update', 'PUT TEMPLATE'), - this.update() - ); - - app.get( - `${base_url_templated}/:template_id`, - cors(), - userMiddleware, - this.authorizedByAPIKey('get', 'GET TEMPLATE'), - this.retrieve() - ); - - app.delete( - `${base_url_templated}/:template_id`, - cors(), - userMiddleware, - this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'), - this.destroy() - ); - - app.get( - `${base_url_templated}/`, - cors(), - userMiddleware, - this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'), - this.list() - ); - - app.options( - `${base_url_templated}/:template_id`, - cors('Content-Type') - ); -}; - -NamedMapsAdminController.prototype.authorizedByAPIKey = function (action, label) { - return function authorizedByAPIKeyMiddleware (req, res, next) { - const { user } = res.locals; - - this.authApi.authorizedByAPIKey(user, req, (err, authenticated) => { - if (err) { - return next(err); - } - - if (!authenticated) { - const error = new Error(`Only authenticated user can ${action} templated maps`); - error.http_status = 403; - error.label = label; - return next(error); - } - - next(); - }); - }.bind(this); -}; - -NamedMapsAdminController.prototype.checkContentType = function (action, label) { - return function checkContentTypeMiddleware (req, res, next) { - if (!req.is('application/json')) { - const error = new Error(`template ${action} data must be of type application/json`); - error.label = label; - return next(error); - } - next(); - }; -}; - -NamedMapsAdminController.prototype.create = function () { - return function createTemplateMiddleware (req, res, next) { - const { user } = res.locals; - const template = req.body; - - this.templateMaps.addTemplate(user, template, (err, templateId) => { - if (err) { - return next(err); - } - - res.status(200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method]({ template_id: templateId }); - }); - }.bind(this); -}; - -NamedMapsAdminController.prototype.update = function () { - return function updateTemplateMiddleware (req, res, next) { - const { user } = res.locals; - const template = req.body; - const templateId = templateName(req.params.template_id); - - this.templateMaps.updTemplate(user, templateId, template, (err) => { - if (err) { - return next(err); - } - - res.status(200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method]({ template_id: templateId }); - }); - }.bind(this); -}; - -NamedMapsAdminController.prototype.retrieve = function () { - return function retrieveTemplateMiddleware (req, res, next) { - req.profiler.start('windshaft-cartodb.get_template'); - - const { user } = res.locals; - const templateId = templateName(req.params.template_id); - - this.templateMaps.getTemplate(user, templateId, (err, template) => { - if (err) { - return next(err); - } - - if (!template) { - const error = new Error(`Cannot find template '${templateId}' of user '${user}'`); - error.http_status = 404; - return next(error); - } - // auth_id was added by ourselves, - // so we remove it before returning to the user - delete template.auth_id; - - res.status(200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method]({ template }); - }); - }.bind(this); -}; - -NamedMapsAdminController.prototype.destroy = function () { - return function destroyTemplateMiddleware (req, res, next) { - req.profiler.start('windshaft-cartodb.delete_template'); - - const { user } = res.locals; - const templateId = templateName(req.params.template_id); - - this.templateMaps.delTemplate(user, templateId, (err/* , tpl_val */) => { - if (err) { - return next(err); - } - - res.status(204); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method](''); - }); - }.bind(this); -}; - -NamedMapsAdminController.prototype.list = function () { - return function listTemplatesMiddleware (req, res, next) { - req.profiler.start('windshaft-cartodb.get_template_list'); - - const { user } = res.locals; - - this.templateMaps.listTemplates(user, (err, templateIds) => { - if (err) { - return next(err); - } - - res.status(200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method]({ template_ids: templateIds }); - }); - }.bind(this); -}; diff --git a/lib/cartodb/middleware/allow-query-params.js b/lib/cartodb/middleware/allow-query-params.js deleted file mode 100644 index 7ec31d74..00000000 --- a/lib/cartodb/middleware/allow-query-params.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = function allowQueryParams(params) { - if (!Array.isArray(params)) { - throw new Error('allowQueryParams must receive an Array of params'); - } - return function allowQueryParamsMiddleware(req, res, next) { - res.locals.allowedQueryParams = params; - next(); - }; -}; diff --git a/lib/cartodb/middleware/context/index.js b/lib/cartodb/middleware/context/index.js deleted file mode 100644 index 640aafd6..00000000 --- a/lib/cartodb/middleware/context/index.js +++ /dev/null @@ -1,15 +0,0 @@ -const locals = require('./locals'); -const cleanUpQueryParams = require('./clean-up-query-params'); -const layergroupToken = require('./layergroup-token'); -const authorize = require('./authorize'); -const dbConnSetup = require('./db-conn-setup'); - -module.exports = function prepareContextMiddleware(authApi, pgConnection) { - return [ - locals, - cleanUpQueryParams(), - layergroupToken, - authorize(authApi), - dbConnSetup(pgConnection) - ]; -}; diff --git a/lib/cartodb/middleware/context/layergroup-token.js b/lib/cartodb/middleware/context/layergroup-token.js deleted file mode 100644 index 026d0806..00000000 --- a/lib/cartodb/middleware/context/layergroup-token.js +++ /dev/null @@ -1,32 +0,0 @@ -var LayergroupToken = require('../../models/layergroup-token'); - -module.exports = function layergroupTokenMiddleware(req, res, next) { - if (!res.locals.token) { - return next(); - } - - var user = res.locals.user; - - var layergroupToken = LayergroupToken.parse(res.locals.token); - res.locals.token = layergroupToken.token; - res.locals.cache_buster = layergroupToken.cacheBuster; - - if (layergroupToken.signer) { - res.locals.signer = layergroupToken.signer; - if (!res.locals.signer) { - res.locals.signer = user; - } else if (res.locals.signer !== user) { - var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`); - err.type = 'auth'; - err.http_status = 403; - if (req.query && req.query.callback) { - err.http_status = 200; - } - - req.profiler.done('req2params'); - return next(err); - } - } - - return next(); -}; diff --git a/lib/cartodb/middleware/context/locals.js b/lib/cartodb/middleware/context/locals.js deleted file mode 100644 index 0fdcce50..00000000 --- a/lib/cartodb/middleware/context/locals.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = function localsMiddleware(req, res, next) { - // save req.params in res.locals - res.locals = Object.assign(req.params || {}, res.locals); - - next(); -}; diff --git a/lib/cartodb/middleware/cors.js b/lib/cartodb/middleware/cors.js deleted file mode 100644 index 227bb477..00000000 --- a/lib/cartodb/middleware/cors.js +++ /dev/null @@ -1,11 +0,0 @@ -module.exports = function cors(extraHeaders) { - return function(req, res, next) { - var baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token"; - if(extraHeaders) { - baseHeaders += ", " + extraHeaders; - } - res.set("Access-Control-Allow-Origin", "*"); - res.set("Access-Control-Allow-Headers", baseHeaders); - next(); - }; -}; diff --git a/lib/cartodb/middleware/lzma.js b/lib/cartodb/middleware/lzma.js deleted file mode 100644 index 6655cdeb..00000000 --- a/lib/cartodb/middleware/lzma.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict'; - -const LZMA = require('lzma').LZMA; - -const lzmaWorker = new LZMA(); - -module.exports = function lzmaMiddleware(req, res, next) { - if (!req.query.hasOwnProperty('lzma')) { - return next(); - } - - // Decode (from base64) - var lzma = new Buffer(req.query.lzma, 'base64') - .toString('binary') - .split('') - .map(function(c) { - return c.charCodeAt(0) - 128; - }); - - // Decompress - lzmaWorker.decompress(lzma, function(result) { - try { - delete req.query.lzma; - Object.assign(req.query, JSON.parse(result)); - next(); - } catch (err) { - next(new Error('Error parsing lzma as JSON: ' + err)); - } - }); -}; diff --git a/lib/cartodb/middleware/user.js b/lib/cartodb/middleware/user.js deleted file mode 100644 index adf06203..00000000 --- a/lib/cartodb/middleware/user.js +++ /dev/null @@ -1,8 +0,0 @@ -var CdbRequest = require('../models/cdb_request'); -var cdbRequest = new CdbRequest(); - -module.exports = function userMiddleware(req, res, next) { - res.locals.user = cdbRequest.userByReq(req); - - next(); -}; diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index 50e4dcfe..abd53fe3 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -1,25 +1,19 @@ +'use strict'; + const MapConfig = require('windshaft').model.MapConfig; const aggregationQuery = require('./aggregation-query'); const aggregationValidator = require('./aggregation-validator'); const { createPositiveNumberValidator, createIncludesValueValidator, - createAggregationColumnsValidator + createAggregationColumnsValidator, + createAggregationFiltersValidator } = aggregationValidator; -const SubstitutionTokens = require('../../utils/substitution-tokens'); +const queryUtils = require('../../utils/query-utils'); const removeDuplicates = arr => [...new Set(arr)]; -function prepareSql(sql) { - return sql && SubstitutionTokens.replace(sql, { - bbox: 'ST_MakeEnvelope(0,0,0,0)', - scale_denominator: '0', - pixel_width: '1', - pixel_height: '1' - }); -} - module.exports = class AggregationMapConfig extends MapConfig { static get AGGREGATIONS () { return aggregationQuery.SUPPORTED_AGGREGATE_FUNCTIONS; @@ -43,6 +37,18 @@ module.exports = class AggregationMapConfig extends MapConfig { ]; } + static get FILTER_PARAMETERS () { + return [ + // TODO: valid combinations of parameters: + // * Except for less/greater params, only one parameter allowed per filter. + // * Any less parameter can be combined with one of the greater paramters. (to define a range) + 'less_than', 'less_than_or_equal_to', + 'greater_than', 'greater_than_or_equal_to', + 'equal', 'not_equal', + 'between', 'in', 'not_in' + ]; + } + static supportsGeometryType(geometryType) { return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType); } @@ -58,11 +64,15 @@ module.exports = class AggregationMapConfig extends MapConfig { const positiveNumberValidator = createPositiveNumberValidator(this); const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS); const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS); + const aggregationFiltersValidator = createAggregationFiltersValidator( + this, AggregationMapConfig.FILTER_PARAMETERS + ); validate('resolution', positiveNumberValidator); validate('placement', includesValidPlacementsValidator); validate('threshold', positiveNumberValidator); validate('columns', aggregationColumnsValidator); + validate('filters', aggregationFiltersValidator); this.user = user; this.pgConnection = connection; @@ -77,7 +87,8 @@ module.exports = class AggregationMapConfig extends MapConfig { threshold = AggregationMapConfig.THRESHOLD, placement, columns = {}, - dimensions = {} + dimensions = {}, + filters = {} } = this.getAggregation(index); return aggregationQuery({ @@ -87,6 +98,7 @@ module.exports = class AggregationMapConfig extends MapConfig { placement, columns, dimensions, + filters, isDefaultAggregation: this._isDefaultLayerAggregation(index) }); } @@ -152,21 +164,19 @@ module.exports = class AggregationMapConfig extends MapConfig { _getLayerAggregationRequiredColumns (index) { const { columns, dimensions } = this.getAggregation(index); + let finalColumns = ['cartodb_id', '_cdb_feature_count']; + let aggregatedColumns = []; if (columns) { - aggregatedColumns = Object.keys(columns) - .map(key => columns[key].aggregated_column) - .filter(aggregatedColumn => typeof aggregatedColumn === 'string'); + aggregatedColumns = Object.keys(columns); } let dimensionsColumns = []; if (dimensions) { - dimensionsColumns = Object.keys(dimensions) - .map(key => dimensions[key]) - .filter(dimension => typeof dimension === 'string'); + dimensionsColumns = Object.keys(dimensions); } - return removeDuplicates(aggregatedColumns.concat(dimensionsColumns)); + return removeDuplicates(finalColumns.concat(aggregatedColumns).concat(dimensionsColumns)); } doesLayerReachThreshold(index, featureCount) { @@ -188,7 +198,7 @@ module.exports = class AggregationMapConfig extends MapConfig { } const sql = limitedQuery({ - query: prepareSql(layer.options.sql) + query: queryUtils.substituteDummyTokens(layer.options.sql) }); connection.query(sql, (err, result) => { @@ -220,7 +230,8 @@ module.exports = class AggregationMapConfig extends MapConfig { _isDefaultAggregation (aggregation) { return aggregation.placement === undefined && aggregation.columns === undefined && - this._isEmptyParameter(aggregation.dimensions); + this._isEmptyParameter(aggregation.dimensions) && + this._isEmptyParameter(aggregation.filters); } _isEmptyParameter(parameter) { diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 16e897ab..1bddf02d 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -1,5 +1,10 @@ +'use strict'; + +const timeDimension = require('./time-dimension'); + const DEFAULT_PLACEMENT = 'point-sample'; + /** * Returns a template function (function that accepts template parameters and returns a string) * to generate an aggregation query. @@ -24,6 +29,16 @@ const templateForOptions = (options) => { return templateFn; }; +function optionsToParams (options) { + return { + sourceQuery: options.query, + res: 256/options.resolution, + columns: options.columns, + dimensions: options.dimensions, + filters: options.filters + }; +} + /** * Generates an aggregation query given the aggregation options: * - query @@ -38,15 +53,23 @@ const templateForOptions = (options) => { * When placement, columns or dimensions are specified, columns are aggregated as requested * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement. */ -const queryForOptions = (options) => templateForOptions(options)({ - sourceQuery: options.query, - res: 256/options.resolution, - columns: options.columns, - dimensions: options.dimensions -}); +const queryForOptions = (options) => templateForOptions(options)(optionsToParams(options)); module.exports = queryForOptions; +module.exports.infoForOptions = (options) => { + const params = optionsToParams(options); + const dimensions = {}; + dimensionNamesAndExpressions(params).forEach(([dimensionName, info]) => { + dimensions[dimensionName] = { + sql: info.sql, + params: info.effectiveParams, + type: info.type + }; + }); + return dimensions; +}; + const SUPPORTED_AGGREGATE_FUNCTIONS = { 'count': { sql: (column_name, params) => `count(${params.aggregated_column || '*'})` @@ -93,73 +116,253 @@ const aggregateColumnNames = (ctx, table) => { return sep(Object.keys(columns)); }; +const aggregateExpression = (column_name, column_parameters) => { + const aggregate_function = column_parameters.aggregate_function || 'count'; + const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; + if (!aggregate_definition) { + throw new Error("Invalid Aggregate function: '" + aggregate_function + "'"); + } + return aggregate_definition.sql(column_name, column_parameters); +}; + const aggregateColumnDefs = ctx => { let columns = aggregateColumns(ctx); return sep(Object.keys(columns).map(column_name => { - const aggregate_function = columns[column_name].aggregate_function || 'count'; - const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; - if (!aggregate_definition) { - throw new Error("Invalid Aggregate function: '" + aggregate_function + "'"); - } - const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]); + const aggregate_expression = aggregateExpression(column_name, columns[column_name]); return `${aggregate_expression} AS ${column_name}`; })); }; - const aggregateDimensions = ctx => ctx.dimensions || {}; -const dimensionNames = (ctx, table) => { - let dimensions = aggregateDimensions(ctx); - if (table) { - return sep(Object.keys(dimensions).map( - dimension_name => `${table}.${dimension_name}` - )); - } - return sep(Object.keys(dimensions)); +const timeDimensionParameters = definition => { + // definition.column should correspond to a wrapped date column + const group = definition.group || {}; + return { + time: `to_timestamp("${definition.column}")`, + timezone: group.timezone || 'utc', + units: group.units, + count: group.count || 1, + starting: group.starting, + format: definition.format + }; }; -const dimensionDefs = ctx => { +// Adapt old-style dimension definitions for backwards compatibility +const adaptDimensionDefinition = definition => { + if (typeof(definition) === 'string') { + return { column: definition }; + } + return definition; +}; + +const dimensionExpression = definition => { + if (definition.group) { + // Currently only time dimensions are supported with parameters + return Object.assign({ type: 'timeDimension' }, timeDimension(timeDimensionParameters(definition))); + } else { + return { sql: `"${definition.column}"` }; + } +}; + +const dimensionNamesAndExpressions = (ctx) => { let dimensions = aggregateDimensions(ctx); - return sep(Object.keys(dimensions).map(dimension_name => { - const expression = dimensions[dimension_name]; - return `${expression} AS ${dimension_name}`; + return Object.keys(dimensions).map(dimensionName => { + const dimension = adaptDimensionDefinition(dimensions[dimensionName]); + const expression = dimensionExpression(dimension); + return [dimensionName, expression]; + }); +}; + +const dimensionNames = (ctx, table) => { + return sep(dimensionNamesAndExpressions(ctx).map(([dimensionName]) => { + return table ? `${table}."${dimensionName}"` : `"${dimensionName}"`; })); }; +const dimensionDefs = ctx => { + return sep( + dimensionNamesAndExpressions(ctx) + .map(([dimensionName, expression]) => `${expression.sql} AS "${dimensionName}"`) + ); +}; + +const aggregateFilters = ctx => ctx.filters || {}; + +const filterConditionSQL = (expr, filter) => { + // TODO: validate filter parameters (e.g. cannot have both greater_than and greater_than or equal to) + + if (filter) { + if (!Array.isArray(filter)) { + filter = [filter]; + } + if (filter.length > 0) { + return filter.map(f => filterSingleConditionSQL(expr, f)).join(' OR '); + } + } +}; + +const filterSingleConditionSQL = (expr, filter) => { + let cond; + Object.keys(FILTERS).some(f => { + cond = FILTERS[f](expr, filter); + return cond; + }); + return cond; +}; + +const sqlQ = (value) => { + if (isFinite(value)) { + return String(value); + } + return `'${value}'`; // TODO: escape single quotes! (by doubling them) +}; + +/* jshint eqeqeq: false */ +/* x != null is used to check for both null and undefined; triple !== wouldn't do the trick */ + +const FILTERS = { + between: (expr, filter) => { + const lo = filter.greater_than_or_equal_to, hi = filter.less_than_or_equal_to; + if (lo != null && hi != null) { + return `(${expr} BETWEEN ${sqlQ(lo)} AND ${sqlQ(hi)})`; + } + }, + in: (expr, filter) => { + if (filter.in != null) { + return `(${expr} IN (${filter.in.map(v => sqlQ(v)).join(',')}))`; + } + }, + notin: (expr, filter) => { + if (filter.not_in != null) { + return `(${expr} NOT IN (${filter.not_in.map(v => sqlQ(v)).join(',')}))`; + } + }, + equal: (expr, filter) => { + if (filter.equal != null) { + return `(${expr} = ${sqlQ(filter.equal)})`; + } + }, + not_equal: (expr, filter) => { + if (filter.not_equal != null) { + return `(${expr} <> ${sqlQ(filter.not_equal)})`; + } + }, + range: (expr, filter) => { + let conds = []; + if (filter.greater_than_or_equal_to != null) { + conds.push(`(${expr} >= ${sqlQ(filter.greater_than_or_equal_to)})`); + } + if (filter.greater_than != null) { + conds.push(`(${expr} > ${sqlQ(filter.greater_than)})`); + } + if (filter.less_than_or_equal_to != null) { + conds.push(`(${expr} <= ${sqlQ(filter.less_than_or_equal_to)})`); + } + if (filter.less_than != null) { + conds.push(`(${expr} < ${sqlQ(filter.less_than)})`); + } + if (conds.length > 0) { + return conds.join(' AND '); + } + } +}; + +const filterConditions = ctx => { + let columns = aggregateColumns(ctx); + let dimensions = aggregateDimensions(ctx); + let filters = aggregateFilters(ctx); + return Object.keys(filters).map(filtered_column => { + let filtered_expr; + if (columns[filtered_column]) { + filtered_expr = aggregateExpression(filtered_column, columns[filtered_column]); + } + else if (dimensions[filtered_column]) { + filtered_expr = dimensions[filtered_column]; + } + if (!filtered_expr) { + throw new Error("Invalid filtered column: '" + filtered_column + "'"); + } + return filterConditionSQL(filtered_expr, filters[filtered_column]); + }).join(' AND '); +}; + +const havingClause = ctx => { + let cond = filterConditions(ctx); + return cond ? `HAVING ${cond}` : ''; +}; + // SQL expression to compute the aggregation resolution (grid cell size). -// This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))` // This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension // (i.e. each tile is divided into ctx.res*ctx.res cells). // We limit the the minimum resolution to avoid division by zero problems. The limit used is // the pixel size of zoom level 30 (i.e. 1/2*(30+8) of the full earth web-mercator extent), which is about 0.15 mm. +// Computing this using !scale_denominator!, !pixel_width! or !pixel_height! produces +// inaccurate results due to rounding present in those values. const gridResolution = ctx => { const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38); - return `GREATEST(${256*0.00028/ctx.res}*!scale_denominator!, ${minimumResolution})::double precision`; + const pixelSize = 'CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))'; + return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`; }; +// Each aggregation cell is defined by the cell coordinates Floor(x/res), Floor(y/res), +// i.e. they include the West and South borders but not the East and North ones. +// So, to avoid picking points that don't belong to cells in the tile, given the tile +// limits Xmin, Ymin, Xmax, Ymax (bbox), we should select points that satisfy +// Xmin <= x < Xmax and Ymin <= y < Ymax (with x, y from the_geom_webmercator) +// On the other hand we can efficiently filter spatially (relying on spatial indexing) +// with `the_geom_webmercator && bbox` which is equivalent to +// Xmin <= x <= Xmax and Ymin <= y <= Ymax +// So, in order to be both efficient and accurate we will need to use both +// conditions for spatial filtering. +const spatialFilter = ` + (_cdb_query.the_geom_webmercator && _cdb_params.bbox) AND + ST_X(_cdb_query.the_geom_webmercator) >= _cdb_params.xmin AND + ST_X(_cdb_query.the_geom_webmercator) < _cdb_params.xmax AND + ST_Y(_cdb_query.the_geom_webmercator) >= _cdb_params.ymin AND + ST_Y(_cdb_query.the_geom_webmercator) < _cdb_params.ymax +`; + // Notes: // * We need to filter spatially using !bbox! to make the queries efficient because // the filter added by Mapnik (wrapping the query) // is only applied after the aggregation. // * This queries are used for rendering and the_geom is omitted in the results for better performance +// * If the MVT extent or tile buffer was 0 or a multiple of the resolution we could use directly +// the bbox for them, but in general we need to find the nearest cell limits inside the bbox. +// * bbox coordinates can have an error in the last digits; we apply a small correction before +// applying CEIL or FLOOR to compensate for this, so that coordinates closer than a small (`eps`) +// fraction of the cell size to a cell limit are moved to the exact limit. +const sqlParams = (ctx) => ` +_cdb_res AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox, + (1E-6::double precision) AS eps +), +_cdb_params AS ( + SELECT + res, + bbox, + CEIL((ST_XMIN(bbox) - eps*res)/res)*res AS xmin, + FLOOR((ST_XMAX(bbox) + eps*res)/res)*res AS xmax, + CEIL((ST_YMIN(bbox) - eps*res)/res)*res AS ymin, + FLOOR((ST_YMAX(bbox) + eps*res)/res)*res AS ymax + FROM _cdb_res +) +`; // The special default aggregation includes all the columns of a sample row per grid cell and // the count (_cdb_feature_count) of the aggregated rows. const defaultAggregationQueryTemplate = ctx => ` - WITH - _cdb_params AS ( - SELECT - ${gridResolution(ctx)} AS res, - !bbox! AS bbox - ), + WITH ${sqlParams(ctx)}, _cdb_clusters AS ( SELECT MIN(cartodb_id) AS cartodb_id ${dimensionDefs(ctx)} ${aggregateColumnDefs(ctx)} FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params - WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + WHERE ${spatialFilter} GROUP BY Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) @@ -174,14 +377,9 @@ const defaultAggregationQueryTemplate = ctx => ` const aggregationQueryTemplates = { 'centroid': ctx => ` - WITH - _cdb_params AS ( - SELECT - ${gridResolution(ctx)} AS res, - !bbox! AS bbox - ) + WITH ${sqlParams(ctx)} SELECT - row_number() over() AS cartodb_id, + MIN(_cdb_query.cartodb_id) AS cartodb_id, ST_SetSRID( ST_MakePoint( AVG(ST_X(_cdb_query.the_geom_webmercator)), @@ -191,32 +389,30 @@ const aggregationQueryTemplates = { ${dimensionDefs(ctx)} ${aggregateColumnDefs(ctx)} FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params - WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + WHERE ${spatialFilter} GROUP BY Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} + ${havingClause(ctx)} `, 'point-grid': ctx => ` - WITH - _cdb_params AS ( - SELECT - ${gridResolution(ctx)} AS res, - !bbox! AS bbox - ), + WITH ${sqlParams(ctx)}, _cdb_clusters AS ( SELECT + MIN(_cdb_query.cartodb_id) AS cartodb_id, Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx, Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy ${dimensionDefs(ctx)} ${aggregateColumnDefs(ctx)} FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params - WHERE the_geom_webmercator && _cdb_params.bbox + WHERE ${spatialFilter} GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)} + ${havingClause(ctx)} ) SELECT - row_number() over() AS cartodb_id, + _cdb_clusters.cartodb_id AS cartodb_id, ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator ${dimensionNames(ctx)} ${aggregateColumnNames(ctx)} @@ -224,28 +420,24 @@ const aggregationQueryTemplates = { `, 'point-sample': ctx => ` - WITH - _cdb_params AS ( - SELECT - ${gridResolution(ctx)} AS res, - !bbox! AS bbox - ), + WITH ${sqlParams(ctx)}, _cdb_clusters AS ( SELECT MIN(cartodb_id) AS cartodb_id ${dimensionDefs(ctx)} ${aggregateColumnDefs(ctx)} FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params - WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + WHERE ${spatialFilter} GROUP BY Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} + ${havingClause(ctx)} ) SELECT _cdb_clusters.cartodb_id, - the_geom, the_geom_webmercator - ${dimensionNames(ctx, '_cdb_query')} + the_geom_webmercator + ${dimensionNames(ctx, '_cdb_clusters')} ${aggregateColumnNames(ctx, '_cdb_clusters')} FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query diff --git a/lib/cartodb/models/aggregation/aggregation-validator.js b/lib/cartodb/models/aggregation/aggregation-validator.js index d0dc24c2..257e7980 100644 --- a/lib/cartodb/models/aggregation/aggregation-validator.js +++ b/lib/cartodb/models/aggregation/aggregation-validator.js @@ -1,3 +1,5 @@ +'use strict'; + module.exports = function aggregationValidator (mapconfig) { return function validateProperty (key, validator) { for (let index = 0; index < mapconfig.getLayers().length; index++) { @@ -42,6 +44,38 @@ module.exports.createAggregationColumnsValidator = function (mapconfig, validAgg }; }; +module.exports.createAggregationFiltersValidator = function (mapconfig, validParameters) { + return function validateAggregationFilters (value, key, index) { + const dims = mapconfig.getAggregation(index).dimensions || {}; + const cols = mapconfig.getAggregation(index).columns || {}; + const validKeys = Object.keys(dims).concat(Object.keys(cols)); + Object.keys(value).forEach((filteredName) => { + // filteredName must be the name of either an aggregated column or a dimension in the same layer + if (!validKeys.includes(filteredName)) { + const message = `Invalid filtered column: ${filteredName}`; + throw createLayerError(message, mapconfig, index); + } + // The filter parameters must be valid + let filters = value[filteredName]; + // a single filter or an array of filters (to be OR-combined) are accepted + if (!Array.isArray(filters)) { + filters = [filters]; + } + filters.forEach(params => { + Object.keys(params).forEach(paramName => { + if (!validParameters.includes(paramName)) { + const message = `Invalid filter parameter name: ${paramName}`; + throw createLayerError(message, mapconfig, index); + } + }); + // TODO: check parameter value (params[paramName]) to be of the correct type + }); + // TODO: if multiple parameters within params check the combination is valid, + // i.e. one of the *less* parameters and one of the *greater* parameters. + }); + }; +}; + function createAggregationColumnNamesValidator(mapconfig) { return function validateAggregationColumnNames (value, key, index) { Object.keys(value).forEach((columnName) => { diff --git a/lib/cartodb/models/aggregation/time-dimension.js b/lib/cartodb/models/aggregation/time-dimension.js new file mode 100644 index 00000000..bd14bbf2 --- /dev/null +++ b/lib/cartodb/models/aggregation/time-dimension.js @@ -0,0 +1,267 @@ +'use strict'; + +// timezones can be defined either by an numeric offset in seconds or by +// a valid (case-insensitive) tz/PG name; +// they include abbreviations defined by PG (which have precedence and +// are fixed offsets, not handling DST) or general names that can handle DST. +function timezone(tz) { + if (isFinite(tz)) { + return `INTERVAL '${tz} seconds'`; + } + return `'${tz}'`; +} + +// We assume t is a TIMESTAMP WITH TIME ZONE. +// If this was to be used with a t which is a TIMESTAMP or TIME (no time zone) +// it should be converted with `timezone('utc',t)` to a type with time zone. +// Note that by default CARTO uses timestamp with time zone columns for dates +// and VectorMapConfigAdapter converts them to epoch numbers. +// So, for using this with aggregations, relying on dates & times +// converted to UTC UNIX epoch numbers, apply `to_timestamp` to the +// (converted) column. +function timeExpression(t, tz) { + if (tz !== undefined) { + return `timezone(${timezone(tz)}, ${t})`; + } + return t; +} + +function epochWithDefaults(epoch) { + /* jshint maxcomplexity:9 */ // goddammit linter, I like this as is!! + const format = /^(\d\d\d\d)(?:\-?(\d\d)(?:\-?(\d\d)(?:[T\s]?(\d\d)(?:(\d\d)(?:\:(\d\d))?)?)?)?)?$/; + const match = (epoch || '').match(format) || []; + const year = match[1] || '0001'; + const month = match[2] || '01'; + const day = match[3] || '01'; + const hour = match[4] || '00'; + const minute = match[5] || '00'; + const second = match[6] || '00'; + return `${year}-${month}-${day}T${hour}:${minute}:${second}`; +} + +// Epoch should be an ISO timestamp literal without time zone +// (it is interpreted as in the defined timzezone for the input time) +// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc. +// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00 +// It returns a timestamp without time zone +function epochExpression(epoch) { + return `TIMESTAMP '${epoch}'`; + } + +const YEARSPAN = "(date_part('year', $t)-date_part('year', $epoch))"; +// Note that SECONDSPAN is not a UTC epoch, but an epoch in the specified TZ, +// so we can use it to compute any multiple of seconds with it without using date_part or date_trunc +const SECONDSPAN = "(date_part('epoch', $t) - date_part('epoch', $epoch))"; + +const serialParts = { + second: { + sql: `FLOOR(${SECONDSPAN})`, + zeroBased: true + }, + minute: { + sql: `FLOOR(${SECONDSPAN}/60)`, + zeroBased: true + }, + hour: { + sql: `FLOOR(${SECONDSPAN}/3600)`, + zeroBased: true + }, + day: { + sql: `1 + FLOOR(${SECONDSPAN}/86400)`, + zeroBased: false + }, + week: { + sql: `1 + FLOOR(${SECONDSPAN}/(7*86400))`, + zeroBased: false + }, + month: { + sql: `1 + date_part('month', $t) - date_part('month', $epoch) + 12*${YEARSPAN}`, + zeroBased: false + }, + quarter: { + sql: `1 + date_part('quarter', $t) - date_part('quarter', $epoch) + 4*${YEARSPAN}`, + zeroBased: false + }, + semester: { + sql: `1 + FLOOR((date_part('month', $t) - date_part('month', $epoch))/6) + 2*${YEARSPAN}`, + zeroBased: false + }, + trimester: { + sql: `1 + FLOOR((date_part('month', $t) - date_part('month', $epoch))/4) + 3*${YEARSPAN}`, + zeroBased: false + }, + year: { + // for the default epoch this coincides with date_part('year', $t) + sql: `1 + ${YEARSPAN}`, + zeroBased: false + }, + decade: { + // for the default epoch this coincides with date_part('decade', $t) + sql: `FLOOR((${YEARSPAN} + 1)/10)`, + zeroBased: true + }, + century: { + // for the default epoch this coincides with date_part('century', $t) + sql: `1 + FLOOR(${YEARSPAN}/100)`, + zeroBased: false + }, + millennium: { + // for the default epoch this coincides with date_part('millennium', $t) + sql: `1 + FLOOR(${YEARSPAN}/1000)`, + zeroBased: false + } +}; + +function serialSqlExpr(params) { + const { sql, zeroBased } = serialParts[params.units]; + const column = timeExpression(params.time, params.timezone); + const epoch = epochExpression(params.starting); + const serial = sql.replace(/\$t/g, column).replace(/\$epoch/g, epoch); + let expr = serial; + if (params.count !== 1) { + if (zeroBased) { + expr = `FLOOR((${expr})/(${params.count}::double precision))::int`; + } else { + expr = `CEIL((${expr})/(${params.count}::double precision))::int`; + } + } else { + expr = `(${expr})::int`; + } + return expr; +} + +const isoParts = { + second: `to_char($t, 'YYYY-MM-DD"T"HH24:MI:SS')`, + minute: `to_char($t, 'YYYY-MM-DD"T"HH24:MI')`, + hour: `to_char($t, 'YYYY-MM-DD"T"HH24')`, + day: `to_char($t, 'YYYY-MM-DD')`, + month: `to_char($t, 'YYYY-MM')`, + year: `to_char($t, 'YYYY')`, + week: `to_char($t, 'IYYY-"W"IW')`, + quarter: `to_char($t, 'YYYY-"Q"Q')`, + semester: `to_char($t, 'YYYY"S"') || to_char(CEIL(date_part('month', $t)/6), '9')`, + trimester: `to_char($t, 'YYYY"t"') || to_char(CEIL(date_part('month', $t)/4), '9')`, + decade: `to_char(date_part('decade', $t), '"D"999')`, + century: `to_char($t, '"C"CC')`, + millennium: `to_char(date_part('millennium', $t), '"M"999')` +}; + +function isoSqlExpr(params) { + const column = timeExpression(params.time, params.timezone); + if (params.count > 1) { + // TODO: it would be sensible to return the ISO of the first unit in the period + throw new Error('Multiple time units not supported for ISO format'); + } + return isoParts[params.units].replace(/\$t/g, column); +} + +const cyclicParts = { + dayOfWeek: `date_part('isodow', $t)`, // 1 = monday to 7 = sunday; + dayOfMonth: `date_part('day', $t)`, // 1 to 31 + dayOfYear: `date_part('doy', $t)`, // 1 to 366 + hourOfDay: `date_part('hour', $t)`, // 0 to 23 + monthOfYear: `date_part('month', $t)`, // 1 to 12 + quarterOfYear: `date_part('quarter', $t)`, // 1 to 4 + semesterOfYear: `FLOOR((date_part('month', $t)-1)/6.0) + 1`, // 1 to 2 + trimesterOfYear: `FLOOR((date_part('month', $t)-1)/4.0) + 1`, // 1 to 3 + weekOfYear: `date_part('week', $t)`, // 1 to 53 + minuteOfHour: `date_part('minute', $t)` // 0 to 59 +}; + +function cyclicSqlExpr(params) { + const column = timeExpression(params.time, params.timezone); + return cyclicParts[params.units].replace(/\$t/g, column); +} + +const ACCEPTED_PARAMETERS = ['time', 'units', 'timezone', 'count', 'starting', 'format']; +const REQUIRED_PARAMETERS = ['time', 'units']; + +function validateParameters(params, checker) { + const errors = []; + const presentParams = Object.keys(params); + const invalidParams = presentParams.filter(param => !ACCEPTED_PARAMETERS.includes(param)); + if (invalidParams.length) { + errors.push(`Invalid parameters: ${invalidParams.join(', ')}`); + } + const missingParams = REQUIRED_PARAMETERS.filter(param => !presentParams.includes(param)); + if (missingParams.length) { + errors.push(`Missing parameters: ${missingParams.join(', ')}`); + } + const params_errors = checker(params); + errors.push(...params_errors.errors); + if (errors.length) { + throw new Error(`Invalid time dimension:\n${errors.join("\n")}`); + } + return params_errors.params; +} + +const VALID_CYCLIC_UNITS = Object.keys(cyclicParts); +const VALID_SERIAL_UNITS = Object.keys(serialParts); +const VALID_ISO_UNITS = Object.keys(isoParts); + +function cyclicCheckParams(params) { + const errors = []; + if (!VALID_CYCLIC_UNITS.includes(params.units)) { + errors.push(`Invalid units "${params.units}"`); + } + if (params.count && params.count > 1) { + errors.push(`Count ${params.count} not supported for cyclic ${params.units}`); + } + return { errors: errors, params: params }; +} + +function serialCheckParams(params) { + const errors = []; + if (!VALID_SERIAL_UNITS.includes(params.units)) { + errors.push(`Invalid grouping units "${params.units}"`); + } + return { errors: errors, params: Object.assign({}, params, { starting: epochWithDefaults(params.starting) }) }; +} + +function isoCheckParams(params) { + const errors = []; + if (!VALID_ISO_UNITS.includes(params.units)) { + errors.push(`Invalid units "${params.units}"`); + } + if (params.starting) { + errors.push("Parameter 'starting' not supported for ISO format"); + } + return { errors: errors, params: params }; +} + +const CLASSIFIERS = { + cyclic: { + sqlExpr: cyclicSqlExpr, + checkParams: cyclicCheckParams + }, + iso: { + sqlExpr: isoSqlExpr, + checkParams: isoCheckParams + }, + serial: { + sqlExpr: serialSqlExpr, + checkParams: serialCheckParams + } +}; + +function isCyclic(units) { + return VALID_CYCLIC_UNITS.includes(units); +} + +function classifierFor(params) { + let classifier = 'serial'; + if (params.units && isCyclic(params.units)) { + classifier = 'cyclic'; + } else if (params.format === 'iso') { + classifier = 'iso'; + } + return CLASSIFIERS[classifier]; +} + +function classificationSql(params) { + const classifier = classifierFor(params); + params = validateParameters(params, classifier.checkParams); + return { sql: classifier.sqlExpr(params), effectiveParams: params }; +} + +module.exports = classificationSql; diff --git a/lib/cartodb/models/cdb_request.js b/lib/cartodb/models/cdb_request.js index 6650dd08..54593c17 100644 --- a/lib/cartodb/models/cdb_request.js +++ b/lib/cartodb/models/cdb_request.js @@ -1,3 +1,5 @@ +'use strict'; + function CdbRequest() { this.RE_USER_FROM_HOST = new RegExp(global.environment.user_from_host || '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com" diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 568d3cc2..4efbbba2 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -1,3 +1,5 @@ +'use strict'; + const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:aggregation'); @@ -42,7 +44,7 @@ const rankedCategoriesQueryTpl = ctx => ` ${ctx.aggregationFn} AS value, row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank FROM (${filteredQueryTpl(ctx)}) filtered_source - ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''} + WHERE ${ctx.aggregation === "count" ? `${ctx.column}` : `${ctx.aggregationColumn}`} IS NOT NULL GROUP BY ${ctx.column} ORDER BY 2 DESC ) @@ -279,7 +281,7 @@ module.exports = class Aggregation extends BaseDataview { max_val = 0, categories_count = 0 } = result.rows[0] || {}; - + return { aggregation: this.aggregation, count: count, @@ -290,10 +292,10 @@ module.exports = class Aggregation extends BaseDataview { max: max_val, categoriesCount: categories_count, categories: result.rows.map(({ category, value, agg }) => { - return { + return { category: agg ? 'Other' : category, - value, - agg + value, + agg }; }) }; diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 8c1ada5d..8525224f 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -1,3 +1,5 @@ +'use strict'; + const FLOAT_OIDS = { 700: true, 701: true, @@ -21,7 +23,7 @@ function getPGTypeName (pgType) { module.exports = class BaseDataview { getResult (psql, override, callback) { - this.sql(psql, override, (err, query) => { + this.sql(psql, override, (err, query, flags = null) => { if (err) { return callback(err); } @@ -34,8 +36,20 @@ module.exports = class BaseDataview { result = this.format(result, override); result.type = this.getType(); - return callback(null, result); + //Overviews logging + const stats = {}; + if (flags && flags.usesOverviews !== undefined) { + stats.usesOverviews = flags.usesOverviews; + } else { + stats.usesOverviews = false; + } + + if (this.getType) { + stats.dataviewType = this.getType(); + } + + return callback(null, result, stats); }, true); // use read-only transaction }); } @@ -52,6 +66,11 @@ module.exports = class BaseDataview { if (err) { return callback(err); } + + if (!result || !result.rows || !result.rows.length) { + return callback(new Error('The column type could not be determined')); + } + const pgType = result.rows[0].pg_typeof; callback(null, getPGTypeName(pgType)); }, readOnlyTransaction); diff --git a/lib/cartodb/models/dataview/factory.js b/lib/cartodb/models/dataview/factory.js index fa55641c..729305fc 100644 --- a/lib/cartodb/models/dataview/factory.js +++ b/lib/cartodb/models/dataview/factory.js @@ -1,3 +1,5 @@ +'use strict'; + const dataviews = require('./'); module.exports = class DataviewFactory { diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index df009e4b..a0b3278a 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -1,3 +1,5 @@ +'use strict'; + const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:formula'); const utils = require('../../utils/query-utils'); diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index affbbd25..bc51979e 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,3 +1,5 @@ +'use strict'; + const debug = require('debug')('windshaft:dataview:histogram'); const NumericHistogram = require('./histograms/numeric-histogram'); const DateHistogram = require('./histograms/date-histogram'); diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js index ec251d09..29496632 100644 --- a/lib/cartodb/models/dataview/histograms/base-histogram.js +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -1,3 +1,5 @@ +'use strict'; + const BaseDataview = require('../base'); const TYPE = 'histogram'; diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 4c720b3b..632c4438 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -1,7 +1,10 @@ +'use strict'; + const BaseHistogram = require('./base-histogram'); const debug = require('debug')('windshaft:dataview:date-histogram'); const utils = require('../../../utils/query-utils'); + /** * Gets the name of a timezone with the same offset as the required * using the pg_timezone_names table. We do this because it's simpler to pass diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index d3eedad9..ddf3577f 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -1,3 +1,5 @@ +'use strict'; + const BaseHistogram = require('./base-histogram'); const debug = require('debug')('windshaft:dataview:numeric-histogram'); const utils = require('../../../utils/query-utils'); @@ -99,8 +101,14 @@ module.exports = class NumericHistogram extends BaseHistogram { var extra_tables = ``; var extra_queries = ``; var extra_groupby = ``; + var extra_filter = ``; - if (ctx.start >= ctx.end) { + if (ctx.start < ctx.end) { + extra_filter = ` + WHERE __ctx_query.${ctx.column} >= ${ctx.start} + AND __ctx_query.${ctx.column} <= ${ctx.end} + `; + } else { ctx.end = `__cdb_basics.__cdb_max_val`; ctx.start = `__cdb_basics.__cdb_min_val`; extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`; @@ -135,7 +143,7 @@ SELECT END AS bin FROM ( - ${ctx.query} + SELECT * FROM (${ctx.query}) __ctx_query${extra_tables} ${extra_filter} ) __cdb_filtered_source_query${extra_tables} GROUP BY 10${extra_groupby} ORDER BY 10;`; diff --git a/lib/cartodb/models/dataview/index.js b/lib/cartodb/models/dataview/index.js index 05cfaab7..0fc107d3 100644 --- a/lib/cartodb/models/dataview/index.js +++ b/lib/cartodb/models/dataview/index.js @@ -1,3 +1,5 @@ +'use strict'; + module.exports = { Aggregation: require('./aggregation'), Formula: require('./formula'), diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 5df092f4..1d0f7de6 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,3 +1,5 @@ +'use strict'; + var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../aggregation'); var debug = require('debug')('windshaft:widget:aggregation:overview'); @@ -209,7 +211,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { debug(aggregationSql); - return callback(null, aggregationSql); + return callback(null, aggregationSql, { usesOverviews: true }); }; var aggregationFnQueryTpl = { diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index 38b2c119..9b1c7c58 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -1,3 +1,5 @@ +'use strict'; + var _ = require('underscore'); var BaseDataview = require('../base'); diff --git a/lib/cartodb/models/dataview/overviews/factory.js b/lib/cartodb/models/dataview/overviews/factory.js index d7a07d54..1a402f88 100644 --- a/lib/cartodb/models/dataview/overviews/factory.js +++ b/lib/cartodb/models/dataview/overviews/factory.js @@ -1,3 +1,5 @@ +'use strict'; + var parentFactory = require('../factory'); var dataviews = require('./'); diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 2a97061e..73aa2f57 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,3 +1,5 @@ +'use strict'; + var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); var debug = require('debug')('windshaft:widget:formula:overview'); @@ -74,5 +76,5 @@ Formula.prototype.sql = function (psql, override, callback) { debug(formulaSql); - return callback(null, formulaSql); + return callback(null, formulaSql, { usesOverviews: true }); }; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 6674f6a0..91c2818c 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -1,3 +1,5 @@ +'use strict'; + var _ = require('underscore'); var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../histogram'); @@ -178,7 +180,7 @@ Histogram.prototype.sql = function(psql, override, callback) { var histogramSql = this._buildQuery(override); - return callback(null, histogramSql); + return callback(null, histogramSql, { usesOverviews: true }); }; Histogram.prototype._buildQuery = function (override) { @@ -278,5 +280,3 @@ Histogram.prototype._shouldOverride = function (override) { Histogram.prototype._shouldOverrideBins = function (override) { return override && _.has(override, 'bins'); }; - - diff --git a/lib/cartodb/models/dataview/overviews/index.js b/lib/cartodb/models/dataview/overviews/index.js index ac032749..8b5cf9f7 100644 --- a/lib/cartodb/models/dataview/overviews/index.js +++ b/lib/cartodb/models/dataview/overviews/index.js @@ -1,3 +1,5 @@ +'use strict'; + module.exports = { Aggregation: require('./aggregation'), Formula: require('./formula'), diff --git a/lib/cartodb/models/filter/analysis.js b/lib/cartodb/models/filter/analysis.js index a0975662..1da580c6 100644 --- a/lib/cartodb/models/filter/analysis.js +++ b/lib/cartodb/models/filter/analysis.js @@ -1,3 +1,5 @@ +'use strict'; + var filters = { category: require('./analysis/category'), range: require('./analysis/range') diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js index 0f7072ae..1766ba8c 100644 --- a/lib/cartodb/models/filter/bbox.js +++ b/lib/cartodb/models/filter/bbox.js @@ -1,3 +1,5 @@ +'use strict'; + var debug = require('debug')('windshaft:filter:bbox'); var dot = require('dot'); dot.templateSettings.strip = false; diff --git a/lib/cartodb/models/layergroup-token.js b/lib/cartodb/models/layergroup-token.js index 67691bf6..8c51bed5 100644 --- a/lib/cartodb/models/layergroup-token.js +++ b/lib/cartodb/models/layergroup-token.js @@ -1,3 +1,5 @@ +'use strict'; + /** * @param {String} token might match the following pattern: {user}@{tpl_id}@{token}:{cache_buster} */ diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 67c5eac4..ceb3e031 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -1,3 +1,5 @@ +'use strict'; + const AggregationMapConfig = require('../../aggregation/aggregation-mapconfig'); const queryUtils = require('../../../utils/query-utils'); @@ -95,12 +97,25 @@ module.exports = class AggregationMapConfigAdapter { const sqlQueryWrap = layer.options.sql_wrap; - let aggregationSql = mapConfig.getAggregatedQuery(index); + let aggregationSql; + + try { + aggregationSql = mapConfig.getAggregatedQuery(index); + } + catch (error) { + return reject(error); + } if (sqlQueryWrap) { aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); } + if (!layer.options.sql_raw) { + // if sql_wrap is present, the original query should already be + // in sql_raw (with sql being the wrapped query); + // otherwise we keep the now the original query in sql_raw + layer.options.sql_raw = layer.options.sql; + } layer.options.sql = aggregationSql; mapConfig.getLayerAggregationColumns(index, (err, columns) => { @@ -159,6 +174,7 @@ module.exports = class AggregationMapConfigAdapter { } _getAggregationMetadata (mapConfig, layer, adapted) { + // also: pre-aggr query, columns, ... if (!adapted) { return { png: false, mvt: false }; } diff --git a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js index 6cd0241d..c4a71272 100644 --- a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js @@ -1,3 +1,5 @@ +'use strict'; + var queue = require('queue-async'); var debug = require('debug')('windshaft:analysis'); @@ -107,24 +109,32 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi return sourceId2Query; }, {}); - var missingNodesErrors = []; + var analysesErrors = []; requestMapConfig.layers = requestMapConfig.layers.map(function(layer, layerIndex) { if (getLayerSourceId(layer)) { var layerSourceId = getLayerSourceId(layer); var layerNode = sourceId2Node[layerSourceId]; if (layerNode) { - var analysisSql = layerQuery(layerNode); - var sqlQueryWrap = layer.options.sql_wrap; - if (sqlQueryWrap) { - layer.options.sql_raw = analysisSql; - analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql); + try { + var analysisSql; + + // it might throw error: Range filter from camshaft, for instance. + analysisSql = layerQuery(layerNode); + + var sqlQueryWrap = layer.options.sql_wrap; + if (sqlQueryWrap) { + layer.options.sql_raw = analysisSql; + analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql); + } + layer.options.sql = analysisSql; + layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews)); + layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode); + } catch (error) { + analysesErrors.push(error); } - layer.options.sql = analysisSql; - layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews)); - layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode); } else { - missingNodesErrors.push( + analysesErrors.push( new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex) ); } @@ -133,8 +143,8 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi }); var missingDataviewsNodesErrors = getMissingDataviewsSourceIds(dataviews, sourceId2Node); - if (missingNodesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) { - return callback(missingNodesErrors.concat(missingDataviewsNodesErrors)); + if (analysesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) { + return callback(analysesErrors.concat(missingDataviewsNodesErrors)); } // Augment dataviews with sql from analyses diff --git a/lib/cartodb/models/mapconfig/adapter/dataviews-widgets-adapter.js b/lib/cartodb/models/mapconfig/adapter/dataviews-widgets-adapter.js index 8ae13272..f59c0d27 100644 --- a/lib/cartodb/models/mapconfig/adapter/dataviews-widgets-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/dataviews-widgets-adapter.js @@ -1,3 +1,5 @@ +'use strict'; + function DataviewsWidgetsMapConfigAdapter() { } diff --git a/lib/cartodb/models/mapconfig/adapter/index.js b/lib/cartodb/models/mapconfig/adapter/index.js index 70704bea..dd82df32 100644 --- a/lib/cartodb/models/mapconfig/adapter/index.js +++ b/lib/cartodb/models/mapconfig/adapter/index.js @@ -11,16 +11,21 @@ MapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, param var i = 0; var tasksLeft = this.adapters.length; - function next(err, _requestMapConfig) { + let mapConfigStats = {}; + + function next(err, _requestMapConfig, adapterStats = {}) { if (err) { return callback(err); } + + mapConfigStats = Object.assign(mapConfigStats, adapterStats); + if (tasksLeft-- === 0) { - return callback(null, _requestMapConfig); + return callback(null, _requestMapConfig, mapConfigStats); } var nextAdapter = self.adapters[i++]; nextAdapter.getMapConfig(user, _requestMapConfig, params, context, next); } - next(null, requestMapConfig); + next(null, requestMapConfig, mapConfigStats); }; diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js index aead2d91..07fc45bc 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js @@ -1,3 +1,5 @@ +'use strict'; + function MapConfigBufferSizeAdapter() { this.formats = ['png', 'png32', 'mvt', 'grid.json']; } diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js index 17c2059f..68d2a950 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js @@ -1,3 +1,5 @@ +'use strict'; + var queue = require('queue-async'); var _ = require('underscore'); var Datasource = require('windshaft').model.Datasource; @@ -108,8 +110,7 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC var dbAuth = {}; if (_.some(layers, isNamedTypeLayer)) { - // Lazy load dbAuth - this.pgConnection.setDBAuth(user, dbAuth, function(err) { + this.pgConnection.setDBAuth(user, dbAuth, 'master', function(err) { if (err) { return callback(err); } diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js index 72c29593..3d0f6580 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js @@ -1,101 +1,136 @@ -var step = require('step'); +'use strict'; + var queue = require('queue-async'); var _ = require('underscore'); -function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) { - this.overviewsMetadataApi = overviewsMetadataApi; - this.filterStatsApi = filterStatsApi; +function MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend) { + this.overviewsMetadataBackend = overviewsMetadataBackend; + this.filterStatsBackend = filterStatsBackend; } module.exports = MapConfigOverviewsAdapter; -MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) { - var self = this; - - var layers = requestMapConfig.layers; - var analysesResults = context.analysesResults; - - if (!layers || layers.length === 0) { - return callback(null, requestMapConfig); - } - - var augmentLayersQueue = queue(layers.length); - - function augmentLayer(layer, done) { - if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) { - return done(null, layer); - } - self.overviewsMetadataApi.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){ - if (err) { - done(err, layer); - } else { - var query_rewrite_data = { overviews: metadata }; - step( - function collectFiltersData() { - var filters, unfiltered_query; - if ( layer.options.source && analysesResults && !layer.options.sql_wrap) { - var sourceId = layer.options.source.id; - var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; }); - if ( node ) { - node = node.rootNode; - filters = node.getFilters(); - var filters_disabler = Object.keys(filters).reduce( - function(disabler, filter_id){ disabler[filter_id] = false; return disabler; }, - {} - ); - unfiltered_query = node.getQuery(filters_disabler); - query_rewrite_data.filters = filters; - query_rewrite_data.unfiltered_query = unfiltered_query; - } - } - this(null, filters, unfiltered_query); - }, - function collectStatsData(err, filters, unfiltered_query) { - var next_step = this; - if ( filters ) { - self.filterStatsApi.getFilterStats( - user, - unfiltered_query, filters, - function(err, stats) { - if ( !err ) { - query_rewrite_data.filter_stats = stats; - } - return next_step(err); - } - ); - } else { - return next_step(null); - } - }, - function addDataToLayer(err) { - if ( !err && !_.isEmpty(metadata) ) { - layer = _.extend({}, layer); - layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data }); - } - done(null, layer); - } - ); - } - }); - } - - function layersAugmentQueueFinish(err, layers) { - if (err) { - return callback(err); - } +MapConfigOverviewsAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { + var layers = requestMapConfig.layers; + var analysesResults = context.analysesResults; if (!layers || layers.length === 0) { - return callback(new Error('Missing layers array from layergroup config')); + return callback(null, requestMapConfig); } - requestMapConfig.layers = layers; + var augmentLayersQueue = queue(layers.length); - return callback(null, requestMapConfig); - } + layers.forEach(layer => augmentLayersQueue.defer(this._augmentLayer.bind(this), user, layer, analysesResults)); - layers.forEach(function(layer) { - augmentLayersQueue.defer(augmentLayer, layer); - }); - augmentLayersQueue.awaitAll(layersAugmentQueueFinish); + augmentLayersQueue.awaitAll(function layersAugmentQueueFinish (err, results) { + if (err) { + return callback(err); + } + const layers = results.map(result => result.layer); + const overviewsAddedToMapconfig = results.some(result => result.overviewsAddedToMapconfig); + + if (!layers || layers.length === 0) { + return callback(new Error('Missing layers array from layergroup config')); + } + + requestMapConfig.layers = layers; + + const stats = { overviewsAddedToMapconfig }; + + return callback(null, requestMapConfig, stats); + }); }; + +MapConfigOverviewsAdapter.prototype._augmentLayer = function (user, layer, analysesResults, callback) { + let overviewsAddedToMapconfig = false; + if (layer.type !== 'mapnik' && layer.type !== 'cartodb') { + return callback(null, { layer, overviewsAddedToMapconfig }); + } + + this.overviewsMetadataBackend.getOverviewsMetadata(user, layer.options.sql, (err, metadata) => { + if (err) { + return callback(err, { layer, overviewsAddedToMapconfig }); + } + + if (_.isEmpty(metadata)) { + return callback(null, { layer, overviewsAddedToMapconfig }); + } + + var filters = getFilters(analysesResults, layer); + + overviewsAddedToMapconfig = true; + + if (!filters) { + layer.options = Object.assign({}, layer.options, getQueryRewriteData(layer, analysesResults, { + overviews: metadata + })); + + return callback(null, { layer, overviewsAddedToMapconfig }); + } + + var unfilteredQuery = getUnfilteredQuery(analysesResults, layer); + + this.filterStatsBackend.getFilterStats(user, unfilteredQuery, filters, function (err, stats) { + if (err) { + return callback(null, { layer, overviewsAddedToMapconfig }); + } + + layer.options = Object.assign({}, layer.options, getQueryRewriteData(layer, analysesResults, { + overviews: metadata, + filter_stats: stats + })); + + return callback(null, { layer, overviewsAddedToMapconfig }); + }); + }); +}; + +function getRootNode (analysesResults, sourceId) { + var node = _.find(analysesResults, function (a) { + return a.rootNode.params.id === sourceId; + }); + + return node ? node.rootNode : undefined; +} + +function getFilters (analysesResults, layer) { + if (layer.options.source && analysesResults && !layer.options.sql_wrap) { + var sourceId = layer.options.source.id; + var node = getRootNode(analysesResults, sourceId); + + if (node) { + return node.getFilters(); + } + } +} + +function getUnfilteredQuery (analysesResults, layer) { + if (layer.options.source && analysesResults && !layer.options.sql_wrap) { + var sourceId = layer.options.source.id; + var node = getRootNode(analysesResults, sourceId); + + if (node) { + var filters = node.getFilters(); + var filters_disabler = Object.keys(filters).reduce(function (disabler, filter_id) { + disabler[filter_id] = false; + return disabler; + }, {}); + + return node.getQuery(filters_disabler); + } + } +} + +function getQueryRewriteData (layer, analysesResults, extend = {}) { + var queryRewriteData = {}; + + if (layer.options.source && analysesResults && !layer.options.sql_wrap) { + queryRewriteData.filters = getFilters(analysesResults, layer); + queryRewriteData.unfiltered_query = getUnfilteredQuery(analysesResults, layer); + } + + queryRewriteData = Object.assign({}, queryRewriteData, extend); + + return { query_rewrite_data: queryRewriteData }; +} diff --git a/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js index 8bb3d201..e3f030ce 100644 --- a/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js @@ -1,3 +1,5 @@ +'use strict'; + function SqlWrapMapConfigAdapter() { } diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index b9487095..31c4f7c4 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -11,6 +11,8 @@ var PostgresDatasource = require('../../../backends/turbo-carto-postgres-datasou var MapConfig = require('windshaft').model.MapConfig; +const dbParamsFromReqParams = require('../../../utils/database-params'); + function TurboCartoAdapter() { } @@ -115,7 +117,9 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer, var layerSql = layer.options.sql; var layerRawSql = layer.options.sql_raw; - if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) { + if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql && layer.options.sql_wrap) { + // For wrapped queries we'll derive the tokens from the data extent + // instead of the whole Earth/root tile. var self = this; var tokensQuery = tokensQueryTpl({_sql: layerRawSql}); return pg.query(tokensQuery, function(err, resultSet) { @@ -158,23 +162,3 @@ TurboCartoAdapter.prototype.process = function (psql, cartocss, sql, callback) { function shouldParseLayerCartocss(layer) { return layer && layer.options && layer.options.cartocss && layer.options.sql; } - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/models/mapconfig/adapter/vector-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/vector-mapconfig-adapter.js new file mode 100644 index 00000000..dd03f3fe --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/vector-mapconfig-adapter.js @@ -0,0 +1,65 @@ +'use strict'; + +const queryUtils = require('../../../utils/query-utils'); +const dateWrapper = require('../../../utils/date-wrapper'); + +/** + * This middleware wraps the layer query transforming the date fields into numbers because mvt tiles + * doesnt support dates as primitive type. + * + * - This middleware is ONLY activated when the `dates_as_numbers` option is enabled for some layer in the mapConfig. + */ +class VectorMapConfigAdapter { + constructor(pgConnection) { + this.pgConnection = pgConnection; + } + + getMapConfig(user, requestMapConfig, params, context, callback) { + if (!this._isDatesAsNumbersFlagEnabled(requestMapConfig)) { + return callback(null, requestMapConfig); + } + + this._wrapDates(requestMapConfig, user) + .then(updatedRequestMapConfig => callback(null, updatedRequestMapConfig)) + .catch(callback); + } + + _wrapDates(requestMapConfig, user) { + return Promise.all(requestMapConfig.layers.map(layer => this._wrapLayer(layer, user))) + .then(() => requestMapConfig); + } + + _wrapLayer(layer, user) { + if (!layer.options.dates_as_numbers || !layer.options.sql) { + return Promise.resolve(layer); + } + const originalQuery = layer.options.sql; + return this._getColumns(user, originalQuery) + .then(result => { + const newSqlQuery = dateWrapper.wrapDates(originalQuery, result.fields); + layer.options.sql = newSqlQuery; + return layer; + }); + } + + _getColumns(user, originalQuery) { + return new Promise((resolve, reject) => { + this.pgConnection.getConnection(user, (err, connection) => { + if (err) { + return reject(err); + } + const query = queryUtils.getQueryLimited(queryUtils.substituteDummyTokens(originalQuery), 0); + queryUtils.queryPromise(connection, query) + .then(resolve) + .catch(reject); + }); + }); + } + + _isDatesAsNumbersFlagEnabled(requestMapConfig) { + return requestMapConfig.layers && requestMapConfig.layers.some(layer => layer.options.dates_as_numbers); + } +} + + +module.exports = VectorMapConfigAdapter; diff --git a/lib/cartodb/models/mapconfig/provider/base-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/provider/base-mapconfig-adapter.js new file mode 100644 index 00000000..de156217 --- /dev/null +++ b/lib/cartodb/models/mapconfig/provider/base-mapconfig-adapter.js @@ -0,0 +1,67 @@ +'use strict'; + +const QueryTables = require('cartodb-query-tables'); + +module.exports = class BaseMapConfigProvider { + createAffectedTables (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + const queries = []; + + this.mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(null, affectedTables); + }); + }); + }); + } + + getAffectedTables (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + return this.createAffectedTables(callback); + }); + } +}; diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 340073b5..15e2fc6f 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -1,48 +1,29 @@ -var assert = require('assert'); -var step = require('step'); +'use strict'; -var MapStoreMapConfigProvider = require('./map-store-provider'); +const MapStoreMapConfigProvider = require('./map-store-provider'); -/** - * @param {MapConfig} mapConfig - * @param {String} user - * @param {UserLimitsApi} userLimitsApi - * @param {Object} params - * @constructor - * @type {CreateLayergroupMapConfigProvider} - */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) { - this.mapConfig = mapConfig; - this.user = user; - this.userLimitsApi = userLimitsApi; - this.params = params; - this.cacheBuster = params.cache_buster || 0; -} +module.exports = class CreateLayergroupMapConfigProvider extends MapStoreMapConfigProvider { + constructor (mapConfig, user, userLimitsBackend, pgConnection, affectedTablesCache, params) { + super(null, user, userLimitsBackend, pgConnection, affectedTablesCache, params); + this.mapConfig = mapConfig; + } -module.exports = CreateLayergroupMapConfigProvider; - -CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { - var self = this; - var context = {}; - step( - function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); - }, - function handleRenderLimits(err, renderLimits) { - assert.ifError(err); - context.limits = renderLimits; - return null; - }, - function finish(err) { - return callback(err, self.mapConfig, self.params, context); + getMapConfig (callback) { + if (this.mapConfig && this.params && this.context) { + return callback(null, this.mapConfig, this.params, this.context); } - ); + + const context = {}; + + this.userLimitsBackend.getRenderLimits(this.user, this.params.api_key, (err, renderLimits) => { + if (err) { + return callback(err); + } + + context.limits = renderLimits; + this.context = context; + + return callback(null, this.mapConfig, this.params, context); + }); + } }; - -CreateLayergroupMapConfigProvider.prototype.getKey = MapStoreMapConfigProvider.prototype.getKey; - -CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigProvider.prototype.getCacheBuster; - -CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter; - -CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey; diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index 177322d4..b0dda9d1 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -1,77 +1,82 @@ -var _ = require('underscore'); -var assert = require('assert'); -var dot = require('dot'); -var step = require('step'); +'use strict'; -/** - * @param {MapStore} mapStore - * @param {String} user - * @param {UserLimitsApi} userLimitsApi - * @param {Object} params - * @constructor - * @type {MapStoreMapConfigProvider} - */ -function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) { - this.mapStore = mapStore; - this.user = user; - this.userLimitsApi = userLimitsApi; - this.params = params; - this.token = params.token; - this.cacheBuster = params.cache_buster || 0; -} - -module.exports = MapStoreMapConfigProvider; - -MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { - var self = this; - var context = {}; - step( - function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); - }, - function handleRenderLimits(err, renderLimits) { - assert.ifError(err); - context.limits = renderLimits; - return null; - }, - function loadMapConfig(err) { - assert.ifError(err); - self.mapStore.load(self.token, this); - }, - function finish(err, mapConfig) { - return callback(err, mapConfig, self.params, context); - } - ); -}; - -MapStoreMapConfigProvider.prototype.getKey = function() { - return this.createKey(false); -}; - -MapStoreMapConfigProvider.prototype.getCacheBuster = function() { - return this.cacheBuster; -}; - -MapStoreMapConfigProvider.prototype.filter = function(key) { - var regex = new RegExp('^' + this.createKey(true) + '.*'); - return key && key.match(regex); -}; +const BaseMapConfigProvider = require('./base-mapconfig-adapter'); +const dot = require('dot'); // Configure bases for cache keys suitable for string interpolation -var baseKey = '{{=it.dbname}}:{{=it.token}}'; -var rendererKey = baseKey + ':{{=it.dbuser}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}'; +const baseKey = '{{=it.dbname}}:{{=it.token}}'; +const rendererKey = baseKey + ':{{=it.dbuser}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}'; -var baseKeyTpl = dot.template(baseKey); -var rendererKeyTpl = dot.template(rendererKey); +const baseKeyTpl = dot.template(baseKey); +const rendererKeyTpl = dot.template(rendererKey); -MapStoreMapConfigProvider.prototype.createKey = function(base) { - var tplValues = _.defaults({}, this.params, { - dbname: '', - token: '', - dbuser: '', - format: '', - layer: '', - scale_factor: 1 - }); - return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); -}; \ No newline at end of file +module.exports = class MapStoreMapConfigProvider extends BaseMapConfigProvider { + /** + * @param {MapStore} mapStore + * @param {String} user + * @param {UserLimitsBackend} userLimitsBackend + * @param {Object} params + * @constructor + * @type {MapStoreMapConfigProvider} + */ + constructor (mapStore, user, userLimitsBackend, pgConnection, affectedTablesCache, params) { + super(); + this.mapStore = mapStore; + this.user = user; + this.userLimitsBackend = userLimitsBackend; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; + this.params = params; + this.token = params.token; + this.cacheBuster = params.cache_buster || 0; + this.mapConfig = null; + this.context = null; + } + + getMapConfig (callback) { + if (this.mapConfig !== null) { + return callback(null, this.mapConfig, this.params, this.context); + } + + const context = {}; + + this.userLimitsBackend.getRenderLimits(this.user, this.params.api_key, (err, renderLimits) => { + if (err) { + return callback(err); + } + + context.limits = renderLimits; + + this.mapStore.load(this.token, (err, mapConfig) => { + if (err) { + return callback(err); + } + + this.mapConfig = mapConfig; + this.context = context; + + return callback(null, mapConfig, this.params, context); + }); + }); + } + + getKey () { + return this.createKey(false); + } + + getCacheBuster () { + return this.cacheBuster; + } + + filter (key) { + const regex = new RegExp('^' + this.createKey(true) + '.*'); + return key && key.match(regex); + } + + createKey (base) { + const { dbname = '', token = '', dbuser = '', format = '', layer = '', scale_factor = 1 } = this.params; + const tplValues = { dbname, token, dbuser, format, layer, scale_factor }; + + return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); + } +}; diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 17c66c29..513fb7b7 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -1,289 +1,282 @@ -var _ = require('underscore'); -var assert = require('assert'); -var crypto = require('crypto'); -var dot = require('dot'); -var step = require('step'); -var MapConfig = require('windshaft').model.MapConfig; -var templateName = require('../../../backends/template_maps').templateName; -var QueryTables = require('cartodb-query-tables'); +'use strict'; -/** - * @constructor - * @type {NamedMapMapConfigProvider} - */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, - owner, templateId, config, authToken, params) { - this.templateMaps = templateMaps; - this.pgConnection = pgConnection; - this.metadataBackend = metadataBackend; - this.userLimitsApi = userLimitsApi; - this.mapConfigAdapter = mapConfigAdapter; - - this.owner = owner; - this.templateName = templateName(templateId); - this.config = config; - this.authToken = authToken; - this.params = params; - - this.cacheBuster = Date.now(); - - // use template after call to mapConfig - this.template = null; - - this.affectedTablesAndLastUpdate = null; - - // providing - this.err = null; - this.mapConfig = null; - this.rendererParams = null; - this.context = {}; - this.analysesResults = []; -} - -module.exports = NamedMapMapConfigProvider; - -NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { - if (!!this.err || this.mapConfig !== null) { - return callback(this.err, this.mapConfig, this.rendererParams, this.context); - } - - var self = this; - - var mapConfig = null; - var rendererParams; - var apiKey; - - var context = {}; - - step( - function getTemplate() { - self.getTemplate(this); - }, - function prepareDbParams(err, tpl) { - assert.ifError(err); - self.template = tpl; - - rendererParams = _.extend({}, self.params, { - user: self.owner - }); - self.setDBParams(self.owner, rendererParams, this); - }, - function getUserApiKey(err) { - assert.ifError(err); - self.metadataBackend.getUserMapKey(self.owner, this); - }, - function prepareParams(err, _apiKey) { - assert.ifError(err); - - apiKey = _apiKey; - - var templateParams = {}; - if (self.config) { - try { - templateParams = _.isString(self.config) ? JSON.parse(self.config) : self.config; - } catch (e) { - throw new Error('malformed config parameter, should be a valid JSON'); - } - } - - return templateParams; - }, - function instantiateTemplate(err, templateParams) { - assert.ifError(err); - context.templateParams = templateParams; - return self.templateMaps.instance(self.template, templateParams); - }, - function prepareAdapterMapConfig(err, requestMapConfig) { - assert.ifError(err); - context.analysisConfiguration = { - user: self.owner, - db: { - host: rendererParams.dbhost, - port: rendererParams.dbport, - dbname: rendererParams.dbname, - user: rendererParams.dbuser, - pass: rendererParams.dbpassword - }, - batch: { - username: self.owner, - apiKey: apiKey - } - }; - self.mapConfigAdapter.getMapConfig(self.owner, requestMapConfig, rendererParams, context, this); - }, - function prepareContextLimits(err, _mapConfig) { - assert.ifError(err); - mapConfig = _mapConfig; - self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this); - }, - function cacheAndReturnMapConfig(err, renderLimits) { - self.err = err; - self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource); - self.analysesResults = context.analysesResults || []; - self.rendererParams = rendererParams; - self.context = context; - self.context.limits = renderLimits || {}; - return callback(self.err, self.mapConfig, self.rendererParams, self.context); - } - ); -}; - -NamedMapMapConfigProvider.prototype.getTemplate = function(callback) { - var self = this; - - if (!!this.err || this.template !== null) { - return callback(this.err, this.template); - } - - step( - function getTemplate() { - self.templateMaps.getTemplate(self.owner, self.templateName, this); - }, - function checkExists(err, tpl) { - assert.ifError(err); - if (!tpl) { - var notFoundErr = new Error( - "Template '" + self.templateName + "' of user '" + self.owner + "' not found" - ); - notFoundErr.http_status = 404; - throw notFoundErr; - } - return tpl; - }, - function checkAuthorized(err, tpl) { - assert.ifError(err); - - var authorized = false; - try { - authorized = self.templateMaps.isAuthorized(tpl, self.authToken); - } catch (err) { - // we catch to add http_status - var authorizationFailedErr = new Error('Failed to authorize template'); - authorizationFailedErr.http_status = 403; - throw authorizationFailedErr; - } - if ( ! authorized ) { - var unauthorizedErr = new Error('Unauthorized template instantiation'); - unauthorizedErr.http_status = 403; - throw unauthorizedErr; - } - - return tpl; - }, - function cacheAndReturnTemplate(err, template) { - self.err = err; - self.template = template; - return callback(self.err, self.template); - } - ); -}; - -NamedMapMapConfigProvider.prototype.getKey = function() { - return this.createKey(false); -}; - -NamedMapMapConfigProvider.prototype.getCacheBuster = function() { - return this.cacheBuster; -}; - -NamedMapMapConfigProvider.prototype.reset = function() { - this.template = null; - - this.affectedTablesAndLastUpdate = null; - - this.err = null; - this.mapConfig = null; - - this.cacheBuster = Date.now(); -}; - -NamedMapMapConfigProvider.prototype.filter = function(key) { - var regex = new RegExp('^' + this.createKey(true) + '.*'); - return key && key.match(regex); -}; +const BaseMapConfigProvider = require('./base-mapconfig-adapter'); +const crypto = require('crypto'); +const dot = require('dot'); +const MapConfig = require('windshaft').model.MapConfig; +const templateName = require('../../../backends/template_maps').templateName; // Configure bases for cache keys suitable for string interpolation -var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}'; -var rendererKey = baseKey + ':{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}'; +const baseKey = '{{=it.dbname}}:{{=it.user}}:{{=it.templateName}}'; +const rendererKey = baseKey + ':{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}'; -var baseKeyTpl = dot.template(baseKey); -var rendererKeyTpl = dot.template(rendererKey); +const baseKeyTpl = dot.template(baseKey); +const rendererKeyTpl = dot.template(rendererKey); -NamedMapMapConfigProvider.prototype.createKey = function(base) { - var tplValues = _.defaults({}, this.params, { - dbname: '', - owner: this.owner, - templateName: this.templateName, - authToken: this.authToken || '', - configHash: configHash(this.config), - layer: '', - scale_factor: 1 - }); - return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); +module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider { + constructor ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsBackend, + mapConfigAdapter, + affectedTablesCache, + user, + templateId, + config, + authToken, + params + ) { + super(); + this.templateMaps = templateMaps; + this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; + this.userLimitsBackend = userLimitsBackend; + this.mapConfigAdapter = mapConfigAdapter; + + this.user = user; + this.templateName = templateName(templateId); + this.config = config; + this.authToken = authToken; + this.params = params; + + this.cacheBuster = Date.now(); + + // use template after call to mapConfig + this.template = null; + + this.affectedTablesCache = affectedTablesCache; + + // providing + this.err = null; + this.mapConfig = null; + this.rendererParams = null; + this.context = {}; + this.analysesResults = []; + } + + getMapConfig (callback) { + if (!!this.err || this.mapConfig !== null) { + return callback(this.err, this.mapConfig, this.rendererParams, this.context); + } + + this.getContext((err, context) => { + if (err) { + this.err = err; + return callback(err); + } + + let templateParams = {}; + + if (this.config) { + try { + templateParams = Object.prototype.toString.call(this.config) === '[object String]' ? + JSON.parse(this.config) : + this.config; + } catch (e) { + const error = new Error('malformed config parameter, should be a valid JSON'); + this.err = error; + + return callback(err); + } + } + + context.templateParams = templateParams; + + this.getTemplate((err, template) => { + if (err) { + this.err = err; + return callback(err); + } + + let requestMapConfig; + + try { + requestMapConfig = this.templateMaps.instance(template, templateParams); + } catch (err) { + this.err = err; + return callback(err); + } + + const { user, rendererParams } = this; + + this.mapConfigAdapter.getMapConfig( + user, requestMapConfig, rendererParams, context, (err, mapConfig, stats = {}) => { + if (err) { + this.err = err; + return callback(err); + } + + this.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource); + this.analysesResults = context.analysesResults || []; + + return callback(null, this.mapConfig, this.rendererParams, this.context, stats); + }); + }); + }); + } + + getContext (callback) { + this.getDBParams(this.user, (err, rendererParams) => { + if (err) { + return callback(err); + } + + this.rendererParams = rendererParams; + + this.metadataBackend.getUserMapKey(this.user, (err, apiKey) => { + if (err) { + return callback(err); + } + + const context = {}; + + context.analysisConfiguration = { + user: this.user, + db: { + host: rendererParams.dbhost, + port: rendererParams.dbport, + dbname: rendererParams.dbname, + user: rendererParams.dbuser, + pass: rendererParams.dbpassword + }, + batch: { + username: this.user, + apiKey: apiKey + } + }; + + this.userLimitsBackend.getRenderLimits(this.user, this.params.api_key, (err, renderLimits) => { + if (err) { + this.err = err; + return callback(err); + } + + context.limits = renderLimits || {}; + + this.context = context; + + return callback(null, context); + }); + }); + }); + } + + getTemplate (callback) { + if (!!this.err || this.template !== null) { + return callback(this.err, this.template); + } + + this.templateMaps.getTemplate(this.user, this.templateName, (err, tpl) => { + if (err) { + this.err = err; + return callback(err); + } + + if (!tpl) { + const error = new Error(`Template '${this.templateName}' of user '${this.user}' not found`); + error.http_status = 404; + + this.err = error; + + return callback(error); + } + + let authorized = false; + + try { + authorized = this.templateMaps.isAuthorized(tpl, this.authToken); + } catch (err) { + const error = new Error('Failed to authorize template'); + error.http_status = 403; + + this.err = error; + + return callback(error); + } + + if (!authorized) { + const error = new Error('Unauthorized template instantiation'); + error.http_status = 403; + this.err = error; + + return callback(error); + } + + this.template = tpl; + + return callback(null, this.template); + }); + } + + getKey () { + return this.createKey(false); + } + + getCacheBuster () { + return this.cacheBuster; + } + + reset () { + this.template = null; + + this.affectedTables = null; + + this.err = null; + this.mapConfig = null; + + this.cacheBuster = Date.now(); + } + + filter (key) { + const regex = new RegExp('^' + this.createKey(true) + '.*'); + return key && key.match(regex); + } + + createKey (base) { + const { + dbname = '', + user = this.user, + templateName = this.templateName, + authToken = this.authToken || '', + configHash = createConfigHash(this.config), + layer = '', + scale_factor = 1 + } = this.params; + const tplValues = { dbname, user, templateName, authToken, configHash, layer, scale_factor }; + + return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); + } + + getDBParams (cdbuser, callback) { + const dbParams = Object.assign({ user: this.user }, this.params); + + this.pgConnection.getDatabaseParams(cdbuser, (err, databaseParams) => { + if (err) { + return callback(err); + } + + dbParams.dbuser = databaseParams.dbuser; + dbParams.dbpass = databaseParams.dbpass; + dbParams.dbhost = databaseParams.dbhost; + dbParams.dbport = databaseParams.dbport; + dbParams.dbname = databaseParams.dbname; + + return callback(null, dbParams); + }); + } + + getTemplateName () { + return this.templateName; + } }; -function configHash(config) { +function createConfigHash(config) { if (!config) { return ''; } + return crypto.createHash('md5').update(JSON.stringify(config)).digest('hex').substring(0,8); } -module.exports.configHash = configHash; - -NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, callback) { - var self = this; - step( - function setAuth() { - self.pgConnection.setDBAuth(cdbuser, params, this); - }, - function setConn(err) { - assert.ifError(err); - self.pgConnection.setDBConn(cdbuser, params, this); - }, - function finish(err) { - callback(err); - } - ); -}; - -NamedMapMapConfigProvider.prototype.getTemplateName = function() { - return this.templateName; -}; - -NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) { - var self = this; - - if (this.affectedTablesAndLastUpdate !== null) { - return callback(null, this.affectedTablesAndLastUpdate); - } - - step( - function getMapConfig() { - self.getMapConfig(this); - }, - function getSql(err, mapConfig) { - assert.ifError(err); - return mapConfig.getLayers().map(function(layer) { - return layer.options.sql; - }).join(';'); - }, - function getAffectedTables(err, sql) { - assert.ifError(err); - step( - function getConnection() { - self.pgConnection.getConnection(self.owner, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function finish(err, result) { - self.affectedTablesAndLastUpdate = result; - return callback(err, result); - } - ); -}; +module.exports.configHash = createConfigHash; diff --git a/lib/cartodb/models/resource-locator.js b/lib/cartodb/models/resource-locator.js index f9b1c1ad..1eba6638 100644 --- a/lib/cartodb/models/resource-locator.js +++ b/lib/cartodb/models/resource-locator.js @@ -1,3 +1,5 @@ +'use strict'; + var dot = require('dot'); dot.templateSettings.strip = false; @@ -73,7 +75,7 @@ ResourceLocator.prototype.getUrls = function(username, resourcePath) { } }; -function urlForTemplate(tpl, username, cdnDomain, resource, templated) { +ResourceLocator.prototype.urlForTemplate = function (tpl, username, cdnDomain, resource, templated) { cdnDomain = cdnDomain || {}; if (templated) { return { @@ -101,16 +103,28 @@ function urlForTemplate(tpl, username, cdnDomain, resource, templated) { resource: resource.getPath() }); } -} +}; ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource, templated) { var urls = {}; var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {}; if (this.resourcesUrlTemplates.http) { - urls.http = urlForTemplate(this.resourcesUrlTemplates.http, username, cdnDomain.http, resource, templated); + urls.http = this.urlForTemplate( + this.resourcesUrlTemplates.http, + username, + cdnDomain.http, + resource, + templated + ); } if (this.resourcesUrlTemplates.https) { - urls.https = urlForTemplate(this.resourcesUrlTemplates.https, username, cdnDomain.https, resource, templated); + urls.https = this.urlForTemplate( + this.resourcesUrlTemplates.https, + username, + cdnDomain.https, + resource, + templated + ); } return urls; diff --git a/lib/cartodb/monitoring/health_check.js b/lib/cartodb/monitoring/health_check.js index 294390cf..31cc8245 100644 --- a/lib/cartodb/monitoring/health_check.js +++ b/lib/cartodb/monitoring/health_check.js @@ -1,3 +1,5 @@ +'use strict'; + var fs = require('fs'); function HealthCheck(disableFile) { diff --git a/lib/cartodb/controllers/server_info.js b/lib/cartodb/server-info-controller.js similarity index 79% rename from lib/cartodb/controllers/server_info.js rename to lib/cartodb/server-info-controller.js index c70de9fa..d81051cb 100644 --- a/lib/cartodb/controllers/server_info.js +++ b/lib/cartodb/server-info-controller.js @@ -1,4 +1,6 @@ -var HealthCheck = require('../monitoring/health_check'); +'use strict'; + +var HealthCheck = require('./monitoring/health_check'); var WELCOME_MSG = "This is the CartoDB Maps API, " + "see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html"; @@ -12,10 +14,10 @@ function ServerInfoController(versions) { module.exports = ServerInfoController; -ServerInfoController.prototype.register = function(app) { - app.get('/health', this.health.bind(this)); - app.get('/', this.welcome.bind(this)); - app.get('/version', this.version.bind(this)); +ServerInfoController.prototype.register = function(monitorRouter) { + monitorRouter.get('/health', this.health.bind(this)); + monitorRouter.get('/', this.welcome.bind(this)); + monitorRouter.get('/version', this.version.bind(this)); }; ServerInfoController.prototype.welcome = function(req, res) { diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 46ff7122..3e1b3c4c 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -1,275 +1,41 @@ -var express = require('express'); -var bodyParser = require('body-parser'); -var RedisPool = require('redis-mpool'); -var cartodbRedis = require('cartodb-redis'); -var _ = require('underscore'); +'use strict'; -var controller = require('./controllers'); +const _ = require('underscore'); +const express = require('express'); +const windshaft = require('windshaft'); +const { mapnik } = windshaft; -var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); -var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); -var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); -var FastlyCacheBackend = require('./cache/backend/fastly'); +const jsonReplacer = require('./utils/json-replacer'); -var StatsClient = require('./stats/client'); -const stats = require('./middleware/stats'); +const ApiRouter = require('./api/api-router'); +const ServerInfoController = require('./server-info-controller'); -var RendererStatsReporter = require('./stats/reporter/renderer'); +const StatsClient = require('./stats/client'); -var windshaft = require('windshaft'); -var mapnik = windshaft.mapnik; +module.exports = function createServer (serverOptions) { + validateOptions(serverOptions); -var TemplateMaps = require('./backends/template_maps.js'); -var OverviewsMetadataApi = require('./api/overviews_metadata_api'); -var FilterStatsApi = require('./api/filter_stats_api'); -var UserLimitsApi = require('./api/user_limits_api'); -var AuthApi = require('./api/auth_api'); -var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); -var NamedMapProviderCache = require('./cache/named_map_provider_cache'); -var PgQueryRunner = require('./backends/pg_query_runner'); -var PgConnection = require('./backends/pg_connection'); - -var AnalysisBackend = require('./backends/analysis'); - -var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; -var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); - -var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); -var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter'); -var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter'); -var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter'); -var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); -var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); -var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter'); -var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter'); -var MapConfigAdapter = require('./models/mapconfig/adapter'); - -var StatsBackend = require('./backends/stats'); - -const lzmaMiddleware = require('./middleware/lzma'); -const errorMiddleware = require('./middleware/error-middleware'); - -const prepareContextMiddleware = require('./middleware/context'); - -module.exports = function(serverOptions) { // Make stats client globally accessible global.statsClient = StatsClient.getInstance(serverOptions.statsd); - var redisPool = new RedisPool(_.defaults(global.environment.redis, { - name: 'windshaft-server', - unwatchOnRelease: false, - noReadyCheck: true - })); - - redisPool.on('status', function(status) { - var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.'; - global.statsClient.gauge(keyPrefix + 'count', status.count); - global.statsClient.gauge(keyPrefix + 'unused', status.unused); - global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); - }); - - var metadataBackend = cartodbRedis({pool: redisPool}); - var pgConnection = new PgConnection(metadataBackend); - var pgQueryRunner = new PgQueryRunner(pgConnection); - var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); - var filterStatsApi = new FilterStatsApi(pgQueryRunner); - var userLimitsApi = new UserLimitsApi(metadataBackend, { - limits: { - cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, - render: serverOptions.renderer.mapnik.limits.render || 0 - } - }); - - var templateMaps = new TemplateMaps(redisPool, { - max_user_templates: global.environment.maxUserTemplates - }); - - var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions)); - - function invalidateNamedMap (owner, templateName) { - var startTime = Date.now(); - surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { - var logMessage = JSON.stringify({ - username: owner, - type: 'named_map_invalidation', - elapsed: Date.now() - startTime, - error: !!err ? JSON.stringify(err.message) : undefined - }); - if (err) { - global.logger.warn(logMessage); - } else { - global.logger.info(logMessage); - } - }); - } - ['update', 'delete'].forEach(function(eventType) { - templateMaps.on(eventType, invalidateNamedMap); - }); - serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions); - validateOptions(serverOptions); - bootstrapFonts(serverOptions); - // initialize express server - var app = bootstrap(serverOptions); - // Extend windshaft with all the elements of the options object - _.extend(app, serverOptions); + const app = express(); - var mapStore = new windshaft.storage.MapStore({ - pool: redisPool, - expire_time: serverOptions.grainstore.default_layergroup_ttl - }); + app.enable('jsonp callback'); + app.disable('x-powered-by'); + app.disable('etag'); + app.set('json replacer', jsonReplacer()); - var onTileErrorStrategy; - if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { - onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { + const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment }); + apiRouter.register(app); - function isRenderTimeoutError (err) { - return err.message === 'Render timed out'; - } + const versions = getAndValidateVersions(serverOptions); - function isDatasourceTimeoutError (err) { - return err.message && err.message.match(/canceling statement due to statement timeout/i); - } - - function isTimeoutError (err) { - return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); - } - - function isRasterFormat (format) { - return format === 'png' || format === 'jpg'; - } - - if (isTimeoutError(err) && isRasterFormat(format)) { - return callback(null, timeoutErrorTile, { - 'Content-Type': 'image/png', - }, {}); - } else { - return callback(err, tile, headers, stats); - } - }; - } - - var rendererFactory = new windshaft.renderer.Factory({ - onTileErrorStrategy: onTileErrorStrategy, - mapnik: { - redisPool: redisPool, - grainstore: serverOptions.grainstore, - mapnik: serverOptions.renderer.mapnik - }, - http: serverOptions.renderer.http, - mvt: serverOptions.renderer.mvt - }); - - // initialize render cache - var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, { - ttl: 60000, // 60 seconds TTL by default - statsInterval: 60000 // reports stats every milliseconds defined here - }); - var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); - var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); - rendererStatsReporter.start(); - - var attributesBackend = new windshaft.backend.Attributes(); - var previewBackend = new windshaft.backend.Preview(rendererCache); - var tileBackend = new windshaft.backend.Tile(rendererCache); - var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); - var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); - - var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); - - var statsBackend = new StatsBackend(); - - var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); - app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - - var mapConfigAdapter = new MapConfigAdapter( - new MapConfigNamedLayersAdapter(templateMaps, pgConnection), - new MapConfigBufferSizeAdapter(), - new SqlWrapMapConfigAdapter(), - new DataviewsWidgetsAdapter(), - new AnalysisMapConfigAdapter(analysisBackend), - new AggregationMapConfigAdapter(pgConnection), - new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi), - new TurboCartoAdapter() - ); - - var namedMapProviderCache = new NamedMapProviderCache( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter - ); - - ['update', 'delete'].forEach(function(eventType) { - templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); - }); - - var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); - - var TablesExtentApi = require('./api/tables_extent_api'); - var tablesExtentApi = new TablesExtentApi(pgQueryRunner); - - var versions = getAndValidateVersions(serverOptions); - - const prepareContext = typeof serverOptions.req2params === 'function' ? - serverOptions.req2params : - prepareContextMiddleware(authApi, pgConnection); - - /******************************************************************************************************************* - * Routing - ******************************************************************************************************************/ - - new controller.Layergroup( - prepareContext, - pgConnection, - mapStore, - tileBackend, - previewBackend, - attributesBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - analysisBackend - ).register(app); - - new controller.Map( - prepareContext, - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - mapConfigAdapter, - statsBackend - ).register(app); - - new controller.NamedMaps( - prepareContext, - namedMapProviderCache, - tileBackend, - previewBackend, - surrogateKeysCache, - tablesExtentApi, - metadataBackend - ).register(app); - - new controller.NamedMapsAdmin(authApi, templateMaps).register(app); - - new controller.Analyses(prepareContext).register(app); - - new controller.ServerInfo(versions).register(app); - - /******************************************************************************************************************* - * END Routing - ******************************************************************************************************************/ - - app.use(errorMiddleware()); + const serverInfoController = new ServerInfoController(versions); + serverInfoController.register(app); return app; }; @@ -280,10 +46,24 @@ function validateOptions(opts) { } } +function bootstrapFonts(opts) { + // Set carto renderer configuration for MMLStore + opts.grainstore.carto_env = opts.grainstore.carto_env || {}; + var cenv = opts.grainstore.carto_env; + cenv.validation_data = cenv.validation_data || {}; + if ( ! cenv.validation_data.fonts ) { + mapnik.register_system_fonts(); + mapnik.register_default_fonts(); + cenv.validation_data.fonts = _.keys(mapnik.fontFiles()); + } +} + +function mapnikVersion(opts) { + return opts.grainstore.mapnik_version || mapnik.versions.mapnik; +} + function getAndValidateVersions(options) { - // jshint undef:false - var warn = console.warn.bind(console); - // jshint undef:true + var warn = console.warn.bind(console); // jshint ignore:line var packageDefinition = require('../../package.json'); @@ -311,127 +91,10 @@ function getAndValidateVersions(options) { }); // Be nice and warn if configured mapnik version is != installed mapnik version - if (mapnik.versions.mapnik !== options.grainstore.mapnik_version) { - warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' + + if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) { + warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' + ' != configured mapnik version (' + options.grainstore.mapnik_version + ')'); } return installedDependenciesVersions; } - -function bootstrapFonts(opts) { - // Set carto renderer configuration for MMLStore - opts.grainstore.carto_env = opts.grainstore.carto_env || {}; - var cenv = opts.grainstore.carto_env; - cenv.validation_data = cenv.validation_data || {}; - if ( ! cenv.validation_data.fonts ) { - mapnik.register_system_fonts(); - mapnik.register_default_fonts(); - cenv.validation_data.fonts = _.keys(mapnik.fontFiles()); - } -} - -function bootstrap(opts) { - var app; - if (_.isObject(opts.https)) { - // use https if possible - app = express.createServer(opts.https); - } else { - // fall back to http by default - app = express(); - } - app.enable('jsonp callback'); - app.disable('x-powered-by'); - app.disable('etag'); - - // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 - // See: http://expressjs.com/en/4x/api.html#app.set - app.set('json replacer', function (key, value) { - if (value !== value) { - return 'NaN'; - } - - if (value === Infinity) { - return 'Infinity'; - } - - if (value === -Infinity) { - return '-Infinity'; - } - - return value; - }); - - app.use(bodyParser.json()); - - app.use(function bootstrap$prepareRequestResponse(req, res, next) { - if (global.environment && global.environment.api_hostname) { - res.set('X-Served-By-Host', global.environment.api_hostname); - } - - next(); - }); - - app.use(stats({ - enabled: opts.useProfiler, - statsClient: global.statsClient - })); - - app.use(lzmaMiddleware); - - // temporary measure until we upgrade to newer version expressjs so we can check err.status - app.use(function(err, req, res, next) { - if (err) { - if (err.name === 'SyntaxError') { - res.status(400).json({ errors: [err.name + ': ' + err.message] }); - } else { - next(err); - } - } else { - next(); - } - }); - - setupLogger(app, opts); - - return app; -} - -function setupLogger(app, opts) { - if (global.log4js && opts.log_format) { - var loggerOpts = { - // Allowing for unbuffered logging is mainly - // used to avoid hanging during unit testing. - // TODO: provide an explicit teardown function instead, - // releasing any event handler or timer set by - // this component. - buffer: !opts.unbuffered_logging, - // optional log format - format: opts.log_format - }; - app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'}))); - } -} - -function surrogateKeysCacheBackends(serverOptions) { - var cacheBackends = []; - - if (serverOptions.varnish_purge_enabled) { - cacheBackends.push( - new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) - ); - } - - if (serverOptions.fastly && - !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { - cacheBackends.push( - new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) - ); - } - - return cacheBackends; -} - -function mapnikVersion(opts) { - return opts.grainstore.mapnik_version || mapnik.versions.mapnik; -} diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 1891e208..5c9da8c6 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,34 +1,48 @@ -var os = require('os'); +'use strict'; + +const fqdn = require('@carto/fqdn-sync'); var _ = require('underscore'); var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter'); -var overviewsQueryRewriter = new OverviewsQueryRewriter({ - zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' -}); - var rendererConfig = _.defaults(global.environment.renderer || {}, { cache_ttl: 60000, // milliseconds statsInterval: 60000, mapnik: { poolSize: 8, + poolMaxWaitingClients: 64, metatile: 2, bufferSize: 64, snapToGrid: false, clipByBox2d: false, metrics: false, + postgis: { + simplify_geometries: false, + extent: '-20037508.3,-20037508.3,20037508.3,20037508.3', + row_limit: 65535, + persist_connection: false, + use_overviews: true, + max_size: 500, + twkb_encoding: true + }, limits: {} }, - http: {} + http: {}, + mvt: {} }); -rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter; +rendererConfig.mapnik.queryRewriter = new OverviewsQueryRewriter({ + zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' +}); + +rendererConfig.mvt.queryRewriter = new OverviewsQueryRewriter({ + zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' +}); // Perform keyword substitution in statsd // See https://github.com/CartoDB/Windshaft-cartodb/issues/153 if (global.environment.statsd) { if (global.environment.statsd.prefix) { - var host_token = os.hostname().split('.').reverse().join('.'); - global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token); + global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, fqdn.reverse()); } } @@ -49,12 +63,14 @@ module.exports = { port: global.environment.port, host: global.environment.host }, + // FIXME: Remove it. This is no longer needed, paths are defined in routers // This is for inline maps and table maps base_url: global.environment.base_url_legacy || '/tiles/:table', /// @deprecated with Windshaft-0.17.0 ///base_url_notable: '/tiles', + // FIXME: Remove it. This is no longer needed, paths are defined in routers // This is for Detached maps // // "maps" is the official, while @@ -62,14 +78,61 @@ module.exports = { // base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)', + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)', + // Base URLs for the APIs + // + // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + routes: global.environment.routes || { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + }, + grainstore: { map: { // TODO: allow to specify in configuration srid: 3857 }, - datasource: global.environment.postgres, + datasource: rendererConfig.mapnik.postgis || global.environment.postgres || {}, cachedir: global.environment.millstone.cache_basedir, use_workers: rendererConfig.mapnik.useCartocssWorkers || false, mapnik_version: global.environment.mapnik_version, @@ -82,18 +145,9 @@ module.exports = { statsInterval: rendererConfig.statsInterval }, renderer: { - mvt: rendererConfig.mvt, - mapnik: _.defaults(rendererConfig.mapnik, { - geojson: { - dbPoolParams: { - size: 16, - idleTimeout: 3000, - reapInterval: 1000 - }, - clipByBox2d: false - } - }), - torque: rendererConfig.torque, + mvt: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.mvt), + mapnik: rendererConfig.mapnik, + torque: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.torque), http: rendererConfig.http }, diff --git a/lib/cartodb/stats/client.js b/lib/cartodb/stats/client.js index 614344cb..be6f6c13 100644 --- a/lib/cartodb/stats/client.js +++ b/lib/cartodb/stats/client.js @@ -1,3 +1,5 @@ +'use strict'; + var _ = require('underscore'); var debug = require('debug')('windshaft:stats_client'); var StatsD = require('node-statsd').StatsD; @@ -70,4 +72,4 @@ module.exports = { return this.instance; } -}; \ No newline at end of file +}; diff --git a/lib/cartodb/stats/profiler_proxy.js b/lib/cartodb/stats/profiler_proxy.js index 1e1f91e7..5ac60d57 100644 --- a/lib/cartodb/stats/profiler_proxy.js +++ b/lib/cartodb/stats/profiler_proxy.js @@ -1,3 +1,5 @@ +'use strict'; + var Profiler = require('step-profiler'); /** diff --git a/lib/cartodb/stats/reporter/renderer.js b/lib/cartodb/stats/reporter/renderer.js index 18545202..60899918 100644 --- a/lib/cartodb/stats/reporter/renderer.js +++ b/lib/cartodb/stats/reporter/renderer.js @@ -1,3 +1,5 @@ +'use strict'; + // - Reports stats about: // * Total number of renderers // * For mapnik renderers: diff --git a/lib/cartodb/stats/timer.js b/lib/cartodb/stats/timer.js index 0ef725ab..f15c423a 100644 --- a/lib/cartodb/stats/timer.js +++ b/lib/cartodb/stats/timer.js @@ -1,3 +1,5 @@ +'use strict'; + function Timer() { this.times = {}; } diff --git a/lib/cartodb/utils/database-params.js b/lib/cartodb/utils/database-params.js new file mode 100644 index 00000000..425b7616 --- /dev/null +++ b/lib/cartodb/utils/database-params.js @@ -0,0 +1,27 @@ +'use strict'; + +module.exports = function getDatabaseConnectionParams (params) { + const dbParams = {}; + + if (params.dbuser) { + dbParams.user = params.dbuser; + } + + if (params.dbpassword) { + dbParams.pass = params.dbpassword; + } + + if (params.dbhost) { + dbParams.host = params.dbhost; + } + + if (params.dbport) { + dbParams.port = params.dbport; + } + + if (params.dbname) { + dbParams.dbname = params.dbname; + } + + return dbParams; +}; diff --git a/lib/cartodb/utils/date-wrapper.js b/lib/cartodb/utils/date-wrapper.js new file mode 100644 index 00000000..4cf4fd31 --- /dev/null +++ b/lib/cartodb/utils/date-wrapper.js @@ -0,0 +1,61 @@ +'use strict'; + +const DATE_OIDS = Object.freeze({ + 1082: 'DATE', + 1083: 'TIME', + 1114: 'TIMESTAMP', + 1184: 'TIMESTAMPTZ', + 1266: 'TIMETZ' +}); + +/** + * Wrap a query transforming all date columns into a unix epoch + * @param {*} originalQuery + * @param {*} fields + */ +function wrapDates(originalQuery, fields) { + return ` + SELECT + ${fields.map(field => _isDateType(field) ? _castColumnToEpoch(field.name) : `"${field.name}"`).join(',')} + FROM + (${originalQuery}) _cdb_epoch_transformation `; +} + +/** + * @param {object} field + */ +function _isDateType(field) { + return DATE_OIDS.hasOwnProperty(field.dataTypeID); +} + +/** + * Return a sql query to transform a date column into a unix epoch + * @param {string} column - The name of the date column + */ +function _castColumnToEpoch(columnName) { + return `date_part('epoch', "${columnName}") as "${columnName}"`; +} + +function getColumnsWithWrappedDates(query) { + if(!query){ + return; + } + if (!query.match(/\b_cdb_epoch_transformation\b/)) { + return; + } + const columns = []; + const fieldMatcher = /\bdate_part\('epoch', "([^"]+)"\) as "([^"]+)"/gmi; + let match; + do { + match = fieldMatcher.exec(query); + if (match && match[1] === match[2]) { + columns.push(match[1]); + } + } while (match); + return columns; +} + +module.exports = { + wrapDates, + getColumnsWithWrappedDates +}; diff --git a/lib/cartodb/utils/json-replacer.js b/lib/cartodb/utils/json-replacer.js new file mode 100644 index 00000000..7ec959b3 --- /dev/null +++ b/lib/cartodb/utils/json-replacer.js @@ -0,0 +1,21 @@ +'use strict'; + +module.exports = function jsonReplacerFactory () { + // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 + // See: http://expressjs.com/en/4x/api.html#app.set + return function jsonReplacer (key, value) { + if (value !== value) { + return 'NaN'; + } + + if (value === Infinity) { + return 'Infinity'; + } + + if (value === -Infinity) { + return '-Infinity'; + } + + return value; + }; +}; diff --git a/lib/cartodb/utils/layergroup-metadata.js b/lib/cartodb/utils/layergroup-metadata.js new file mode 100644 index 00000000..1a62f701 --- /dev/null +++ b/lib/cartodb/utils/layergroup-metadata.js @@ -0,0 +1,192 @@ +'use strict'; + +const dateWrapper = require('./date-wrapper'); +const querystring = require('querystring'); + +module.exports = class LayergroupMetadata { + constructor (resourceLocator) { + this.resourceLocator = resourceLocator; + } + + // TODO this should take into account several URL patterns + addDataviewsAndWidgetsUrls (username, layergroup, mapConfig) { + this._addDataviewsUrls(username, layergroup, mapConfig); + this._addWidgetsUrl(username, layergroup, mapConfig); + } + + _addDataviewsUrls (username, layergroup, mapConfig) { + layergroup.metadata.dataviews = layergroup.metadata.dataviews || {}; + var dataviews = mapConfig.dataviews || {}; + + Object.keys(dataviews).forEach((dataviewName) => { + var resource = layergroup.layergroupid + '/dataview/' + dataviewName; + layergroup.metadata.dataviews[dataviewName] = { + url: this.resourceLocator.getUrls(username, resource) + }; + }); + } + + _addWidgetsUrl (username, layergroup, mapConfig) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map((layer, layerIndex) => { + var mapConfigLayer = mapConfig.layers[layerIndex]; + if (mapConfigLayer.options && mapConfigLayer.options.widgets) { + layer.widgets = layer.widgets || {}; + Object.keys(mapConfigLayer.options.widgets).forEach((widgetName) => { + var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName; + layer.widgets[widgetName] = { + type: mapConfigLayer.options.widgets[widgetName].type, + url: this.resourceLocator.getUrls(username, resource) + }; + }); + } + + return layer; + }); + } + } + + addAnalysesMetadata (username, layergroup, analysesResults, includeQuery) { + includeQuery = includeQuery || false; + analysesResults = analysesResults || []; + layergroup.metadata.analyses = []; + + analysesResults.forEach((analysis) => { + var nodes = analysis.getNodes(); + layergroup.metadata.analyses.push({ + nodes: nodes.reduce((nodesIdMap, node) => { + if (node.params.id) { + var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); + var nodeRepr = { + status: node.getStatus(), + url: this.resourceLocator.getUrls(username, nodeResource) + }; + if (includeQuery) { + nodeRepr.query = node.getQuery(); + } + if (node.getStatus() === 'failed') { + nodeRepr.error_message = node.getErrorMessage(); + } + nodesIdMap[node.params.id] = nodeRepr; + } + + return nodesIdMap; + }, {}) + }); + }); + } + + addAggregationContextMetadata (layergroup, mapConfig, context) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + if (context.aggregation && Array.isArray(context.aggregation.layers)) { + layer.meta.aggregation = context.aggregation.layers[layerIndex]; + } + return layer; + }); + } + } + + addTileJsonMetadata (layergroup, user, mapconfig, userApiKey = null) { + const isVectorOnlyMapConfig = mapconfig.isVectorOnlyMapConfig(); + let hasMapnikLayers = false; + const apiKey = userApiKey ? '?' + querystring.stringify({api_key: userApiKey}) : ''; + + layergroup.metadata.layers.forEach((layerMetadata, index) => { + const layerId = mapconfig.getLayerId(index); + const rasterResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.png${apiKey}`; + + if (mapconfig.layerType(index) === 'mapnik') { + hasMapnikLayers = true; + const vectorResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.mvt${apiKey}`; + const layerTilejson = { + vector: this._getTilejson(this.resourceLocator.getTileUrls(user, vectorResource)) + }; + if (!isVectorOnlyMapConfig) { + let grids = null; + const layer = mapconfig.getLayer(index); + if (layer.options.interactivity) { + const gridResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.grid.json${apiKey}`; + grids = this.resourceLocator.getTileUrls(user, gridResource); + } + layerTilejson.raster = this._getTilejson( + this.resourceLocator.getTileUrls(user, rasterResource), + grids + ); + } + layerMetadata.tilejson = layerTilejson; + } else { + layerMetadata.tilejson = { + raster: this._getTilejson(this.resourceLocator.getTileUrls(user, rasterResource)) + }; + } + }); + + const tilejson = {}; + const url = {}; + + if (hasMapnikLayers) { + const vectorResource = `${layergroup.layergroupid}/{z}/{x}/{y}.mvt${apiKey}`; + tilejson.vector = this._getTilejson( + this.resourceLocator.getTileUrls(user, vectorResource) + ); + url.vector = this._getTemplateUrl(this.resourceLocator.getTemplateUrls(user, vectorResource)); + + if (!isVectorOnlyMapConfig) { + const rasterResource = `${layergroup.layergroupid}/{z}/{x}/{y}.png${apiKey}`; + tilejson.raster = this._getTilejson( + this.resourceLocator.getTileUrls(user, rasterResource) + ); + url.raster = this._getTemplateUrl(this.resourceLocator.getTemplateUrls(user, rasterResource)); + } + } + + layergroup.metadata.tilejson = tilejson; + layergroup.metadata.url = url; + } + + _getTilejson(tiles, grids) { + const tilejson = { + tilejson: '2.2.0', + tiles: tiles.https || tiles.http + }; + + if (grids) { + tilejson.grids = grids.https || grids.http; + } + + return tilejson; + } + + _getTemplateUrl(url) { + return url.https || url.http; + } + + addTurboCartoContextMetadata(layergroup, mapConfig, context) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { + layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; + } + return layer; + }); + } + } + + addDateWrappingMetadata (layergroup, mapConfig) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + const mapConfigLayer = mapConfig.layers[layerIndex]; + const layerOptions = mapConfigLayer.options; + if (layerOptions.dates_as_numbers && layerOptions.sql) { + const wrappedColumns = dateWrapper.getColumnsWithWrappedDates(layerOptions.sql); + if (wrappedColumns) { + layer.meta.dates_as_numbers = wrappedColumns; + } + } + return layer; + }); + } + } + +}; diff --git a/lib/cartodb/utils/overviews_query_rewriter.js b/lib/cartodb/utils/overviews_query_rewriter.js index 8dcc2f08..cf61b070 100644 --- a/lib/cartodb/utils/overviews_query_rewriter.js +++ b/lib/cartodb/utils/overviews_query_rewriter.js @@ -1,3 +1,5 @@ +'use strict'; + var _ = require('underscore'); var TableNameParser = require('./table_name_parser'); diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index c1b56276..2fc00c5a 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -1,16 +1,20 @@ -function prepareQuery(sql) { - var affectedTableRegexCache = { - bbox: /!bbox!/g, - scale_denominator: /!scale_denominator!/g, - pixel_width: /!pixel_width!/g, - pixel_height: /!pixel_height!/g - }; +'use strict'; - return sql - .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') - .replace(affectedTableRegexCache.scale_denominator, '0') - .replace(affectedTableRegexCache.pixel_width, '1') - .replace(affectedTableRegexCache.pixel_height, '1'); +const SubstitutionTokens = require('./substitution-tokens'); + +function prepareQuery(sql) { + var affectedTableRegexCache = { + bbox: /!bbox!/g, + scale_denominator: /!scale_denominator!/g, + pixel_width: /!pixel_width!/g, + pixel_height: /!pixel_height!/g + }; + + return sql + .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') + .replace(affectedTableRegexCache.scale_denominator, '0') + .replace(affectedTableRegexCache.pixel_width, '1') + .replace(affectedTableRegexCache.pixel_height, '1'); } module.exports.extractTableNames = function extractTableNames(query) { @@ -21,10 +25,15 @@ module.exports.extractTableNames = function extractTableNames(query) { ].join(''); }; +module.exports.getQueryActualRowCount = function (query) { + return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`; +}; + function getQueryRowEstimation(query) { return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows'; } -module.exports.getQueryRowCount = getQueryRowEstimation; + +module.exports.getQueryRowEstimation = getQueryRowEstimation; module.exports.getAggregationMetadata = ctx => ` WITH @@ -54,20 +63,153 @@ module.exports.handleFloatColumn = function handleFloatColumn(ctx) { }; /** Count NULL appearances */ -module.exports.countNULLs= function countNULLs(ctx) { +module.exports.countNULLs = function countNULLs(ctx) { return `sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END)`; }; /** Count only infinity (positive and negative) appearances */ module.exports.countInfinites = function countInfinites(ctx) { - return `${!ctx.isFloatColumn ? `0` : + return `${!ctx.isFloatColumn ? '0' : `sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)` }`; }; /** Count only NaNs appearances*/ module.exports.countNaNs = function countNaNs(ctx) { - return `${!ctx.isFloatColumn ? `0` : + return `${!ctx.isFloatColumn ? '0' : `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` }`; }; + +module.exports.getQueryTopCategories = function (query, column, topN, includeNulls = false) { + const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`; + return ` + SELECT ${column} AS category, COUNT(*) AS frequency + FROM (${query}) AS __cdb_query + ${where} + GROUP BY ${column} ORDER BY 2 DESC + LIMIT ${topN} + `; +}; + +function columnSelector(columns) { + if (!columns) { + return '*'; + } + if (typeof columns === 'string') { + return columns; + } + if (Array.isArray(columns)) { + return columns.map(name => `"${name}"`).join(', '); + } + throw new TypeError(`Bad argument type for columns: ${typeof columns}`); +} + +module.exports.getQuerySample = function (query, sampleProb, limit = null, randomSeed = 0.5, columns = null) { + const singleTable = simpleQueryTable(query); + if (singleTable) { + return getTableSample(singleTable.table, columns || singleTable.columns, sampleProb, limit, randomSeed); + } + const limitClause = limit ? `LIMIT ${limit}` : ''; + return ` + WITH __cdb_rndseed AS ( + SELECT setseed(${randomSeed}) + ) + SELECT ${columnSelector(columns)} + FROM (${query}) AS __cdb_query + WHERE random() < ${sampleProb} + ${limitClause} + `; +}; + +function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) { + const limitClause = limit ? `LIMIT ${limit}` : ''; + sampleProb *= 100; + randomSeed *= Math.pow(2, 31) - 1; + return ` + SELECT ${columnSelector(columns)} + FROM ${table} + TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause} + `; +} + +function simpleQueryTable(sql) { + const basicQuery = + /\s*SELECT\s+([\*a-z0-9_,\s]+?)\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; + const unwrappedQuery = new RegExp('^' + basicQuery.source + '$', 'i'); + // queries for named maps are wrapped like this: + var wrappedQuery = new RegExp( + '^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(' + + basicQuery.source + + '\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$', + 'i' + ); + let match = sql.match(unwrappedQuery); + if (!match) { + match = sql.match(wrappedQuery); + } + if (match) { + const columns = match[1]; + const schema = match[3]; + const table = match[4]; + return { table: schema ? `${schema}.${table}` : table, columns }; + } + return false; +} + +module.exports.getQueryGeometryType = function (query, geometryColumn) { + return ` + SELECT ST_GeometryType(${geometryColumn}) AS geom_type + FROM (${query}) AS __cdb_query + WHERE ${geometryColumn} IS NOT NULL + LIMIT 1 + `; +}; + +function getQueryLimited(query, limit = 0) { + return ` + SELECT * + FROM (${query}) AS __cdb_query + LIMIT ${limit} + `; +} + +function queryPromise(dbConnection, query) { + return new Promise((resolve, reject) => { + dbConnection.query(query, (err, res) => err ? reject(err) : resolve(res)); + }); +} + +function substituteDummyTokens(sql) { + return sql && SubstitutionTokens.replace(sql, { + bbox: 'ST_MakeEnvelope(0,0,0,0)', + scale_denominator: '0', + pixel_width: '1', + pixel_height: '1' + }); +} + +function subsituteTokensForZoom(sql, zoom, singleTile=false) { + const tileRes = 256; + const wmSize = 6378137.0*2*Math.PI; + const nTiles = Math.pow(2, zoom); + const tileSize = wmSize / nTiles; + const resolution = tileSize / tileRes; + const scaleDenominator = resolution / 0.00028; + const x0 = -wmSize/2, y0 = -wmSize/2; + let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`; + if (singleTile) { + bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`; + } + return SubstitutionTokens.replace(sql, { + bbox: bbox, + scale_denominator: scaleDenominator, + pixel_width: resolution, + pixel_height: resolution + }); +} + +module.exports.queryPromise = queryPromise; +module.exports.getQueryLimited = getQueryLimited; +module.exports.substituteDummyTokens = substituteDummyTokens; +module.exports.subsituteTokensForZoom = subsituteTokensForZoom; diff --git a/lib/cartodb/utils/substitution-tokens.js b/lib/cartodb/utils/substitution-tokens.js index 5905fa4d..b6bd3cf4 100644 --- a/lib/cartodb/utils/substitution-tokens.js +++ b/lib/cartodb/utils/substitution-tokens.js @@ -1,3 +1,5 @@ +'use strict'; + var SUBSTITUTION_TOKENS = { bbox: /!bbox!/g, scale_denominator: /!scale_denominator!/g, diff --git a/lib/cartodb/utils/table_name_parser.js b/lib/cartodb/utils/table_name_parser.js index f71c3f34..f2550ad6 100644 --- a/lib/cartodb/utils/table_name_parser.js +++ b/lib/cartodb/utils/table_name_parser.js @@ -1,3 +1,5 @@ +'use strict'; + // Quote an PostgreSQL identifier if ncecessary function quote_identifier_if_needed(txt) { if ( txt && !txt.match(/^[a-z_][a-z_0-9]*$/)) { diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..fb770817 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,4375 @@ +{ + "name": "windshaft-cartodb", + "version": "7.0.1", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@carto/fqdn-sync": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@carto/fqdn-sync/-/fqdn-sync-0.2.2.tgz", + "integrity": "sha512-hvD8HlzgWkFsCswIHs+wtrIK8HEGC2L8jny3JY00lQ/xwurLxWR+xMYpgjI821INmjEPRFtajFQkniB1HxNBkQ==" + }, + "@carto/mapnik": { + "version": "3.6.2-carto.11", + "resolved": "https://registry.npmjs.org/@carto/mapnik/-/mapnik-3.6.2-carto.11.tgz", + "integrity": "sha1-8vC8TQBRCAFpJnxccpuQxryTRmE=", + "requires": { + "mapnik-vector-tile": "github:cartodb/mapnik-vector-tile#e7ca5471f9e5de81243e6035e70444321fc0a82f", + "nan": "2.10.0", + "node-pre-gyp": "0.10.0" + }, + "dependencies": { + "nan": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" + } + } + }, + "@carto/tilelive-bridge": { + "version": "github:cartodb/tilelive-bridge#e35ae36a6e2d555a6b312440f7e1904c5ad03664", + "from": "github:cartodb/tilelive-bridge#2.5.1-cdb11", + "requires": { + "@carto/mapnik": "3.6.2-carto.11", + "@mapbox/sphericalmercator": "~1.0.1", + "mapnik-pool": "~0.1.3" + }, + "dependencies": { + "@mapbox/sphericalmercator": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz", + "integrity": "sha1-cCN7l3QJXtHP286nqP0fyCsmkfI=" + } + } + }, + "abaculus": { + "version": "github:cartodb/abaculus#3283c523d8bd4bdec78d90166d9c23ff09865ca8", + "from": "github:cartodb/abaculus#2.0.3-cdb13", + "requires": { + "@carto/mapnik": "3.6.2-carto.11", + "d3-queue": "^2.0.2", + "sphericalmercator": "1.0.5" + } + }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, + "accepts": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", + "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "requires": { + "mime-types": "~2.1.18", + "negotiator": "0.6.1" + } + }, + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "requires": { + "co": "^4.6.0", + "fast-deep-equal": "^1.0.0", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.3.0" + } + }, + "amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", + "dev": true, + "optional": true + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" + }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "assertion-error": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", + "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", + "dev": true + }, + "async": { + "version": "1.5.2", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "basic-auth": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.0.tgz", + "integrity": "sha1-AV2z81PgLlY3d1X5YnQuiYHnu7o=", + "requires": { + "safe-buffer": "5.1.1" + } + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "bindings": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz", + "integrity": "sha512-DpLh5EzMR2kzvX1KIlVC0VkC3iZtHKTgdtZ0a3pglBZdaQFjt5S9g9xd1lE+YvXyfd6mtCeRnrUfOLYiTMlNSw==" + }, + "body-parser": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", + "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "~1.6.3", + "iconv-lite": "0.4.23", + "on-finished": "~2.3.0", + "qs": "6.5.2", + "raw-body": "2.3.3", + "type-is": "~1.6.16" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, + "boom": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", + "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", + "requires": { + "hoek": "4.x.x" + } + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", + "dev": true + }, + "buffer-writer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz", + "integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg=" + }, + "bunyan": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.1.tgz", + "integrity": "sha1-aMakpQLVYgvJ9y1nNoEMGxiYCX8=", + "requires": { + "dtrace-provider": "~0.6", + "moment": "^2.10.6", + "mv": "~2", + "safe-json-stringify": "~1" + } + }, + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + }, + "camelcase": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz", + "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=" + }, + "camshaft": { + "version": "0.63.4", + "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.63.4.tgz", + "integrity": "sha512-v6CpIyL2lCjCCUhksTrUSsmCUcDfjn9BD0ULs9lkz0TUwHfEMy3w2Q14NMsBXCCu0OjuiivnbPNm5XGjQe5Awg==", + "requires": { + "async": "^1.5.2", + "bunyan": "1.8.1", + "cartodb-psql": "0.13.1", + "debug": "^3.1.0", + "dot": "^1.0.3", + "request": "2.85.0" + }, + "dependencies": { + "request": { + "version": "2.85.0", + "resolved": "http://registry.npmjs.org/request/-/request-2.85.0.tgz", + "integrity": "sha512-8H7Ehijd4js+s6wuVPLjwORxD4zeuyjYugprdOXlPSqaApmL/QOy+EB/beICHVCHkGMKNh5rvihb5ov+IDw4mg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.6.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.5", + "extend": "~3.0.1", + "forever-agent": "~0.6.1", + "form-data": "~2.3.1", + "har-validator": "~5.0.3", + "hawk": "~6.0.2", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.17", + "oauth-sign": "~0.8.2", + "performance-now": "^2.1.0", + "qs": "~6.5.1", + "safe-buffer": "^5.1.1", + "stringstream": "~0.0.5", + "tough-cookie": "~2.3.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.1.0" + } + } + } + }, + "canvas": { + "version": "github:cartodb/node-canvas#45bd3c7cc5e6f99ba75b0d417b6ac6384b43083d", + "from": "github:cartodb/node-canvas#1.6.2-cdb3", + "requires": { + "nan": "^2.4.0" + } + }, + "carto": { + "version": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", + "from": "github:cartodb/carto#0.15.1-cdb5", + "requires": { + "mapnik-reference": "~6.0.2", + "optimist": "~0.6.0", + "underscore": "1.8.3" + }, + "dependencies": { + "underscore": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", + "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=" + } + } + }, + "cartocolor": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cartocolor/-/cartocolor-4.0.0.tgz", + "integrity": "sha1-hBoyIti1sicY2dVFseW5cssm6zY=", + "requires": { + "colorbrewer": "1.0.0" + } + }, + "cartodb-psql": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.13.1.tgz", + "integrity": "sha512-1z3Dk9G8KQlNGurbcmGBvNj8DVCh1Keue9uzyyvB6hKOYzBHMxixAMG0D+8nSsA7oQmWUsx/xkZZ5ZxT9toEHA==", + "requires": { + "debug": "^3.1.0", + "pg": "github:cartodb/node-postgres#5417d7b29b7272ca2e71bb396899ab3f177a9ae6", + "underscore": "~1.6.0" + } + }, + "cartodb-query-tables": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.4.0.tgz", + "integrity": "sha512-abhhz2juO1IO9hZvv10WAtgt77UTk2Fc/6BK7pml+0Cnd0HMfDC+XKzfY+r/wEjyEa20IXTc//cPsP0BgevyMw==" + }, + "cartodb-redis": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-2.1.0.tgz", + "integrity": "sha512-VT9SWWad5eyLkDZKO7W6jjPuboJB3mZSf29WTJqLVRVh9IYzwgGBfpXFBtXaRfVTaNMg/TplBL1+qffMsQtUfA==", + "requires": { + "dot": "~1.0.2", + "redis-mpool": "0.7.0", + "underscore": "~1.6.0" + }, + "dependencies": { + "dot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/dot/-/dot-1.0.3.tgz", + "integrity": "sha1-+HUL+2sDx2ZOsObLHrTGZBmvlCc=" + } + } + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "chai": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", + "integrity": "sha512-XQU3bhBukrOsQCuwZndwGcCVQHyZi53fQ6Ys1Fym7E4olpIqqZZhhoFJoaKVvV17lWQoXYwgWN2nF5crA8J2jw==", + "dev": true, + "requires": { + "assertion-error": "^1.1.0", + "check-error": "^1.0.2", + "deep-eql": "^3.0.1", + "get-func-name": "^2.0.0", + "pathval": "^1.1.0", + "type-detect": "^4.0.5" + } + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "dependencies": { + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" + } + } + }, + "check-error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", + "dev": true + }, + "chownr": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==" + }, + "chroma-js": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/chroma-js/-/chroma-js-1.1.1.tgz", + "integrity": "sha1-m7lDSVkzbs51cAqq3+7ccYBtjAU=" + }, + "cli": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cli/-/cli-1.0.1.tgz", + "integrity": "sha1-IoF1NPJL+klQw01TLUjsvGIbjBQ=", + "dev": true, + "requires": { + "exit": "0.1.2", + "glob": "^7.1.1" + } + }, + "cliui": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", + "requires": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wrap-ansi": "^2.0.0" + } + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" + }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" + }, + "colorbrewer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/colorbrewer/-/colorbrewer-1.0.0.tgz", + "integrity": "sha1-T5czO5abp2Ejgr5LwzlLNB+0yKI=" + }, + "combined-stream": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "commander": { + "version": "2.17.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", + "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==", + "dev": true, + "optional": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "console-browserify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.1.0.tgz", + "integrity": "sha1-8CQcRXMKn8YyOyBtvzjtx0HQuxA=", + "dev": true, + "requires": { + "date-now": "^0.1.4" + } + }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, + "content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + }, + "content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + }, + "cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "cryptiles": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.4.tgz", + "integrity": "sha512-8I1sgZHfVwcSOY6mSGpVU3lw/GSIZvusg8dD2+OGehCJpOhQRLNcH0qb9upQnOH4XhgxxFJSg6E2kx95deb1Tw==", + "requires": { + "boom": "5.x.x" + }, + "dependencies": { + "boom": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", + "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", + "requires": { + "hoek": "4.x.x" + } + } + } + }, + "d3": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz", + "integrity": "sha1-vEZ0gAQ3iyGjYMn8fPUjF5B2L7g=" + }, + "d3-queue": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/d3-queue/-/d3-queue-2.0.3.tgz", + "integrity": "sha1-B/vaOsrlNYqcUpmq+ICt8JU+0sI=" + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "date-now": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz", + "integrity": "sha1-6vQ5/U1ISK105cx9vvIAZyueNFs=", + "dev": true + }, + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + }, + "deep-eql": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", + "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", + "dev": true, + "requires": { + "type-detect": "^4.0.0" + } + }, + "deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", + "dev": true + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" + }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + }, + "destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + }, + "diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true + }, + "dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "dev": true, + "requires": { + "domelementtype": "~1.1.1", + "entities": "~1.1.1" + }, + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", + "dev": true + }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "dev": true + } + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true + }, + "domhandler": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.3.0.tgz", + "integrity": "sha1-LeWaCCLVAn+r/28DLCsloqir5zg=", + "dev": true, + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "dev": true, + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "dot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/dot/-/dot-1.1.2.tgz", + "integrity": "sha1-xzdwGfxOVQeYkosrmv62ar+h8vk=" + }, + "double-ended-queue": { + "version": "2.1.0-0", + "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", + "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" + }, + "dtrace-provider": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "integrity": "sha1-CweNVReTfYcxAUUtkUZzdVe3XlE=", + "optional": true, + "requires": { + "nan": "^2.0.8" + } + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + }, + "entities": { + "version": "1.0.0", + "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", + "dev": true + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "es6-promise": { + "version": "3.1.2", + "resolved": "http://registry.npmjs.org/es6-promise/-/es6-promise-3.1.2.tgz", + "integrity": "sha1-eV4lzrR/e6uyY9FRr77dktGOagc=" + }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, + "escodegen": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.8.1.tgz", + "integrity": "sha1-WltTr0aTEQvrsIZ6o0MN07cKEBg=", + "dev": true, + "requires": { + "esprima": "^2.7.1", + "estraverse": "^1.9.1", + "esutils": "^2.0.2", + "optionator": "^0.8.1", + "source-map": "~0.2.0" + }, + "dependencies": { + "esprima": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", + "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", + "dev": true + }, + "source-map": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.2.0.tgz", + "integrity": "sha1-2rc/vPwrqBm03gO9b26qSBZLP50=", + "dev": true, + "optional": true, + "requires": { + "amdefine": ">=0.0.4" + } + } + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, + "estraverse": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz", + "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=", + "dev": true + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "dev": true + }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + }, + "execa": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", + "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", + "requires": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + } + }, + "exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=", + "dev": true + }, + "express": { + "version": "4.16.3", + "resolved": "http://registry.npmjs.org/express/-/express-4.16.3.tgz", + "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=", + "requires": { + "accepts": "~1.3.5", + "array-flatten": "1.1.1", + "body-parser": "1.18.2", + "content-disposition": "0.5.2", + "content-type": "~1.0.4", + "cookie": "0.3.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.1.1", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.3", + "qs": "6.5.1", + "range-parser": "~1.2.0", + "safe-buffer": "5.1.1", + "send": "0.16.2", + "serve-static": "1.13.2", + "setprototypeof": "1.1.0", + "statuses": "~1.4.0", + "type-is": "~1.6.16", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "body-parser": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", + "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.1", + "http-errors": "~1.6.2", + "iconv-lite": "0.4.19", + "on-finished": "~2.3.0", + "qs": "6.5.1", + "raw-body": "2.3.2", + "type-is": "~1.6.15" + } + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "iconv-lite": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", + "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" + }, + "qs": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + }, + "raw-body": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", + "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.2", + "iconv-lite": "0.4.19", + "unpipe": "1.0.0" + }, + "dependencies": { + "depd": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", + "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" + }, + "http-errors": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", + "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", + "requires": { + "depd": "1.1.1", + "inherits": "2.0.3", + "setprototypeof": "1.0.3", + "statuses": ">= 1.3.1 < 2" + } + }, + "setprototypeof": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", + "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" + } + } + }, + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "http://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, + "fastly-purge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fastly-purge/-/fastly-purge-1.0.1.tgz", + "integrity": "sha1-O9/p6h0PvyplcS8vX+Lspj/LWWA=", + "requires": { + "request": "^2.55.0" + } + }, + "finalhandler": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "statuses": "~1.4.0", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "requires": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + }, + "fs-minipass": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz", + "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", + "requires": { + "minipass": "^2.2.1" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, + "gc-stats": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/gc-stats/-/gc-stats-1.2.1.tgz", + "integrity": "sha512-CPQfMBQPGkqG4upxCn4zHxYZo20woPClSeqnC/WK8pFqlfAtz6zpxbOfnmxOIDYiC26H/pYlWQfdoPVGoqxFUA==", + "requires": { + "nan": "^2.10.0", + "node-pre-gyp": "^0.11.0" + }, + "dependencies": { + "abbrev": { + "version": "1.1.1", + "bundled": true + }, + "ansi-regex": { + "version": "2.1.1", + "bundled": true + }, + "aproba": { + "version": "1.2.0", + "bundled": true + }, + "are-we-there-yet": { + "version": "1.1.5", + "bundled": true, + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "balanced-match": { + "version": "1.0.0", + "bundled": true + }, + "brace-expansion": { + "version": "1.1.11", + "bundled": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "chownr": { + "version": "1.1.1", + "bundled": true + }, + "code-point-at": { + "version": "1.1.0", + "bundled": true + }, + "concat-map": { + "version": "0.0.1", + "bundled": true + }, + "console-control-strings": { + "version": "1.1.0", + "bundled": true + }, + "core-util-is": { + "version": "1.0.2", + "bundled": true + }, + "debug": { + "version": "2.6.9", + "bundled": true, + "requires": { + "ms": "2.0.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "bundled": true + }, + "delegates": { + "version": "1.0.0", + "bundled": true + }, + "detect-libc": { + "version": "1.0.3", + "bundled": true + }, + "fs-minipass": { + "version": "1.2.5", + "bundled": true, + "requires": { + "minipass": "^2.2.1" + } + }, + "fs.realpath": { + "version": "1.0.0", + "bundled": true + }, + "gauge": { + "version": "2.7.4", + "bundled": true, + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, + "glob": { + "version": "7.1.2", + "bundled": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "has-unicode": { + "version": "2.0.1", + "bundled": true + }, + "iconv-lite": { + "version": "0.4.24", + "bundled": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "ignore-walk": { + "version": "3.0.1", + "bundled": true, + "requires": { + "minimatch": "^3.0.4" + } + }, + "inflight": { + "version": "1.0.6", + "bundled": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.3", + "bundled": true + }, + "ini": { + "version": "1.3.5", + "bundled": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "bundled": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "isarray": { + "version": "1.0.0", + "bundled": true + }, + "minimatch": { + "version": "3.0.4", + "bundled": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "0.0.8", + "bundled": true + }, + "minipass": { + "version": "2.3.5", + "bundled": true, + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "minizlib": { + "version": "1.1.1", + "bundled": true, + "requires": { + "minipass": "^2.2.1" + } + }, + "mkdirp": { + "version": "0.5.1", + "bundled": true, + "requires": { + "minimist": "0.0.8" + } + }, + "ms": { + "version": "2.0.0", + "bundled": true + }, + "needle": { + "version": "2.2.4", + "bundled": true, + "requires": { + "debug": "^2.1.2", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "dependencies": { + "sax": { + "version": "1.2.4", + "bundled": true + } + } + }, + "node-pre-gyp": { + "version": "0.11.0", + "bundled": true, + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + } + }, + "nopt": { + "version": "4.0.1", + "bundled": true, + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, + "npm-bundled": { + "version": "1.0.5", + "bundled": true + }, + "npm-packlist": { + "version": "1.1.12", + "bundled": true, + "requires": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1" + } + }, + "npmlog": { + "version": "4.1.2", + "bundled": true, + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "bundled": true + }, + "object-assign": { + "version": "4.1.1", + "bundled": true + }, + "once": { + "version": "1.4.0", + "bundled": true, + "requires": { + "wrappy": "1" + } + }, + "os-homedir": { + "version": "1.0.2", + "bundled": true + }, + "os-tmpdir": { + "version": "1.0.2", + "bundled": true + }, + "osenv": { + "version": "0.1.5", + "bundled": true, + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "bundled": true + }, + "process-nextick-args": { + "version": "2.0.0", + "bundled": true + }, + "rc": { + "version": "1.2.8", + "bundled": true, + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "bundled": true + } + } + }, + "readable-stream": { + "version": "2.3.6", + "bundled": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "rimraf": { + "version": "2.6.2", + "bundled": true, + "requires": { + "glob": "^7.0.5" + } + }, + "safe-buffer": { + "version": "5.1.2", + "bundled": true + }, + "safer-buffer": { + "version": "2.1.2", + "bundled": true + }, + "semver": { + "version": "5.5.0", + "bundled": true + }, + "set-blocking": { + "version": "2.0.0", + "bundled": true + }, + "signal-exit": { + "version": "3.0.2", + "bundled": true + }, + "string-width": { + "version": "1.0.2", + "bundled": true, + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "string_decoder": { + "version": "1.1.1", + "bundled": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "bundled": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "bundled": true + }, + "tar": { + "version": "4.4.6", + "bundled": true, + "requires": { + "chownr": "^1.0.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.3.3", + "minizlib": "^1.1.0", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.2" + } + }, + "util-deprecate": { + "version": "1.0.2", + "bundled": true + }, + "wide-align": { + "version": "1.1.3", + "bundled": true, + "requires": { + "string-width": "^1.0.2 || 2" + } + }, + "wrappy": { + "version": "1.0.2", + "bundled": true + }, + "yallist": { + "version": "3.0.2", + "bundled": true + } + } + }, + "gdal": { + "version": "0.9.8", + "resolved": "https://registry.npmjs.org/gdal/-/gdal-0.9.8.tgz", + "integrity": "sha512-qSG8/W8qyJbKkmc1/B9mJwRZPbKEGkqlipsAfVMch/R2Qi9Wl1bpkLv7cAHd1urUA1d3MowpovHmSwQq4ZyvUw==", + "requires": { + "nan": "~2.10.0", + "node-pre-gyp": "^0.11.0" + }, + "dependencies": { + "abbrev": { + "version": "1.1.1", + "bundled": true + }, + "ansi-regex": { + "version": "2.1.1", + "bundled": true + }, + "aproba": { + "version": "1.2.0", + "bundled": true + }, + "are-we-there-yet": { + "version": "1.1.5", + "bundled": true, + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "balanced-match": { + "version": "1.0.0", + "bundled": true + }, + "brace-expansion": { + "version": "1.1.8", + "bundled": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "chownr": { + "version": "1.0.1", + "bundled": true + }, + "code-point-at": { + "version": "1.1.0", + "bundled": true + }, + "concat-map": { + "version": "0.0.1", + "bundled": true + }, + "console-control-strings": { + "version": "1.1.0", + "bundled": true + }, + "core-util-is": { + "version": "1.0.2", + "bundled": true + }, + "debug": { + "version": "2.6.8", + "bundled": true, + "requires": { + "ms": "2.0.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "bundled": true + }, + "delegates": { + "version": "1.0.0", + "bundled": true + }, + "detect-libc": { + "version": "1.0.3", + "bundled": true + }, + "fs-minipass": { + "version": "1.2.5", + "bundled": true, + "requires": { + "minipass": "^2.2.1" + } + }, + "fs.realpath": { + "version": "1.0.0", + "bundled": true + }, + "gauge": { + "version": "2.7.4", + "bundled": true, + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, + "glob": { + "version": "7.1.2", + "bundled": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "has-unicode": { + "version": "2.0.1", + "bundled": true + }, + "iconv-lite": { + "version": "0.4.18", + "bundled": true + }, + "ignore-walk": { + "version": "3.0.1", + "bundled": true, + "requires": { + "minimatch": "^3.0.4" + } + }, + "inflight": { + "version": "1.0.6", + "bundled": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.3", + "bundled": true + }, + "ini": { + "version": "1.3.5", + "bundled": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "bundled": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "isarray": { + "version": "1.0.0", + "bundled": true + }, + "minimatch": { + "version": "3.0.4", + "bundled": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "0.0.8", + "bundled": true + }, + "minipass": { + "version": "2.3.4", + "bundled": true, + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "bundled": true + }, + "yallist": { + "version": "3.0.2", + "bundled": true + } + } + }, + "minizlib": { + "version": "1.1.0", + "bundled": true, + "requires": { + "minipass": "^2.2.1" + } + }, + "mkdirp": { + "version": "0.5.1", + "bundled": true, + "requires": { + "minimist": "0.0.8" + } + }, + "ms": { + "version": "2.0.0", + "bundled": true + }, + "nan": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" + }, + "needle": { + "version": "2.2.2", + "bundled": true, + "requires": { + "debug": "^2.1.2", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "dependencies": { + "sax": { + "version": "1.2.4", + "bundled": true + } + } + }, + "node-pre-gyp": { + "version": "0.11.0", + "bundled": true, + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "bundled": true + }, + "tar": { + "version": "4.4.6", + "bundled": true, + "requires": { + "chownr": "^1.0.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.3.3", + "minizlib": "^1.1.0", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.0.2", + "bundled": true + } + } + }, + "nopt": { + "version": "4.0.1", + "bundled": true, + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, + "npm-bundled": { + "version": "1.0.5", + "bundled": true + }, + "npm-packlist": { + "version": "1.1.11", + "bundled": true, + "requires": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1" + } + }, + "npmlog": { + "version": "4.1.2", + "bundled": true, + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "bundled": true + }, + "object-assign": { + "version": "4.1.1", + "bundled": true + }, + "once": { + "version": "1.4.0", + "bundled": true, + "requires": { + "wrappy": "1" + } + }, + "os-homedir": { + "version": "1.0.2", + "bundled": true + }, + "os-tmpdir": { + "version": "1.0.2", + "bundled": true + }, + "osenv": { + "version": "0.1.5", + "bundled": true, + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "bundled": true + }, + "process-nextick-args": { + "version": "1.0.7", + "bundled": true + }, + "rc": { + "version": "1.2.8", + "bundled": true, + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "bundled": true + } + } + }, + "readable-stream": { + "version": "2.3.3", + "bundled": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.0.3", + "util-deprecate": "~1.0.1" + } + }, + "rimraf": { + "version": "2.6.1", + "bundled": true, + "requires": { + "glob": "^7.0.5" + } + }, + "safe-buffer": { + "version": "5.1.1", + "bundled": true + }, + "semver": { + "version": "5.4.1", + "bundled": true + }, + "set-blocking": { + "version": "2.0.0", + "bundled": true + }, + "signal-exit": { + "version": "3.0.2", + "bundled": true + }, + "string-width": { + "version": "1.0.2", + "bundled": true, + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "string_decoder": { + "version": "1.0.3", + "bundled": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "bundled": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "bundled": true + }, + "util-deprecate": { + "version": "1.0.2", + "bundled": true + }, + "wide-align": { + "version": "1.1.3", + "bundled": true, + "requires": { + "string-width": "^1.0.2 || 2" + } + }, + "wrappy": { + "version": "1.0.2", + "bundled": true + } + } + }, + "generic-pool": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.2.2.tgz", + "integrity": "sha1-eon0kdV1tC+fBpoOjixtuqPCQb4=" + }, + "get-caller-file": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", + "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==" + }, + "get-func-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", + "dev": true + }, + "get-stream": { + "version": "3.0.0", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "glob": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" + }, + "grainstore": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/grainstore/-/grainstore-1.10.0.tgz", + "integrity": "sha512-mHBznPEqsM7g11ycVOrWBO7/1VAE4i/G8rFHTw7O5Ru79cbJdl0hJrnuG2ILBY207VNjSPxBC80uN1Pm1tx0Bg==", + "requires": { + "carto": "0.16.3", + "debug": "~3.1.0", + "generic-pool": "~2.2.0", + "millstone": "github:cartodb/millstone#f201885250d2d7ea6c870a1e1060ca6bcfdef186", + "postcss": "~5.2.8", + "postcss-scss": "0.4.0", + "postcss-strip-inline-comments": "0.1.5", + "semver": "~5.0.3", + "underscore": "~1.6.0" + }, + "dependencies": { + "carto": { + "version": "0.16.3", + "resolved": "https://registry.npmjs.org/carto/-/carto-0.16.3.tgz", + "integrity": "sha1-9C34RVf/KKkDavcWZgsechztyMA=", + "requires": { + "chroma-js": "~1.1.1", + "husl": "^6.0.1", + "js-yaml": "^3.4.6", + "lodash": "^4.5.1", + "mapnik-reference": "~8.5.3", + "semver": "^5.1.0", + "yargs": "^4.2.0" + }, + "dependencies": { + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" + } + } + }, + "mapnik-reference": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/mapnik-reference/-/mapnik-reference-8.5.6.tgz", + "integrity": "sha1-XoB5ghQJMi1slYzhgjxniD67dlU=", + "requires": { + "semver": "^5.1.0" + }, + "dependencies": { + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" + } + } + }, + "postcss": { + "version": "5.2.18", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz", + "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==", + "requires": { + "chalk": "^1.1.3", + "js-base64": "^2.1.9", + "source-map": "^0.5.6", + "supports-color": "^3.2.3" + } + }, + "semver": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz", + "integrity": "sha1-d0Zt5YnNXTyV8TiqeLxWmjy10no=" + }, + "yargs": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-4.8.1.tgz", + "integrity": "sha1-wMQpJMpKqmsObaFznfshZDn53cA=", + "requires": { + "cliui": "^3.2.0", + "decamelize": "^1.1.1", + "get-caller-file": "^1.0.1", + "lodash.assign": "^4.0.3", + "os-locale": "^1.4.0", + "read-pkg-up": "^1.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^1.0.1", + "set-blocking": "^2.0.0", + "string-width": "^1.0.1", + "which-module": "^1.0.0", + "window-size": "^0.2.0", + "y18n": "^3.2.1", + "yargs-parser": "^2.4.1" + } + } + } + }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true + }, + "handlebars": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.12.tgz", + "integrity": "sha512-RhmTekP+FZL+XNhwS1Wf+bTTZpdLougwt5pcgA1tuz6Jcx0fpH/7z0qd71RKnZHBCxIRBHfBOnio4gViPemNzA==", + "dev": true, + "requires": { + "async": "^2.5.0", + "optimist": "^0.6.1", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4" + }, + "dependencies": { + "async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", + "integrity": "sha512-fNEiL2+AZt6AlAw/29Cr0UDe4sRAHCpEHh54WMz+Bb7QfNcFw4h3loofyJpLeQs4Yx7yuqu/2dLgM5hKOs6HlQ==", + "dev": true, + "requires": { + "lodash": "^4.17.10" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", + "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", + "requires": { + "ajv": "^5.1.0", + "har-schema": "^2.0.0" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "has-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", + "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=" + }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, + "hawk": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", + "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", + "requires": { + "boom": "4.x.x", + "cryptiles": "3.x.x", + "hoek": "4.x.x", + "sntp": "2.x.x" + } + }, + "he": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", + "dev": true + }, + "hiredis": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.5.0.tgz", + "integrity": "sha1-2wOpi+zXAD0TwmAEOs7s+s31m4c=", + "requires": { + "bindings": "^1.2.1", + "nan": "^2.3.4" + } + }, + "hoek": { + "version": "4.2.1", + "resolved": "http://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", + "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" + }, + "hosted-git-info": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", + "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==" + }, + "htmlparser2": { + "version": "3.8.3", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", + "dev": true, + "requires": { + "domelementtype": "1", + "domhandler": "2.3", + "domutils": "1.5", + "entities": "1.0", + "readable-stream": "1.1" + }, + "dependencies": { + "readable-stream": { + "version": "1.1.14", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + } + } + }, + "http-errors": { + "version": "1.6.3", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "husl": { + "version": "6.0.6", + "resolved": "https://registry.npmjs.org/husl/-/husl-6.0.6.tgz", + "integrity": "sha1-9xs+RdIADWQGQyqcwXpLfgxbgA0=" + }, + "iconv-lite": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", + "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "ignore-walk": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz", + "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", + "requires": { + "minimatch": "^3.0.4" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + }, + "invert-kv": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", + "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" + }, + "ipaddr.js": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", + "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=" + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=" + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "istanbul": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/istanbul/-/istanbul-0.4.5.tgz", + "integrity": "sha1-ZcfXPUxNqE1POsMQuRj7C4Azczs=", + "dev": true, + "requires": { + "abbrev": "1.0.x", + "async": "1.x", + "escodegen": "1.8.x", + "esprima": "2.7.x", + "glob": "^5.0.15", + "handlebars": "^4.0.1", + "js-yaml": "3.x", + "mkdirp": "0.5.x", + "nopt": "3.x", + "once": "1.x", + "resolve": "1.1.x", + "supports-color": "^3.1.0", + "which": "^1.1.1", + "wordwrap": "^1.0.0" + }, + "dependencies": { + "abbrev": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", + "integrity": "sha1-kbR5JYinc4wl813W9jdSovh3YTU=", + "dev": true + }, + "esprima": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", + "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", + "dev": true + }, + "glob": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", + "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=", + "dev": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "nopt": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", + "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "dev": true, + "requires": { + "abbrev": "1" + } + }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "dev": true + } + } + }, + "js-base64": { + "version": "2.4.9", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.4.9.tgz", + "integrity": "sha512-xcinL3AuDJk7VSzsHgb9DvvIXayBbadtMZ4HFPx8rUszbW1MuNMlwYVC4zzCZ6e1sqZpnNS5ZFYOhXqA39T7LQ==" + }, + "js-string-escape": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", + "integrity": "sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8=" + }, + "js-yaml": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz", + "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "jshint": { + "version": "2.9.7", + "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.9.7.tgz", + "integrity": "sha512-Q8XN38hGsVQhdlM+4gd1Xl7OB1VieSuCJf+fEJjpo59JH99bVJhXRXAh26qQ15wfdd1VPMuDWNeSWoNl53T4YA==", + "dev": true, + "requires": { + "cli": "~1.0.0", + "console-browserify": "1.1.x", + "exit": "0.1.x", + "htmlparser2": "3.8.x", + "lodash": "~4.17.10", + "minimatch": "~3.0.2", + "shelljs": "0.3.x", + "strip-json-comments": "1.0.x" + }, + "dependencies": { + "strip-json-comments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz", + "integrity": "sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E=", + "dev": true + } + } + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "lcid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", + "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", + "requires": { + "invert-kv": "^1.0.0" + } + }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "strip-bom": "^2.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "dependencies": { + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" + } + } + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + }, + "lodash.assign": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", + "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=" + }, + "log4js": { + "version": "github:cartodb/log4js-node#145d5f91e35e7fb14a6278cbf7a711ced6603727", + "from": "github:cartodb/log4js-node#cdb", + "requires": { + "async": "~0.2.0", + "readable-stream": "~1.0.2", + "semver": "~4.3.3", + "underscore": "1.8.2" + }, + "dependencies": { + "async": { + "version": "0.2.10", + "resolved": "http://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=" + }, + "semver": { + "version": "4.3.6", + "resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz", + "integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=" + }, + "underscore": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.2.tgz", + "integrity": "sha1-ZN8utZCJnelQeC83NRkLpC6/MR0=" + } + } + }, + "lru-cache": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz", + "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==", + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "lzma": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/lzma/-/lzma-2.3.2.tgz", + "integrity": "sha1-N4OySFi5wOdHoN88vx+1/KqSxEE=" + }, + "mapnik-pool": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/mapnik-pool/-/mapnik-pool-0.1.3.tgz", + "integrity": "sha1-m9rRyzTYehm8eOy0y/uqIDsvPys=", + "requires": { + "generic-pool": "~2.2.1", + "xtend": "~4.0.0" + } + }, + "mapnik-reference": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/mapnik-reference/-/mapnik-reference-6.0.5.tgz", + "integrity": "sha1-1SMZ4d+I5aB6gR6E3vKhpCO28F8=" + }, + "mapnik-vector-tile": { + "version": "github:cartodb/mapnik-vector-tile#e7ca5471f9e5de81243e6035e70444321fc0a82f", + "from": "github:cartodb/mapnik-vector-tile#v1.6.1-carto.2" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + }, + "mem": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", + "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=", + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + }, + "millstone": { + "version": "github:cartodb/millstone#f201885250d2d7ea6c870a1e1060ca6bcfdef186", + "from": "github:cartodb/millstone#v0.6.17-carto.2", + "requires": { + "generic-pool": "~2.4.1", + "mime": "~2.3.1", + "mkdirp": "~0.5.0", + "optimist": "0.6.1", + "request": "2.x", + "sqlite3": "~4.0.1", + "srs": "^1.2.0", + "step": "~1.0.0", + "underscore": "~1.9.1", + "zipfile": "~0.5.11" + }, + "dependencies": { + "generic-pool": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.6.tgz", + "integrity": "sha1-8bVeVyFn26L+ddWqkeux6fcmQtc=" + }, + "mime": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz", + "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg==" + }, + "underscore": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz", + "integrity": "sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==" + } + } + }, + "mime": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", + "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + }, + "mime-db": { + "version": "1.37.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", + "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" + }, + "mime-types": { + "version": "2.1.21", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", + "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", + "requires": { + "mime-db": "~1.37.0" + } + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "0.0.8", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + }, + "minipass": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + } + } + }, + "minizlib": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", + "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", + "requires": { + "minipass": "^2.2.1" + } + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + } + }, + "mocha": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", + "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", + "dev": true, + "requires": { + "browser-stdout": "1.3.1", + "commander": "2.15.1", + "debug": "3.1.0", + "diff": "3.5.0", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.5", + "he": "1.1.1", + "minimatch": "3.0.4", + "mkdirp": "0.5.1", + "supports-color": "5.4.0" + }, + "dependencies": { + "commander": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", + "dev": true + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "supports-color": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", + "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "moment": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz", + "integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ==" + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "mv": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", + "optional": true, + "requires": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + } + }, + "nan": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.1.tgz", + "integrity": "sha512-iji6k87OSXa0CcrLl9z+ZiYSuR2o+c0bGuNmXdrhTQTakxytAFsC56SArGYoiHlJlFoHSnvmhpceZJaXkVuOtA==" + }, + "ncp": { + "version": "2.0.0", + "resolved": "http://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", + "optional": true + }, + "needle": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz", + "integrity": "sha512-HyoqEb4wr/rsoaIDfTH2aVL9nWtQqba2/HvMv+++m8u0dz808MaagKILxtfeSN7QU7nvbQ79zk3vYOJp9zsNEA==", + "requires": { + "debug": "^2.1.2", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, + "negotiator": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", + "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + }, + "nock": { + "version": "9.2.6", + "resolved": "https://registry.npmjs.org/nock/-/nock-9.2.6.tgz", + "integrity": "sha512-loyWSNToPH416mENYcqN/ORNwJvfMs+n1+kk9HS3zO/Eb/ci3TZqoNIWp5oYW9VIFh18jXHxnzYSsrXvA63RYA==", + "dev": true, + "requires": { + "chai": "^4.1.2", + "debug": "^3.1.0", + "deep-equal": "^1.0.0", + "json-stringify-safe": "^5.0.1", + "lodash": "^4.17.5", + "mkdirp": "^0.5.0", + "propagate": "^1.0.0", + "qs": "^6.5.1", + "semver": "^5.5.0" + } + }, + "node-pre-gyp": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.10.0.tgz", + "integrity": "sha512-G7kEonQLRbcA/mOoFoxvlMrw6Q6dPf92+t/l0DFSMuSlDoWaI9JWIyPwK0jyE1bph//CUEL65/Fz1m2vJbmjQQ==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.0", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.1.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + }, + "dependencies": { + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "node-statsd": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.1.1.tgz", + "integrity": "sha1-J6WTSHY9CvegN6wqAx/vPwUQE9M=" + }, + "nopt": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", + "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + }, + "dependencies": { + "resolve": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz", + "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==", + "requires": { + "path-parse": "^1.0.6" + } + } + } + }, + "npm-bundled": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz", + "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==" + }, + "npm-packlist": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.3.0.tgz", + "integrity": "sha512-qPBc6CnxEzpOcc4bjoIBJbYdy0D/LFFPUdxvfwor4/w3vxeE0h6TiOVurCEPpQ6trjN77u/ShyfeJGsbAfB3dA==", + "requires": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1" + } + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "requires": { + "path-key": "^2.0.0" + } + }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" + }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "object-keys": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-0.4.0.tgz", + "integrity": "sha1-KKaq50KN0sOpLz2V8hM13SBOAzY=" + }, + "on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } + }, + "on-headers": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz", + "integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c=" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + } + }, + "optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "dev": true, + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + }, + "dependencies": { + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "dev": true + } + } + }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" + }, + "os-locale": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", + "requires": { + "lcid": "^1.0.0" + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "requires": { + "p-try": "^1.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "requires": { + "p-limit": "^1.1.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=" + }, + "packet-reader": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz", + "integrity": "sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc=" + }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "requires": { + "error-ex": "^1.2.0" + } + }, + "parseurl": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + }, + "path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "requires": { + "pinkie-promise": "^2.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" + }, + "path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" + }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "requires": { + "graceful-fs": "^4.1.2", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + } + }, + "pathval": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", + "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", + "dev": true + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "pg": { + "version": "github:cartodb/node-postgres#5417d7b29b7272ca2e71bb396899ab3f177a9ae6", + "from": "github:cartodb/node-postgres#6.4.2-cdb2", + "requires": { + "buffer-writer": "1.0.1", + "js-string-escape": "1.0.1", + "packet-reader": "0.3.1", + "pg-connection-string": "0.1.3", + "pg-pool": "1.*", + "pg-types": "1.*", + "pgpass": "1.*", + "semver": "4.3.2" + }, + "dependencies": { + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + } + } + }, + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-1.8.0.tgz", + "integrity": "sha1-9+xzgkw3oD8Hb1G/33DjQBR8Tzc=", + "requires": { + "generic-pool": "2.4.3", + "object-assign": "4.1.0" + }, + "dependencies": { + "generic-pool": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.4.3.tgz", + "integrity": "sha1-eAw29p360FpaBF3Te+etyhGk9v8=" + }, + "object-assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.0.tgz", + "integrity": "sha1-ejs9DpgGPUP0wD8uiubNUahog6A=" + } + } + }, + "pg-types": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz", + "integrity": "sha512-lfKli0Gkl/+za/+b6lzENajczwZHc7D5kiUCZfgm914jipD2kIOIvEkAhZ8GrW3/TUoP9w8FHjwpPObBye5KQQ==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~1.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.0", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", + "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "requires": { + "split": "^1.0.0" + } + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "requires": { + "pinkie": "^2.0.0" + } + }, + "postcss": { + "version": "5.0.19", + "resolved": "http://registry.npmjs.org/postcss/-/postcss-5.0.19.tgz", + "integrity": "sha1-tjQqAdx1uMq36Wiv2pau/Gf4iK8=", + "requires": { + "js-base64": "^2.1.9", + "source-map": "^0.5.1", + "supports-color": "^3.1.2" + } + }, + "postcss-scss": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/postcss-scss/-/postcss-scss-0.4.0.tgz", + "integrity": "sha1-CHwFLFKbknDZWAvRJIoPk9O0DVc=", + "requires": { + "postcss": "^5.2.5" + }, + "dependencies": { + "postcss": { + "version": "5.2.18", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz", + "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==", + "requires": { + "chalk": "^1.1.3", + "js-base64": "^2.1.9", + "source-map": "^0.5.6", + "supports-color": "^3.2.3" + } + } + } + }, + "postcss-strip-inline-comments": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/postcss-strip-inline-comments/-/postcss-strip-inline-comments-0.1.5.tgz", + "integrity": "sha1-f/a83BTmM+1M36AguuPt2tT4S5A=", + "requires": { + "postcss": "^5.0.18" + } + }, + "postcss-value-parser": { + "version": "3.3.0", + "resolved": "http://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.0.tgz", + "integrity": "sha1-h/OPnxj3dKSrTIojL1xc6IcqnRU=" + }, + "postgres-array": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz", + "integrity": "sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz", + "integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g=" + }, + "postgres-interval": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz", + "integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, + "progress-stream": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/progress-stream/-/progress-stream-0.5.0.tgz", + "integrity": "sha1-zEdZFnpv9PBYdheThPC2rg0ddYc=", + "requires": { + "single-line-log": "~0.3.1", + "speedometer": "~0.1.2", + "through2": "~0.2.3" + } + }, + "propagate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-1.0.0.tgz", + "integrity": "sha1-AMLa7t2iDofjeCs0Stuhzd1q1wk=", + "dev": true + }, + "proxy-addr": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", + "integrity": "sha512-5erio2h9jp5CHGwcybmxmVqHmnCBZeewlfJ0pex+UW7Qny7OOZXTtH56TGNyBizkgiOwhJtMKrVzDTeKcySZwA==", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.8.0" + } + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "queue-async": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.1.0.tgz", + "integrity": "sha1-1fiOMv0B/uT22UbSHRer0WrGXzU=" + }, + "range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + }, + "raw-body": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", + "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.3", + "iconv-lite": "0.4.23", + "unpipe": "1.0.0" + } + }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + } + } + }, + "read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "requires": { + "load-json-file": "^1.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^1.0.0" + } + }, + "read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "requires": { + "find-up": "^1.0.0", + "read-pkg": "^1.0.0" + } + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "redis": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", + "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", + "requires": { + "double-ended-queue": "^2.1.0-0", + "redis-commands": "^1.2.0", + "redis-parser": "^2.6.0" + } + }, + "redis-commands": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.4.0.tgz", + "integrity": "sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw==" + }, + "redis-mpool": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/redis-mpool/-/redis-mpool-0.7.0.tgz", + "integrity": "sha512-+Tk3LXK5YkmseZjsz6Bx6/uRHy0CBGVvl6GU5tlzugYRn1qArlxK9wSEhadIfWh96Qx9yngvtacb7g8GNi78rA==", + "requires": { + "generic-pool": "~2.1.1", + "hiredis": "~0.5.0", + "redis": "^2.8.0", + "underscore": "~1.6.0" + }, + "dependencies": { + "generic-pool": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz", + "integrity": "sha1-rwTcLDJc/Ll1Aj+lK/zpYXp0Nf0=" + } + } + }, + "redis-parser": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", + "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" + }, + "request": { + "version": "2.87.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.87.0.tgz", + "integrity": "sha512-fcogkm7Az5bsS6Sl0sibkbhcKsnyon/jV1kF3ajGmF0c8HrttdKTPRT9hieOaQHA5HEq6r8OyWOo/o781C1tNw==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.6.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.5", + "extend": "~3.0.1", + "forever-agent": "~0.6.1", + "form-data": "~2.3.1", + "har-validator": "~5.0.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.17", + "oauth-sign": "~0.8.2", + "performance-now": "^2.1.0", + "qs": "~6.5.1", + "safe-buffer": "^5.1.1", + "tough-cookie": "~2.3.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.1.0" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "require-main-filename": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", + "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" + }, + "resolve": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", + "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=", + "dev": true + }, + "rimraf": { + "version": "2.4.5", + "resolved": "http://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "optional": true, + "requires": { + "glob": "^6.0.1" + }, + "dependencies": { + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "optional": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } + } + }, + "safe-buffer": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" + }, + "safe-json-stringify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", + "optional": true + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, + "semver": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", + "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==" + }, + "send": { + "version": "0.16.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", + "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.6.2", + "mime": "1.4.1", + "ms": "2.0.0", + "on-finished": "~2.3.0", + "range-parser": "~1.2.0", + "statuses": "~1.4.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "statuses": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + } + } + }, + "serve-static": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", + "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.2", + "send": "0.16.2" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" + }, + "shelljs": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", + "dev": true + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" + }, + "simple-statistics": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/simple-statistics/-/simple-statistics-0.9.2.tgz", + "integrity": "sha1-PjXLEDCPx2ljqk7nJS5qbqENKOQ=" + }, + "single-line-log": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/single-line-log/-/single-line-log-0.3.1.tgz", + "integrity": "sha1-p61lB/IYzl3+FsS/LWWSRkGeegY=" + }, + "sntp": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", + "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", + "requires": { + "hoek": "4.x.x" + } + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" + }, + "spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==" + }, + "spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.3.tgz", + "integrity": "sha512-uBIcIl3Ih6Phe3XHK1NqboJLdGfwr1UN3k6wSD1dZpmPsIkb8AGNbZYJ1fOBk834+Gxy8rpfDxrS6XLEMZMY2g==" + }, + "speedometer": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/speedometer/-/speedometer-0.1.4.tgz", + "integrity": "sha1-mHbb0qFp0xFUAtSObqYynIgWpQ0=" + }, + "sphericalmercator": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sphericalmercator/-/sphericalmercator-1.0.5.tgz", + "integrity": "sha1-3cWgSeNg4ADQ+tn8IsQHGIJYSYA=" + }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + }, + "sqlite3": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.0.6.tgz", + "integrity": "sha512-EqBXxHdKiwvNMRCgml86VTL5TK1i0IKiumnfxykX0gh6H6jaKijAXvE9O1N7+omfNSawR2fOmIyJZcfe8HYWpw==", + "requires": { + "nan": "~2.10.0", + "node-pre-gyp": "^0.11.0", + "request": "^2.87.0" + }, + "dependencies": { + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "nan": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" + }, + "node-pre-gyp": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", + "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "srs": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/srs/-/srs-1.2.0.tgz", + "integrity": "sha1-7LRHhU33bOsStFKQMr4j6eGO7Nw=", + "requires": { + "gdal": "~0.9.2" + } + }, + "sshpk": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz", + "integrity": "sha512-Ra/OXQtuh0/enyl4ETZAfTaeksa6BXks5ZcjpSUNrjBr0DvrJKX+1fsKDPpT9TBXgHAFsa4510aNVgI8g/+SzA==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + }, + "step": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/step/-/step-1.0.0.tgz", + "integrity": "sha1-swDp0q6QV9TXhjOq4jA4E6lL3/I=" + }, + "step-profiler": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz", + "integrity": "sha1-hBNozkTyMwyGLt1exg5/vsFI8LM=", + "requires": { + "debug": "^2.2.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } + } + }, + "strftime": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.0.tgz", + "integrity": "sha1-s/D6QZKVICpaKJ9ta+n0kJphcZM=", + "dev": true + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "stringstream": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.6.tgz", + "integrity": "sha512-87GEBAkegbBcweToUrdzf3eLhWNg06FJTebl4BVJz/JgWy8CvEr9dRtX5qWphiynMSQlxxi+QqN0z5T32SLlhA==" + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "requires": { + "is-utf8": "^0.2.0" + } + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + }, + "supports-color": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz", + "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=", + "requires": { + "has-flag": "^1.0.0" + } + }, + "tar": { + "version": "4.4.8", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz", + "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.3.4", + "minizlib": "^1.1.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + } + } + }, + "through": { + "version": "2.3.8", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "through2": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/through2/-/through2-0.2.3.tgz", + "integrity": "sha1-6zKE2k6jEbbMis42U3SKUqvyWj8=", + "requires": { + "readable-stream": "~1.1.9", + "xtend": "~2.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "xtend": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.1.2.tgz", + "integrity": "sha1-bv7MKk2tjmlixJAbM3znuoe10os=", + "requires": { + "object-keys": "~0.4.0" + } + } + } + }, + "tilelive": { + "version": "5.12.3", + "resolved": "https://registry.npmjs.org/tilelive/-/tilelive-5.12.3.tgz", + "integrity": "sha1-nIx3DhGUqo01PZodAxR0BDB83r0=", + "requires": { + "minimist": "~0.2.0", + "progress-stream": "~0.5.x", + "queue-async": "~1.0.7", + "sphericalmercator": "~1.0.1" + }, + "dependencies": { + "minimist": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.2.0.tgz", + "integrity": "sha1-Tf/lJdriuGTGbC4jxicdev3s784=" + }, + "queue-async": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz", + "integrity": "sha1-Iq4KHaxKkvW81GNPmTxoKiqBCUU=" + } + } + }, + "tilelive-mapnik": { + "version": "github:cartodb/tilelive-mapnik#a89d4147b3288804b679a018271893dda18373c2", + "from": "github:cartodb/tilelive-mapnik#0.6.18-cdb18", + "requires": { + "@carto/mapnik": "3.6.2-carto.11", + "generic-pool": "3.5.0", + "mime": "2.3.1" + }, + "dependencies": { + "generic-pool": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.5.0.tgz", + "integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w==" + }, + "mime": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz", + "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg==" + } + } + }, + "torque.js": { + "version": "2.17.1", + "resolved": "https://registry.npmjs.org/torque.js/-/torque.js-2.17.1.tgz", + "integrity": "sha512-jZuw5P3muNoILsmiN2kaclzczYW6h9HBSQo5riEB1ub/oNzmj7FqvFJOJKG3dbsxuAqOMn0+AwcQUC3Fb0YTRw==", + "requires": { + "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", + "d3": "3.5.17", + "turbo-carto": "^0.21.1", + "turf-jenks": "~1.0.1" + }, + "dependencies": { + "carto": { + "version": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", + "from": "github:cartodb/carto#master", + "requires": { + "mapnik-reference": "~6.0.2", + "optimist": "~0.6.0", + "underscore": "1.8.3" + } + }, + "turbo-carto": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/turbo-carto/-/turbo-carto-0.21.1.tgz", + "integrity": "sha512-MijaCzgz4cRjKMLaNxhMBq/lw+bb8cs+ZwGsHMAeJ8nTVWN7LY4b5LDBMIyxALmGiRkY8ziglmAoMU+xYnSTMA==", + "requires": { + "cartocolor": "4.0.0", + "colorbrewer": "1.0.0", + "debug": "^3.1.0", + "es6-promise": "3.1.2", + "postcss": "5.0.19", + "postcss-value-parser": "3.3.0" + } + }, + "underscore": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", + "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=" + } + } + }, + "tough-cookie": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", + "requires": { + "punycode": "^1.4.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "turbo-carto": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/turbo-carto/-/turbo-carto-0.21.0.tgz", + "integrity": "sha512-ARe+bG5vqns2q4tsKn8X3gglu75Jl3sTguJPIyYyDRby8COmFTPbMuV/qVd104E+PF2dDMD5fRcy5PqynsFmVg==", + "requires": { + "cartocolor": "4.0.0", + "colorbrewer": "1.0.0", + "debug": "^3.1.0", + "es6-promise": "3.1.2", + "postcss": "5.0.19", + "postcss-value-parser": "3.3.0" + } + }, + "turf-jenks": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/turf-jenks/-/turf-jenks-1.0.1.tgz", + "integrity": "sha1-XHKuoTpxbv8vACOnLPpUmgAgLX8=", + "requires": { + "simple-statistics": "^0.9.0" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2" + } + }, + "type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true + }, + "type-is": { + "version": "1.6.16", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", + "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.18" + } + }, + "uglify-js": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.9.tgz", + "integrity": "sha512-8CJsbKOtEbnJsTyv6LE6m6ZKniqMiFWmm9sRbopbkGs3gMPPfd3Fh8iIA4Ykv5MgaTbqHr4BaoGLJLZNhsrW1Q==", + "dev": true, + "optional": true, + "requires": { + "commander": "~2.17.1", + "source-map": "~0.6.1" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "optional": true + } + } + }, + "underscore": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" + }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "requires": { + "isexe": "^2.0.0" + } + }, + "which-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz", + "integrity": "sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8=" + }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "requires": { + "string-width": "^1.0.2 || 2" + } + }, + "window-size": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz", + "integrity": "sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU=" + }, + "windshaft": { + "version": "4.13.1", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-4.13.1.tgz", + "integrity": "sha512-sjk8D6gOZTp2jb7iuF38FyhPIcAAOUsOQtS+SmTLLIk1AipEK+xQU11IVl9cOvhiaCYjs7dRLscM54qoMqKVxw==", + "requires": { + "@carto/mapnik": "3.6.2-carto.11", + "@carto/tilelive-bridge": "github:cartodb/tilelive-bridge#e35ae36a6e2d555a6b312440f7e1904c5ad03664", + "abaculus": "github:cartodb/abaculus#3283c523d8bd4bdec78d90166d9c23ff09865ca8", + "canvas": "github:cartodb/node-canvas#45bd3c7cc5e6f99ba75b0d417b6ac6384b43083d", + "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", + "cartodb-psql": "0.13.1", + "debug": "3.1.0", + "dot": "1.1.2", + "grainstore": "1.10.0", + "queue-async": "1.1.0", + "redis-mpool": "0.7.0", + "request": "2.87.0", + "semver": "5.5.0", + "sphericalmercator": "1.0.5", + "tilelive": "5.12.3", + "tilelive-mapnik": "github:cartodb/tilelive-mapnik#a89d4147b3288804b679a018271893dda18373c2", + "torque.js": "2.17.1", + "underscore": "1.6.0" + } + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + }, + "wrap-ansi": { + "version": "2.1.0", + "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "requires": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, + "y18n": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", + "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=" + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + }, + "yargs": { + "version": "11.1.0", + "resolved": "http://registry.npmjs.org/yargs/-/yargs-11.1.0.tgz", + "integrity": "sha512-NwW69J42EsCSanF8kyn5upxvjp5ds+t3+udGBeTbFnERA+lF541DDpMawzo4z6W/QrzNM18D+BPMiOBibnFV5A==", + "requires": { + "cliui": "^4.0.0", + "decamelize": "^1.1.1", + "find-up": "^2.1.0", + "get-caller-file": "^1.0.1", + "os-locale": "^2.0.0", + "require-directory": "^2.1.1", + "require-main-filename": "^1.0.1", + "set-blocking": "^2.0.0", + "string-width": "^2.0.0", + "which-module": "^2.0.0", + "y18n": "^3.2.1", + "yargs-parser": "^9.0.2" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=" + }, + "cliui": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz", + "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==", + "requires": { + "string-width": "^2.1.1", + "strip-ansi": "^4.0.0", + "wrap-ansi": "^2.0.0" + } + }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "requires": { + "locate-path": "^2.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "os-locale": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz", + "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==", + "requires": { + "execa": "^0.7.0", + "lcid": "^1.0.0", + "mem": "^1.1.0" + } + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, + "yargs-parser": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-9.0.2.tgz", + "integrity": "sha1-nM9qQ0YP5O1Aqbto9I1DuKaMwHc=", + "requires": { + "camelcase": "^4.1.0" + } + } + } + }, + "yargs-parser": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-2.4.1.tgz", + "integrity": "sha1-hVaN488VD/SfpRgl8DqMiA3cxcQ=", + "requires": { + "camelcase": "^3.0.0", + "lodash.assign": "^4.0.6" + } + }, + "zipfile": { + "version": "0.5.12", + "resolved": "https://registry.npmjs.org/zipfile/-/zipfile-0.5.12.tgz", + "integrity": "sha512-zA60gW+XgQBu/Q4qV3BCXNIDRald6Xi5UOPj3jWGlnkjmBHaKDwIz7kyXWV3kq7VEsQN/2t/IWjdXdKeVNm6Eg==", + "requires": { + "nan": "~2.10.0", + "node-pre-gyp": "~0.10.2" + }, + "dependencies": { + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "nan": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" + }, + "node-pre-gyp": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.10.3.tgz", + "integrity": "sha512-d1xFs+C/IPS8Id0qPTZ4bUT8wWryfR/OzzAFxweG+uLN85oPzyo2Iw6bVlLQ/JOdgNonXLCoRyqDzDWq4iw72A==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "requires": { + "glob": "^7.1.3" + } + } + } + } + } +} diff --git a/package.json b/package.json index 09bfb4a8..f2ce3a36 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "5.3.2", + "version": "7.0.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -24,53 +24,54 @@ "Simon Martin " ], "dependencies": { - "body-parser": "^1.18.2", - "camshaft": "0.61.2", - "cartodb-psql": "0.10.2", - "cartodb-query-tables": "0.3.0", - "cartodb-redis": "0.15.0", - "debug": "^3.1.0", - "dot": "~1.0.2", - "express": "~4.16.0", - "fastly-purge": "~1.0.1", - "glob": "^7.1.2", - "log4js": "cartodb/log4js-node#cdb", - "lru-cache": "2.6.5", - "lzma": "~2.3.2", - "node-statsd": "~0.0.7", - "on-headers": "^1.0.1", - "queue-async": "~1.0.7", - "redis-mpool": "0.5.0", - "request": "^2.83.0", - "semver": "~5.3.0", - "step": "~0.0.6", - "step-profiler": "~0.3.0", - "turbo-carto": "0.20.2", - "underscore": "~1.6.0", - "windshaft": "4.5.3", - "yargs": "~5.0.0" + "@carto/fqdn-sync": "0.2.2", + "basic-auth": "2.0.0", + "body-parser": "1.18.3", + "camshaft": "0.63.4", + "cartodb-psql": "0.13.1", + "cartodb-query-tables": "0.4.0", + "cartodb-redis": "2.1.0", + "debug": "3.1.0", + "dot": "1.1.2", + "express": "4.16.3", + "fastly-purge": "1.0.1", + "gc-stats": "1.2.1", + "glob": "7.1.2", + "log4js": "github:cartodb/log4js-node#cdb", + "lru-cache": "4.1.3", + "lzma": "2.3.2", + "node-statsd": "0.1.1", + "on-headers": "1.0.1", + "queue-async": "1.1.0", + "redis-mpool": "0.7.0", + "request": "2.87.0", + "semver": "5.5.0", + "step-profiler": "0.3.0", + "turbo-carto": "0.21.0", + "underscore": "1.6.0", + "windshaft": "4.13.1", + "yargs": "11.1.0" }, "devDependencies": { - "istanbul": "~0.4.3", - "jshint": "~2.9.4", - "mocha": "~3.4.1", - "moment": "~2.18.1", - "nock": "~2.11.0", + "istanbul": "0.4.5", + "jshint": "2.9.7", + "mocha": "5.2.0", + "moment": "2.22.1", + "nock": "9.2.6", "redis": "2.8.0", - "strftime": "~0.8.2" + "step": "1.0.0", + "strftime": "0.10.0" }, "scripts": { - "lint": "jshint lib test", + "lint": "jshint lib test app.js", "preinstall": "make pre-install", "test": "make test-all", - "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn", - "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", - "docker-pull": "docker pull cartoimages/windshaft-testing", - "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh && docker ps --filter status=dead --filter status=exited -aq | xargs -r docker rm -v", - "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-testing bash" + "update-internal-deps": "rm -rf node_modules && npm install", + "docker-test": "./docker-test.sh", + "docker-bash": "./docker-bash.sh" }, "engines": { - "node": ">=6.9", - "yarn": ">=0.27.5 <1.0.0" + "node": "^10.15.1", + "npm": "^6.4.1" } } diff --git a/run_tests.sh b/run_tests.sh index d1c63614..9703ea69 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash OPT_CREATE_REDIS=yes # create the redis test environment OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment @@ -6,6 +6,7 @@ OPT_DROP_REDIS=yes # drop the redis test environment OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment OPT_COVERAGE=no # run tests with coverage OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files +OPT_REDIS_CELL=yes # download redis cell export PGAPPNAME=cartodb_tiler_tester @@ -88,6 +89,10 @@ while [ -n "$1" ]; do OPT_CREATE_PGSQL=no shift continue + elif test "$1" = "--norediscell"; then + OPT_REDIS_CELL=no + shift + continue else break fi @@ -99,6 +104,7 @@ if [ -z "$1" ]; then echo " --nocreate do not create the test environment on start" >&2 echo " --nodrop do not drop the test environment on exit" >&2 echo " --with-coverage use istanbul to determine code coverage" >&2 + echo " --norediscell do not download redis-cell" >&2 exit 1 fi @@ -106,7 +112,12 @@ TESTS=$@ if test x"$OPT_CREATE_REDIS" = xyes; then echo "Starting redis on port ${REDIS_PORT}" - echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log & + REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.so" + if [[ "$OSTYPE" == "darwin"* ]]; then + REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.dylib" + fi + + echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${REDIS_CELL_PATH} > ${BASEDIR}/test.log & PID_REDIS=$! echo ${PID_REDIS} > ${BASEDIR}/redis.pid fi @@ -124,7 +135,7 @@ fi echo "Preparing the environment" cd ${BASEDIR}/test/support -sh prepare_db.sh ${PREPARE_DB_OPTS} || die "database preparation failure" +source prepare_db.sh "${PREPARE_DB_OPTS}" || die "database preparation failure" cd - PATH=node_modules/.bin/:$PATH @@ -134,10 +145,10 @@ PATH=node_modules/.bin/:$PATH if test x"$OPT_COVERAGE" = xyes; then echo "Running tests with coverage" - ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS} + ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 --exit ${TESTS} else echo "Running tests" - ./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS} + ./node_modules/.bin/_mocha -c -u tdd -t 5000 --exit ${TESTS} fi ret=$? diff --git a/run_tests_docker.sh b/run_tests_docker.sh new file mode 100644 index 00000000..ee347b30 --- /dev/null +++ b/run_tests_docker.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +/etc/init.d/postgresql start + +source /src/nodejs-install.sh + +echo "Node.js version: " +node -v + +echo "npm version: " +npm -v + +echo "Clean install: " +npm ci +npm ls + +# run tests +npm test diff --git a/scripts/install.sh b/scripts/install.sh index 6f6e1fb1..f596aa12 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -4,4 +4,4 @@ if [[ "$OSTYPE" == "darwin"* ]]; then export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/opt/X11/lib/pkgconfig fi -yarn +npm install diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index d8870c8a..bd03aa41 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -1,19 +1,50 @@ +'use strict'; + require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); const serverOptions = require('../../lib/cartodb/server_options'); -const suites = [{ - desc: 'mvt (mapnik)', - usePostGIS: false -}]; - -if (process.env.POSTGIS_VERSION === '2.4') { - suites.push({ +const suites = [ + { + desc: 'mvt (mapnik)', + usePostGIS: false + }, + { desc: 'mvt (postgis)', usePostGIS: true - }); + } +]; + +// Generate points with values and times. +// The point location is spanned over a given length, by default it is 0 so +// all points have the same location, which can be used to test aggregation dimensions +// the default point is in tile +function pointsWithTimeSQL(n, startTime, endTime, span = 0, x0 = 0.1, y0 = 0.1) { + return ` + WITH params AS ( + SELECT + '${startTime}'::timestamp with time zone AS min_t, + '${endTime}'::timestamp with time zone AS max_t, + ${x0} AS x0, ${y0} AS y0, + ${span} AS length, + ${n} AS n + ), + positions AS ( + SELECT + step::float8/n AS s, + x0 + (step::float8/n - 0.5)*length AS x, y0 AS y + FROM params, generate_series(1, n) AS step + ) + SELECT + row_number() over () AS cartodb_id, + n*10 AS value, + min_t + (max_t - min_t)*s AS date, + ST_SetSRID(ST_MakePoint(x, y), 4326) AS the_geom, + ST_Transform(ST_SetSRID(ST_MakePoint(x, y), 4326), 3857) AS the_geom_webmercator + FROM params, positions + `; } describe('aggregation', function () { @@ -27,6 +58,15 @@ describe('aggregation', function () { from generate_series(-3, 3) x `; + const POINTS_SQL_0 = ` + select + x + 4 as cartodb_id, + st_setsrid(st_makepoint(x*10+1, x*10+1), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10+1, x*10+1), 4326), 3857) as the_geom_webmercator, + x as value + from generate_series(-3, 3) x + `; + const POINTS_SQL_TIMESTAMP_1 = ` select row_number() over() AS cartodb_id, @@ -100,6 +140,125 @@ describe('aggregation', function () { from generate_series(-3, 3) x `; + const POINTS_SQL_PAIRS = ` + -- Generate pairs of near points + select + x + 1 as cartodb_id, + st_setsrid( + st_makepoint( + Floor((x-6)/2)*10 + 9E-3*(x % 2 + 1), + Floor((x-6)/2)*10 + 9E-3*(x % 2 + 1) + ), + 4326 + ) as the_geom, + st_transform( + st_setsrid( + st_makepoint( + Floor((x-6)/2)*10 + 9E-3*(x % 2 + 1), + Floor((x-6)/2)*10 + 9E-3*(x % 2 + 1) + ), + 4326 + ), + 3857 + ) as the_geom_webmercator, + x as value + from generate_series(0, 13) x + `; + + // Some points at corners and centers of aggregation grids (zoom=1, resolution=1) + // @ = point location + // --+---+---+---+---+---+---+-- + // | | | | | | | + // --+---+---@---@---@---+---+-- + // | | | @ | @ | | | + // --+---+---@---@---@---+---+--- Y = 0 + // | | | @ | @ | | | + // --+---+---@---@---@---+---+-- + // | | | | | | | + // --+---+---+---+---+---+---+-- + // | + // X = 0 + // Point identifiers (cartodb_id) + // --+---+---+---+---+---+---+-- + // | | | | | | | + // --+---+---7---8---9---+---+-- + // | | |12 |13 | | | + // --+---+---4---5---6---+---+--- + // | | |10 |11 | | | + // --+---+---1---2---3---+---+-- + // | | | | | | | + // --+---+---+---+---+---+---+-- + // Point count per aggregation cell and Z=1 tiles + // + // Tile 0,0 -+---+---+---+- Tile 1,0 + // | | | 1 | 1 | 1 | | + // --+---+---@---@---@---+---+-- + // | | | 2 | 2 | 1 | | + // --+---+---@---@---@---+---+--- + // | | | 2 | 2 | 1 | | + // --+---+---@---@---@---+---+-- + // | | | | | | | + // Tile 0, 1 -+---+---+---+- Tile 1,1 + // + const POINTS_SQL_GRID = ` + WITH params AS ( + SELECT CDB_XYZ_Resolution($Z)*$resolution AS l -- cell size for Z, resolution + ) + SELECT + row_number() OVER () AS cartodb_id, + ST_SetSRID(ST_MakePoint(x*l, y*l), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(x*l, y*l), 3857), 4326) AS the_geom + FROM params, generate_series(-1,1) x, generate_series(-1,1) y + UNION ALL + SELECT + row_number() OVER () + 9 AS cartodb_id, + ST_SetSRID(ST_MakePoint(x*l-l/2, y*l-l/2), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(x*l-l/2, y*l-l/2), 3857), 4326) AS the_geom + FROM params, generate_series(0,1) x, generate_series(0,1) y + `; + + // Points positioned inside one cell of Z=20, X=1000000, X=1000000 (the SW corner) + // The center of the cell is x = 18181005.874444414, y = -18181043.94366749 + const POINTS_SQL_CELL = ` + SELECT + 1 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857), 4326) AS the_geom + UNION ALL SELECT + 2 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181005.9, -18181044.0), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181005.9, -18181044.0), 3857), 4326) AS the_geom + UNION ALL SELECT + 3 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181005.87, -18181043.94), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181005.87, -18181043.94), 3857), 4326) AS the_geom + UNION ALL SELECT + 4 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181005.8, -18181043.9), 3857), 4326) AS the_geom + `; + + // Points positioned inside one cell of Z=20, X=1000000, X=1000000 (inner cell not on border) + // The center of the cell is x = 18181006.023735486, y = -18181043.794376418 + const POINTS_SQL_CELL_INNER = ` + SELECT + 1 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181005.95, -18181043.8), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181005.95, -18181043.8), 3857), 4326) AS the_geom + UNION ALL SELECT + 2 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181006.09, -18181043.72), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181006.09, -18181043.72), 3857), 4326) AS the_geom + UNION ALL SELECT + 3 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181006.02, -18181043.79), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181006.02, -18181043.79), 3857), 4326) AS the_geom + UNION ALL SELECT + 4 AS cartodb_id, + ST_SetSRID(ST_MakePoint(18181006.01, -18181043.75), 3857) AS the_geom_webmercator, + ST_Transform(ST_SetSRID(ST_MakePoint(18181006.01, -18181043.75), 3857), 4326) AS the_geom + `; + function createVectorMapConfig (layers = [ { type: 'cartodb', @@ -130,6 +289,7 @@ describe('aggregation', function () { before(function () { serverOptions.renderer.mvt.usePostGIS = usePostGIS; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; }); after(function (){ @@ -138,6 +298,7 @@ describe('aggregation', function () { afterEach(function (done) { this.testClient.drain(done); + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); it('should return a layergroup indicating the mapconfig was aggregated', function (done) { @@ -357,6 +518,103 @@ describe('aggregation', function () { }); }); + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it('should provide all the requested columns in non-default aggregation ', + function (done) { + const response = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + placement: placement, + columns: { + 'first_column': { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + dimensions: { + second_column: 'sqrt_value' + }, + threshold: 1 + }, + cartocss: '#layer { marker-width: [first_column]; line-width: [second_column]; }', + cartocss_version: '2.3.0' + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup({ response }, (err, body) => { + if (err) { + return done(err); + } + + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png)); + done(); + }); + }); + + it('should provide only the requested columns in non-default aggregation ', + function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + placement: placement, + columns: { + 'first_column': { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + dimensions: { + second_column: 'sqrt_value' + }, + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(0, 0, 0, { format: 'mvt' }, function (err, res, mvt) { + if (err) { + return done(err); + } + + const geojsonTile = JSON.parse(mvt.toGeoJSONSync(0)); + let columns = new Set(); + geojsonTile.features.forEach(f => { + Object.keys(f.properties).forEach(p => columns.add(p)); + }); + columns = Array.from(columns); + const expected_columns = [ + '_cdb_feature_count', 'cartodb_id', 'first_column', 'second_column' + ]; + assert.deepEqual(columns.sort(), expected_columns.sort()); + + done(); + }); + }); + }); + it('should skip aggregation to create a layergroup with aggregation defined already', function (done) { const mapConfig = createVectorMapConfig([ { @@ -651,6 +909,440 @@ describe('aggregation', function () { }); }); + it('time dimensions', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_TIMESTAMP_1, + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + dow: { + column: 'date', + group: { + units: 'dayOfWeek' + } + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(feature => assert.equal(typeof feature.properties.dow, 'number')); + + done(); + }); + }); + + it('aggregation dimensions only used if present', function (done) { + const nPoints = 50; + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(nPoints, '2000-01-01T00:00:00+00', '2019-12-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + // Everything's aggregated into a single feature because the only + // dimension is space and all points are in the same place. + assert.deepEqual(tileJSON[0].features.map(f => f.properties._cdb_feature_count), [nPoints]); + done(); + }); + }); + + it('aggregation dimension year used', function (done) { + const nPoints = 50; + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(nPoints, '2000-01-01T00:00:00+00', '2019-12-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + year: { + column: 'date', + group: { + units: 'year' + } + } + } + + }, + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + // Now all features have same location, but the year is an additional dimension + // with 20 different values, so we'll have an aggregated feature for each. + const expectedYears = Array.from({length: 20}, (_, k) => 2000 + k); // 2000 to 2019 + const resultYears = tileJSON[0].features.map(f => f.properties.year).sort((a, b) => a - b); + assert.deepEqual(resultYears, expectedYears); + + done(); + }); + }); + + it('aggregation dimension month with count', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(50, '2018-01-01T00:00:00+00', '2018-12-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + month: { + column: 'date', + group: { + units: 'month', + count: 5, + starting: '2018-01' + } + } + } + + }, + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + assert.equal(tileJSON[0].features.length, 3); + const resultQuimesters = tileJSON[0].features.map(f => f.properties.month).sort((a, b) => a - b); + assert.deepEqual(resultQuimesters, [1, 2, 3]); + + done(); + }); + }); + + it('aggregation dimension month with starting', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(50, '2018-01-01T00:00:00+00', '2018-12-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + month: { + column: 'date', + group: { + units: 'month', + starting: '2017-01' + } + } + } + + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + const resultMonths = tileJSON[0].features.map(f => f.properties.month).sort((a, b) => a - b); + assert.deepEqual(resultMonths, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]); + + done(); + }); + }); + + it('aggregation dimension month by default UTC', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(50, '2018-01-01T00:00:00+00', '2018-01-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + dow: { + column: 'date', + group: { + units: 'month', + timezone: '+00' + } + } + } + + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + // In UTC all times are in the same month 2018-01 + assert.equal(tileJSON[0].features.length, 1); + + done(); + }); + }); + + it('aggregation dimension month with timezone', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(50, '2018-01-01T00:00:00+00', '2018-01-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + dow: { + column: 'date', + group: { + units: 'month', + timezone: '+7200' + } + } + } + + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + // In UTC+2 some times are in a different month + assert.equal(tileJSON[0].features.length, 2); + done(); + }); + }); + + it('time dimensions stats', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_TIMESTAMP_1, + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + dow: { + column: 'date', + group: { + units: 'dayOfWeek' + } + } + } + }, + metadata: { + dimensions: true + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + const expectedDimensions = { + dow: + { params: + { time: 'to_timestamp("date")', + timezone: 'utc', + units: 'dayOfWeek', + count: 1 }, + min: 4, + max: 7, + type: 'number' } + }; + assert.deepEqual(layergroup.metadata.layers[0].meta.stats.dimensions, expectedDimensions); + done(); + }); + }); + + it('no time dimensions stats by default', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_TIMESTAMP_1, + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + dow: { + column: 'date', + group: { + units: 'dayOfWeek' + } + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert(!layergroup.metadata.layers[0].meta.stats.dimensions); + done(); + }); + }); + + it('aggregation dimension month iso format', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(50, '2018-01-01T00:00:00+00', '2018-12-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + month: { + column: 'date', + group: { + units: 'month', + }, + format: 'iso' + } + } + + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + const resultMonths = tileJSON[0].features.map(f => f.properties.month).sort(); + assert.deepEqual(resultMonths, [ + '2018-01', '2018-02', '2018-03', '2018-04', '2018-05','2018-06', + '2018-07', '2018-08', '2018-09', '2018-10', '2018-11', '2018-12' + ]); + done(); + }); + }); + + it('aggregation dimension month iso format with timezone', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: pointsWithTimeSQL(50, '2018-01-01T00:00:00+00', '2018-12-31T23:59:59+00', 0), + dates_as_numbers: true, + aggregation: { + threshold: 1, + dimensions: { + month: { + column: 'date', + group: { + units: 'month', + timezone: '+7200' + }, + format: 'iso', + } + } + + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + const tileJSON = tile.toJSON(); + const resultMonths = tileJSON[0].features.map(f => f.properties.month).sort(); + assert.deepEqual(resultMonths, [ + '2018-01', '2018-02', '2018-03', '2018-04', '2018-05', '2018-06', + '2018-07', '2018-08', '2018-09', '2018-10', '2018-11', '2018-12', + '2019-01' + ]); + done(); + }); + }); + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { it(`dimensions should work for ${placement} placement`, function(done) { this.mapConfig = createVectorMapConfig([ @@ -689,6 +1381,45 @@ describe('aggregation', function () { }); }); + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`dimensions with alias should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + dimensions: { + value2: "value" + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.value2, 'number') + ); + + done(); + }); + }); + }); + + it(`dimensions should trigger non-default aggregation`, function(done) { this.mapConfig = createVectorMapConfig([ { @@ -979,7 +1710,6 @@ describe('aggregation', function () { type: 'cartodb', options: { sql: POINTS_SQL_1, - resolution: 256, aggregation: { threshold: 1 } @@ -1375,7 +2105,7 @@ describe('aggregation', function () { }); }); - ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + ['centroid', 'point-sample', 'point-grid', 'default'].forEach(placement => { it(`cartodb_id should be present in ${placement} aggregation`, function(done) { this.mapConfig = createVectorMapConfig([ { @@ -1383,8 +2113,8 @@ describe('aggregation', function () { options: { sql: POINTS_SQL_1, aggregation: { - placement: placement, - threshold: 1 + threshold: 1, + placement: placement !== 'default' ? placement : undefined }, cartocss: '#layer { marker-width: 1; }', cartocss_version: '2.3.0', @@ -1408,73 +2138,1213 @@ describe('aggregation', function () { done(); }); }); - }); - it('should only require the_geom_webmercator for aggregation', function (done) { - this.mapConfig = createVectorMapConfig([ - { - type: 'cartodb', - options: { - sql: POINTS_SQL_ONLY_WEBMERCATOR, - aggregation: { - threshold: 1 + it(`should only require the_geom_webmercator for ${placement} aggregation`, function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_ONLY_WEBMERCATOR, + aggregation: { + threshold: 1, + placement: placement !== 'default' ? placement : undefined + } } } - } - ]); - this.testClient = new TestClient(this.mapConfig); + ]); + this.testClient = new TestClient(this.mapConfig); - this.testClient.getLayergroup((err, body) => { - if (err) { - return done(err); - } + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } - assert.equal(typeof body.metadata, 'object'); - assert.ok(Array.isArray(body.metadata.layers)); + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); - body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); - body.metadata.layers.forEach(layer => assert.ok(!layer.meta.aggregation.png)); + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(!layer.meta.aggregation.png)); - done(); + done(); + }); + }); + + it(`${placement} aggregation should work with attributes`, function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + cartocss: '#layer { marker-width: 7; }', + cartocss_version: '2.3.0', + aggregation: { + threshold: 1 + }, + attributes: { + id: 'cartodb_id', + columns: [ + 'value' + ] + } + } + } + ]); + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png)); + + done(); + }); }); }); - it('aggregation should work with attributes', function (done) { + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than_or_equal_to: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value >= 0); + }); + + done(); + }); + }); + }); + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`multiple ORed filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: [ + { greater_than: 0 }, + { less_than: -2 } + ] + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > 0 || row.properties.value < -2); + }); + + done(); + }); + }); + }); + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`multiple ANDed filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + }, + value2: { + aggregate_function: 'sum', + aggregated_column: 'sqrt_value' + } + }, + filters: { + value: { greater_than: 0 }, + value2: { less_than: 9 } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > 0 && row.properties.value2 < 9); + }); + + done(); + }); + }); + }); + + it(`supports IN filters`, function(done) { this.mapConfig = createVectorMapConfig([ { type: 'cartodb', options: { sql: POINTS_SQL_1, - cartocss: '#layer { marker-width: 7; }', - cartocss_version: '2.3.0', aggregation: { - threshold: 1 - }, - attributes: { - id: 'cartodb_id', - columns: [ - 'value' - ] + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { in: [1, 3] } + } } } } ]); - this.testClient = new TestClient(this.mapConfig); - this.testClient.getLayergroup((err, body) => { + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { if (err) { return done(err); } - assert.equal(typeof body.metadata, 'object'); - assert.ok(Array.isArray(body.metadata.layers)); + const tileJSON = tile.toJSON(); - body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); - body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png)); + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value === 1 || row.properties.value === 3); + }); done(); }); }); + + it(`supports NOT IN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { not_in: [1, 3] } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value !== 1 && row.properties.value !== 3); + }); + + done(); + }); + }); + + it(`supports EQUAL filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: [{ equal: 1}, { equal: 3}] + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value === 1 || row.properties.value === 3); + }); + + done(); + }); + }); + + it(`supports NOT EQUAL filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { not_equal: 1 } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value !== 1); + }); + + done(); + }); + }); + + it(`supports BETWEEN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than_or_equal_to: -1, + less_than_or_equal_to: 2 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value >= -1 || row.properties.value <= 2); + }); + + done(); + }); + }); + + it(`supports RANGE filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than: -1, + less_than_or_equal_to: 2 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > -1 || row.properties.value <= 2); + }); + + done(); + }); + }); + + it(`invalid filters cause errors`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + not_a_valid_parameter: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + const options = { + response: { + status: 400 + } + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ 'Invalid filter parameter name: not_a_valid_parameter'], + errors_with_context:[{ + type: 'layer', + message: 'Invalid filter parameter name: not_a_valid_parameter', + layer: { + id: "layer0", + index: 0, + type: "mapnik", + } + }] + }); + + done(); + }); + }); + + it(`filters on invalid columns cause errors`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value_sum: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + not_a_valid_parameter: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + const options = { + response: { + status: 400 + } + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ 'Invalid filtered column: value'], + errors_with_context:[{ + type: 'layer', + message: 'Invalid filtered column: value', + layer: { + id: "layer0", + index: 0, + type: "mapnik", + } + }] + }); + + done(); + }); + }); + + ['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`for ${placement} and no points between tiles has unique ids`, function (done) { + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_0, + aggregation: { + threshold: 1, + resolution: 1, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(1, 0, 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + + const tile1 = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(tile1.features)); + assert.ok(tile1.features.length > 0); + + this.testClient.getTile(1, 1, 0, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + + const tile2 = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(tile2.features)); + assert.ok(tile2.features.length > 0); + + const tile1Ids = tile1.features.map(f => f.properties.cartodb_id); + const tile2Ids = tile2.features.map(f => f.properties.cartodb_id); + const repeatedIds = tile1Ids.filter(id => tile2Ids.includes(id)); + + assert.equal(repeatedIds.length, 0); + + done(); + }); + + }); + }); + it(`for ${placement} has unique ids save between tiles`, function (done) { + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_1, + resolution: 1, + aggregation: { + threshold: 1, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(1, 0, 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + + const tile1 = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(tile1.features)); + assert.ok(tile1.features.length > 0); + + this.testClient.getTile(1, 1, 0, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + + const tile2 = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(tile2.features)); + assert.ok(tile2.features.length > 0); + + const tile1Ids = tile1.features.map(f => f.properties.cartodb_id); + const tile2Ids = tile2.features.map(f => f.properties.cartodb_id); + const repeatedIds = tile1Ids.filter(id => tile2Ids.includes(id)); + + // It is not guaranteed that features appear in a single tile: + // features on the border of tiles can appear in multiple tiles + if (repeatedIds.length > 0) { + repeatedIds.forEach(id => { + const tile1Features = tile1.features.filter(f => f.properties.cartodb_id === id); + const tile2Features = tile2.features.filter(f => f.properties.cartodb_id === id); + // repetitions cannot occur inside a tile + assert.equal(tile1Features.length, 1); + assert.equal(tile2Features.length, 1); + const feature1 = tile1Features[0]; + const feature2 = tile2Features[0]; + // features should be identical (geometry and properties) + assert.deepEqual(feature1.properties, feature2.properties); + assert.deepEqual(feature1.geometry, feature2.geometry); + // and geometry should be on the border; + // for the dataset and zoom 1, only point with cartodb_id=4 (0,0) + assert.equal(feature1.properties.cartodb_id, 4); + assert.equal(feature2.properties.cartodb_id, 4); + }); + } + done(); + }); + + }); + }); + + it(`for ${placement} each aggr. cell is in a single tile`, function (done) { + const z = 1; + const resolution = 1; + const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution); + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: query, + aggregation: { + threshold: 1, + resolution: resolution, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + + const c = Math.pow(2, z - 1) - 1; // center tile coordinates + + this.testClient.getTile(z, c + 0, c + 0, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile00 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c + 0, c + 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile01 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c + 1, c + 0, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile10 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c + 1, c + 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile11 = JSON.parse(mvt.toGeoJSONSync(0)); + + const tile00Expected = [ + { cartodb_id: 4, _cdb_feature_count: 2 }, + { cartodb_id: 7, _cdb_feature_count: 1 } + ]; + const tile10Expected = [ + { cartodb_id: 5, _cdb_feature_count: 2 }, + { cartodb_id: 6, _cdb_feature_count: 1 }, + { cartodb_id: 8, _cdb_feature_count: 1 }, + { cartodb_id: 9, _cdb_feature_count: 1 } + ]; + const tile01Expected = [ + { cartodb_id: 1, _cdb_feature_count: 2 } + ]; + const tile11Expected = [ + { cartodb_id: 2, _cdb_feature_count: 2 }, + { cartodb_id: 3, _cdb_feature_count: 1 } + ]; + const tile00Actual = tile00.features.map(f => f.properties); + const tile10Actual = tile10.features.map(f => f.properties); + const tile01Actual = tile01.features.map(f => f.properties); + const tile11Actual = tile11.features.map(f => f.properties); + const orderById = (a, b) => a.cartodb_id - b.cartodb_id; + assert.deepEqual(tile00Actual.sort(orderById), tile00Expected); + assert.deepEqual(tile10Actual.sort(orderById), tile10Expected); + assert.deepEqual(tile01Actual.sort(orderById), tile01Expected); + assert.deepEqual(tile11Actual.sort(orderById), tile11Expected); + + done(); + }); + }); + }); + + }); + }); + + it(`for ${placement} no partially aggregated cells`, function (done) { + // Use level 1 with resolution 2 tiles and buffersize 1 (half the cell size) + // Only the cells completely inside the buffer are aggregated + const z = 1; + const resolution = 2; + // space the test points by half the resolution: + const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution/2); + + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 1 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: query, + aggregation: { + threshold: 1, + resolution: resolution, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + + const c = Math.pow(2, z - 1) - 1; // center tile coordinates + + this.testClient.getTile(z, c, c, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile00 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c, c + 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile01 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c + 1, c, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile10 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c + 1, c + 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile11 = JSON.parse(mvt.toGeoJSONSync(0)); + + const tile00Expected = [ + { cartodb_id: 4, _cdb_feature_count: 3 } + ]; + const tile10Expected = [ + { cartodb_id: 5, _cdb_feature_count: 5 } + ]; + const tile01Expected = [ + { cartodb_id: 1, _cdb_feature_count: 2 } + ]; + const tile11Expected = [ + { cartodb_id: 2, _cdb_feature_count: 3 } + ]; + const tile00Actual = tile00.features.map(f => f.properties); + const tile10Actual = tile10.features.map(f => f.properties); + const tile01Actual = tile01.features.map(f => f.properties); + const tile11Actual = tile11.features.map(f => f.properties); + const orderById = (a, b) => a.cartodb_id - b.cartodb_id; + assert.deepEqual(tile00Actual.sort(orderById), tile00Expected); + assert.deepEqual(tile10Actual.sort(orderById), tile10Expected); + assert.deepEqual(tile01Actual.sort(orderById), tile01Expected); + assert.deepEqual(tile11Actual.sort(orderById), tile11Expected); + + done(); + }); + }); + }); + + }); + }); + + it(`for ${placement} includes complete cells in buffer`, function (done) { + if (!usePostGIS && placement !== 'point-grid') { + // Mapnik seem to filter query results by its (inaccurate) bbox, + // which makes some aggregated clusters get lost here. + // The point-grid placement is resilient to this problem because the result + // coordinates are moved to cluster cell centers, so they're well within + // bbox limits. + this.testClient = new TestClient({}); + return done(); + } + + // use buffersize coincident with resolution, the buffer should include neighbour cells + const z = 2; + const resolution = 1; + const query = POINTS_SQL_GRID.replace('$Z', z).replace('$resolution', resolution); + + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 1 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: query, + aggregation: { + threshold: 1, + resolution: resolution, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + + const c = Math.pow(2, z - 1) - 1; // center tile coordinates + + this.testClient.getTile(z, c, c, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile00 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c, c + 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile01 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c + 1, c, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile10 = JSON.parse(mvt.toGeoJSONSync(0)); + this.testClient.getTile(z, c + 1, c + 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile11 = JSON.parse(mvt.toGeoJSONSync(0)); + + const tile00Expected = [ + { _cdb_feature_count: 2, cartodb_id: 1 }, + { _cdb_feature_count: 2, cartodb_id: 2 }, + { _cdb_feature_count: 2, cartodb_id: 4 }, + { _cdb_feature_count: 2, cartodb_id: 5 }, + { _cdb_feature_count: 1, cartodb_id: 7 }, + { _cdb_feature_count: 1, cartodb_id: 8 } + ]; + const tile10Expected = [ + { _cdb_feature_count: 2, cartodb_id: 1 }, + { _cdb_feature_count: 2, cartodb_id: 2 }, + { _cdb_feature_count: 1, cartodb_id: 3 }, + { _cdb_feature_count: 2, cartodb_id: 4 }, + { _cdb_feature_count: 2, cartodb_id: 5 }, + { _cdb_feature_count: 1, cartodb_id: 6 }, + { _cdb_feature_count: 1, cartodb_id: 7 }, + { _cdb_feature_count: 1, cartodb_id: 8 }, + { _cdb_feature_count: 1, cartodb_id: 9 } + ]; + const tile01Expected = [ + { _cdb_feature_count: 2, cartodb_id: 1 }, + { _cdb_feature_count: 2, cartodb_id: 2 }, + { _cdb_feature_count: 2, cartodb_id: 4 }, + { _cdb_feature_count: 2, cartodb_id: 5 } + ]; + const tile11Expected = [ + { _cdb_feature_count: 2, cartodb_id: 1 }, + { _cdb_feature_count: 2, cartodb_id: 2 }, + { _cdb_feature_count: 1, cartodb_id: 3 }, + { _cdb_feature_count: 2, cartodb_id: 4 }, + { _cdb_feature_count: 2, cartodb_id: 5 }, + { _cdb_feature_count: 1, cartodb_id: 6 } + ]; + const tile00Actual = tile00.features.map(f => f.properties); + const tile10Actual = tile10.features.map(f => f.properties); + const tile01Actual = tile01.features.map(f => f.properties); + const tile11Actual = tile11.features.map(f => f.properties); + const orderById = (a, b) => a.cartodb_id - b.cartodb_id; + assert.deepEqual(tile00Actual.sort(orderById), tile00Expected); + assert.deepEqual(tile10Actual.sort(orderById), tile10Expected); + assert.deepEqual(tile01Actual.sort(orderById), tile01Expected); + assert.deepEqual(tile11Actual.sort(orderById), tile11Expected); + + done(); + }); + }); + }); + + }); + }); + + it(`for ${placement} points aggregated into corner cluster`, function (done) { + // this test will fail due to !bbox! lack of accuracy if strict cell filtering is in place + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_CELL, + aggregation: { + threshold: 1, + resolution: 1, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(20, 1000000, 1000000, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile = JSON.parse(mvt.toGeoJSONSync(0)); + assert.equal(tile.features.length, 1); + assert.equal(tile.features[0].properties._cdb_feature_count, 4); + if (placement === 'point-grid') { + // check geometry x = 18181005.874444414, y = -18181043.94366749 + const expectedPoint = [ 163.322754576802, -83.3823797469878 ]; + assert.deepEqual(tile.features[0].geometry.coordinates, expectedPoint); + } + done(); + }); + }); + + it(`for ${placement} points aggregated into correct cluster`, function (done) { + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_CELL_INNER, + aggregation: { + threshold: 1, + resolution: 1, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(20, 1000000, 1000000, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + const tile = JSON.parse(mvt.toGeoJSONSync(0)); + assert.equal(tile.features.length, 1); + assert.equal(tile.features[0].properties._cdb_feature_count, 4); + if (placement === 'point-grid') { + // check geometry x = 18181006.023735486, y = -18181043.794376418 + const expectedPoint = [ 163.322755917907, -83.3823795924354 ]; + assert.deepEqual(tile.features[0].geometry.coordinates, expectedPoint); + } + done(); + }); + }); + }); + + ['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`default pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + aggregation: { + threshold: 1, + resolution: 1, + placement: placement !== 'default' ? placement : undefined + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.ok(body.metadata.layers[0].meta.stats.estimatedFeatureCount > 0); + + done(); + }); + }); + + it(`on demand post-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + aggregation: { + threshold: 1, + resolution: 1, + placement: placement !== 'default' ? placement : undefined + }, + metadata: { + aggrFeatureCount: 10 + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 13); + + done(); + }); + }); + + it(`post-aggregation count adapts to zoom level with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + aggregation: { + threshold: 1, + resolution: 1, + placement: placement !== 'default' ? placement : undefined + }, + metadata: { + aggrFeatureCount: 0 + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 7); + + done(); + }); + }); + + it(`on-demand pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + aggregation: { + threshold: 1, + resolution: 1, + placement: placement !== 'default' ? placement : undefined + }, + metadata: { + featureCount: true + } + } + } + ] + }; + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.featureCount, 14); + + done(); + }); + }); + }); + }); }); }); diff --git a/test/acceptance/analysis/analyses-controller.js b/test/acceptance/analysis/analyses-controller.js index dd1d138c..6ed0808b 100644 --- a/test/acceptance/analysis/analyses-controller.js +++ b/test/acceptance/analysis/analyses-controller.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/analysis/analyses-filters-params.js b/test/acceptance/analysis/analyses-filters-params.js index c4a17ae4..21fd8f01 100644 --- a/test/acceptance/analysis/analyses-filters-params.js +++ b/test/acceptance/analysis/analyses-filters-params.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); const assert = require('../../support/assert'); diff --git a/test/acceptance/analysis/analyses-filters.js b/test/acceptance/analysis/analyses-filters.js index 152b29e6..ff86855f 100644 --- a/test/acceptance/analysis/analyses-filters.js +++ b/test/acceptance/analysis/analyses-filters.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); const assert = require('../../support/assert'); diff --git a/test/acceptance/analysis/analysis-layers-dataviews.js b/test/acceptance/analysis/analysis-layers-dataviews.js index 8e95096a..7a35607f 100644 --- a/test/acceptance/analysis/analysis-layers-dataviews.js +++ b/test/acceptance/analysis/analysis-layers-dataviews.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 2f087626..cc10c7dc 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index 1eb3d4e6..b7eeac48 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/analysis/error-cases.js b/test/acceptance/analysis/error-cases.js index 376d7606..5c610dde 100644 --- a/test/acceptance/analysis/error-cases.js +++ b/test/acceptance/analysis/error-cases.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -188,7 +190,7 @@ describe('analysis-layers error cases', function() { ] ); - var testClient = new TestClient(mapConfig, 11111); + var testClient = new TestClient(mapConfig); //No apikey provided -> using default public apikey testClient.getLayergroup({ response: AUTH_ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js index b9c93e00..dd34b7dd 100644 --- a/test/acceptance/analysis/named-maps.js +++ b/test/acceptance/analysis/named-maps.js @@ -1,15 +1,22 @@ +'use strict'; + var assert = require('../../support/assert'); var helper = require('../../support/test_helper'); var CartodbWindshaft = require('../../../lib/cartodb/server'); var serverOptions = require('../../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var TestClient = require('../../support/test-client'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('named-maps analysis', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + var IMAGE_TOLERANCE_PER_MIL = 20; @@ -256,6 +263,62 @@ describe('named-maps analysis', function() { ); }); + it('should fail to retrieve static map preview via layergroup ' + + 'when filtering by invalid layers', function(done) { + assert.response( + server, + { + url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.png?layer=1', + method: 'GET', + encoding: 'binary', + headers: { + host: username + } + }, + { + status: 400 + }, + function(res, err) { + done(err); + } + ); + }); + + it('should return and an error requesting unsupported image format', function(done) { + assert.response( + server, + { + url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.gif', + method: 'GET', + encoding: 'binary', + headers: { + host: username + } + }, + { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + assert.ifError(err); + assert.deepEqual( + JSON.parse(res.body), + { + errors:['Unsupported image format \"gif\"'], + errors_with_context:[{ + type: 'unknown', + message: 'Unsupported image format \"gif\"' + }] + } + ); + done(); + + } + ); + }); + }); describe('auto-instantiation', function() { diff --git a/test/acceptance/analysis/regressions.js b/test/acceptance/analysis/regressions.js index 94089ac0..72fd9a35 100644 --- a/test/acceptance/analysis/regressions.js +++ b/test/acceptance/analysis/regressions.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/auth/authorization-basic-use-cases.js b/test/acceptance/auth/authorization-basic-use-cases.js new file mode 100644 index 00000000..d0c3a17c --- /dev/null +++ b/test/acceptance/auth/authorization-basic-use-cases.js @@ -0,0 +1,183 @@ +'use strict'; + +const assert = require('../../support/assert'); +const testHelper = require('../../support/test_helper'); +const CartodbWindshaft = require('../../../lib/cartodb/server'); +const serverOptions = require('../../../lib/cartodb/server_options'); +var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); + +function singleLayergroupConfig(sql, cartocss) { + return { + version: '1.7.0', + layers: [ + { + type: 'mapnik', + options: { + sql: sql, + cartocss: cartocss, + cartocss_version: '2.3.0' + } + } + ] + }; +} + +function createRequest(layergroup, userHost, apiKey) { + var url = layergroupUrl; + if (apiKey) { + url += '?api_key=' + apiKey; + } + return { + url: url, + method: 'POST', + headers: { + host: userHost || 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }; +} + +var layergroupUrl = '/api/v1/map'; +var pointSqlMaster = "select * from test_table_private_1"; +var pointSqlPublic = "select * from test_table"; +var keysToDelete; + +describe('Basic authorization use cases', function () { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + + beforeEach(function () { + keysToDelete = {}; + }); + + afterEach(function (done) { + testHelper.deleteRedisKeys(keysToDelete, done); + }); + + it("succeed with master", function (done) { + var layergroup = singleLayergroupConfig(pointSqlMaster, '#layer { marker-fill:red; }'); + + assert.response(server, + createRequest(layergroup, 'localhost', '1234'), + { + status: 200 + }, + function (res, err) { + assert.ifError(err); + + var parsed = JSON.parse(res.body); + assert.ok(parsed.layergroupid); + assert.equal(res.headers['x-layergroup-id'], parsed.layergroupid); + + keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + done(); + } + ); + }); + + + it("succeed with default - sending default_public", function (done) { + var layergroup = singleLayergroupConfig(pointSqlPublic, '#layer { marker-fill:red; }'); + + assert.response(server, + createRequest(layergroup, 'localhost', 'default_public'), + { + status: 200 + }, + function (res, err) { + assert.ifError(err); + + var parsed = JSON.parse(res.body); + assert.ok(parsed.layergroupid); + assert.equal(res.headers['x-layergroup-id'], parsed.layergroupid); + + keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + done(); + } + ); + }); + + it("fail with non-existent api key", function (done) { + var layergroup = singleLayergroupConfig(pointSqlPublic, '#layer { marker-fill:red; }'); + + assert.response(server, + createRequest(layergroup, 'localhost', 'THIS-API-KEY-DOESNT-EXIST'), + { + status: 401 + }, + function (res, err) { + assert.ifError(err); + var parsed = JSON.parse(res.body); + assert.ok(parsed.hasOwnProperty('errors')); + assert.equal(parsed.errors.length, 1); + assert.ok(parsed.errors[0].match(/Unauthorized/)); + done(); + } + ); + }); + + it("fail with default", function (done) { + var layergroup = singleLayergroupConfig(pointSqlMaster, '#layer { marker-fill:red; }'); + + assert.response(server, + createRequest(layergroup, 'localhost', 'default_public'), + { + status: 403 + }, + function (res, err) { + assert.ifError(err); + + done(); + } + ); + }); + + describe('No api key provided - fallback to default_public', function () { + it("succeed with default - public dataset", function (done) { + var layergroup = singleLayergroupConfig(pointSqlPublic, '#layer { marker-fill:red; }'); + + assert.response(server, + createRequest(layergroup, 'localhost'), + { + status: 200 + }, + function (res, err) { + assert.ifError(err); + + var parsed = JSON.parse(res.body); + assert.ok(parsed.layergroupid); + assert.equal(res.headers['x-layergroup-id'], parsed.layergroupid); + + keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + done(); + } + ); + }); + + it("fail with default - private dataset", function (done) { + var layergroup = singleLayergroupConfig(pointSqlMaster, '#layer { marker-fill:red; }'); + + assert.response(server, + createRequest(layergroup, 'localhost'), + { + status: 403 + }, + function (res, err) { + assert.ifError(err); + + done(); + } + ); + }); + }); +}); diff --git a/test/acceptance/auth/authorization.js b/test/acceptance/auth/authorization.js new file mode 100644 index 00000000..7fff7aa3 --- /dev/null +++ b/test/acceptance/auth/authorization.js @@ -0,0 +1,934 @@ +'use strict'; + +require('../../support/test_helper'); + +const assert = require('../../support/assert'); +const TestClient = require('../../support/test-client'); +const mapnik = require('windshaft').mapnik; + +const PERMISSION_DENIED_RESPONSE = { + status: 403, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } +}; + +describe('authorization', function() { + it('should create a layergroup with regular apikey token', function(done) { + const apikeyToken = 'regular1'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + options: { + sql: 'select * FROM test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup(function (err, layergroupResult) { + assert.ifError(err); + + assert.ok(layergroupResult.layergroupid); + + testClient.drain(done); + }); + }); + + + it('should fail getting a named map tile with default apikey token', function (done) { + const apikeyTokenCreate = 'regular1'; + const apikeyTokenGet = 'default_public'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + options: { + sql: 'select * FROM test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + + const testClientCreate = new TestClient(mapConfig, apikeyTokenCreate); + testClientCreate.getLayergroup(function (err, layergroupResult) { + assert.ifError(err); + const layergroupId = layergroupResult.layergroupid; + + const testClientGet = new TestClient({}, apikeyTokenGet); + + const params = { + layergroupid: layergroupId, + response: PERMISSION_DENIED_RESPONSE + }; + + testClientGet.getTile(0, 0, 0, params, function(err, res, body) { + + assert.ifError(err); + + assert.ok(body.hasOwnProperty('errors')); + assert.equal(body.errors.length, 1); + assert.ok(body.errors[0].match(/permission denied/), body.errors[0]); + + testClientGet.drain(done); + }); + }); + }); + + it('should fail creating a layergroup with default apikey token', function (done) { + const apikeyToken = 'default_public'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + options: { + sql: 'select * FROM test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup({ response: { status: 403 } }, function (err, layergroupResult) { + assert.ifError(err); + + assert.ok(layergroupResult.hasOwnProperty('errors')); + assert.equal(layergroupResult.errors.length, 1); + assert.ok(layergroupResult.errors[0].match(/permission denied/), layergroupResult.errors[0]); + + testClient.drain(done); + }); + }); + + it('should create a layergroup with default apikey token', function (done) { + const apikeyToken = 'default_public'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + options: { + sql: 'select * FROM test_table', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup(function (err, layergroupResult) { + assert.ifError(err); + + assert.ok(layergroupResult.layergroupid); + + testClient.drain(done); + }); + }); + + it('should create and get a tile with default apikey token', function (done) { + const apikeyToken = 'default_public'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + options: { + sql: 'select * FROM test_table', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getTile(0, 0, 0, function (err, res, tile) { + assert.ifError(err); + + assert.equal(res.statusCode, 200); + assert.ok(tile instanceof mapnik.Image); + + testClient.drain(done); + }); + }); + + it('should fail if apikey does not grant access to table', function (done) { + const mapConfig = { + version: '1.7.0', + layers: [ + { + options: { + sql: 'select * FROM test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + const testClient = new TestClient(mapConfig); //no apikey provided, using default + + testClient.getLayergroup({ response: { status: 403 } }, function (err, layergroupResult) { //TODO 401 + assert.ifError(err); + + assert.ok(layergroupResult.hasOwnProperty('errors')); + assert.equal(layergroupResult.errors.length, 1); + assert.ok(layergroupResult.errors[0].match(/permission denied/), layergroupResult.errors[0]); + + testClient.drain(done); + }); + }); + + it('should forbide access to API if API key does not grant access', function (done) { + const apikeyToken = 'regular2'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + options: { + sql: 'select * FROM test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup({ response: { status: 403 } }, function (err, layergroupResult) { + assert.ifError(err); + + assert.ok(layergroupResult.hasOwnProperty('errors')); + assert.equal(layergroupResult.errors.length, 1); + assert.ok(layergroupResult.errors[0].match(/Forbidden/), layergroupResult.errors[0]); + + testClient.drain(done); + }); + }); + + it('should create a layergroup with a source analysis using a default apikey token', function (done) { + const apikeyToken = 'default_public'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + type: 'cartodb', + options: { + source: { + id: 'HEAD' + }, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ], + analyses: [ + { + id: 'HEAD', + type: 'source', + params: { + query: 'select * from populated_places_simple_reduced' + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup(function (err, layergroupResult) { + assert.ifError(err); + + assert.ok(layergroupResult.layergroupid); + + testClient.drain(done); + }); + }); + + it('should create a layergroup with a source analysis using a regular apikey token', function (done) { + const apikeyToken = 'regular1'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + type: 'cartodb', + options: { + source: { + id: 'HEAD' + }, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ], + analyses: [ + { + id: 'HEAD', + type: 'source', + params: { + query: 'select * from test_table_localhost_regular1' + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup(function (err, layergroupResult) { + assert.ifError(err); + + assert.ok(layergroupResult.layergroupid); + + testClient.drain(done); + }); + }); + + // Warning: TBA + it('should create a layergroup with a buffer analysis using a regular apikey token', function (done) { + const apikeyToken = 'regular1'; + const mapConfig = { + version: '1.7.0', + layers: [ + { + type: 'cartodb', + options: { + source: { + id: 'HEAD1' + }, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ], + analyses: [ + { + id: "HEAD1", + type: "buffer", + params: { + source: { + id: 'HEAD2', + type: 'source', + params: { + query: 'select * from test_table_localhost_regular1' + } + }, + radius: 50000 + } + } + ] + }; + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup(function (err, layergroupResult) { + assert.ifError(err); + + assert.ok(layergroupResult.layergroupid); + + testClient.drain(done); + }); + }); + + describe('Named maps', function () { + describe('LIST Named maps', function () { + + it('should fail while listing named maps with a regular apikey token', function (done) { + const apikeyToken = 'regular1'; + + const testClient = new TestClient({}, apikeyToken); + + testClient.getNamedMapList({ response: { status: 403 } }, function (err, res, body) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(body.errors.length, 1); + assert.ok(body.errors[0].match(/Forbidden/), body.errors[0]); + + testClient.drain(done); + }); + }); + + it('should fail while listing named maps with the default apikey token', function (done) { + const apikeyToken = 'default_public'; + + const testClient = new TestClient({}, apikeyToken); + + testClient.getNamedMapList({ response: { status: 403 } }, function (err, res, body) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(body.errors.length, 1); + assert.ok(body.errors[0].match(/Forbidden/), body.errors[0]); + + testClient.drain(done); + }); + }); + + it('should fail while listing named maps with non-existent apikey token', function (done) { + const apikeyToken = 'wadus-wadus'; + + const testClient = new TestClient({}, apikeyToken); + + testClient.getNamedMapList({ response: { status: 401 } }, function (err, res, body) { + assert.ifError(err); + + assert.equal(res.statusCode, 401); + + assert.equal(body.errors.length, 1); + assert.ok(body.errors[0].match(/Unauthorized/), body.errors[0]); + + testClient.drain(done); + }); + }); + + it('should list named maps with master apikey token', function (done) { + const apikeyToken = 1234; + + const testClient = new TestClient({}, apikeyToken); + + testClient.getNamedMapList({}, function (err, res, body) { + assert.ifError(err); + + assert.equal(res.statusCode, 200); + assert.ok(Array.isArray(body.template_ids)); + + testClient.drain(done); + }); + }); + }); + + describe('CREATE Named Map', function () { + const template = { + version: '0.0.1', + name: 'auth-api-template', + placeholders: { + buffersize: { + type: 'number', + default: 0 + } + }, + layergroup: { + version: '1.7.0', + layers: [{ + type: 'cartodb', + options: { + sql: 'select * from test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + } + }] + } + }; + + it('should create and get a named map tile using the master apikey token', function (done) { + const apikeyToken = 1234; + + const testClient = new TestClient(template, apikeyToken); + + testClient.getTile(0, 0, 0, function (err, res, tile) { + assert.ifError(err); + + assert.equal(res.statusCode, 200); + assert.ok(tile instanceof mapnik.Image); + + testClient.drain(done); + }); + }); + + it('should fail creating a named map using a regular apikey token', function (done) { + const apikeyToken = 'regular1'; + + const testClient = new TestClient(template, apikeyToken); + + testClient.createTemplate({ response: { status: 403 } }, function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClient.drain(done); + }); + }); + + it('should fail creating a named map using the default apikey token', function (done) { + const apikeyToken = 'default_public'; + + const testClient = new TestClient(template, apikeyToken); + + testClient.createTemplate({ response: { status: 403 } }, function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClient.drain(done); + }); + }); + + it('should fail creating a named map using a non-existent apikey token', function (done) { + const apikeyToken = 'wadus-wadus'; + + const testClient = new TestClient(template, apikeyToken); + + testClient.createTemplate({ response: { status: 401 } }, function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 401); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Unauthorized/), response.errors[0]); + + testClient.drain(done); + }); + }); + }); + + describe('DELETE Named Map', function () { + const templateBase = { + version: '0.0.1', + name: 'auth-api-template', + placeholders: { + buffersize: { + type: 'number', + default: 0 + } + }, + layergroup: { + version: '1.7.0', + layers: [{ + type: 'cartodb', + options: { + sql: 'select * from test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + } + }] + } + }; + + it('should delete a named map using the master apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenDelete = 1234; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-delete-master' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient(template, apikeyTokenDelete); + testClientDelete.deleteTemplate( + { + templateId: template.template_id, + response: { status: 204 } + }, + function (err, res) { + assert.ifError(err); + + assert.equal(res.statusCode, 204); + + testClientDelete.drain(done); + } + ); + }); + }); + + it('should fail deleting a named map using a regular apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenDelete = 'regular1'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-delete-regular' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient({}, apikeyTokenDelete); + testClientDelete.deleteTemplate( + { + templateId: template.template_id, + response: { status: 403 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClientDelete.drain(done); + } + ); + }); + }); + + it('should fail deleting a named map using the default apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenDelete = 'default_public'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-delete-default' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient(template, apikeyTokenDelete); + testClientDelete.deleteTemplate( + { + templateId: template.template_id, + response: { status: 403 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClientDelete.drain(done); + } + ); + }); + }); + + it('should fail deleting a named map using a non-existent apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenDelete = 'wadus'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-delete-wadus' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient(template, apikeyTokenDelete); + testClientDelete.deleteTemplate( + { + templateId: template.template_id, + response: { status: 401 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 401); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Unauthorized/), response.errors[0]); + + testClientDelete.drain(done); + } + ); + }); + }); + }); + + describe('GET Named Map', function () { + const templateBase = { + version: '0.0.1', + name: 'auth-api-template', + placeholders: { + buffersize: { + type: 'number', + default: 0 + } + }, + layergroup: { + version: '1.7.0', + layers: [{ + type: 'cartodb', + options: { + sql: 'select * from test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + } + }] + } + }; + + it('should get a named map using the master apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenGet = 1234; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-get-master' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient({}, apikeyTokenGet); + testClientDelete.getTemplate( + { + templateId: template.template_id, + response: { status: 200 } + }, + function (err, res) { + assert.ifError(err); + + assert.equal(res.statusCode, 200); + + testClientDelete.drain(done); + } + ); + }); + }); + + it('should fail getting a named map using a regular apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenGet = 'regular1'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-get-regular' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientGet = new TestClient({}, apikeyTokenGet); + testClientGet.getTemplate( + { + templateId: template.template_id, + response: { status: 403 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClientGet.drain(done); + } + ); + }); + }); + + it('should fail getting a named map using the default apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenGet = 'default_public'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-get-default' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientGet = new TestClient(template, apikeyTokenGet); + testClientGet.getTemplate( + { + templateId: template.template_id, + response: { status: 403 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClientGet.drain(done); + } + ); + }); + }); + + it('should fail getting a named map using a non-existent apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenGet = 'wadus'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-get-wadus' }); + + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientGet = new TestClient(template, apikeyTokenGet); + testClientGet.getTemplate( + { + templateId: template.template_id, + response: { status: 401 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 401); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Unauthorized/), response.errors[0]); + + testClientGet.drain(done); + } + ); + }); + }); + }); + + describe('UPDATE Named Map', function () { + const templateBase = { + version: '0.0.1', + name: 'auth-api-template', + placeholders: { + buffersize: { + type: 'number', + default: 0 + } + }, + layergroup: { + version: '1.7.0', + layers: [{ + type: 'cartodb', + options: { + sql: 'select * from test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + } + }] + } + }; + + it('should update a named map using the master apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenUpdate = 1234; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-update-master' }); + const templateUpdate = Object.assign({}, template, { zoom: 3 }); + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient({}, apikeyTokenUpdate); + testClientDelete.updateTemplate( + { + templateId: template.template_id, + templateData: templateUpdate, + response: { status: 200 } + }, + function (err, res) { + assert.ifError(err); + + assert.equal(res.statusCode, 200); + + testClientDelete.drain(done); + } + ); + }); + }); + + it('should fail updating a named map using a regular apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenUpdate = 'regular1'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-update-regular' }); + const templateUpdate = Object.assign({}, template, { zoom: 3 }); + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient({}, apikeyTokenUpdate); + testClientDelete.updateTemplate( + { + templateId: template.template_id, + templateData: templateUpdate, + response: { status: 403 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClientDelete.drain(done); + } + ); + }); + }); + + it('should fail updating a named map using the default apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenUpdate = 'default_public'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-update-default' }); + const templateUpdate = Object.assign({}, template, { zoom: 3 }); + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient({}, apikeyTokenUpdate); + testClientDelete.updateTemplate( + { + templateId: template.template_id, + templateData: templateUpdate, + response: { status: 403 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Forbidden/), response.errors[0]); + + testClientDelete.drain(done); + } + ); + }); + }); + + it('should fail updating a named map using a non-existent apikey token', function (done) { + const apikeyTokenCreate = 1234; + const apikeyTokenUpdate = 'wadus'; + + const template = Object.assign({}, templateBase, { name: templateBase.name + '-update-wadus' }); + const templateUpdate = Object.assign({}, template, { zoom: 3 }); + const testClientCreate = new TestClient(template, apikeyTokenCreate); + + testClientCreate.createTemplate({}, function (err, res, template) { + assert.ifError(err); + + const testClientDelete = new TestClient({}, apikeyTokenUpdate); + testClientDelete.updateTemplate( + { + templateId: template.template_id, + templateData: templateUpdate, + response: { status: 401 } + }, + function (err, res, response) { + assert.ifError(err); + + assert.equal(res.statusCode, 401); + + assert.equal(response.errors.length, 1); + assert.ok(response.errors[0].match(/Unauthorized/), response.errors[0]); + + testClientDelete.drain(done); + } + ); + }); + }); + + }); + }); +}); diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index ba8955e7..e3daf626 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var fs = require('fs'); @@ -54,6 +56,8 @@ function createMapConfig (bufferSize, cartocss) { } describe('buffer size per format', function () { + let testClient; + var testCases = [ { desc: 'should get png tile using buffer-size 0', @@ -126,36 +130,54 @@ describe('buffer size per format', function () { ]; afterEach(function(done) { - if (this.testClient) { - return this.testClient.drain(done); + if (testClient) { + return testClient.drain(done); } return done(); }); const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; - testCases.forEach(function (test) { - var testFn = (usePostGIS) => { - it(test.desc, function (done) { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - this.testClient = new TestClient(test.mapConfig, 1234); - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - var coords = test.coords; - var options = { - format: test.format, - layers: test.layers - }; - this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - // tile.save(test.fixturePath); - test.assert(tile, done); - }); + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + var testFn = (test) => { + it(test.desc, function (done) { + testClient = new TestClient(test.mapConfig, 1234); + var coords = test.coords; + var options = { + format: test.format, + layers: test.layers + }; + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + // tile.save(test.fixturePath); + test.assert(tile, done); }); - }; - if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ - testFn(true); - } - testFn(false); + }); + }; + + testCases.filter(test => test.format !== 'mvt').forEach(function (test) { + testFn(test); + }); + + describe('using mapnik mvt renderer', function() { + before(function () { + serverOptions.renderer.mvt.usePostGIS = false; + }); + testCases.filter(test => test.format === 'mvt').forEach(function (test) { + testFn(test); + }); + }); + + describe('using postgis mvt renderer', function() { + before(function () { + serverOptions.renderer.mvt.usePostGIS = true; + }); + testCases.filter(test => test.format === 'mvt').forEach(function (test) { + testFn(test); + }); }); }); @@ -302,6 +324,8 @@ describe('buffer size per format for named maps', function () { describe('buffer size per format for named maps w/o placeholders', function () { + let testClient; + var testCases = [ { desc: 'should get png tile using buffer-size 0 overriden by template params', @@ -435,39 +459,57 @@ describe('buffer size per format for named maps w/o placeholders', function () { ]; afterEach(function(done) { - if (this.testClient) { - return this.testClient.drain(done); + if (testClient) { + return testClient.drain(done); } return done(); }); const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; - testCases.forEach(function (test) { - var testFn = (usePostGIS) => { - it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - test.template.name += '_1'; - this.testClient = new TestClient(test.template, 1234); - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - var coords = test.coords; - var options = { - format: test.format, - placeholders: test.placeholders, - layers: test.layers - }; - this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - //tile.save(test.fixturePath); - // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); - // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); - test.assert(tile, done); - }); - }); - }; - if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ - testFn(true); - } - testFn(false); + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + var testFn = (test) => { + it(test.desc, function (done) { + test.template.name += '_1'; + testClient = new TestClient(test.template, 1234); + var coords = test.coords; + var options = { + format: test.format, + placeholders: test.placeholders, + layers: test.layers + }; + testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + //tile.save(test.fixturePath); + // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); + // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); + test.assert(tile, done); + }); + }); + }; + + testCases.filter(test => test.format !== 'mvt').forEach(function (test) { + testFn(test); + }); + + describe('using mapnik mvt renderer', function() { + before(function () { + serverOptions.renderer.mvt.usePostGIS = false; + }); + testCases.filter(test => test.format === 'mvt').forEach(function (test) { + testFn(test); + }); + }); + + describe('using postgis mvt renderer', function() { + before(function () { + serverOptions.renderer.mvt.usePostGIS = true; + }); + testCases.filter(test => test.format === 'mvt').forEach(function (test) { + testFn(test); + }); }); }); diff --git a/test/acceptance/cache/cache_headers.js b/test/acceptance/cache/cache_headers.js index e7d8caf3..46a22026 100644 --- a/test/acceptance/cache/cache_headers.js +++ b/test/acceptance/cache/cache_headers.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -5,12 +7,17 @@ var qs = require('querystring'); var CartodbWindshaft = require('../../../lib/cartodb/server'); var serverOptions = require('../../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); -server.setMaxListeners(0); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('get requests with cache headers', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + server.setMaxListeners(0); + }); + var keysToDelete; beforeEach(function() { @@ -155,6 +162,7 @@ describe('get requests with cache headers', function() { assert.ok(res.headers['x-cache-channel']); assert.ok(res.headers['surrogate-key']); + assert.equal(res.headers.vary, 'Authorization'); if (expectedCacheHeaders) { validateXChannelHeaders(res.headers, expectedCacheHeaders); assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys); diff --git a/test/acceptance/cache/surrogate_keys_invalidation.js b/test/acceptance/cache/surrogate_keys_invalidation.js index fbe2ff7d..aa22892b 100644 --- a/test/acceptance/cache/surrogate_keys_invalidation.js +++ b/test/acceptance/cache/surrogate_keys_invalidation.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -7,7 +9,7 @@ var _ = require('underscore'); var NamedMapsCacheEntry = require(__dirname + '/../../../lib/cartodb/cache/model/named_maps_entry'); var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/server'); - +var nock = require('nock'); describe('templates surrogate keys', function() { @@ -30,7 +32,13 @@ describe('templates surrogate keys', function() { serviceId: FAKE_FASTLY_SERVICE_ID }; - var server = new CartodbWindshaft(serverOptions); + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + nock.disableNetConnect(); + nock.enableNetConnect(/(127.0.0.1|cartocdn.com)/); + }); var templateOwner = 'localhost'; var templateName = 'acceptance'; @@ -70,9 +78,6 @@ describe('templates surrogate keys', function() { var invalidationMatchHeader = '\\b' + cacheEntryKey + '\\b'; var fastlyPurgePath = '/service/' + FAKE_FASTLY_SERVICE_ID + '/purge/' + cacheEntryKey; - var nock = require('nock'); - nock.enableNetConnect(/(127.0.0.1:5555|cartocdn.com)/); - after(function(done) { serverOptions.varnish_purge_enabled = false; serverOptions.varnish_host = varnishHost; @@ -80,7 +85,9 @@ describe('templates surrogate keys', function() { serverOptions.fastly = fastlyConfig; - nock.restore(); + nock.cleanAll(); + nock.enableNetConnect(); + done(); }); diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index e29d9ff4..c6df346b 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); @@ -70,12 +72,8 @@ describe('aggregations happy cases', function() { ].join(' UNION ALL '); operations.forEach(function (operation) { - var not = operation === 'count' ? ' not ' : ' '; - var description = 'should' + - not + - 'handle NULL values in category and aggregation columns using "' + - operation + - '" as aggregation operation'; + var description = 'should handle NULL values in category and aggregation columns using "' + + operation + '" as aggregation operation'; it(description, function (done) { this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val')); @@ -96,12 +94,7 @@ describe('aggregations happy cases', function() { } }); - if (operation === 'count') { - assert.ok(hasNullCategory, 'aggregation has not a category NULL'); - } else { - assert.ok(!hasNullCategory, 'aggregation has category NULL'); - } - + assert.ok(!hasNullCategory, 'aggregation has category NULL'); done(); }); }); @@ -297,7 +290,7 @@ describe('aggregation-dataview: special float values', function() { filters.forEach(function (filter) { it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { + this.testClient.getDataview('val_aggregation', filter, function(err, dataview) { assert.ifError(err); assert.ok(dataview.infinities === (250 + 250)); assert.ok(dataview.nans === 250); @@ -312,7 +305,7 @@ describe('aggregation-dataview: special float values', function() { it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) { this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) { + this.testClient.getDataview('sum_aggregation_numeric', filter, function(err, dataview) { assert.ifError(err); assert.ok(dataview.nans === 333); assert.ok(dataview.categories.length === 2); @@ -425,3 +418,79 @@ describe('aggregation dataview tuned by categories query param', function () { }); }); }); + + + +describe('Count aggregation', function () { + const mapConfig = { + version: '1.5.0', + layers: [ + { + type: "cartodb", + options: { + source: { + "id": "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + dataviews: { + categories: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'count' + } + } + }, + analyses: [ + { + id: "a0", + type: "source", + params: { + query: ` + SELECT + null::geometry the_geom_webmercator, + CASE + WHEN x % 4 = 0 THEN 1 + WHEN x % 4 = 1 THEN 2 + WHEN x % 4 = 2 THEN 3 + ELSE null + END AS val, + CASE + WHEN x % 4 = 0 THEN 'category_1' + WHEN x % 4 = 1 THEN 'category_2' + WHEN x % 4 = 2 THEN 'category_3' + ELSE null + END AS cat + FROM generate_series(1, 1000) x + ` + } + } + ] + }; + + it(`should handle null values correctly when aggregationColumn isn't provided`, function (done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => { + assert.ifError(err); + assert.equal(dataview.categories.length, 3); + this.testClient.drain(done); + }); + }); + + it(`should handle null values correctly when aggregationColumn is provided`, function (done) { + mapConfig.dataviews.categories.options.aggregationColumn = 'val'; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('categories', { own_filter: 0, categories: 0 }, (err, dataview) => { + assert.ifError(err); + assert.equal(dataview.categories.length, 3); + this.testClient.drain(done); + }); + }); +}); diff --git a/test/acceptance/dataviews/error-cases.js b/test/acceptance/dataviews/error-cases.js index 3e6cc2a8..835efa53 100644 --- a/test/acceptance/dataviews/error-cases.js +++ b/test/acceptance/dataviews/error-cases.js @@ -1,87 +1,215 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); -describe('histogram-dataview', function() { +describe('dataview error cases', function() { - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); + describe('generic errors', function() { - var ERROR_RESPONSE = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); - function createMapConfig(dataviews) { - return { - version: '1.5.0', - layers: [ - { - "type": "cartodb", - "options": { - "source": { - "id": "HEAD" - }, - "cartocss": "#points { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0" - } - } - ], - dataviews: dataviews, - analyses: [ - { - "id": "HEAD", - "type": "source", - "params": { - "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x" - } - } - ] + var ERROR_RESPONSE = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } }; - } - it('should fail when invalid dataviews object is provided, string case', function(done) { - var mapConfig = createMapConfig("wadus-string"); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { - assert.ok(!err, err); + function createMapConfig(dataviews) { + return { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + dataviews: dataviews, + analyses: [ + { + "id": "HEAD", + "type": "source", + "params": { + "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x" + } + } + ] + }; + } - assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]); + it('should fail when invalid dataviews object is provided, string case', function(done) { + var mapConfig = createMapConfig("wadus-string"); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { + assert.ok(!err, err); - done(); + assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]); + + done(); + }); + }); + + it('should fail when invalid dataviews object is provided, array case', function(done) { + var mapConfig = createMapConfig([]); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { + assert.ok(!err, err); + + assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]); + + done(); + }); + }); + + it('should work with empty but valid objects', function(done) { + var mapConfig = createMapConfig({}); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err, err); + + assert.ok(layergroup); + assert.ok(layergroup.layergroupid); + + done(); + }); }); }); - it('should fail when invalid dataviews object is provided, array case', function(done) { - var mapConfig = createMapConfig([]); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { - assert.ok(!err, err); + describe('pg_typeof', function() { - assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]); - - done(); + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } }); - }); - it('should work with empty but valid objects', function(done) { - var mapConfig = createMapConfig({}); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err, err); + function createMapConfig(query) { + query = query || 'select * from populated_places_simple_reduced'; - assert.ok(layergroup); - assert.ok(layergroup.layergroupid); + return { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: query, + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', + cartocss_version: '2.0.1', + source: { id: "a0" } + } + } + ], + analyses: [{ + id: "a0", + type: "source", + params: { + query: query + } + }], + dataviews: { + aggregation_count_dataview: { + type: "aggregation", + source: { id: "a0" }, + options: { + column: "adm0name", + aggregation: "count", + aggregationColumn: "adm0name" + } + } + } + }; + } - done(); + it('should work without filters', function(done) { + this.testClient = new TestClient(createMapConfig()); + this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err) { + assert.ifError(err); + done(); + }); }); + + it('should work with filters', function(done) { + var params = { + filters: { + dataviews: {aggregation_count_dataview: {accept: ['Canada']}} + } + }; + + this.testClient = new TestClient(createMapConfig()); + this.testClient.getDataview('aggregation_count_dataview', params, function(err) { + assert.ifError(err); + done(); + }); + }); + + it('should return an error if the column used by dataview does not exist', function(done) { + const query = 'select cartodb_id, the_geom, the_geom_webmercator from populated_places_simple_reduced'; + + const params = { + response: { + status: 404, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + const expectedResponseBody = { + errors:['column "adm0name" does not exist'], + errors_with_context:[{ + type:'unknown', + message:'column "adm0name" does not exist' + }] + }; + + this.testClient = new TestClient(createMapConfig(query)); + this.testClient.getDataview('aggregation_count_dataview', params, function(err, result) { + assert.ifError(err); + assert.deepEqual(result, expectedResponseBody); + done(); + }); + }); + + it('should not fail if query does not return rows', function(done) { + const query = 'select * from populated_places_simple_reduced limit 0'; + + const expectedResponseBody = { + aggregation: 'count', + count: 0, + nulls: 0, + nans: 0, + infinities: 0, + min: 0, + max: 0, + categoriesCount: 0, + categories: [], + type: 'aggregation' + }; + + this.testClient = new TestClient(createMapConfig(query)); + this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { + assert.ifError(err); + assert.deepEqual(result, expectedResponseBody); + done(); + }); + }); + }); }); diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js index 941dc07e..028b6368 100644 --- a/test/acceptance/dataviews/formula.js +++ b/test/acceptance/dataviews/formula.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); @@ -11,70 +13,125 @@ function createMapConfig(layers, dataviews, analysis) { }; } -describe('formula-dataview: special float values', function() { - - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); +function getMapConfig(operation, lastNumber) { + return createMapConfig([ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } } + ], + { + val_formula: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: 'val', + operation: operation + } + } + }, + [ + { + id: "a0", + type: "source", + params: { + query: ` + SELECT + null::geometry the_geom_webmercator, + CASE + WHEN x % 5 = 1 THEN 'infinity'::float + WHEN x % 5 = 2 THEN '-infinity'::float + WHEN x % 5 = 3 THEN 'NaN'::float + WHEN x % 5 = 4 THEN NULL + ELSE x + END AS val + FROM generate_series(1, ${lastNumber}) x + ` + } + } + ]); +} + +describe('formula-dataview: special float values', function() { + describe('easy numbers', function() { // not needed, but I keep it here to help human calc + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + const lastNumber = 10; + + [ + {operation: 'count', result: 2}, + {operation: 'avg', result: 7.5}, + {operation: 'sum', result: 15}, + {operation: 'min', result: 5}, + {operation: 'max', result: 10} + ] + .forEach(operationData => { + it(operationData.operation, function(done) { + this.testClient = new TestClient(getMapConfig(operationData.operation, lastNumber), 1234); + this.testClient.getDataview('val_formula', {}, function(err, dataview) { + assert.ok(!err, err); + assert.deepStrictEqual(dataview, { + type: 'formula', + operation: operationData.operation, + result: operationData.result, + nulls: lastNumber / 5, + nans: lastNumber / 5, + infinities: 2 * lastNumber / 5 + }); + done(); + }); + }); + }); }); - var mapConfig = createMapConfig( - [ - { - "type": "cartodb", - "options": { - "source": { - "id": "a0" - }, - "cartocss": "#points { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0" - } + describe('bigger numbers', function() { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); } - ], - { - val_formula: { - source: { - id: 'a0' - }, - type: 'formula', - options: { - column: 'val', - operation: 'avg' - } - } - }, - [ - { - "id": "a0", - "type": "source", - "params": { - "query": [ - 'SELECT', - ' null::geometry the_geom_webmercator,', - ' CASE', - ' WHEN x % 4 = 0 THEN \'infinity\'::float', - ' WHEN x % 4 = 1 THEN \'-infinity\'::float', - ' WHEN x % 4 = 2 THEN \'NaN\'::float', - ' ELSE x', - ' END AS val', - 'FROM generate_series(1, 1000) x' - ].join('\n') - } - } - ] - ); + }); - it('should filter infinities out and count them in the summary', function(done) { - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('val_formula', {}, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.result, 501); - assert.ok(dataview.infinities === (250 + 250)); - assert.ok(dataview.nans === 250); - done(); + const lastNumber = 1000; + + [ + {operation: 'count', result: 200}, + {operation: 'avg', result: 502.5}, + {operation: 'sum', result: 100500}, + {operation: 'min', result: 5}, + {operation: 'max', result: 1000} + ] + .forEach(operationData => { + it(operationData.operation, function(done) { + this.testClient = new TestClient(getMapConfig(operationData.operation, lastNumber), 1234); + this.testClient.getDataview('val_formula', {}, function(err, dataview) { + assert.ok(!err, err); + assert.deepStrictEqual(dataview, { + type: 'formula', + operation: operationData.operation, + result: operationData.result, + nulls: lastNumber / 5, + nans: lastNumber / 5, + infinities: 2 * lastNumber / 5 + }); + done(); + }); + }); }); }); }); diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 161fdef8..aa89d53e 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -67,6 +69,27 @@ describe('histogram-dataview', function() { ] ); + it('should get bins with min >= start and max <= end', function(done) { + var params = { + bins: 3, + start: 50, + end: 500 + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('pop_max_histogram', params, function(err, dataview) { + assert.ok(!err, err); + + assert.ok(3 === dataview.bins_count, 'Unexpected bin count: ' + dataview.bins_count); + assert.ok(3 === dataview.bins.length, 'Unexpected number of bins: ' + dataview.bins.length); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min >= params.start, 'bin min < start: ' + JSON.stringify(bin)); + assert.ok(bin.max <= params.end, 'bin max > end: ' + JSON.stringify(bin)); + }); + done(); + }); + }); + it('should get bin_width right when max > min in filter', function(done) { var params = { bins: 10, @@ -921,8 +944,8 @@ describe('histogram-dataview for date column type', function() { this.testClient.getDataview('datetime_histogram_tz', {}, function (err, dataview) { assert.ok(!err, err); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('datetime_histogram_tz', params, function (err, filteredDataview) { + const _testClient = new TestClient(mapConfig, 1234); + _testClient.getDataview('datetime_histogram_tz', params, function (err, filteredDataview) { assert.ok(!err, err); assert.deepEqual(dataview, filteredDataview); diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index 552b9ac6..7642dc73 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -48,11 +50,12 @@ describe('dataviews using tables without overviews', function() { it("should expose a formula", function(done) { var testClient = new TestClient(nonOverviewsMapConfig); - testClient.getDataview('country_places_count', { own_filter: 0 }, function(err, formula_result) { + testClient.getDataview('country_places_count', { own_filter: 0 }, function(err, formula_result, headers) { if (err) { return done(err); } assert.deepEqual(formula_result, { operation: 'count', result: 7313, nulls: 0, type: 'formula' }); + assert(getUsesOverviewsFromHeaders(headers) === false); //Overviews logging testClient.drain(done); }); @@ -257,7 +260,7 @@ describe('dataviews using tables with overviews', function() { it("should expose a sum formula", function(done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_sum', { own_filter: 0 }, function(err, formula_result) { + testClient.getDataview('test_sum', { own_filter: 0 }, function(err, formula_result, headers) { if (err) { return done(err); } @@ -269,6 +272,8 @@ describe('dataviews using tables with overviews', function() { "nulls":0, "type":"formula" }); + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging + assert(getDataviewTypeFromHeaders(headers) === 'formula'); //Overviews logging testClient.drain(done); }); @@ -276,7 +281,7 @@ describe('dataviews using tables with overviews', function() { it("should expose an avg formula", function(done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_avg', { own_filter: 0 }, function(err, formula_result) { + testClient.getDataview('test_avg', { own_filter: 0 }, function(err, formula_result, headers) { if (err) { return done(err); } @@ -288,6 +293,8 @@ describe('dataviews using tables with overviews', function() { "infinities": 0, "nans": 0 }); + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging + assert(getDataviewTypeFromHeaders(headers) === 'formula'); //Overviews logging testClient.drain(done); }); @@ -295,7 +302,7 @@ describe('dataviews using tables with overviews', function() { it("should expose a count formula", function(done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_count', { own_filter: 0 }, function(err, formula_result) { + testClient.getDataview('test_count', { own_filter: 0 }, function(err, formula_result, headers) { if (err) { return done(err); } @@ -307,6 +314,8 @@ describe('dataviews using tables with overviews', function() { "infinities": 0, "nans": 0 }); + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging + assert(getDataviewTypeFromHeaders(headers) === 'formula'); //Overviews logging testClient.drain(done); }); @@ -374,13 +383,16 @@ describe('dataviews using tables with overviews', function() { it("should expose a histogram", function (done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_histogram', function (err, histogram) { + testClient.getDataview('test_histogram', function (err, histogram, headers) { if (err) { return done(err); } assert.ok(histogram); assert.equal(histogram.type, 'histogram'); assert.ok(Array.isArray(histogram.bins)); + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging + assert(getDataviewTypeFromHeaders(headers) === 'histogram'); //Overviews logging + testClient.drain(done); }); }); @@ -462,7 +474,7 @@ describe('dataviews using tables with overviews', function() { it("should expose a filtered sum formula", function (done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_sum', params, function (err, formula_result) { + testClient.getDataview('test_sum', params, function (err, formula_result, headers) { if (err) { return done(err); } @@ -474,13 +486,14 @@ describe('dataviews using tables with overviews', function() { "nans": 0, "type":"formula" }); + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging testClient.drain(done); }); }); it("should expose a filtered avg formula", function(done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_avg', params, function(err, formula_result) { + testClient.getDataview('test_avg', params, function(err, formula_result, headers) { if (err) { return done(err); } @@ -492,6 +505,7 @@ describe('dataviews using tables with overviews', function() { "nans": 0, "type":"formula" }); + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging testClient.drain(done); }); @@ -499,7 +513,7 @@ describe('dataviews using tables with overviews', function() { it("should expose a filtered count formula", function(done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_count', params, function(err, formula_result) { + testClient.getDataview('test_count', params, function(err, formula_result, headers) { if (err) { return done(err); } @@ -511,6 +525,7 @@ describe('dataviews using tables with overviews', function() { "nulls":0, "type":"formula" }); + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging testClient.drain(done); }); @@ -586,10 +601,11 @@ describe('dataviews using tables with overviews', function() { it("should expose an aggregation dataview filtering special float values out", function (done) { var testClient = new TestClient(overviewsMapConfig); - testClient.getDataview('test_categories_special_values', params, function (err, dataview) { + testClient.getDataview('test_categories_special_values', params, function (err, dataview, headers) { if (err) { return done(err); } + assert.deepEqual(dataview, { aggregation: 'sum', count: 5, @@ -602,6 +618,10 @@ describe('dataviews using tables with overviews', function() { categories: [ { category: 'Hawai', value: 6, agg: false } ], type: 'aggregation' }); + + assert.ok(getUsesOverviewsFromHeaders(headers)); //Overviews logging + assert(getDataviewTypeFromHeaders(headers) === 'aggregation'); //Overviews logging + testClient.drain(done); }); }); @@ -647,3 +667,11 @@ describe('dataviews using tables with overviews', function() { }); }); }); + +function getUsesOverviewsFromHeaders(headers) { + return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).usesOverviews; +} + +function getDataviewTypeFromHeaders(headers) { + return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).dataviewType; +} diff --git a/test/acceptance/date-wrapping.spec.js b/test/acceptance/date-wrapping.spec.js new file mode 100644 index 00000000..f26caf1e --- /dev/null +++ b/test/acceptance/date-wrapping.spec.js @@ -0,0 +1,319 @@ +'use strict'; + +/* eslint-env mocha */ +const assert = require('assert'); +const TestClient = require('../support/test-client'); +const mapConfigFactory = require('../fixtures/test_mapconfigFactory'); +const serverOptions = require('../../lib/cartodb/server_options'); + +const usePgMvtRenderer = serverOptions.renderer.mvt.usePostGIS; +const describe_mvt = !usePgMvtRenderer ? describe : describe.skip; + +describe_mvt('date-wrapping', () => { + let testClient; + + describe('when a map instantiation has one single layer', () => { + describe('and the layer has the "dates_as_numbers" option enabled', () => { + beforeEach(() => { + const mapConfig = mapConfigFactory.getVectorMapConfig({ layerOptions: [{ dates_as_numbers: true }]}); + testClient = new TestClient(mapConfig); + }); + + afterEach(done => testClient.drain(done)); + + it('should return date columns casted as numbers', done => { + + testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, mvt) => { + const expected = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0, date: 1527810000 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1, date: 1527900000 } + } + ]; + const actual = JSON.parse(mvt.toGeoJSONSync(0)).features; + + assert.deepEqual(actual, expected); + done(); + }); + }); + + it('should return metadata with casted columns', done => { + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.deepEqual(layergroup.metadata.layers[0].meta.dates_as_numbers, ['date']); + done(); + }); + + }); + + }); + + describe('and the layer has the "dates_as_numbers" option disabled', () => { + beforeEach(() => { + const mapConfig = mapConfigFactory.getVectorMapConfig({ layerOptions: [{ dates_as_numbers: false }]}); + testClient = new TestClient(mapConfig); + }); + + afterEach(done => testClient.drain(done)); + + it('should return date columns as dates', done => { + + testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, mvt) => { + const expected = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1 } + } + ]; + const actual = JSON.parse(mvt.toGeoJSONSync(0)).features; + + assert.deepEqual(actual, expected); + done(); + }); + }); + }); + }); + + describe('when a map instantiation has multiple layers', () => { + afterEach(done => testClient.drain(done)); + + describe('and both layers have the "dates_as_numbers" option enabled', () => { + beforeEach(() => { + const mapConfig = mapConfigFactory.getVectorMapConfig({ + numberOfLayers: 2, + layerOptions: [ + { dates_as_numbers: true }, + { dates_as_numbers: true } + ] + }); + testClient = new TestClient(mapConfig); + }); + + it('should return dates as numbers for every layer', done => { + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.deepEqual(layergroup.metadata.layers[0].meta.dates_as_numbers, ['date']); + assert.deepEqual(layergroup.metadata.layers[1].meta.dates_as_numbers, ['date']); + }); + + testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, mvt) => { + const expected0 = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0, date: 1527810000 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1, date: 1527900000 } + } + ]; + const expected1 = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0, date: 1527810000 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1, date: 1527900000 } + } + ]; + const actual0 = JSON.parse(mvt.toGeoJSONSync(0)).features; + const actual1 = JSON.parse(mvt.toGeoJSONSync(1)).features; + + assert.deepEqual(actual0, expected0); + assert.deepEqual(actual1, expected1); + done(); + }); + }); + }); + + describe('and only one layers has the "dates_as_numbers" option enabled', () => { + beforeEach(() => { + const mapConfig = mapConfigFactory.getVectorMapConfig({ + numberOfLayers: 2, + layerOptions: [ + { dates_as_numbers: false }, + { dates_as_numbers: true } + ] + }); + testClient = new TestClient(mapConfig); + }); + + it('should return dates as numbers only for the layer with the "dates_as_numbers" flag enabled', done => { + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.deepEqual(layergroup.metadata.layers[0].meta.dates_as_numbers || [], []); + assert.deepEqual(layergroup.metadata.layers[1].meta.dates_as_numbers, ['date']); + }); + + testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, mvt) => { + const expected0 = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1 } + } + ]; + const expected1 = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0, date: 1527810000 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1, date: 1527900000 } + } + ]; + const actual0 = JSON.parse(mvt.toGeoJSONSync(0)).features; + const actual1 = JSON.parse(mvt.toGeoJSONSync(1)).features; + + assert.deepEqual(actual0, expected0); + assert.deepEqual(actual1, expected1); + done(); + }); + }); + }); + + describe('and none of the layers has the "dates_as_numbers" option enabled', () => { + beforeEach(() => { + const mapConfig = mapConfigFactory.getVectorMapConfig({ + numberOfLayers: 2, + layerOptions: [ + { dates_as_numbers: false }, + { dates_as_numbers: false } + ] + }); + testClient = new TestClient(mapConfig); + }); + + it('should return dates as dates for both layers', done => { + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.deepEqual(layergroup.metadata.layers[0].meta.dates_as_numbers || [], []); + assert.deepEqual(layergroup.metadata.layers[1].meta.dates_as_numbers || [], []); + }); + + testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, mvt) => { + const expected0 = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1 } + } + ]; + const expected1 = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 0 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { _cdb_feature_count: 1, cartodb_id: 1 } + } + ]; + const actual0 = JSON.parse(mvt.toGeoJSONSync(0)).features; + const actual1 = JSON.parse(mvt.toGeoJSONSync(1)).features; + + assert.deepEqual(actual0, expected0); + assert.deepEqual(actual1, expected1); + done(); + }); + }); + }); + }); + + describe('when sql queries use mapnik tokens', () => { + beforeEach(() => { + const mapConfig = mapConfigFactory.getVectorMapConfig({ + layerOptions: [{ + dates_as_numbers: true, + additionalColumns: [ + '!scale_denominator! AS sc' + ] + }] + }); + testClient = new TestClient(mapConfig); + }); + + afterEach(done => testClient.drain(done)); + + it('should work', done => { + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.deepEqual(layergroup.metadata.layers[0].meta.dates_as_numbers, ['date']); + done(); + }); + + }); + + it('should return correct tiles', done => { + testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, mvt) => { + const expected = [ + { + type: 'Feature', + id: 0, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { cartodb_id: 0, date: 1527810000, sc: 559082264.0287178839788058162356 } + }, + { + type: 'Feature', + id: 1, + geometry: { type: 'Point', coordinates: [0, 0] }, + properties: { cartodb_id: 1, date: 1527900000, sc: 559082264.0287178839788058162356 } + } + ]; + const actual = JSON.parse(mvt.toGeoJSONSync(0)).features; + assert.deepEqual(actual, expected); + done(); + }); + }); + }); + +}); diff --git a/test/acceptance/dynamic-styling-named-maps.js b/test/acceptance/dynamic-styling-named-maps.js index 87796c5e..994e347f 100644 --- a/test/acceptance/dynamic-styling-named-maps.js +++ b/test/acceptance/dynamic-styling-named-maps.js @@ -1,12 +1,18 @@ +'use strict'; + var assert = require('../support/assert'); var step = require('step'); var LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); var testHelper = require(__dirname + '/../support/test_helper'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); describe('dynamic styling for named maps', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); var keysToDelete; diff --git a/test/acceptance/error-middleware.js b/test/acceptance/error-middleware.js index 4df5b7e6..88211e09 100644 --- a/test/acceptance/error-middleware.js +++ b/test/acceptance/error-middleware.js @@ -1,3 +1,5 @@ +'use strict'; + const assert = require('../support/assert'); const TestClient = require('../support/test-client'); diff --git a/test/acceptance/errors-with-context.js b/test/acceptance/errors-with-context.js index e6e88f8b..1adc877a 100644 --- a/test/acceptance/errors-with-context.js +++ b/test/acceptance/errors-with-context.js @@ -1,9 +1,17 @@ +'use strict'; + var assert = require('../support/assert'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); describe('error with context', function () { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + + var layerOK = { options: { sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator ' + diff --git a/test/acceptance/health_check.js b/test/acceptance/health_check.js index 576a6e6f..112cc88d 100644 --- a/test/acceptance/health_check.js +++ b/test/acceptance/health_check.js @@ -1,3 +1,5 @@ +'use strict'; + require(__dirname + '/../support/test_helper'); var fs = require('fs'); @@ -96,7 +98,7 @@ describe('health checks', function () { }); }); - it('not err if disabled file does not exists', function(done) { + it('not err if disabled file does not exist', function(done) { global.environment.disabled_file = '/tmp/ftreftrgtrccre'; var server = new CartodbWindshaft(serverOptions); diff --git a/test/acceptance/label-wrap.js b/test/acceptance/label-wrap.js index d4482b01..914b2842 100644 --- a/test/acceptance/label-wrap.js +++ b/test/acceptance/label-wrap.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var TestClient = require('../support/test-client'); @@ -7,10 +9,10 @@ var IMAGE_TOLERANCE = 5; describe('CartoCSS wrap', function () { const options = { sql: ` - SELECT - 5 as cartodb_id, - ST_Transform(ST_SetSRID(ST_MakePoint(-57.65625,-15.6230368),4326),3857) as the_geom_webmercator, - ST_SetSRID(ST_MakePoint(-57.65625,-15.62303683),4326) as the_geom, + SELECT + 5 as cartodb_id, + ST_Transform(ST_SetSRID(ST_MakePoint(-57.65625,-15.6230368),4326),3857) as the_geom_webmercator, + ST_SetSRID(ST_MakePoint(-57.65625,-15.62303683),4326) as the_geom, 'South America' as continent `, cartocss: ` diff --git a/test/acceptance/layergroup-metadata.js b/test/acceptance/layergroup-metadata.js new file mode 100644 index 00000000..50e30895 --- /dev/null +++ b/test/acceptance/layergroup-metadata.js @@ -0,0 +1,59 @@ +'use strict'; + +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); + +describe('layergroup metadata', function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + before(function () { + serverOptions.renderer.mvt.usePostGIS = true; + }); + + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + [1234, 'default_public', false].forEach(api_key => { + it(`tiles base urls ${api_key ? `with api key: ${api_key}` : 'without api key'}`, function (done) { + const mapConfig = { + version: '1.7.0', + layers: [ + { + type: 'cartodb', + options: { + sql: 'select * from populated_places_simple_reduced', + } + } + ] + }; + + const host = `https://localhost.localhost.lan:${global.environment.port}`; + + const testClient = new TestClient(mapConfig, api_key); + testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + let urlLayer = `${host}/api/v1/map/${body.layergroupid}/layer0/{z}/{x}/{y}.mvt`; + let urlNoLayer = `${host}/api/v1/map/${body.layergroupid}/{z}/{x}/{y}.mvt`; + + if (api_key) { + urlLayer += `?api_key=${api_key}`; + urlNoLayer += `?api_key=${api_key}`; + } + + assert.ok(body.layergroupid); + assert.equal(body.metadata.layers[0].tilejson.vector.tiles[0], urlLayer); + assert.equal(body.metadata.tilejson.vector.tiles[0], urlNoLayer); + assert.equal(body.metadata.url.vector.urlTemplate, urlNoLayer); + + testClient.drain(done); + }); + }); + }); +}); diff --git a/test/acceptance/layers-filters.js b/test/acceptance/layers-filters.js index fa42c900..2767300f 100644 --- a/test/acceptance/layers-filters.js +++ b/test/acceptance/layers-filters.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var TestClient = require('../support/test-client'); @@ -51,4 +53,4 @@ describe('layers filters', function() { }); }); -}); \ No newline at end of file +}); diff --git a/test/acceptance/max-waiting-workers.js b/test/acceptance/max-waiting-workers.js new file mode 100644 index 00000000..adf259de --- /dev/null +++ b/test/acceptance/max-waiting-workers.js @@ -0,0 +1,128 @@ +'use strict'; + +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); +const createMapConfig = ({ + version = '1.8.0', + type = 'cartodb', + sql = TestClient.SQL.ONE_POINT, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ] +}); +const coords = [ + [0, 0, 0], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + [1, 1, 1], + [2, 0, 0], + [2, 0, 1], + [2, 0, 2], + [2, 0, 3], + [2, 1, 0], + [2, 1, 1], + [2, 1, 2], + [2, 1, 3], + [2, 2, 0], + [2, 2, 1], + [2, 2, 2], + [2, 2, 3], + [2, 3, 0], + [2, 3, 1], + [2, 3, 2], + [2, 3, 3] +]; + +function getTiles ({ testClient, layergroupid, coords }) { + return Promise.all(coords.map((coord) => getTile({ testClient, layergroupid, coord }))); +} + +function getTile ({ testClient, layergroupid, coord }) { + return new Promise((resolve, reject) => { + const [ z, x, y ] = coord; + const params = { + layergroupid, + format: 'png', + response: { + status: [ 200, 429 ], + headers: { + 'Content-Type': /^(image\/png|application\/json; charset=utf-8)$/ + } + } + }; + + testClient.getTile(z, x, y, params, (err, res, tile) => { + if (err) { + return reject(err); + } + + return resolve({ res, tile }); + }); + }); +} + +describe('exceeding max waiting workers', function () { + const originalPoolSize = global.environment.renderer.mapnik.poolSize; + const poolMaxWaitingClients = global.environment.renderer.mapnik.poolMaxWaitingClients; + const apikey = 1234; + const testClient = new TestClient(createMapConfig(), apikey); + let layergroupid; + + before(function (done) { + global.environment.renderer.mapnik.poolSize = 1; + global.environment.renderer.mapnik.poolMaxWaitingClients = 1; + + testClient.getLayergroup({ status: 200 }, (err, res) => { + if (err) { + return done(err); + } + + layergroupid = res.layergroupid; + done(); + }); + }); + + after(function () { + global.environment.renderer.mapnik.poolSize = originalPoolSize; + global.environment.renderer.mapnik.poolMaxWaitingClients = poolMaxWaitingClients; + }); + + it('should get 429: You are over platform\'s limits', function (done) { + const testClient = new TestClient(createMapConfig(), apikey); + + getTiles({ testClient, layergroupid, coords }) + .then((results) => { + const errs = results + .filter(({ res }) => res.headers['content-type'] === 'application/json; charset=utf-8') + .filter(({ tile }) => tile.errors && tile.errors_with_context[0].subtype === 'render-capacity'); + + assert.ok(errs.length > 0); + testClient.drain(done); + }); + }); +}); diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index a1a93e40..1d9b2422 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require('../support/assert'); var _ = require('underscore'); var redis = require('redis'); @@ -18,8 +20,6 @@ var IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL = 25; var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); -server.setMaxListeners(0); var QueryTables = require('cartodb-query-tables'); @@ -27,7 +27,12 @@ var QueryTables = require('cartodb-query-tables'); var suiteName = 'multilayer:postgres=layergroup_url=' + layergroup_url; describe(suiteName, function() { + var server; + before(function () { + server = new CartodbWindshaft(serverOptions); + server.setMaxListeners(0); + }); var keysToDelete; @@ -1272,6 +1277,8 @@ describe(suiteName, function() { it("cache control for layergroup default value", function(done) { global.environment.varnish.layergroupTtl = null; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=86400,must-revalidate'); @@ -1287,6 +1294,8 @@ describe(suiteName, function() { var layergroupTtl = 300; global.environment.varnish.layergroupTtl = layergroupTtl; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=' + layergroupTtl + ',must-revalidate'); @@ -1336,7 +1345,7 @@ describe(suiteName, function() { status: 403 }, function(res) { - assert.ok(res.body.match(/permission denied for relation test_table_private_1/)); + assert.ok(res.body.match(/permission denied for .+?test_table_private_1/)); done(); } ); diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index b599cf9c..f13c8f0a 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../support/test_helper'); var assert = require('../support/assert'); @@ -10,10 +12,14 @@ var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); var QueryTables = require('cartodb-query-tables'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); -server.setMaxListeners(0); describe('tests from old api translated to multilayer', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + server.setMaxListeners(0); + }); var layergroupUrl = '/api/v1/map'; @@ -230,7 +236,7 @@ describe('tests from old api translated to multilayer', function() { var parsed = JSON.parse(res.body); assert.ok(parsed.errors); assert.equal(parsed.errors.length, 1); - assert.ok(parsed.errors[0].match(/^Unexpected token W/)); + assert.ok(parsed.errors[0].match(/Unexpected token W/)); done(); } @@ -369,6 +375,7 @@ describe('tests from old api translated to multilayer', function() { }; // reset internal cacheChannel cache + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); assert.response(server, diff --git a/test/acceptance/mvt-regressions.js b/test/acceptance/mvt-regressions.js new file mode 100644 index 00000000..c660e23d --- /dev/null +++ b/test/acceptance/mvt-regressions.js @@ -0,0 +1,180 @@ +'use strict'; + +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); + +const suites = [ + { + desc: 'mapnik', + usePostGIS: false + }, + { + desc: 'postgis', + usePostGIS: true + } +]; + +describe('mvt regressions', function () { + + suites.forEach((suite) => { + const { desc, usePostGIS } = suite; + + describe(desc, function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('should not error on missing column from aggregation', function (done) { + const mapConfig = { + version: '1.7.0', + layers: [ + { + type: 'cartodb', + options: { + sql: 'select * from populated_places_simple_reduced', + aggregation: { + placement: 'point-sample', + columns: { + pop_max_avg: { + aggregate_function: 'avg', + aggregated_column: 'pop_max' + } + }, + threshold: 1 + } + } + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + + done(); + }); + }); + }); + }); +}); + +describe('MVT Mapnik', function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + before(function () { + serverOptions.renderer.mvt.usePostGIS = false; + }); + + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('invalid properties', function (done) { + const query = ` + select ldc, cartodb_id, _2016_6_partcntry, the_geom_webmercator, country + from countries_null_values + where country = 'Kenya' OR country = 'Sudan' LIMIT 3 + `; + + const mapConfig = { + version: '1.7.0', + layers: [ + { + type: 'cartodb', + options: { + sql: query + } + } + ] + }; + + const handler = (resolve, reject) => { + return (err, res, MVT) => { + if (err) { + return reject(err); + } + try { + const geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + const sudanFeature = geojsonTile.features.filter(_ => { + return _.properties.country === 'Sudan'; + })[0]; + if (!sudanFeature) { + return reject(new Error(`Missing country='Sudan'`)); + } + + resolve(sudanFeature); + } catch (err) { + resolve(err); + } + }; + }; + + this.testClient = new TestClient(mapConfig); + const tile487 = new Promise((resolve, reject) => { + this.testClient.getTile(4, 8, 7, { format: 'mvt', layer: 0 }, handler(resolve, reject)); + }); + const tile497 = new Promise((resolve, reject) => { + this.testClient.getTile(4, 9, 7, { format: 'mvt', layer: 0 }, handler(resolve, reject)); + }); + Promise.all([tile487, tile497]) + .then(features => { + const [tile487SudanFeature, tile497SudanFeature] = features; + assert.strictEqual(tile487SudanFeature.properties._2016_6_partcntry, 0); + assert.strictEqual(tile497SudanFeature.properties._2016_6_partcntry, 0); + return done(); + }) + .catch(err => done(err)); + }); + + it('should not convert boolean to numeric', function (done) { + const geomWebmercator = 'st_transform(st_setsrid(st_makepoint(10, 10), 4326), 3857) as the_geom_webmercator'; + const sql = `SELECT ${geomWebmercator}, FALSE as status, 0 as data`; + + const mapConfig = { + version: '1.7.0', + layers: [ + { + type: 'cartodb', + options: { sql } + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, { format: 'mvt', layer: 0 }, (err, res, MVT) => { + if (err) { + return done(err); + } + + const geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.strictEqual(geojsonTile.features[0].properties.status, false); + assert.strictEqual(geojsonTile.features[0].properties.data, 0); + done(); + }); + }); + +}); diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index fa9189f2..bb446680 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('../support/assert'); @@ -20,488 +22,271 @@ function createMapConfig(sql = TestClient.SQL.ONE_POINT) { } describe('mvt (mapnik)', mvt(false)); -if (process.env.POSTGIS_VERSION === '2.4') { - describe('mvt (postgis)', mvt(true)); -} +describe('mvt (postgis)', mvt(true)); function mvt(usePostGIS) { -return function () { - const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; - before(function () { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - }); - after(function (){ - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - }); - - describe('analysis-layers-dataviews-mvt', function () { - - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - - var CARTOCSS = [ - "#points {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 1.0;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: red;", - " marker-allow-overlap: true;", - "}" - ].join('\n'); - - var mapConfig = createMapConfig( - [ - { - "type": "cartodb", - "options": { - "source": { - "id": "2570e105-7b37-40d2-bdf4-1af889598745" - }, - "cartocss": CARTOCSS, - "cartocss_version": "2.3.0" - } - } - ], - { - pop_max_histogram: { - source: { - id: '2570e105-7b37-40d2-bdf4-1af889598745' - }, - type: 'histogram', - options: { - column: 'pop_max' - } - } - }, - [ - { - "id": "2570e105-7b37-40d2-bdf4-1af889598745", - "type": "source", - "params": { - "query": "select * from populated_places_simple_reduced" - } - } - ] - ); - - it('should get pop_max column from dataview', function (done) { - var testClient = new TestClient(mapConfig); - - testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) { - var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); - assert.ok(!err, err); - - assert.ok(Array.isArray(geojsonTile.features)); - assert.ok(geojsonTile.features.length > 0); - var feature = geojsonTile.features[0]; - assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); - - testClient.drain(done); - }); + return function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; }); - }); - - - const testCases = [ - { - desc: 'should get empty mvt with code 204 (no content)', - coords: { z: 0, x: 0, y: 0 }, - format: 'mvt', - response: { - status: 204, - headers: { - 'Content-Type': undefined - } - }, - mapConfig: createMapConfig(TestClient.SQL.EMPTY) - }, - { - desc: 'should get mvt tile with code 200 (ok)', - coords: { z: 0, x: 0, y: 0 }, - format: 'mvt', - response: { - status: 200, - headers: { - 'Content-Type': 'application/x-protobuf' - } - }, - mapConfig: createMapConfig() - } - ]; - - testCases.forEach(function (test) { - it(test.desc, done => { - var testClient = new TestClient(test.mapConfig); - const { z, x, y } = test.coords; - const { format, response } = test; - - testClient.getTile(z, x, y, { format, response }, err => { - assert.ifError(err); - testClient.drain(done); - }); - }); - }); - - - if (usePostGIS){ - describe('use only needed columns', onlyNeededColumns); - }else{ - describe.skip('use only needed columns', onlyNeededColumns); - } - - function onlyNeededColumns() { - function getFeatureByCartodbId(features, cartodbId) { - for (var i = 0, len = features.length; i < len; i++) { - if (features[i].properties.cartodb_id === cartodbId) { - return features[i]; - } - } - return {}; - } - - var options = { format: 'mvt', layer: 0 }; - - afterEach(function (done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - - it('with aggregation widget, interactivity and cartocss columns', function (done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: - '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', - cartocss_version: '2.0.1', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_min" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { - var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373, - pop_min: 57586 - }); - done(); - }); - }); - - it('should not duplicate columns', function (done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: ['#layer0 {', - 'marker-fill: red;', - 'marker-width: 10;', - '[name="Madrid"] { marker-fill: green; } ', - '[pop_max>100000] { marker-fill: black; } ', - '}'].join('\n'), - cartocss_version: '2.3.0', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_max" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { - var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373 - }); - done(); - }); - }); - - it('with formula widget, no interactivity and no cartocss columns', function (done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id', - widgets: { - pop_max_f: { - type: 'formula', - options: { - column: 'pop_max', - operation: 'count' - } - } + describe('named map tile', function () { + it('should get default named vector tile', function (done) { + const apikeyToken = 1234; + const templateName = `mvt-template-${usePostGIS ? 'postgis' : 'mapnik'}`; + const template = { + version: '0.0.1', + name: templateName, + placeholders: { + buffersize: { + type: 'number', + default: 0 } + }, + layergroup: { + version: '1.7.0', + layers: [{ + type: 'cartodb', + options: { + sql: 'select * from populated_places_simple_reduced limit 10', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0', + } + }] } - }] - }; - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { - var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max: 71373 - }); - done(); - }); - }); - it('with cartocss with multiple expressions', function (done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' + - '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id' - } - }] - }; + }; - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { - var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max: 71373, - name: "Mardin", - adm0name: "Turkey" + const testClient = new TestClient(template, apikeyToken); + testClient.keysToDelete['map_tpl|localhost'] = 0; + + testClient.getNamedTile(templateName, 0, 0, 0, 'mvt', {}, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + assert.equal(tileJSON[0].features.length, 10); + + testClient.drain(done); }); - done(); }); }); - var skipOnPostGIS = usePostGIS ? it.skip: it; - skipOnPostGIS('should work with mapnik substitution tokens', function (done) { - var cartocss = [ - "#layer {", - " line-width: 2;", - " line-color: #3B3B58;", - " line-opacity: 1;", - " polygon-opacity: 0.7;", - " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", + describe('analysis-layers-dataviews-mvt', function () { + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var CARTOCSS = [ + "#points {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 1.0;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: red;", + " marker-allow-overlap: true;", "}" ].join('\n'); - var sql = [ - 'WITH hgrid AS (', - ' SELECT CDB_HexagonGrid(', - ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', - ' greatest(!pixel_width!,!pixel_height!) * 100', - ' ) as cell', - ')', - 'SELECT', - ' hgrid.cell as the_geom_webmercator,', - ' count(1) as points_count,', - ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', - ' 1 as cartodb_id', - 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', - 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', - 'GROUP BY hgrid.cell' - ].join('\n'); - - var mapConfig = { - "version": "1.4.0", - "layers": [ + var mapConfig = createMapConfig( + [ { - "type": 'mapnik', + "type": "cartodb", "options": { - "cartocss_version": '2.3.0', - "sql": sql, - "cartocss": cartocss + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": CARTOCSS, + "cartocss_version": "2.3.0" + } + } + ], + { + pop_max_histogram: { + source: { + id: '2570e105-7b37-40d2-bdf4-1af889598745' + }, + type: 'histogram', + options: { + column: 'pop_max' + } + } + }, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" } } ] - }; + ); - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { - var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); - assert.ok(!err, err); + it('should get pop_max column from dataview', function (done) { + var testClient = new TestClient(mapConfig); - assert.ok(geojsonTile); - assert.equal(geojsonTile.features.length, 5); + testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); - done(); + assert.ok(Array.isArray(geojsonTile.features)); + assert.ok(geojsonTile.features.length > 0); + var feature = geojsonTile.features[0]; + assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); + + testClient.drain(done); + }); }); + }); - it('should skip empty and null columns for geojson tiles', function (done) { - var mapConfig = { - "analyses": [ - { - "id": "a0", - "params": { - "query": "SELECT * FROM test_table" - }, - "type": "source" - } - ], - "dataviews": { - "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "74f590f8-625c-4e95-922f-34ad3e9919c0": { - "options": { - "aggregation": "sum", - "aggregationColumn": "cartodb_id", - "column": "name" - }, - "source": { - "id": "a0" - }, - "type": "aggregation" - }, - "98a75757-3006-400a-b028-fb613a6c0b69": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "sum" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "ebbc97b2-87d2-4895-9e1f-2f012df3679d": { - "options": { - "aggregationColumn": null, - "bins": "12", - "column": "cartodb_id" - }, - "source": { - "id": "a0" - }, - "type": "histogram" - }, - "ebc0653f-3581-469c-8b31-c969e440a865": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" + const testCases = [ + { + desc: 'should get empty mvt with code 204 (no content)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + response: { + status: 204, + headers: { + 'Content-Type': undefined } }, - "layers": [ - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png" - }, - "type": "http" - }, - { - "options": { - "attributes": { - "columns": [ - "name", - "address" - ], - "id": "cartodb_id" - }, - "cartocss": "#layer { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0", - "interactivity": "cartodb_id", - "layer_name": "wadus", - "source": { - "id": "a0" - } - }, - "type": "cartodb" - }, - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png" - }, - "type": "http" + mapConfig: createMapConfig(TestClient.SQL.EMPTY) + }, + { + desc: 'should get mvt tile with code 200 (ok)', + coords: { z: 0, x: 0, y: 0 }, + format: 'mvt', + response: { + status: 200, + headers: { + 'Content-Type': 'application/x-protobuf' } - ] - }; + }, + mapConfig: createMapConfig() + } + ]; - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { - var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); - assert.ok(!err, err); + testCases.forEach(function (test) { + it(test.desc, done => { + var testClient = new TestClient(test.mapConfig); + const { z, x, y } = test.coords; + const { format, response } = test; - assert.ok(geojsonTile); - assert.equal(geojsonTile.features.length, 5); - - assert.deepEqual(Object.keys(geojsonTile.features[0].properties), ['cartodb_id', 'name']); - - done(); + testClient.getTile(z, x, y, { format, response }, err => { + assert.ifError(err); + testClient.drain(done); + }); }); }); - } -}; + describe('overviews', function () { + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.8.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + it('should use overviews to fetch mvt data', function (done) { + const mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "sql": 'SELECT * FROM test_table_overviews', + "cartocss": TestClient.CARTOCSS.POINTS, + "cartocss_version": "2.3.0" + } + } + ] + ); + + const testClient = new TestClient(mapConfig); + const [ z, x, y ] = [ 0, 0, 0 ]; + const options = { format: 'mvt' }; + + testClient.getTile(z, x, y, options, function (err, res, mvt) { + assert.ifError(err); + + const geojsonTile = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(geojsonTile.features)); + assert.ok(geojsonTile.features.length > 0); + + const feature = geojsonTile.features[0]; + + assert.ok(feature.properties.hasOwnProperty('_feature_count'), 'Missing _feature_count property'); + assert.equal(feature.properties.cartodb_id, 1); + assert.equal(feature.properties.name, 'Hawai'); + assert.equal(feature.properties._feature_count, 5); // original table has _feature_count = 1 + assert.equal(feature.properties.value, 3); // original table has value = 1.0 + + testClient.drain(done); + }); + }); + + it('first layer should use overviews, second layer shouldn\'t', function (done) { + const mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "sql": 'SELECT * FROM test_table_overviews', + "cartocss": TestClient.CARTOCSS.POINTS, + "cartocss_version": "2.3.0" + } + }, + { + "type": "cartodb", + "options": { + "sql": 'SELECT * FROM test_table', + "cartocss": TestClient.CARTOCSS.POINTS, + "cartocss_version": "2.3.0" + } + } + ] + ); + + const testClient = new TestClient(mapConfig); + const [ z, x, y ] = [ 0, 0, 0 ]; + const options = { format: 'mvt' }; + + testClient.getTile(z, x, y, options, function (err, res, mvt) { + assert.ifError(err); + + const tileWithOverviews = JSON.parse(mvt.toGeoJSONSync(0)); + const tileWithoutOverviews = JSON.parse(mvt.toGeoJSONSync(1)); + + assert.ok(Array.isArray(tileWithOverviews.features)); + assert.equal(tileWithOverviews.features.length, 1); + assert.equal(tileWithOverviews.features[0].properties._feature_count, 5); + + assert.ok(Array.isArray(tileWithoutOverviews.features)); + assert.equal(tileWithoutOverviews.features.length, 5); + assert.equal(tileWithoutOverviews.features[0].properties._feature_count, undefined); + + testClient.drain(done); + }); + }); + }); + }; } diff --git a/test/acceptance/named_layers.js b/test/acceptance/named_layers.js index 9c0a9966..b80bfacc 100644 --- a/test/acceptance/named_layers.js +++ b/test/acceptance/named_layers.js @@ -1,9 +1,10 @@ +'use strict'; + var test_helper = require('../support/test_helper'); var assert = require('../support/assert'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); @@ -13,6 +14,12 @@ var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); var step = require('step'); describe('named_layers', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); @@ -108,7 +115,7 @@ describe('named_layers', function() { }); beforeEach(function(done) { - global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true}; + global.environment.enabledFeatures.cdbQueryTablesFromPostgres = true; templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) { if (err) { return done(err); @@ -125,7 +132,7 @@ describe('named_layers', function() { }); afterEach(function(done) { - global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false}; + global.environment.enabledFeatures.cdbQueryTablesFromPostgres = false; templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) { if (err) { return done(err); @@ -659,7 +666,7 @@ describe('named_layers', function() { } var parsedBody = JSON.parse(response.body); - assert.ok(parsedBody.errors[0].match(/permission denied for relation test_table_private_1/)); + assert.ok(parsedBody.errors[0].match(/permission denied for .+?test_table_private_1/)); return null; }, diff --git a/test/acceptance/named_layers_visibility.js b/test/acceptance/named_layers_visibility.js index 01e10126..0df92ed6 100644 --- a/test/acceptance/named_layers_visibility.js +++ b/test/acceptance/named_layers_visibility.js @@ -1,10 +1,11 @@ +'use strict'; + var step = require('step'); var test_helper = require('../support/test_helper'); var assert = require('../support/assert'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var RedisPool = require('redis-mpool'); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); @@ -13,6 +14,12 @@ var mapnik = require('windshaft').mapnik; var IMAGE_TOLERANCE = 20; describe('layers visibility for previews', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); var templateMaps = new TemplateMaps(redisPool, { diff --git a/test/acceptance/named_maps_authentication.js b/test/acceptance/named_maps_authentication.js index ba791f37..4f8e343c 100644 --- a/test/acceptance/named_maps_authentication.js +++ b/test/acceptance/named_maps_authentication.js @@ -1,3 +1,5 @@ +'use strict'; + var test_helper = require('../support/test_helper'); var RedisPool = require('redis-mpool'); var querystring = require('querystring'); @@ -6,11 +8,16 @@ var assert = require('../support/assert'); var mapnik = require('windshaft').mapnik; var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry'); describe('named maps authentication', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); diff --git a/test/acceptance/named_maps_cache.js b/test/acceptance/named_maps_cache.js index 5d121afa..44cfc84e 100644 --- a/test/acceptance/named_maps_cache.js +++ b/test/acceptance/named_maps_cache.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var RedisPool = require('redis-mpool'); @@ -5,10 +7,15 @@ var assert = require('../support/assert'); var mapnik = require('windshaft').mapnik; var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); describe('named maps provider cache', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index 9844c667..a6283d61 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -1,3 +1,5 @@ +'use strict'; + var qs = require('querystring'); var testHelper = require('../support/test_helper'); var RedisPool = require('redis-mpool'); @@ -282,4 +284,57 @@ describe('named maps static view', function() { }); }); + it('should return an error requesting unsupported image format', function (done) { + var view = { + zoom: 4, + center: { + lng: 40, + lat: 20 + } + }; + + templateMaps.addTemplate(username, createTemplate(view), function (err) { + if (err) { + return done(err); + } + + var url = `/api/v1/map/static/named/${templateName}/640/480.gif`; + + + var requestOptions = { + url: url, + method: 'GET', + headers: { + host: username + }, + encoding: 'binary' + }; + + var expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + // this could be removed once named maps are invalidated, otherwise you hits the cache + var server = new CartodbWindshaft(serverOptions); + + assert.response(server, requestOptions, expectedResponse, function (res, err) { + assert.ifError(err); + assert.deepEqual( + JSON.parse(res.body), + { + errors:['Unsupported image format \"gif\"'], + errors_with_context:[{ + type: 'unknown', + message: 'Unsupported image format \"gif\"' + }] + } + ); + done(); + }); + }); + }); + }); diff --git a/test/acceptance/named_maps_stats.js b/test/acceptance/named_maps_stats.js index 9813a402..fc80cbbc 100644 --- a/test/acceptance/named_maps_stats.js +++ b/test/acceptance/named_maps_stats.js @@ -1,3 +1,5 @@ +'use strict'; + var test_helper = require('../support/test_helper'); var RedisPool = require('redis-mpool'); var querystring = require('querystring'); @@ -6,11 +8,16 @@ var assert = require('../support/assert'); var mapnik = require('windshaft').mapnik; var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry'); describe('named maps preview stats', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + var redisPool = new RedisPool(global.environment.redis); var templateMaps = new TemplateMaps(redisPool, { diff --git a/test/acceptance/overviews_metadata.js b/test/acceptance/overviews_metadata.js index 8af7b2a4..90642777 100644 --- a/test/acceptance/overviews_metadata.js +++ b/test/acceptance/overviews_metadata.js @@ -1,9 +1,10 @@ +'use strict'; + var test_helper = require('../support/test_helper'); var assert = require('../support/assert'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); @@ -15,6 +16,12 @@ var windshaft = require('windshaft'); describe('overviews metadata', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); @@ -67,6 +74,7 @@ describe('overviews metadata', function() { data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); + var parsedBody = JSON.parse(res.body); assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid); expected_token = parsedBody.layergroupid; @@ -108,9 +116,92 @@ describe('overviews metadata', function() { } ); }); + + describe('Overviews Flags', function () { + it("Overviews used", function (done) { + var layergroup = { + version: '1.0.0', + layers: [overviews_layer, non_overviews_layer] + }; + + var layergroup_url = '/api/v1/map'; + + var expected_token; + step( + function do_post() { + var next = this; + assert.response(server, { + url: layergroup_url, + method: 'POST', + headers: { host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function (res) { + assert.equal(res.statusCode, 200, res.body); + + const headers = JSON.parse(res.headers['x-tiler-profiler']); + + assert.ok(headers.overviewsAddedToMapconfig); + assert.equal(headers.mapType, 'anonymous'); + + const parsedBody = JSON.parse(res.body); + expected_token = parsedBody.layergroupid; + next(); + }); + }, + function finish(err) { + keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + done(err); + } + ); + }); + it("Overviews NOT used", function (done) { + var layergroup = { + version: '1.0.0', + layers: [non_overviews_layer] + }; + + var layergroup_url = '/api/v1/map'; + + var expected_token; + step( + function do_post() { + var next = this; + assert.response(server, { + url: layergroup_url, + method: 'POST', + headers: { host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function (res) { + assert.equal(res.statusCode, 200, res.body); + + const headers = JSON.parse(res.headers['x-tiler-profiler']); + + assert.equal(headers.overviewsAddedToMapconfig, false); + assert.equal(headers.mapType, 'anonymous'); + + const parsedBody = JSON.parse(res.body); + expected_token = parsedBody.layergroupid; + next(); + }); + }, + function finish(err) { + keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + done(err); + } + ); + }); + }); }); describe('overviews metadata with filters', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); diff --git a/test/acceptance/overviews_metadata_named_maps.js b/test/acceptance/overviews_metadata_named_maps.js index a6557910..9e66cf07 100644 --- a/test/acceptance/overviews_metadata_named_maps.js +++ b/test/acceptance/overviews_metadata_named_maps.js @@ -1,9 +1,10 @@ +'use strict'; + var test_helper = require('../support/test_helper'); var assert = require('../support/assert'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); @@ -13,8 +14,13 @@ var step = require('step'); var windshaft = require('windshaft'); - describe('overviews metadata for named maps', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); + // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); @@ -172,4 +178,129 @@ describe('overviews metadata for named maps', function() { } ); }); + + describe('Overviews Flags', function() { + it("Overviews used", function (done) { + step( + function postTemplate() { + var next = this; + + assert.response(server, { + url: '/api/v1/map/named?api_key=1234', + method: 'POST', + headers: { host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(template) + }, {}, function (res, err) { + next(err, res); + }); + }, + function instantiateTemplate(err) { + assert.ifError(err); + + var next = this; + assert.response(server, { + url: '/api/v1/map/named/' + templateId, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } + }, {}, + function (res, err) { + return next(err, res); + }); + + }, + function checkFlags(err, res) { + assert.ifError(err); + + var next = this; + + var parsedBody = JSON.parse(res.body); + + keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + const headers = JSON.parse(res.headers['x-tiler-profiler']); + + assert.ok(headers.overviewsAddedToMapconfig); + assert.equal(headers.mapType, 'named'); + + next(); + }, + + function finish(err) { + done(err); + } + ); + }); + + it("Overviews NOT used", function (done) { + + const nonOverviewsTemplateId = 'non-overviews-template'; + + var nonOverviewsTemplate = { + version: '0.0.1', + name: nonOverviewsTemplateId, + auth: { method: 'open' }, + layergroup: { + version: '1.0.0', + layers: [non_overviews_layer] + } + }; + + step( + function postTemplate() { + var next = this; + + assert.response(server, { + url: '/api/v1/map/named?api_key=1234', + method: 'POST', + headers: { host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(nonOverviewsTemplate) + }, {}, function (res, err) { + next(err, res); + }); + }, + function instantiateTemplate(err) { + assert.ifError(err); + + var next = this; + assert.response(server, { + url: '/api/v1/map/named/' + nonOverviewsTemplateId, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } + }, {}, + function (res, err) { + return next(err, res); + }); + + }, + function checkFlags(err, res) { + assert.ifError(err); + + var next = this; + + var parsedBody = JSON.parse(res.body); + + keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + const headers = JSON.parse(res.headers['x-tiler-profiler']); + + assert.equal(headers.overviewsAddedToMapconfig, false); + assert.equal(headers.mapType, 'named'); + + next(); + }, + + function finish(err) { + done(err); + } + ); + }); + }); }); diff --git a/test/acceptance/overviews_queries.js b/test/acceptance/overviews_queries.js index ad7599a6..c3726a61 100644 --- a/test/acceptance/overviews_queries.js +++ b/test/acceptance/overviews_queries.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../support/test_helper'); var assert = require('../support/assert'); @@ -6,9 +8,12 @@ var ServerOptions = require('./ported/support/ported_server_options'); var testClient = require('./ported/support/test_client'); describe('overviews_queries', function() { + var server; - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2; diff --git a/test/acceptance/ported/attributes.js b/test/acceptance/ported/attributes.js index abd4e5ab..6f899c64 100644 --- a/test/acceptance/ported/attributes.js +++ b/test/acceptance/ported/attributes.js @@ -1,15 +1,20 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); -var assert = require('../../support/assert'); -var step = require('step'); +var assert = require('../../support/assert'); +var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var PortedServerOptions = require('./support/ported_server_options'); - var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('attributes', function() { - var server = cartodbServer(PortedServerOptions); - server.setMaxListeners(0); + var server; + + before(function () { + server = cartodbServer(PortedServerOptions); + server.setMaxListeners(0); + }); var test_mapconfig_1 = { version: '1.1.0', @@ -31,7 +36,7 @@ describe('attributes', function() { function checkCORSHeaders(res) { assert.equal( res.headers['access-control-allow-headers'], - 'X-Requested-With, X-Prototype-Version, X-CSRF-Token' + 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization' ); assert.equal(res.headers['access-control-allow-origin'], '*'); } @@ -47,132 +52,147 @@ describe('attributes', function() { }); it("can only be fetched from layer having an attributes spec", function(done) { - - var expected_token; - step( - function do_post() - { - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(test_mapconfig_1) - }, {}, function(res, err) { next(err, res); }); - }, - function checkPost(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - // CORS headers should be sent with response - // from layergroup creation via POST - checkCORSHeaders(res); - var parsedBody = JSON.parse(res.body); - if ( expected_token ) { - assert.deepEqual(parsedBody, {layergroupid: expected_token, layercount: 2}); - } else { - expected_token = parsedBody.layergroupid; - } - return null; - }, - function do_get_attr_0(err) - { - assert.ifError(err); - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1', - method: 'GET' - }, {}, function(res, err) { next(err, res); }); - }, - function check_error_0(err, res) { - assert.ifError(err); - assert.equal( - res.statusCode, - 400, - res.statusCode + ( res.statusCode !== 200 ? (': ' + res.body) : '' ) - ); - var parsed = JSON.parse(res.body); - assert.equal(parsed.errors[0], "Layer 0 has no exposed attributes"); - return null; - }, - function do_get_attr_1(err) - { - assert.ifError(err); - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1', - method: 'GET' - }, {}, function(res, err) { next(err, res); }); - }, - function check_attr_1(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed, {"n":6}); - return null; - }, - function do_get_attr_1_404(err) - { - assert.ifError(err); - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/-666', - method: 'GET' - }, {}, function(res, err) { next(err, res); }); - }, - function check_attr_1_404(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body); - var parsed = JSON.parse(res.body); - assert.ok(parsed.errors); - var msg = parsed.errors[0]; - assert.ok(msg.match(/0 features.*identified by fid -666/), msg); - return null; - }, - function finish(err) { - keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - - done(err); + var expected_token; + step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(test_mapconfig_1) + }, {}, function(res, err) { next(err, res); }); + }, + function checkPost(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + // CORS headers should be sent with response + // from layergroup creation via POST + checkCORSHeaders(res); + var parsedBody = JSON.parse(res.body); + if ( expected_token ) { + assert.deepEqual(parsedBody, {layergroupid: expected_token, layercount: 2}); + } else { + expected_token = parsedBody.layergroupid; } - ); - }); + return null; + }, + function do_get_attr_0(err) + { + assert.ifError(err); + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1', + method: 'GET', + headers: { + host: 'localhost' + }, + }, {}, function(res, err) { next(err, res); }); + }, + function check_error_0(err, res) { + assert.ifError(err); + assert.equal( + res.statusCode, + 400, + res.statusCode + ( res.statusCode !== 200 ? (': ' + res.body) : '' ) + ); + var parsed = JSON.parse(res.body); + assert.equal(parsed.errors[0], "Layer 0 has no exposed attributes"); + return null; + }, + function do_get_attr_1(err) + { + assert.ifError(err); + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1', + method: 'GET', + headers: { + host: 'localhost' + } + }, {}, function(res, err) { next(err, res); }); + }, + function check_attr_1(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var parsed = JSON.parse(res.body); + assert.deepEqual(parsed, {"n":6}); + return null; + }, + function do_get_attr_1_404(err) + { + assert.ifError(err); + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/-666', + method: 'GET', + headers: { + host: 'localhost' + } + }, {}, function(res, err) { next(err, res); }); + }, + function check_attr_1_404(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body); + var parsed = JSON.parse(res.body); + assert.ok(parsed.errors); + var msg = parsed.errors[0]; + assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1"); + return null; + }, + function finish(err) { + keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + + done(err); + } + ); + }); // See https://github.com/CartoDB/Windshaft/issues/131 it("are checked at map creation time", function(done) { - // clone the mapconfig test - var mapconfig = JSON.parse(JSON.stringify(test_mapconfig_1)); - // append unexistant attribute name - mapconfig.layers[1].options.sql = 'SELECT * FROM test_table'; - mapconfig.layers[1].options.attributes.id = 'unexistant'; - mapconfig.layers[1].options.attributes.columns = ['cartodb_id']; + // clone the mapconfig test + var mapconfig = JSON.parse(JSON.stringify(test_mapconfig_1)); + // append unexistant attribute name + mapconfig.layers[1].options.sql = 'SELECT * FROM test_table'; + mapconfig.layers[1].options.attributes.id = 'unexistant'; + mapconfig.layers[1].options.attributes.columns = ['cartodb_id']; - step( - function do_post() - { - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(mapconfig) - }, {}, function(res, err) { next(err, res); }); - }, - function checkPost(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 404, res.statusCode + ': ' + (res.statusCode===200?'...':res.body)); - var parsed = JSON.parse(res.body); - assert.ok(parsed.errors); - assert.equal(parsed.errors.length, 1); - var msg = parsed.errors[0]; - assert.equal(msg, 'column "unexistant" does not exist'); - return null; - }, - function finish(err) { - done(err); - } - ); - }); + step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(mapconfig) + }, {}, function(res, err) { next(err, res); }); + }, + function checkPost(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 404, res.statusCode + ': ' + (res.statusCode===200?'...':res.body)); + var parsed = JSON.parse(res.body); + assert.ok(parsed.errors); + assert.equal(parsed.errors.length, 1); + var msg = parsed.errors[0]; + assert.equal(msg, 'column "unexistant" does not exist'); + return null; + }, + function finish(err) { + done(err); + } + ); + }); it("can be used with jsonp", function(done) { @@ -184,7 +204,10 @@ describe('attributes', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(test_mapconfig_1) }, {}, function(res, err) { next(err, res); }); }, @@ -209,7 +232,10 @@ describe('attributes', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1?callback=test', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_error_0(err, res) { @@ -232,7 +258,10 @@ describe('attributes', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_attr_1(err, res) { @@ -270,7 +299,10 @@ describe('attributes', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, diff --git a/test/acceptance/ported/blend.js b/test/acceptance/ported/blend.js index 52273568..2f4a8714 100644 --- a/test/acceptance/ported/blend.js +++ b/test/acceptance/ported/blend.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/ported/blend_filtering.js b/test/acceptance/ported/blend_filtering.js index 2ada716e..fcb0d41f 100644 --- a/test/acceptance/ported/blend_filtering.js +++ b/test/acceptance/ported/blend_filtering.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/ported/blend_http_fallback.js b/test/acceptance/ported/blend_http_fallback.js index f86d3dfc..13fac245 100644 --- a/test/acceptance/ported/blend_http_fallback.js +++ b/test/acceptance/ported/blend_http_fallback.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/ported/blend_http_timeout.js b/test/acceptance/ported/blend_http_timeout.js index 54481fd0..7edd8996 100644 --- a/test/acceptance/ported/blend_http_timeout.js +++ b/test/acceptance/ported/blend_http_timeout.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/ported/external_resources.js b/test/acceptance/ported/external_resources.js index 0aae14c3..6f4f13b3 100644 --- a/test/acceptance/ported/external_resources.js +++ b/test/acceptance/ported/external_resources.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -11,12 +13,11 @@ describe('external resources', function() { var res_serv; // resources server var res_serv_status = { numrequests:0 }; // status of resources server - var res_serv_port = 8033; // FIXME: make configurable ? + var res_serv_port; var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; before(function(done) { - nock.enableNetConnect('127.0.0.1'); // Start a server to test external resources res_serv = http.createServer( function(request, response) { ++res_serv_status.numrequests; @@ -32,12 +33,24 @@ describe('external resources', function() { response.end(); }); }); - res_serv.listen(res_serv_port, done); + + const host = '127.0.0.1'; + const markersServer = res_serv.listen(0); + + res_serv_port = markersServer.address().port; + + nock.disableNetConnect(); + nock.enableNetConnect(host); + + markersServer.on('listening', done); }); after(function(done) { testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); + nock.cleanAll(); + nock.enableNetConnect(); + // Close the resources server res_serv.close(done); }); diff --git a/test/acceptance/ported/limits.js b/test/acceptance/ported/limits.js index 0a8ee378..ea549fa3 100644 --- a/test/acceptance/ported/limits.js +++ b/test/acceptance/ported/limits.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var fs = require('fs'); diff --git a/test/acceptance/ported/multilayer.js b/test/acceptance/ported/multilayer.js index 0ad9f5de..d7a0b0c7 100644 --- a/test/acceptance/ported/multilayer.js +++ b/test/acceptance/ported/multilayer.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -10,17 +12,24 @@ var ServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('multilayer', function() { + var server; + + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); mapnik.register_system_fonts(); var available_system_fonts = _.keys(mapnik.fontFiles()); var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20; function checkCORSHeaders(res) { - assert.equal(res.headers['access-control-allow-headers'], 'X-Requested-With, X-Prototype-Version, X-CSRF-Token'); - assert.equal(res.headers['access-control-allow-origin'], '*'); + assert.equal( + res.headers['access-control-allow-headers'], + 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization' + ); + assert.equal(res.headers['access-control-allow-origin'], '*'); } // See https://github.com/Vizzuality/Windshaft/issues/70 @@ -44,7 +53,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json; charset=utf-8' }, + headers: { host: 'localhost', 'Content-Type': 'application/json; charset=utf-8' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -85,7 +94,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -101,7 +110,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -154,7 +164,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); @@ -177,7 +187,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -194,7 +205,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -212,7 +224,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -266,7 +279,7 @@ describe('multilayer', function() { config: JSON.stringify(layergroup) }), method: 'GET', - headers: {'Content-Type': 'application/json' } + headers: { host: 'localhost', 'Content-Type': 'application/json' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); // CORS headers should be sent with response @@ -289,7 +302,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -307,7 +321,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -325,7 +340,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -380,7 +396,7 @@ describe('multilayer', function() { callback: 'jsonp_test' }), method: 'GET', - headers: {'Content-Type': 'application/json' } + headers: { host: 'localhost', 'Content-Type': 'application/json' } }, {}, function(res, err) { next(err, res); }); }, function do_check_token(err, res) { @@ -413,7 +429,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -431,7 +448,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -449,7 +467,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -510,7 +529,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err, res); @@ -530,7 +549,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -548,7 +568,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -566,7 +587,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -583,7 +605,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/4', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); @@ -602,7 +625,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/2/0/0/0.json.torque', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); }, function do_check_torque2(err, res) { @@ -624,7 +648,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.json.torque', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); }, function do_check_torque1(err, res) { @@ -684,7 +709,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup1) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -700,7 +725,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup2) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -717,7 +742,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -734,7 +760,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -752,7 +779,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + token2 + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -769,7 +797,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + token2 + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -826,7 +855,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -848,7 +877,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -893,7 +923,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); @@ -922,7 +952,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); @@ -957,7 +987,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: _.template(tpl, {font:'bogus'}) }, function(res) { next(null, res); }); }, @@ -977,7 +1007,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: _.template(tpl, {font:available_system_fonts[0]}) }, function(res) { next(null, res); }); }, @@ -1000,7 +1030,8 @@ describe('multilayer', function() { // //////////////////////////////////////////////////////////////////// - it("get'ing options on layergroup should return CORS headers", function(done){ + it("geting options on layergroup should return CORS headers", function(done){ + const allowHeaders = 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization, Content-Type'; assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'OPTIONS' @@ -1008,7 +1039,7 @@ describe('multilayer', function() { status: 200, // TODO: use checkCORSHeaders() function headers: { - 'Access-Control-Allow-Headers': 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Content-Type', + 'Access-Control-Allow-Headers': allowHeaders, 'Access-Control-Allow-Origin': '*' } }, function() { done(); }); @@ -1040,7 +1071,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -1061,7 +1092,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { next(null, res); }); @@ -1093,63 +1125,8 @@ describe('multilayer', function() { ); }); - // See http://github.com/CartoDB/Windshaft/issues/157 - it("req2params is called only once for a multilayer post", function(done) { - - var layergroup = { - version: '1.0.1', - layers: [ - { options: { - sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2', - cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', - cartocss_version: '2.0.1', - interactivity: [ 'cartodb_id' ], - geom_column: 'the_geom' - } }, - { options: { - sql: 'select cartodb_id, ST_Translate(the_geom, -50, 0) as the_geom from test_table limit 2 offset 2', - cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', - cartocss_version: '2.0.2', - interactivity: [ 'cartodb_id' ], - geom_column: 'the_geom' - } } - ] - }; - - var expected_token; - step( - function do_post() - { - global.req2params_calls = 0; - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(layergroup) - }, {}, function(res, err) { next(err,res); }); - }, - function check_post(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - var parsedBody = JSON.parse(res.body); - expected_token = LayergroupToken.parse(parsedBody.layergroupid).token; - assert.equal(global.req2params_calls, 1); - return null; - }, - function finish(err) { - if (err) { - return done(err); - } - var keysToDelete = {'user:localhost:mapviews:global': 5}; - keysToDelete['map_cfg|' + expected_token] = 0; - testHelper.deleteRedisKeys(keysToDelete, done); - } - ); - }); - // See https://github.com/CartoDB/Windshaft/issues/163 - it("has different token for different database", function(done) { + it.skip("has different token for different database", function(done) { var layergroup = { version: '1.0.1', layers: [ @@ -1170,7 +1147,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err,res); }); }, @@ -1187,7 +1164,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test2/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'cartodb250user', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err,res); }); }, @@ -1233,7 +1210,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err,res); }); }, @@ -1251,7 +1228,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index 6e7f5964..6194879b 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -8,9 +10,12 @@ var testClient = require('./support/test_client'); var TestClient = require('../../support/test-client'); describe('multilayer error cases', function() { + var server; - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); // var client = null; afterEach(function(done) { @@ -24,7 +29,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/x-www-form-urlencoded' } + headers: { + host: 'localhost', + 'Content-Type': 'application/x-www-form-urlencoded' + } }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); var parsedBody = JSON.parse(res.body); @@ -37,7 +45,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' } + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); var parsedBody = JSON.parse(res.body); @@ -50,7 +61,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup?callback=test', method: 'POST', - headers: {'Content-Type': 'application/json' } + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } }, {}, function(res) { assert.equal(res.statusCode, 200); assert.equal( @@ -65,27 +79,30 @@ describe('multilayer error cases', function() { }); it("layergroup with no cartocss_version", function(done) { - var layergroup = { - version: '1.0.0', - layers: [ - { options: { - sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2', - cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', - geom_column: 'the_geom' - } } - ] - }; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(layergroup) - }, {}, function(res) { - assert.equal(res.statusCode, 400, res.body); - var parsedBody = JSON.parse(res.body); - assert.deepEqual(parsedBody.errors, ["Missing cartocss_version for layer 0 options"]); - done(); - }); + var layergroup = { + version: '1.0.0', + layers: [ + { options: { + sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + geom_column: 'the_geom' + } } + ] + }; + assert.response(server, { + url: '/database/windshaft_test/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 400, res.body); + var parsedBody = JSON.parse(res.body); + assert.deepEqual(parsedBody.errors, ["Missing cartocss_version for layer 0 options"]); + done(); + }); }); it("sql/cartocss combination errors", function(done) { @@ -98,17 +115,18 @@ describe('multilayer error cases', function() { geom_column: 'the_geom' }}] }; - ServerOptions.afterLayergroupCreateCalls = 0; assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { try { assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); // See http://github.com/CartoDB/Windshaft/issues/159 - assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); var parsed = JSON.parse(res.body); assert.ok(parsed); assert.equal(parsed.errors.length, 1); @@ -149,12 +167,9 @@ describe('multilayer error cases', function() { }} ] }; - ServerOptions.afterLayergroupCreateCalls = 0; this.client = new TestClient(layergroup); this.client.getLayergroup({ response: { status: 400 } }, function(err, parsed) { assert.ok(!err, err); - // See http://github.com/CartoDB/Windshaft/issues/159 - assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); assert.ok(parsed); assert.equal(parsed.errors.length, 1); var error = parsed.errors[0]; @@ -186,7 +201,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -222,7 +240,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -264,7 +285,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); @@ -367,7 +391,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/deadbeef/0/0/0/0.grid.json', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function checkResponse(err, res) { diff --git a/test/acceptance/ported/multilayer_interactivity.js b/test/acceptance/ported/multilayer_interactivity.js index 4542f710..c2c11fad 100644 --- a/test/acceptance/ported/multilayer_interactivity.js +++ b/test/acceptance/ported/multilayer_interactivity.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -8,9 +10,12 @@ var PortedServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('multilayer interactivity and layers order', function() { + var server; - var server = cartodbServer(PortedServerOptions); - server.setMaxListeners(0); + before(function () { + server = cartodbServer(PortedServerOptions); + server.setMaxListeners(0); + }); function layerType(layer) { return layer.type || 'undefined'; @@ -23,13 +28,12 @@ describe('multilayer interactivity and layers order', function() { layers: testScenario.layers }; - PortedServerOptions.afterLayergroupCreateCalls = 0; - assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) @@ -49,8 +53,6 @@ describe('multilayer interactivity and layers order', function() { '\n\tLayer types: ' + layergroup.layers.map(layerType).join(', ') ); -// assert.equal(PortedServerOptions.afterLayergroupCreateCalls, 1); - var layergroupResponse = JSON.parse(response.body); assert.ok(layergroupResponse); diff --git a/test/acceptance/ported/raster.js b/test/acceptance/ported/raster.js index 448661f2..58a6b2bf 100644 --- a/test/acceptance/ported/raster.js +++ b/test/acceptance/ported/raster.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper =require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -8,12 +10,19 @@ var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('raster', function() { - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); + var server; + + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); function checkCORSHeaders(res) { - assert.equal(res.headers['access-control-allow-headers'], 'X-Requested-With, X-Prototype-Version, X-CSRF-Token'); - assert.equal(res.headers['access-control-allow-origin'], '*'); + assert.equal( + res.headers['access-control-allow-headers'], + 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization' + ); + assert.equal(res.headers['access-control-allow-origin'], '*'); } var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2; @@ -41,7 +50,7 @@ describe('raster', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, @@ -66,7 +75,8 @@ describe('raster', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); }, function check_response(err, res) { @@ -124,6 +134,7 @@ describe('raster', function() { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) diff --git a/test/acceptance/ported/regressions.js b/test/acceptance/ported/regressions.js index dd765b57..5e6a6293 100644 --- a/test/acceptance/ported/regressions.js +++ b/test/acceptance/ported/regressions.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -8,21 +10,6 @@ describe('regressions', function() { testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); - // See https://github.com/Vizzuality/Windshaft/issues/65 - it("#65 catching non-Error exception doesn't kill the backend", function(done) { - var mapConfig = testClient.defaultTableMapConfig('test_table'); - testClient.withLayergroup(mapConfig, function(err, requestTile, finish) { - var options = { - statusCode: 400, - contentType: 'application/json; charset=utf-8' - }; - requestTile('/0/0/0.png?testUnexpectedError=1', options, function(err, res) { - assert.deepEqual(JSON.parse(res.body).errors, ["test unexpected error"]); - finish(done); - }); - }); - }); - // Test that you cannot write to the database from a tile request // // See http://github.com/CartoDB/Windshaft/issues/130 diff --git a/test/acceptance/ported/retina.js b/test/acceptance/ported/retina.js index 7469214b..bdc1188c 100644 --- a/test/acceptance/ported/retina.js +++ b/test/acceptance/ported/retina.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -11,9 +13,12 @@ describe('retina support', function() { var layergroupId = null; - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); + var server; + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); var keysToDelete; beforeEach(function(done) { @@ -38,6 +43,7 @@ describe('retina support', function() { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(retinaSampleMapConfig) @@ -67,7 +73,10 @@ describe('retina support', function() { { url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/0/0' + scaleFactor + '.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }, responseHead, assertFn diff --git a/test/acceptance/ported/server.js b/test/acceptance/ported/server.js index b18bae07..58febf38 100644 --- a/test/acceptance/ported/server.js +++ b/test/acceptance/ported/server.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -6,9 +8,12 @@ var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); describe('server', function() { + var server; - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); after(function() { testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index f3c9ba81..61daa568 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -8,9 +10,12 @@ var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); describe('server_gettile', function() { + var server; - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; @@ -48,19 +53,19 @@ describe('server_gettile', function() { var mapConfig = testClient.defaultTableMapConfig('test_table'); testClient.withLayergroup(mapConfig, function (err, requestTile, finish) { requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); lastXwc = xwc; requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); assert.ok(xwc >= lastXwc); - requestTile(tileUrl + '?cache_buster=wadus', function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) { + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(!xwc); finish(done); @@ -99,18 +104,25 @@ describe('server_gettile', function() { } testClient.withLayergroup(mapConfig, validateLayergroup, function(err, requestTile, finish) { - requestTile(tileUrl, function(err, res) { - assert.ok(res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time"); - assert.ok(res.headers['x-windshaft-cache'] >= 0); + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); + assert.ok(!xwc); - assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, - function(err) { - finish(function(finishErr) { - done(err || finishErr); - }); - } - ); + requestTile(tileUrl, function (err, res) { + assert.ok( + res.headers.hasOwnProperty('x-windshaft-cache'), + "Did not hit renderer cache on second time" + ); + assert.ok(parseInt(res.headers['x-windshaft-cache'], 10) >= 0); + + assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, + function(err) { + finish(function(finishErr) { + done(err || finishErr); + }); + } + ); + }); }); }); }); diff --git a/test/acceptance/ported/server_png8_format.js b/test/acceptance/ported/server_png8_format.js index 18151330..86af4682 100644 --- a/test/acceptance/ported/server_png8_format.js +++ b/test/acceptance/ported/server_png8_format.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -11,17 +13,22 @@ var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); var IMAGE_EQUALS_TOLERANCE_PER_MIL = 85; describe('server_png8_format', function() { - var serverOptionsPng32 = ServerOptions; - serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore); - serverOptionsPng32.grainstore.mapnik_tile_format = 'png32'; - var serverPng32 = cartodbServer(serverOptionsPng32); - serverPng32.setMaxListeners(0); + var serverPng32; + var serverPng8; - var serverOptionsPng8 = ServerOptions; - serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore); - serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h'; - var serverPng8 = cartodbServer(serverOptionsPng8); - serverPng8.setMaxListeners(0); + before(function () { + var serverOptionsPng32 = ServerOptions; + serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore); + serverOptionsPng32.grainstore.mapnik_tile_format = 'png32'; + serverPng32 = cartodbServer(serverOptionsPng32); + serverPng32.setMaxListeners(0); + + var serverOptionsPng8 = ServerOptions; + serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore); + serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h'; + serverPng8 = cartodbServer(serverOptionsPng8); + serverPng8.setMaxListeners(0); + }); var layergroupId; @@ -62,6 +69,7 @@ describe('server_png8_format', function() { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) @@ -81,7 +89,10 @@ describe('server_png8_format', function() { var requestPayload = { url: '/database/windshaft_test/layergroup/' + layergroupId + tilePartialUrl, method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }; var requestHeaders = { @@ -179,4 +190,3 @@ describe('server_png8_format', function() { }); }); }); - diff --git a/test/acceptance/ported/static_maps.js b/test/acceptance/ported/static_maps.js index 1002cdd5..ce3721f0 100644 --- a/test/acceptance/ported/static_maps.js +++ b/test/acceptance/ported/static_maps.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 8b75ec02..b0fddbe2 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -1,15 +1,43 @@ +'use strict'; + var _ = require('underscore'); var serverOptions = require('../../../../lib/cartodb/server_options'); var mapnik = require('windshaft').mapnik; -var LayergroupToken = require('../../../../lib/cartodb/models/layergroup-token'); var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' }); module.exports = _.extend({}, serverOptions, { + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url: '/database/:dbname/table/:table', + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_mapconfig: '/database/:dbname/layergroup', + + routes: { + v0: { + paths: [ + '/tiles', + '/database/:dbname' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + }, + + grainstore: { datasource: { geometry_field: 'the_geom', @@ -47,48 +75,5 @@ module.exports = _.extend({}, serverOptions, { enable_cors: global.environment.enable_cors, unbuffered_logging: true, // for smoother teardown from tests log_format: null, // do not log anything - afterLayergroupCreateCalls: 0, - useProfiler: true, - req2params: function(req, res, callback){ - - if ( req.query.testUnexpectedError ) { - return callback('test unexpected error'); - } - - // this is in case you want to test sql parameters eg ...png?sql=select * from my_table limit 10 - req.params = _.extend({}, req.params); - - if (req.params.token) { - req.params.token = LayergroupToken.parse(req.params.token).token; - } - - _.extend(req.params, req.query); - req.params.user = 'localhost'; - res.locals.user = 'localhost'; - - req.params.dbhost = global.environment.postgres.host; - req.params.dbport = req.params.dbport || global.environment.postgres.port; - - req.params.dbuser = 'test_windshaft_publicuser'; - if (req.params.dbname !== 'windshaft_test2') { - req.params.dbuser = 'test_windshaft_cartodb_user_1'; - } - req.params.dbname = 'test_windshaft_cartodb_user_1_db'; - - // add all params to res.locals - res.locals = _.extend({}, req.params); - - - // increment number of calls counter - global.req2params_calls = global.req2params_calls ? global.req2params_calls + 1 : 1; - - // send the finished req object on - callback(null,req); - }, - afterLayergroupCreate: function(req, cfg, res, callback) { - res.layercount = cfg.layers.length; -// config.afterLayergroupCreateCalls++; - callback(null); - } - + useProfiler: true }); diff --git a/test/acceptance/ported/support/test_client.js b/test/acceptance/ported/support/test_client.js index 640f1b64..01d00bf2 100644 --- a/test/acceptance/ported/support/test_client.js +++ b/test/acceptance/ported/support/test_client.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../../support/test_helper'); var LayergroupToken = require('../../../../lib/cartodb/models/layergroup-token'); @@ -39,9 +41,18 @@ module.exports = { getTileLayer: getTileLayer }; +var server; -var server = new CartodbServer(PortedServerOptions); -server.setMaxListeners(0); +function getServer () { + if (server) { + return server; + } + + server = new CartodbServer(PortedServerOptions); + server.setMaxListeners(0); + + return server; +} var jsonContentType = 'application/json; charset=utf-8'; var jsContentType = 'text/javascript; charset=utf-8'; @@ -72,7 +83,7 @@ function createLayergroup(layergroupConfig, options, callback) { }); }, function validateLayergroup(err, res) { - assert.ok(!err, 'Failed to request layergroup'); + assert.ifError(err); var parsedBody; var layergroupid; @@ -120,7 +131,7 @@ function serverInstance(options) { return otherServer; } - return server; + return getServer(); } function layergroupRequest(layergroupConfig, method, callbackName, extraParams) { @@ -129,6 +140,7 @@ function layergroupRequest(layergroupConfig, method, callbackName, extraParams) var request = { url: '/database/windshaft_test/layergroup', headers: { + host: 'localhost', 'Content-Type': 'application/json' } }; @@ -337,6 +349,7 @@ function getGeneric(layergroupConfig, url, expectedResponse, callback) { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroupConfig) @@ -347,7 +360,7 @@ function getGeneric(layergroupConfig, url, expectedResponse, callback) { 'Content-Type': 'application/json; charset=utf-8' } }; - assert.response(server, request, expectedResponse, function (res, err) { + assert.response(getServer(), request, expectedResponse, function (res, err) { next(err, res); }); }, @@ -372,14 +385,17 @@ function getGeneric(layergroupConfig, url, expectedResponse, callback) { var request = { url: finalUrl, - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }; if (contentType === pngContentType) { request.encoding = 'binary'; } - assert.response(server, request, expectedResponse, function (res, err) { + assert.response(getServer(), request, expectedResponse, function (res, err) { next(err, res); }); }, @@ -449,12 +465,28 @@ function withLayergroup(layergroupConfig, options, callback) { }; } - var baseUrlTpl = '/database/windshaft_test/layergroup/<%= layergroupid %>'; - var finalUrl = _.template(baseUrlTpl, { layergroupid: layergroupid }) + layergroupUrl; + const signerTpl = function ({ signer }) { + return `${signer ? `:${signer}@` : ''}`; + }; + + const cacheTpl = function ({ cache_buster, cacheBuster }) { + return `${cache_buster ? `:${cache_buster}` : `:${cacheBuster}`}`; + }; + + const urlTpl = function ({layergroupid, cache_buster = null, tile }) { + const { signer, token , cacheBuster } = LayergroupToken.parse(layergroupid); + const base = '/database/windshaft_test/layergroup/'; + return `${base}${signerTpl({signer})}${token}${cacheTpl({cache_buster, cacheBuster})}${tile}`; + }; + + const finalUrl = urlTpl({ layergroupid, cache_buster: options.cache_buster, tile: layergroupUrl }); var request = { url: finalUrl, - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }; if (options.contentType === pngContentType) { diff --git a/test/acceptance/ported/torque.js b/test/acceptance/ported/torque.js index 47ae2eee..82f3fda1 100644 --- a/test/acceptance/ported/torque.js +++ b/test/acceptance/ported/torque.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -9,9 +11,12 @@ var ServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('torque', function() { + var server; - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); var keysToDelete; beforeEach(function() { @@ -23,8 +28,11 @@ describe('torque', function() { }); function checkCORSHeaders(res) { - assert.equal(res.headers['access-control-allow-headers'], 'X-Requested-With, X-Prototype-Version, X-CSRF-Token'); - assert.equal(res.headers['access-control-allow-origin'], '*'); + assert.equal( + res.headers['access-control-allow-headers'], + 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization' + ); + assert.equal(res.headers['access-control-allow-origin'], '*'); } it("missing required property from torque layer", function(done) { @@ -48,7 +56,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -71,7 +79,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -94,7 +102,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -136,7 +144,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -161,7 +169,7 @@ describe('torque', function() { version: '1.1.0', layers: [ { type: 'torque', options: { - sql: "select 1 as id, '1970-01-02'::date as d, 'POINT(0 0)'::geometry as the_geom UNION select 2, " + + sql: "select 1 as id, '1970-01-02'::date as d, 'POINT(0 0)'::geometry as the_geom UNION ALL select 2, " + "'1970-01-01'::date, 'POINT(1 1)'::geometry", geom_column: 'the_geom', cartocss: 'Map { -torque-frame-count:2; -torque-resolution:3; -torque-time-attribute:d; ' + @@ -179,7 +187,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, @@ -218,7 +226,10 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_mapnik_error_1(err, res) { @@ -235,7 +246,10 @@ describe('torque', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_mapnik_error_2(err, res) { @@ -252,7 +266,10 @@ describe('torque', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_torque0_response(err, res) { @@ -270,7 +287,10 @@ describe('torque', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.torque.json', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_torque0_response_1(err, res) { @@ -315,7 +335,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, @@ -344,7 +364,7 @@ describe('torque', function() { version: '1.1.0', layers: [ { type: 'torque', options: { - sql: "select 1 as id, '1970-01-03'::date as d, 'POINT(0 0)'::geometry as the_geom UNION select 2, " + + sql: "select 1 as id, '1970-01-03'::date as d, 'POINT(0 0)'::geometry as the_geom UNION ALL select 2, " + "'1970-01-01'::date, 'POINT(1 1)'::geometry", geom_column: 'the_geom', cartocss: 'Map { -torque-frame-count:2; -torque-resolution:3; -torque-time-attribute:d; ' + @@ -354,19 +374,25 @@ describe('torque', function() { ] }; + let defautlPort = global.environment.postgres.port; + step( function do_post() { var next = this; + global.environment.postgres.port = 54777; assert.response(server, { - url: '/database/windshaft_test/layergroup?dbport=54777', + url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, function checkPost(err, res) { assert.ifError(err); + + global.environment.postgres.port = defautlPort; + assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body); var parsed = JSON.parse(res.body); assert.ok(parsed.errors, parsed); @@ -407,9 +433,9 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) - }, {}, function(res) { next(null, res); }); + }, {}, function (res) { next(null, res); }); }, function checkResponse(err, res) { assert.ifError(err); diff --git a/test/acceptance/ported/torque_boundaries.js b/test/acceptance/ported/torque_boundaries.js index 9b4619b1..1f0fcd80 100644 --- a/test/acceptance/ported/torque_boundaries.js +++ b/test/acceptance/ported/torque_boundaries.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -7,6 +9,13 @@ var ServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('torque boundary points', function() { + var server; + + before(function () { + server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + }); + var layergroupIdToDelete = null; beforeEach(function() { @@ -22,9 +31,6 @@ describe('torque boundary points', function() { testHelper.deleteRedisKeys(keysToDelete, done); }); - var server = cartodbServer(ServerOptions); - server.setMaxListeners(0); - var boundaryPointsMapConfig = { version: '1.1.0', layers: [ @@ -237,7 +243,7 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(boundaryPointsMapConfig) }, {}, function (res, err) { @@ -250,7 +256,10 @@ describe('torque boundary points', function() { var partialUrl = tileRequest.z + '/' + tileRequest.x + '/' + tileRequest.y; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/' + partialUrl + '.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function (res, err) { assert.ok(!err, 'Failed to get json'); @@ -351,7 +360,7 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(londonPointMapConfig) }, {}, function (res, err) { assert.ok(!err, 'Failed to create layergroup'); @@ -363,7 +372,10 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/2/1/1.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function (res, err) { assert.ok(!err, 'Failed to request torque.json'); @@ -414,7 +426,7 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(londonPointMapConfig) }, {}, function (res, err) { assert.ok(!err, 'Failed to create layergroup'); @@ -426,7 +438,10 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/13/4255/2765.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function (res, err) { assert.ok(!err, 'Failed to request torque.json'); diff --git a/test/acceptance/ported/torque_png.js b/test/acceptance/ported/torque_png.js index 57b3ffa1..9c8480bd 100644 --- a/test/acceptance/ported/torque_png.js +++ b/test/acceptance/ported/torque_png.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/ported/torque_zero_zero.js b/test/acceptance/ported/torque_zero_zero.js index fe6755ae..2d81f517 100644 --- a/test/acceptance/ported/torque_zero_zero.js +++ b/test/acceptance/ported/torque_zero_zero.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -19,13 +21,15 @@ describe('torque tiles at 0,0 point', function() { what: 'tl', x: 3, y: 3, - expects: [] + expects: [], + expects_fixed: [{"x__uint8":1,"y__uint8":0,"vals__uint8":[1],"dates__uint16":[0]}] }, { what: 'tr', x: 4, y: 3, - expects: [] + expects: [], + expects_fixed: [{"x__uint8":0,"y__uint8":0,"vals__uint8":[1],"dates__uint16":[0]}] }, { what: 'bl', @@ -74,7 +78,13 @@ describe('torque tiles at 0,0 point', function() { }; testClient.getTorque(mapConfig, 0, 3, tile.x, tile.y, function(err, res) { - assert.deepEqual(JSON.parse(res.body), tile.expects); + assert.ok(!err, err); + try { + assert.deepEqual(JSON.parse(res.body), tile.expects); + } catch (ex) { + // With Proj 5.1 this bug has been fixed and the point appears in all tiles + assert.deepEqual(JSON.parse(res.body), tile.expects_fixed); + } done(); }); }); diff --git a/test/acceptance/ported/wrap.js b/test/acceptance/ported/wrap.js index d79e2b56..6434d6fa 100644 --- a/test/acceptance/ported/wrap.js +++ b/test/acceptance/ported/wrap.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js new file mode 100644 index 00000000..f5b6f456 --- /dev/null +++ b/test/acceptance/rate-limit.test.js @@ -0,0 +1,378 @@ +'use strict'; + +require('../support/test_helper'); + +const assert = require('../support/assert'); +const redis = require('redis'); +const RedisPool = require('redis-mpool'); +const cartodbRedis = require('cartodb-redis'); +const TestClient = require('../support/test-client'); +const UserLimitsBackend = require('../../lib/cartodb/backends/user-limits'); +const rateLimitMiddleware = require('../../lib/cartodb/api/middlewares/rate-limit'); +const serverOptions = require('../../lib/cartodb/server_options'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; + +let userLimitsApi; +let rateLimit; +let redisClient; +let testClient; +let keysToDelete = ['user:localhost:mapviews:global']; +const user = 'localhost'; +let layergroupid; + +const query = ` + SELECT + ST_Transform('SRID=4326;POINT(-70 42)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = query, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); + + +function setLimit(count, period, burst, endpoint = RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) { + redisClient.SELECT(8, err => { + if (err) { + return; + } + + const key = `limits:rate:store:${user}:maps:${endpoint}`; + redisClient.rpush(key, burst); + redisClient.rpush(key, count); + redisClient.rpush(key, period); + keysToDelete.push(key); + }); +} + +function getReqAndRes() { + return { + req: {}, + res: { + headers: {}, + set(headers, value) { + if(typeof headers === 'object') { + this.headers = headers; + } else { + this.headers[headers] = value; + } + }, + locals: { + user: 'localhost' + } + } + }; +} + +function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done) { + let response = { + status, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + 'Carto-Rate-Limit-Limit': limit, + 'Carto-Rate-Limit-Remaining': remaining, + 'Carto-Rate-Limit-Reset': reset + } + }; + + if(retry) { + response.headers['Retry-After'] = retry; + } + + testClient.getLayergroup({ response }, err => { + assert.ifError(err); + if (done) { + setTimeout(done, 1000); + } + }); +} + +function assertRateLimitRequest (status, limit, remaining, reset, retry, done) { + const { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + let expectedHeaders = { + "Carto-Rate-Limit-Limit": limit, + "Carto-Rate-Limit-Remaining": remaining, + "Carto-Rate-Limit-Reset": reset + }; + + if(retry) { + expectedHeaders['Retry-After'] = retry; + } + + assert.deepEqual(res.headers, expectedHeaders); + + if(status === 200) { + assert.ifError(err); + } else { + assert.ok(err); + assert.equal(err.message, 'You are over platform\'s limits: too many requests.' + + ' Please contact us to know more details'); + assert.equal(err.http_status, 429); + assert.equal(err.type, 'limit'); + assert.equal(err.subtype, 'rate-limit'); + } + + if (done) { + setTimeout(done, 1000); + } + }); +} + +describe('rate limit', function() { + before(function() { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; + + redisClient = redis.createClient(global.environment.redis.port); + testClient = new TestClient(createMapConfig(), 1234); + }); + + after(function() { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false; + }); + + afterEach(function(done) { + keysToDelete.forEach( key => { + redisClient.del(key); + }); + + redisClient.SELECT(0, () => { + redisClient.del('user:localhost:mapviews:global'); + + redisClient.SELECT(5, () => { + redisClient.del('user:localhost:mapviews:global'); + done(); + }); + }); + }); + + it('should not be rate limited', function (done) { + const count = 1; + const period = 1; + const burst = 1; + setLimit(count, period, burst); + + assertGetLayergroupRequest(200, '2', '1', '1', null, done); + }); + + it("1 req/sec: 2 req/seg should be limited", function(done) { + const count = 1; + const period = 1; + const burst = 1; + setLimit(count, period, burst); + + assertGetLayergroupRequest(200, '2', '1', '1'); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1'), 250); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 500); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 750); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 950); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', null, done), 1050); + }); + +}); + + +describe('rate limit middleware', function () { + before(function (done) { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; + + const redisPool = new RedisPool(global.environment.redis); + const metadataBackend = cartodbRedis({ pool: redisPool }); + userLimitsApi = new UserLimitsBackend(metadataBackend, { + limits: { + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled + } + }); + rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS); + + redisClient = redis.createClient(global.environment.redis.port); + testClient = new TestClient(createMapConfig(), 1234); + + + const count = 1; + const period = 1; + const burst = 0; + setLimit(count, period, burst); + + setTimeout(done, 1000); + }); + + after(function () { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false; + + keysToDelete.forEach(key => { + redisClient.del(key); + }); + }); + + it("1 req/sec: 2 req/seg should be limited", function (done) { + assertRateLimitRequest(200, 1, 0, 1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 250); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050); + }); + + it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) { + userLimitsApi.metadataBackend.redisCmd( + 8, + 'SCRIPT', + ['FLUSH'], + function () { + assertRateLimitRequest(200, 1, 0, 1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050); + } + ); + }); +}); + + +const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + +describe('rate limit and vector tiles (mapnik)', () => rateLimitAndVectorTilesTest(false)); +describe('rate limit and vector tiles (postgis)', () => rateLimitAndVectorTilesTest(true)); + +function rateLimitAndVectorTilesTest(usePostGIS) { + + before(function() { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + before(function(done) { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true; + + redisClient = redis.createClient(global.environment.redis.port); + const count = 1; + const period = 1; + const burst = 0; + setLimit(count, period, burst, RATE_LIMIT_ENDPOINTS_GROUPS.TILE); + + testClient = new TestClient(createMapConfig(), 1234); + testClient.getLayergroup({status: 200}, (err, res) => { + assert.ifError(err); + + layergroupid = res.layergroupid; + + done(); + }); + }); + + after(function() { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.tile = false; + }); + + afterEach(function(done) { + keysToDelete.forEach( key => { + redisClient.del(key); + }); + + redisClient.SELECT(0, () => { + redisClient.del('user:localhost:mapviews:global'); + + redisClient.SELECT(5, () => { + redisClient.del('user:localhost:mapviews:global'); + setTimeout(done, 1000); + }); + }); + }); + + it('mvt rate limited', function (done) { + const tileParams = (status, limit, remaining, reset, retry, contentType) => { + let headers = { + "Content-Type": contentType, + "Carto-Rate-Limit-Limit": limit, + "Carto-Rate-Limit-Remaining": remaining, + "Carto-Rate-Limit-Reset": reset + }; + + if (retry) { + headers['Retry-After'] = retry; + } + + return { + layergroupid: layergroupid, + format: 'mvt', + response: {status, headers} + }; + }; + + testClient.getTile(0, 0, 0, tileParams(200, '1', '0', '1'), (err) => { + assert.ifError(err); + + testClient.getTile( + 0, + 0, + 0, + tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'), + (err, res, tile) => { + assert.ifError(err); + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + + done(); + } + ); + }); + + }); +} diff --git a/test/acceptance/regressions.js b/test/acceptance/regressions.js index 421df02f..21d9c970 100644 --- a/test/acceptance/regressions.js +++ b/test/acceptance/regressions.js @@ -1,7 +1,12 @@ -require('../support/test_helper'); +'use strict'; +require('../support/test_helper'); var assert = require('../support/assert'); +const helper = require('../support/test_helper'); var TestClient = require('../support/test-client'); +const LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); +const CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); +const serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); describe('regressions', function() { @@ -37,4 +42,175 @@ describe('regressions', function() { testClient.drain(done); }); }); + + // See: https://github.com/CartoDB/Windshaft-cartodb/pull/956 + it('"/user/localhost/api/v1/map" should create an anonymous map', function (done) { + const server = new CartodbWindshaft(serverOptions); + const layergroup = { + version: '1.7.0', + layers: [ + { + type: 'mapnik', + options: { + sql: TestClient.SQL.ONE_POINT, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + + const keysToDelete = {}; + + assert.response(server, + { + url: '/user/localhost/api/v1/map', + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + function(res, err) { + if (err) { + return done(err); + } + + const body = JSON.parse(res.body); + assert.ok(body.layergroupid); + + keysToDelete['map_cfg|' + LayergroupToken.parse(body.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + helper.deleteRedisKeys(keysToDelete, done); + } + ); + }); + + describe('map instantiation', function () { + const apikeyToken = 'regular1'; + const mapConfig = { + version: '1.7.0', + layers: [{ + type: 'cartodb', + options: { + sql: 'select * from test_table_localhost_regular1', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + }] + }; + + it('should have distint timestamps when the source was updated', function (done) { + const testClient = new TestClient(mapConfig, apikeyToken); + + testClient.getLayergroup({}, (err, layergroup) => { + if (err) { + return done(err); + } + + const { cacheBuster: cacheBusterA } = LayergroupToken.parse(layergroup.layergroupid); + + const conn = testClient.getDBConnection(); + + const sql = `select CDB_TableMetadataTouch('test_table_localhost_regular1'::regclass)`; + + conn.query(sql, (err) => { + if (err) { + return done(err); + } + + testClient.getLayergroup({}, (err, layergroup) => { + if (err) { + return done(err); + } + + const { cacheBuster: cacheBusterB } = LayergroupToken.parse(layergroup.layergroupid); + + const timestampA = parseInt(cacheBusterA, 10); + const timestampB = parseInt(cacheBusterB, 10); + + assert.notEqual(timestampA, timestampB); + assert.ok(timestampA < timestampB, `timestampA: ${timestampA} > timestampB:${timestampB}`); + + testClient.drain(done); + }); + }); + }); + }); + }); + + it('should create and instantiate a named map with filters', function (done) { + const apikeyToken = '1234'; + + const template = { + version: '0.0.1', + name: 'regression-dataview-filter-template', + placeholders: { + buffersize: { + type: 'number', + default: 0 + } + }, + layergroup: { + version: '1.6.0', + layers: [ + { + type: 'cartodb', + options: { + source: { + id: 'a1' + }, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ], + dataviews: { + country_places_count: { + source: { + id: 'a1' + }, + type: 'aggregation', + options: { + column: 'adm0_a3', + aggregation: 'count' + } + } + }, + analyses: [ + { + id: 'a1', + type: 'source', + params: { + query: 'select * from populated_places_simple_reduced' + } + } + ] + } + }; + + const testClient = new TestClient(template, apikeyToken); + testClient.keysToDelete['map_tpl|localhost'] = 0; + + const params = { + own_filter: 1, + filters: { + dataviews: { + country_places_count: { + accept: ['CAN'] + } + } + } + }; + + testClient.getDataview('country_places_count', params, (err, dataview) => { + assert.ifError(err); + + assert.equal(dataview.type, 'aggregation'); + assert.equal(dataview.categories.length, 1); + assert.deepEqual(dataview.categories[0], { value: 256, category: 'CAN', agg: false }); + + testClient.drain(done); + }); + }); }); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 6c293aa4..183d6bac 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('../support/assert'); @@ -6,11 +8,15 @@ var step = require('step'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); -server.setMaxListeners(0); describe('server', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + server.setMaxListeners(0); + }); // TODO: I guess this should be a 404 instead... it("get call to server returns 200", function(done){ @@ -54,6 +60,12 @@ describe('server', function() { }); describe('server old_api', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + server.setMaxListeners(0); + }); // See https://github.com/CartoDB/Windshaft-cartodb/issues/115 it.skip("get'ing tile with not-strictly-valid style", function(done) { @@ -68,5 +80,4 @@ describe('server old_api', function() { done(); }); }); - }); diff --git a/test/acceptance/special-numeric-values.js b/test/acceptance/special-numeric-values.js index 2b596b8e..6b328286 100644 --- a/test/acceptance/special-numeric-values.js +++ b/test/acceptance/special-numeric-values.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('../support/assert'); @@ -68,4 +70,3 @@ describe('special numeric values', function() { }); }); }); - diff --git a/test/acceptance/sql-wrap.js b/test/acceptance/sql-wrap.js index 7b3c25d8..7b2f9d57 100644 --- a/test/acceptance/sql-wrap.js +++ b/test/acceptance/sql-wrap.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('../support/assert'); diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index cdac7bb1..dde4f059 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -1,15 +1,34 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +const serverOptions = require('../../../lib/cartodb/server_options'); + +const suites = [ + { + desc: 'mvt (mapnik)', + usePostGIS: false + }, + { + desc: 'mvt (postgis)', + usePostGIS: true + } +]; + +suites.forEach(({desc, usePostGIS}) => { +describe(`[${desc}] Create mapnik layergroup`, function() { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; -describe('Create mapnik layergroup', function() { before(function() { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; this.layerStatsConfig = global.environment.enabledFeatures.layerStats; global.environment.enabledFeatures.layerStats = true; }); after(function() { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); @@ -49,7 +68,18 @@ describe('Create mapnik layergroup', function() { sql: [ 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address', ' from test_table t1, test_table_2 t2', - ' where t1.cartodb_id = t2.cartodb_id;' + ' where t1.cartodb_id = t2.cartodb_id' + ].join(''), + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer100 = { + type: 'mapnik', + options: { + sql: [ + 'SELECT * FROM test_table_100' ].join(''), cartocss_version: cartocssVersion, cartocss: cartocss @@ -74,6 +104,26 @@ describe('Create mapnik layergroup', function() { } }; + var mapnikLayerNullCats = { + type: 'mapnik', + options: { + sql: ` + WITH geom AS ( + SELECT + 'SRID=4326;POINT(0 0)'::geometry AS the_geom, + 'SRID=3857;POINT(0 0)'::geometry AS the_geom_webmercator + ) + SELECT 1 AS cartodb_id, 'A' As cat, geom.* FROM geom + UNION + SELECT 2 AS cartodb_id, 'B' As cat, geom.* FROM geom + UNION + SELECT 2 AS cartodb_id, NULL::text As cat, geom.* FROM geom + `, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + function mapnikBasicLayerId(index) { return 'layer' + index; } @@ -90,7 +140,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); @@ -107,7 +157,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -127,7 +177,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -147,7 +197,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); @@ -164,7 +214,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -183,7 +233,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); @@ -204,7 +254,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); @@ -224,7 +274,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function (err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); @@ -245,7 +295,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); @@ -265,7 +315,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); @@ -273,7 +323,313 @@ describe('Create mapnik layergroup', function() { assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats')); assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1)); assert.equal(layergroup.metadata.layers[2].type, 'http'); + global.environment.enabledFeatures.layerStats = true; testClient.drain(done); }); }); + + function layerWithMetadata(layer, metadata) { + return Object.assign(layer, { + options: Object.assign(layer.options, { metadata }) + }); + } + + it('should provide columns as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columns: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { type: 'number' }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { type: 'string' } + }; + assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns); + testClient.drain(done); + }); + }); + + // metadata categories are ordered only partially by descending frequency; + // this orders them completely to avoid ambiguities when comparing + function withSortedCategories(columns) { + function catOrder(a, b) { + if (a.frequency !== b.frequency) { + return b.frequency - a.frequency; + } + if (a.category < b.category) { + return -1; + } + if (a.category > b.category) { + return +1; + } + return 0; + } + let sorted = {}; + Object.keys(columns).forEach(name => { + let data = columns[name]; + if (data.hasOwnProperty('categories')) { + data = Object.assign(data, { categories: data.categories.sort(catOrder)}); + } + sorted[name] = data; + }); + return sorted; + } + + it('should provide column stats as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columnStats: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { + type: 'number', + avg: 3, + max: 5, + min: 1, + sum: 15 + }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { + type: 'string', + categories: [ + { + category: "Calle de la Palma 72, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle de Pérez Galdós 9, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle Divino Pastor 12, Madrid, Spain", + frequency: 1 + }, + { + category: "Manuel Fernández y González 8, Madrid, Spain", + frequency: 1 + }, + { + category: "Plaza Conde de Toreno 2, Madrid, Spain", + frequency: 1 + } + ] + } + }; + assert.deepEqual( + withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns), + withSortedCategories(expectedColumns) + ); + testClient.drain(done); + }); + }); + + it('should limit the number of categories as requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columnStats: { topCategories: 2 } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.address.categories.length, 2); + testClient.drain(done); + }); + }); + + it('should include null categories if requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayerNullCats, { + columnStats: { includeNulls: true } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.cat.categories.length, 3); + testClient.drain(done); + }); + }); + + it('should not include null categories if not requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayerNullCats, { + columnStats: { includeNulls: false } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.cat.categories.length, 2); + testClient.drain(done); + }); + }); + + it('should provide row count as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + testClient.drain(done); + }); + }); + + it('should provide geometry type as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + testClient.drain(done); + }); + }); + + it(`should not fail "TypeError: ... 'geom_type' of undefined" for empty results`, function(done) { + var testClient = new TestClient({ + version: '1.8.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table where false', + metadata: { + geometryType: true + } + } + } + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, undefined); + testClient.drain(done); + }); + }); + + it('should provide a sample as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer100, { + sample: { num_rows: 30 } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 100); + assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); + const expectedCols = [ 'cartodb_id', 'value', 'the_geom', 'the_geom_webmercator' ].sort(); + assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols); + testClient.drain(done); + }); + }); + + it('can specify sample columns', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer100, { + sample: { + num_rows: 30, + include_columns: [ 'cartodb_id', 'the_geom' ] + } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 100); + assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); + const expectedCols = [ 'cartodb_id', 'the_geom' ].sort(); + assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols); + testClient.drain(done); + }); + }); + + + it('should only provide requested optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true, + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.sample, undefined); + assert.equal(layergroup.metadata.layers[0].meta.stats.columns, undefined); + testClient.drain(done); + }); + }); + +}); }); diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js index ced63b0e..dcf9b402 100644 --- a/test/acceptance/stats/multilayer_stats.js +++ b/test/acceptance/stats/multilayer_stats.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 92a95f60..e8038347 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require('../support/assert'); var _ = require('underscore'); var redis = require('redis'); @@ -20,13 +22,18 @@ var helper = require(__dirname + '/../support/test_helper'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); -server.setMaxListeners(0); var LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); describe('template_api', function() { - server.layergroupAffectedTablesCache.cache.reset(); + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + server.setMaxListeners(0); + // FIXME: we need a better way to reset cache while running tests + server.layergroupAffectedTablesCache.cache.reset(); + }); var httpRendererResourcesServer; before(function(done) { @@ -300,16 +307,17 @@ describe('template_api', function() { assert.response(server, post_request, {}, function(res) { next(null, res); }); }, function testCORS() { - assert.response(server, { - url: '/api/v1/map/named/acceptance1', - method: 'OPTIONS' - },{ - status: 200, - headers: { - 'Access-Control-Allow-Headers': 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Content-Type', - 'Access-Control-Allow-Origin': '*' - } - }, function() { done(); }); + const allowHeaders = 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization, Content-Type'; + assert.response(server, { + url: '/api/v1/map/named/acceptance1', + method: 'OPTIONS' + },{ + status: 200, + headers: { + 'Access-Control-Allow-Headers': allowHeaders, + 'Access-Control-Allow-Origin': '*' + } + }, function() { done(); }); }); }); @@ -1056,6 +1064,7 @@ describe('template_api', function() { assert.ok(cc); assert.equal(cc, expectedCC); // hack simulating restart... + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache var get_request = { url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1', diff --git a/test/acceptance/tilejson.js b/test/acceptance/tilejson.js index 0d3bc133..19ef158f 100644 --- a/test/acceptance/tilejson.js +++ b/test/acceptance/tilejson.js @@ -1,9 +1,25 @@ +'use strict'; + require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); -describe('tilejson', function() { +const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + +describe('tilejson via mapnik renderer', () => tileJsonSuite(false)); +describe('tilejson via postgis renderer', () => tileJsonSuite(true)); + +function tileJsonSuite(usePostGIS) { + + before(function() { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); function tilejsonValidation(tilejson, shouldHaveGrid = false) { assert.equal(tilejson.tilejson, '2.2.0'); @@ -215,5 +231,4 @@ describe('tilejson', function() { }); }); - -}); +} diff --git a/test/acceptance/turbo-carto/anonymous-maps.js b/test/acceptance/turbo-carto/anonymous-maps.js index c0fcbda4..80680192 100644 --- a/test/acceptance/turbo-carto/anonymous-maps.js +++ b/test/acceptance/turbo-carto/anonymous-maps.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); @@ -21,13 +23,13 @@ function makeMapconfig(cartocss) { "sql": [ 'SELECT test_table.*, _prices.price FROM test_table JOIN (' + ' SELECT 1 AS cartodb_id, 10.00 AS price', - ' UNION', + ' UNION ALL', ' SELECT 2, 10.50', - ' UNION', + ' UNION ALL', ' SELECT 3, 11.00', - ' UNION', + ' UNION ALL', ' SELECT 4, 12.00', - ' UNION', + ' UNION ALL', ' SELECT 5, 21.00', ') _prices ON _prices.cartodb_id = test_table.cartodb_id' ].join('\n'), diff --git a/test/acceptance/turbo-carto/error-cases.js b/test/acceptance/turbo-carto/error-cases.js index b6424c72..94b07fe0 100644 --- a/test/acceptance/turbo-carto/error-cases.js +++ b/test/acceptance/turbo-carto/error-cases.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/turbo-carto/named-maps.js b/test/acceptance/turbo-carto/named-maps.js index 063118aa..6a302498 100644 --- a/test/acceptance/turbo-carto/named-maps.js +++ b/test/acceptance/turbo-carto/named-maps.js @@ -1,14 +1,20 @@ +'use strict'; + var assert = require('../../support/assert'); var step = require('step'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); var testHelper = require('../../support/test_helper'); var CartodbWindshaft = require('../../../lib/cartodb/server'); var serverOptions = require('../../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var mapnik = require('windshaft').mapnik; var IMAGE_TOLERANCE_PER_MIL = 10; describe('turbo-carto for named maps', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); var keysToDelete; @@ -41,13 +47,13 @@ describe('turbo-carto for named maps', function() { sql: [ 'SELECT ' + table + '.*, _prices.price FROM ' + table + ' JOIN (' + ' SELECT 1 AS cartodb_id, 10.00 AS price', - ' UNION', + ' UNION ALL', ' SELECT 2, 10.50', - ' UNION', + ' UNION ALL', ' SELECT 3, 11.00', - ' UNION', + ' UNION ALL', ' SELECT 4, 12.00', - ' UNION', + ' UNION ALL', ' SELECT 5, 21.00', ') _prices ON _prices.cartodb_id = ' + table + '.cartodb_id' ].join('\n'), diff --git a/test/acceptance/turbo-carto/regressions.js b/test/acceptance/turbo-carto/regressions.js index 9e2d377f..40dec6de 100644 --- a/test/acceptance/turbo-carto/regressions.js +++ b/test/acceptance/turbo-carto/regressions.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -88,7 +90,7 @@ describe('turbo-carto regressions', function() { var turboCartoError = layergroup.errors_with_context[0]; assert.ok(turboCartoError); assert.equal(turboCartoError.type, 'layer'); - assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\srelation\stest_table_private_1/)); + assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\s.+?test_table_private_1/)); done(); }); @@ -380,4 +382,123 @@ describe('turbo-carto regressions', function() { }); }); + describe('Buckets calculation', function () { + afterEach(function (done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + const scenarios = [ + { + numBuckets: 1, + bucketResponse: [ + { + filter: { + type: 'range', + start: 0, + end: 8 + }, + value: 1 + } + ], + }, + { + numBuckets: 2, + bucketResponse: [ + { + filter: { + type: 'range', + start: 0, + end: 3 + }, + value: 1 + }, + { + filter: { + type: 'range', + start: 3, + end: 8 + }, + value: 20 + } + ], + }, + { + numBuckets: 3, + bucketResponse: [ + { + filter: { + type: 'range', + start: 0, + end: 2 + }, + value: 1 + }, + { + filter: { + type: 'range', + start: 2, + end: 5 + }, + value: 10.5 + }, + { + filter: { + type: 'range', + start: 5, + end: 8 + }, + value: 20 + } + ], + }, + ]; + + scenarios.forEach(function (scenario) { + it('Buckets: ' + scenario.numBuckets, function (done) { + const bucketsMapConfig = makeMapconfig({ numQuantiles: scenario.numBuckets }); + + this.testClient = new TestClient(bucketsMapConfig); + this.testClient.getLayergroup({ response: OK_RESPONSE }, function (err, layergroup) { + const rule = layergroup.metadata.layers[0].meta.cartocss_meta.rules[0]; + + assert.ok(!err, err); + assert.equal(rule.buckets.length, scenario.numBuckets); + assert.deepEqual(rule.buckets, scenario.bucketResponse); + + done(); + }); + }); + }); + + function makeMapconfig({numQuantiles = 1}) { + return { + "version": "1.4.0", + "layers": [ + { + "type": 'mapnik', + "options": { + "cartocss_version": '2.3.0', + "sql": 'SELECT * FROM populated_places_simple_reduced', + "cartocss": `#layer {\n + marker-width: ramp([labelrank], range(1, 20), quantiles(${numQuantiles}));\n + marker-fill: #EE4D5A;\n marker-fill-opacity: 0.9;\n + marker-allow-overlap: true;\n marker-line-width: 1;\n + marker-line-color: #FFFFFF;\n marker-line-opacity: 1;\n}`, + } + } + ] + }; + } + + const OK_RESPONSE = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + }); }); diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 52ead026..629d3778 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -1,7 +1,10 @@ +'use strict'; + require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; @@ -21,11 +24,23 @@ const validationPointSleepSql = ` 2 val `; +const cartoCSSPoints = () => ` +// cache buster: ${Date.now()} +#layer{ + marker-placement: point; + marker-allow-overlap: true; + marker-line-opacity: 0.2; + marker-line-width: 0.5; + marker-opacity: 1; + marker-width: 5; + marker-fill: red; +}`; + const createMapConfig = ({ version = '1.6.0', type = 'cartodb', sql = pointSleepSql, - cartocss = TestClient.CARTOCSS.POINTS, + cartocss = cartoCSSPoints(), cartocss_version = '2.3.0', interactivity = 'cartodb_id', countBy = 'cartodb_id', @@ -67,12 +82,15 @@ const createMapConfig = ({ } }); +const db_limit_error_message = 'You are over platform\'s limits: SQL query timeout error.' + + ' Refactor your query before running again or contact CARTO support for more details.'; + const DATASOURCE_TIMEOUT_ERROR = { - errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors: [db_limit_error_message], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details' + message: db_limit_error_message }] }; @@ -140,11 +158,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors: [db_limit_error_message ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details', + message: db_limit_error_message, layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -362,11 +380,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors: [ db_limit_error_message ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details', + message: db_limit_error_message, layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -376,7 +394,21 @@ describe('user database timeout limit', function () { }); }); - describe('fetching vector tiles', function () { + describe('fetching vector tiles via mapnik renderer', () => testFetchingVectorTiles(false)); + describe('fetching vector tiles via postgis renderer', () => testFetchingVectorTiles(true)); + + function testFetchingVectorTiles(usePostGIS) { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + beforeEach(function (done) { const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); @@ -438,7 +470,7 @@ describe('user database timeout limit', function () { }); }); - }); + } }); @@ -469,11 +501,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors: [ db_limit_error_message ], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details', + message: db_limit_error_message, layer: { id: 'layer0', index: 0, type: 'mapnik' } }] }); @@ -573,11 +605,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors: [db_limit_error_message], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details', + message: db_limit_error_message, layer: { id: 'torque-layer0', index: 0, type: 'torque' } }] }); @@ -704,11 +736,11 @@ describe('user database timeout limit', function () { this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { - errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], + errors: [db_limit_error_message], errors_with_context: [{ type: 'limit', subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details', + message: db_limit_error_message, layer: { id: 'layer0', index: 0, diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 93770080..b5722dcf 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); const assert = require('../support/assert'); @@ -23,6 +25,9 @@ const validationPointSleepSql = ` 2 val `; +const renderTimeoutErrorMessage = 'You are over platform\'s limits: Render timeout error.' + + ' Contact CARTO support for more details.'; + const createMapConfig = ({ version = '1.6.0', type = 'cartodb', @@ -96,11 +101,11 @@ describe('user render timeout limit', function () { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform\'s limits. Please contact us to know more details"], + errors: [renderTimeoutErrorMessage], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform\'s limits. Please contact us to know more details", + message: renderTimeoutErrorMessage, layer: { id: "layer0", index: 0, @@ -139,7 +144,7 @@ describe('user render timeout limit', function () { }); it('layergroup creation works but tile request fails due to render timeout', function (done) { - this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { + this.testClient.getTile(0, 0, 0, { cacheBuster: true }, (err, res, tile) => { assert.ifError(err); assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { @@ -180,18 +185,19 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform\'s limits. Please contact us to know more details"], + errors: [renderTimeoutErrorMessage], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform\'s limits. Please contact us to know more details" + message: renderTimeoutErrorMessage }] }); @@ -201,56 +207,52 @@ describe('user render timeout limit', function () { }); }); - if (process.env.POSTGIS_VERSION === '2.4') { - describe('vector (PostGIS)', vector(true)); - } - - describe('vector (mapnik)', vector(false)); - function vector(usePostGIS) { + describe('vector tile via mapnik renderer', function () { + const usePostGIS = false; const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; - return function () { - beforeEach(function (done) { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserDatabaseTimeoutLimit(50, done); - }); - afterEach(function (done) { - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { - if (err) { - return done(err); + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + response: { + status: 429, + headers: { + 'Content-Type': 'application/x-protobuf' } - this.testClient.drain(done); - }); + }, + cacheBuster: true + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + + done(); }); - - it('layergroup creation works but vector tile request fails due to render timeout', function (done) { - const params = { - format: 'mvt', - response: { - status: 429, - headers: { - 'Content-Type': 'application/x-protobuf' - } - } - }; - - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { - assert.ifError(err); - - var tileJSON = tile.toJSON(); - assert.equal(Array.isArray(tileJSON), true); - assert.equal(tileJSON.length, 2); - assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); - assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); - - done(); - }); - }); - }; - } + }); + }); describe('interativity', function () { beforeEach(function (done) { @@ -277,18 +279,19 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); assert.deepEqual(tile, { - errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors: [renderTimeoutErrorMessage], errors_with_context: [{ type: 'limit', subtype: 'render', - message: 'You are over platform\'s limits. Please contact us to know more details' + message: renderTimeoutErrorMessage }] }); @@ -365,7 +368,7 @@ describe('user render timeout limit', function () { }); }); - it('layergroup creation works and render tile fails', function (done) { + it('layergroup creation works and render static center tile fails', function (done) { const params = { zoom: 0, lat: 0, @@ -378,18 +381,19 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getStaticCenter(params, function (err, res, timeoutError) { assert.ifError(err); assert.deepEqual(timeoutError, { - errors: ["You are over platform\'s limits. Please contact us to know more details"], + errors: [renderTimeoutErrorMessage], errors_with_context: [{ type: 'limit', subtype: 'render', - message: "You are over platform\'s limits. Please contact us to know more details" + message: renderTimeoutErrorMessage }] }); diff --git a/test/acceptance/vector-layergroup.js b/test/acceptance/vector-layergroup.js index 0778453e..d23403db 100644 --- a/test/acceptance/vector-layergroup.js +++ b/test/acceptance/vector-layergroup.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); const assert = require('../support/assert'); @@ -67,17 +69,16 @@ const INVALID_FORMAT_ERROR = { ] }; -const suites = [{ - desc: 'mvt (mapnik)', - usePostGIS: false -}]; - -if (process.env.POSTGIS_VERSION === '2.4') { - suites.push({ +const suites = [ + { + desc: 'mvt (mapnik)', + usePostGIS: false + }, + { desc: 'mvt (postgis)', usePostGIS: true - }); -} + } +]; suites.forEach((suite) => { const { desc, usePostGIS } = suite; diff --git a/test/acceptance/widgets/aggregation.js b/test/acceptance/widgets/aggregation.js index 0b58fb65..8977ef19 100644 --- a/test/acceptance/widgets/aggregation.js +++ b/test/acceptance/widgets/aggregation.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/widgets/histogram.js b/test/acceptance/widgets/histogram.js index a5b7a642..503218e9 100644 --- a/test/acceptance/widgets/histogram.js +++ b/test/acceptance/widgets/histogram.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/acceptance/widgets/named-maps.js b/test/acceptance/widgets/named-maps.js index e54ced23..ab136dc0 100644 --- a/test/acceptance/widgets/named-maps.js +++ b/test/acceptance/widgets/named-maps.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require('../../support/assert'); var step = require('step'); @@ -8,11 +10,15 @@ var helper = require('../../support/test_helper'); var CartodbWindshaft = require('../../../lib/cartodb/server'); var serverOptions = require('../../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('named-maps widgets', function() { + var server; + + before(function () { + server = new CartodbWindshaft(serverOptions); + }); var username = 'localhost'; var widgetsTemplateName = 'widgets-template'; diff --git a/test/acceptance/widgets/ported/aggregation.js b/test/acceptance/widgets/ported/aggregation.js index 62c998a5..2f36d3bb 100644 --- a/test/acceptance/widgets/ported/aggregation.js +++ b/test/acceptance/widgets/ported/aggregation.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../support/test_helper'); var assert = require('../../../support/assert'); diff --git a/test/acceptance/widgets/ported/formula.js b/test/acceptance/widgets/ported/formula.js index 17d5c287..c3ca2bc6 100644 --- a/test/acceptance/widgets/ported/formula.js +++ b/test/acceptance/widgets/ported/formula.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../support/test_helper'); var assert = require('../../../support/assert'); diff --git a/test/acceptance/widgets/ported/histogram.js b/test/acceptance/widgets/ported/histogram.js index 90dfe840..510feb9e 100644 --- a/test/acceptance/widgets/ported/histogram.js +++ b/test/acceptance/widgets/ported/histogram.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../support/test_helper'); var assert = require('../../../support/assert'); diff --git a/test/acceptance/widgets/regressions.js b/test/acceptance/widgets/regressions.js index 74f4544c..755e2a96 100644 --- a/test/acceptance/widgets/regressions.js +++ b/test/acceptance/widgets/regressions.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); @@ -260,17 +262,17 @@ describe('widgets-regressions', function() { ]] }; - const query = ` + const query = ` SELECT ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( '${JSON.stringify(notIntersectingLeftTriangle)}' ), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id, 'notIntersectingLeftTriangle' AS name - UNION + UNION ALL SELECT ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( '${JSON.stringify(notIntersectingRightTriangle)}' ), 4326), 3857), 2, 'notIntersectingRightTriangle' - UNION + UNION ALL SELECT ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON( '${JSON.stringify(intersectingTriangle)}' diff --git a/test/acceptance/widgets/widgets.js b/test/acceptance/widgets/widgets.js index 6fee7a29..94ecb0a2 100644 --- a/test/acceptance/widgets/widgets.js +++ b/test/acceptance/widgets/widgets.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('../../support/assert'); diff --git a/test/fixtures/analysis/basic-source-id-mapnik-layer.png b/test/fixtures/analysis/basic-source-id-mapnik-layer.png index bdb315a9..41ee8fd5 100644 Binary files a/test/fixtures/analysis/basic-source-id-mapnik-layer.png and b/test/fixtures/analysis/basic-source-id-mapnik-layer.png differ diff --git a/test/fixtures/test_mapconfigFactory.js b/test/fixtures/test_mapconfigFactory.js new file mode 100644 index 00000000..f1af550c --- /dev/null +++ b/test/fixtures/test_mapconfigFactory.js @@ -0,0 +1,64 @@ +function getVectorMapConfig(opts) { + return { + buffersize: { + mvt: 1 + }, + layers: _generateLayers(opts), + }; +} + +function _generateLayers(opts) { + const numberOfLayers = opts.numberOfLayers || 1; + const layers = []; + for (let index = 0; index < numberOfLayers; index++) { + const layerOptions = (opts.layerOptions || {})[index] || {}; + layers.push(_generateLayerConfig(layerOptions)); + } + return layers; +} + +function _generateLayerConfig(opts) { + const additionalColumns = opts.additionalColumns ? opts.additionalColumns.join(',')+',' : ''; + return { + type: 'mapnik', + options: { + sql: ` + SELECT + ${additionalColumns} + (DATE '2018-06-01' + x) as date, + x as cartodb_id, + st_makepoint(x * 10, x * 10) as the_geom, + st_makepoint(x * 10, x * 10) as the_geom_webmercator + FROM + generate_series(0, 1) x`, + aggregation: { + columns: {}, + dimensions: { + date: 'date' + }, + placement: 'centroid', + resolution: 1, + threshold: 1 + }, + dates_as_numbers: opts.dates_as_numbers, + metadata: { + geometryType: true, + columnStats: { + topCategories: 32768, + includeNulls: true + }, + sample: { + num_rows: 1000, + include_columns: [ + 'date' + ] + } + } + } + }; +} + + + + +module.exports = { getVectorMapConfig }; \ No newline at end of file diff --git a/test/integration/analysis-backend-limits.js b/test/integration/analysis-backend-limits.js index 720bdd30..21e7d9cc 100644 --- a/test/integration/analysis-backend-limits.js +++ b/test/integration/analysis-backend-limits.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../support/test_helper'); var assert = require('assert'); diff --git a/test/integration/mapconfig_named_layers_datasource.js b/test/integration/mapconfig_named_layers_datasource.js index 48f0b65d..d3eefd28 100644 --- a/test/integration/mapconfig_named_layers_datasource.js +++ b/test/integration/mapconfig_named_layers_datasource.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); diff --git a/test/integration/mapconfig_named_layers_expanded.js b/test/integration/mapconfig_named_layers_expanded.js index fd542514..ea3a34a4 100644 --- a/test/integration/mapconfig_named_layers_expanded.js +++ b/test/integration/mapconfig_named_layers_expanded.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js index 87a87935..64a2d0c9 100644 --- a/test/integration/mapconfig_overviews_adapter.js +++ b/test/integration/mapconfig_overviews_adapter.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); @@ -5,18 +7,18 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); -var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api'); +var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata'); +var FilterStatsBackend = require('../../lib/cartodb/backends/filter-stats'); var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter'); var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); -var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); -var filterStatsApi = new FilterStatsApi(pgQueryRunner); +var overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); +var filterStatsBackend = new FilterStatsBackend(pgQueryRunner); -var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi); +var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend); describe('MapConfigOverviewsAdapter', function() { diff --git a/test/integration/overviews-metadata-api.js b/test/integration/overviews-metadata-api.js index 4d015870..c3e241ec 100644 --- a/test/integration/overviews-metadata-api.js +++ b/test/integration/overviews-metadata-api.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); @@ -7,24 +9,24 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); +var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata'); -describe('OverviewsMetadataApi', function() { +describe('OverviewsMetadataBackend', function() { - var overviewsMetadataApi; + var overviewsMetadataBackend; before(function() { var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); - overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); }); it('should return an empty relation for tables that have no overviews', function(done) { var query = 'select * from test_table'; - overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) { assert.ok(!err, err); assert.deepEqual(result, {}); @@ -35,7 +37,7 @@ describe('OverviewsMetadataApi', function() { it('should return overviews metadata', function(done) { var query = 'select * from test_table_overviews'; - overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) { assert.ok(!err, err); assert.deepEqual(result, { diff --git a/test/integration/pg-query-runner.js b/test/integration/pg-query-runner.js index 9cb9fcbe..a979ee81 100644 --- a/test/integration/pg-query-runner.js +++ b/test/integration/pg-query-runner.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); diff --git a/test/integration/profiler.js b/test/integration/profiler.js index 496a9d72..4bc7ee0e 100644 --- a/test/integration/profiler.js +++ b/test/integration/profiler.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); diff --git a/test/integration/query-tables.js b/test/integration/query-tables.js index 9cd76000..6ce50ce7 100644 --- a/test/integration/query-tables.js +++ b/test/integration/query-tables.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); @@ -68,4 +70,4 @@ describe('QueryTables', function() { }); }); -}); \ No newline at end of file +}); diff --git a/test/integration/template-maps-limits.js b/test/integration/template-maps-limits.js index 12150c68..37f3ca9f 100644 --- a/test/integration/template-maps-limits.js +++ b/test/integration/template-maps-limits.js @@ -1,3 +1,5 @@ +'use strict'; + require('../support/test_helper'); var assert = require('assert'); diff --git a/test/support/assert.js b/test/support/assert.js index 6815a63f..1d449ea2 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -1,3 +1,5 @@ +'use strict'; + // Cribbed from the ever prolific Konstantin Kaefer // https://github.com/mapbox/tilelive-mapnik/blob/master/test/support/assert.js @@ -87,30 +89,18 @@ assert.response = function(server, req, res, callback) { res = {}; } - var port = 5555, + var port = 0, // let the OS to choose a free port host = '127.0.0.1'; - var listeningAttempts = 0; - var listener; - function listen() { - if (listeningAttempts > 25) { - return callback(null, new Error('Tried too many ports')); - } - listener = server.listen(port, host); - listener.on('error', function() { - port++; - listeningAttempts++; - listen(); - }); - listener.on('listening', onServerListening); - } + var listener = server.listen(port, host); + listener.on('error', callback); + listener.on('listening', function onServerListening () { + // jshint maxcomplexity:9 + const { address: host, port } = listener.address(); + const address = `${host}:${port}`; - listen(); - - // jshint maxcomplexity:9 - function onServerListening() { var requestParams = { - url: 'http://' + host + ':' + port + req.url, + url: 'http://' + address + req.url, method: req.method || 'GET', headers: req.headers || {}, timeout: req.timeout || 0, @@ -122,14 +112,17 @@ assert.response = function(server, req, res, callback) { } request(requestParams, function assert$response$requestHandler(error, response, body) { - listener.close(function() { + listener.close(function () { + if (error) { + return callback(null, error); + } + response.body = response.body || body; var err = validateResponse(response, res); return callback(response, err); }); }); - - } + }); }; function validateResponseBody(response, expected) { @@ -149,16 +142,18 @@ function validateResponseBody(response, expected) { function validateResponseStatus(response, expected) { var status = expected.status || expected.statusCode; + const message = colorize('[red]{Invalid response status code.}\n' + + ' Expected: [green]{' + status + '}\n' + + ' Got: [red]{' + response.statusCode + '}\n' + + ' Body: ' + response.body); + // Assert response status - if (typeof status === 'number') { - if (response.statusCode !== status) { - return new Error(colorize( - '[red]{Invalid response status code.}\n' + - ' Expected: [green]{' + status + '}\n' + - ' Got: [red]{' + response.statusCode + '}\n' + - ' Body: ' + response.body) - ); - } + if (typeof status === 'number' && response.statusCode !== status) { + return new Error(message); + } + + if (Array.isArray(status) && !status.includes(response.statusCode)) { + return new Error(message); } } diff --git a/test/support/libredis_cell.dylib b/test/support/libredis_cell.dylib new file mode 100755 index 00000000..7c7d6c88 Binary files /dev/null and b/test/support/libredis_cell.dylib differ diff --git a/test/support/libredis_cell.so b/test/support/libredis_cell.so new file mode 100755 index 00000000..1da8aa66 Binary files /dev/null and b/test/support/libredis_cell.so differ diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 03d97fc7..5989b498 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -16,15 +16,18 @@ DOWNLOAD_SQL_FILES=yes PG_PARALLEL=$(pg_config --version | (awk '{$2*=1000; if ($2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) while [ -n "$1" ]; do - if test "$1" = "--skip-pg"; then + OPTION=$(echo "$1" | tr -d '[:space:]') + if [[ "$OPTION" == "--skip-pg" ]]; then PREPARE_PGSQL=no shift; continue - elif test "$1" = "--skip-redis"; then + elif [[ "$OPTION" == "--skip-redis" ]]; then PREPARE_REDIS=no shift; continue - elif test "$1" = "--no-sql-download"; then + elif [[ "$OPTION" == "--no-sql-download" ]]; then DOWNLOAD_SQL_FILES=no shift; continue + else + shift; continue; fi done @@ -76,15 +79,15 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" - LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' + LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 countries_null_values ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount CDB_RectangleGrid' - CURL_ARGS="" - for i in ${REMOTE_SQL_SCRIPTS} - do - CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql " - done if test x"$DOWNLOAD_SQL_FILES" = xyes; then + CURL_ARGS="" + for i in ${REMOTE_SQL_SCRIPTS} + do + CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql " + done echo "Downloading and updating: ${REMOTE_SQL_SCRIPTS}" echo ${CURL_ARGS} | xargs curl -L -s fi @@ -93,13 +96,6 @@ if test x"$PREPARE_PGSQL" = xyes; then ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}" for i in ${ALL_SQL_SCRIPTS} do - # Strip PARALLEL labels for PostgreSQL releases before 9.6 - if [ $PG_PARALLEL -eq 0 ]; then - TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXXXX) - sed -e 's/PARALLEL \= [A-Z]*,/''/g' \ - -e 's/PARALLEL [A-Z]*/''/g' sql/$i.sql > $TMPFILE - mv $TMPFILE sql/$i.sql - fi cat sql/${i}.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" | sed "s/:PUBLICUSER/${PUBLICUSER}/" | @@ -131,6 +127,7 @@ HMSET rails:users:cartodb250user id ${TESTUSERID} \ map_key 4321 EOF + cat < 0) { url += '?' + qs.stringify(extraParams); } @@ -370,17 +380,73 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { }; step( - function createLayergroup() { + function createTemplate () { var next = this; + + if (!self.template) { + return next(); + } + + if (!self.apiKey) { + return next(new Error('apiKey param is mandatory to create a new template')); + } + + params.placeholders = params.placeholders || {}; + assert.response(self.server, { - url: url, + url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), method: 'POST', headers: { host: 'localhost', 'Content-Type': 'application/json' }, - data: JSON.stringify(self.mapConfig) + data: JSON.stringify(self.template) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function (res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).template_id); + } + ); + }, + function createLayergroup(err, templateId) { + assert.ifError(err); + + var next = this; + + var data = templateId ? params.placeholders : self.mapConfig; + + const queryParams = {}; + + if (self.apiKey) { + queryParams.api_key = self.apiKey; + } + + if (params.filters !== undefined) { + queryParams.filters = JSON.stringify(params.filters); + } + + var path = templateId ? + urlNamed + '/' + templateId + '?' + qs.stringify(queryParams) : + url; + + assert.response(self.server, + { + url: path, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(data) }, { status: 200, @@ -414,7 +480,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { var urlParams = {}; if (params.hasOwnProperty('no_filters')) { urlParams.no_filters = params.no_filters; - } + } if (params.hasOwnProperty('own_filter')) { urlParams.own_filter = params.own_filter; } @@ -443,16 +509,15 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { if (err) { return next(err); } - - next(null, JSON.parse(res.body)); + next(null, JSON.parse(res.body), res.headers); } ); }, - function finish(err, dataview) { + function finish(err, dataview, headers = null) { if (err) { return callback(err); } - return callback(null, dataview); + return callback(null, dataview, headers); } ); }; @@ -668,7 +733,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = '/api/v1/map/' + layergroupId + '/'; + url = `/api/v1/map/${layergroupidTemplate(layergroupId, params)}/`; var layers = params.layers; @@ -711,7 +776,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }, params.response); - var isPng = format.match(/png$/); if (isPng) { @@ -847,7 +911,7 @@ TestClient.prototype.getLayergroup = function (params, callback) { TestClient.prototype.getStaticCenter = function (params, callback) { var self = this; - let { layergroupid, z, lat, lng, width, height, format } = params; + let { layergroupid, zoom, lat, lng, width, height, format } = params; var url = `/api/v1/map/`; @@ -896,7 +960,9 @@ TestClient.prototype.getStaticCenter = function (params, callback) { self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = `/api/v1/map/static/center/${layergroupId}/${z}/${lat}/${lng}/${width}/${height}.${format}`; + const layergroupid = layergroupidTemplate(layergroupId, params); + + url = `/api/v1/map/static/center/${layergroupid}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`; if (self.apiKey) { url += '?' + qs.stringify({api_key: self.apiKey}); @@ -1183,14 +1249,29 @@ TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimi helper.configureMetadata('hmset', params, callback); }; +TestClient.prototype.getDBConnection = function () { + const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; + + const psql = new PSQL({ + user: 'postgres', + dbname: dbname, + host: global.environment.postgres.host, + port: global.environment.postgres.port + }); + + return psql; +}; + TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) { const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }); const publicuser = global.environment.postgres.user; - // we need to guarantee all new connections have the new settings - helper.cleanPGPoolConnections(); - + // IMPORTANT: node-postgres uses internallly a singleton, to refresh all pull connections + // you need to ensure that your dependency tree has only one working version of `cartodb-psql` & `node-postgres` + // if not, test using this function cannot ensure that all connections have the new settings (STATEMENT_TIMEOUT) + // + // TODO: upgrade to node-postgres@7.x const psql = new PSQL({ user: 'postgres', dbname: dbname, @@ -1210,7 +1291,10 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb timeoutSQLs.forEach(sql => psql.query(sql, group())); }, - callback + // we need to guarantee all new connections have the new settings + function refreshPoolConnection () { + psql.end(() => callback()); + } ); }; @@ -1253,3 +1337,276 @@ TestClient.prototype.getAnalysesCatalog = function (params, callback) { } ); }; + +TestClient.prototype.getNamedMapList = function(params, callback) { + const request = { + url: `/api/v1/map/named?${qs.stringify({ api_key: this.apiKey })}`, + method: 'GET', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } + }; + + let expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + if (params.response) { + expectedResponse = Object.assign(expectedResponse, params.response); + } + + assert.response(this.server, request, expectedResponse, (res, err) => { + if (err) { + return callback(err); + } + const body = JSON.parse(res.body); + return callback(null, res, body); + }); +}; + +TestClient.prototype.getNamedTile = function (name, z, x, y, format, options, callback) { + const { params } = options; + + if (!this.apiKey) { + return callback(new Error('apiKey param is mandatory to create a new template')); + } + + const createTemplateRequest = { + url: `/api/v1/map/named?${qs.stringify({ api_key: this.apiKey })}`, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(this.template) + }; + + const createTemplateResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + assert.response(this.server, createTemplateRequest, createTemplateResponse, (res, err) => { + if (err) { + return callback(err); + } + + const templateId = JSON.parse(res.body).template_id; + const queryParams = params ? `?${qs.stringify(params)}` : ''; + const url = `/api/v1/map/named/${templateId}/all/${[z,x,y].join('/')}.${format}${queryParams}`; + const namedTileRequest = { + url, + method: 'GET', + headers: { + host: 'localhost' + }, + encoding: 'binary' + }; + + let contentType; + switch (format) { + case 'png': + contentType = 'image/png'; + break; + case 'mvt': + contentType = 'application/x-protobuf'; + break; + default: + contentType = 'application/json'; + break; + } + + const namedTileResponse = Object.assign({ + status: 200, + headers: { + 'content-type': contentType + } + }, options.response); + + assert.response(this.server, namedTileRequest, namedTileResponse, (res, err) => { + let body; + switch (res.headers['content-type']) { + case 'image/png': + body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary')); + break; + case 'application/x-protobuf': + body = new mapnik.VectorTile(z, x, y); + body.setDataSync(new Buffer(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body; + break; + } + + return callback(err, res, body); + }); + }); +}; + +TestClient.prototype.createTemplate = function (params, callback) { + if (!this.apiKey) { + return callback(new Error('apiKey param is mandatory to create a new template')); + } + + const createTemplateRequest = { + url: `/api/v1/map/named?${qs.stringify({ api_key: this.apiKey })}`, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(this.template) + }; + + let createTemplateResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + if (params.response) { + createTemplateResponse = Object.assign(createTemplateResponse, params.response); + } + + assert.response(this.server, createTemplateRequest, createTemplateResponse, (res, err) => { + let body; + switch (res.headers['content-type']) { + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body; + break; + } + + return callback(err, res, body); + }); +}; + +TestClient.prototype.deleteTemplate = function (params, callback) { + if (!this.apiKey) { + return callback(new Error('apiKey param is mandatory to create a new template')); + } + + const deleteTemplateRequest = { + url: `/api/v1/map/named/${params.templateId}?${qs.stringify({ api_key: this.apiKey })}`, + method: 'DELETE', + headers: { + host: 'localhost', + } + }; + + let deleteTemplateResponse = { + status: 204, + headers: {} + }; + + if (params.response) { + deleteTemplateResponse = Object.assign(deleteTemplateResponse, params.response); + } + + assert.response(this.server, deleteTemplateRequest, deleteTemplateResponse, (res, err) => { + let body; + switch (res.headers['content-type']) { + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body; + break; + } + + return callback(err, res, body); + }); +}; + +TestClient.prototype.updateTemplate = function (params, callback) { + if (!this.apiKey) { + return callback(new Error('apiKey param is mandatory to create a new template')); + } + + const updateTemplateRequest = { + url: `/api/v1/map/named/${params.templateId}?${qs.stringify({ api_key: this.apiKey })}`, + method: 'PUT', + headers: { + host: 'localhost', + 'Content-Type': 'application/json; charset=utf-8' + }, + data: JSON.stringify(params.templateData) + }; + + let updateTemplateResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + if (params.response) { + updateTemplateResponse = Object.assign(updateTemplateResponse, params.response); + } + + assert.response(this.server, updateTemplateRequest, updateTemplateResponse, (res, err) => { + let body; + switch (res.headers['content-type']) { + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body; + break; + } + + return callback(err, res, body); + }); +}; + + +TestClient.prototype.getTemplate = function (params, callback) { + if (!this.apiKey) { + return callback(new Error('apiKey param is mandatory to create a new template')); + } + + const getTemplateRequest = { + url: `/api/v1/map/named/${params.templateId}?${qs.stringify({ api_key: this.apiKey })}`, + method: 'GET', + headers: { + host: 'localhost' + } + }; + + let getTemplateResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + if (params.response) { + getTemplateResponse = Object.assign(getTemplateResponse, params.response); + } + + assert.response(this.server, getTemplateRequest, getTemplateResponse, (res, err) => { + let body; + switch (res.headers['content-type']) { + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body; + break; + } + + return callback(err, res, body); + }); +}; diff --git a/test/support/test_helper.js b/test/support/test_helper.js index 7d5c6a7c..55fad97a 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -1,3 +1,5 @@ +'use strict'; + /** * User: simon * Date: 30/08/2011 @@ -12,9 +14,7 @@ var LZMA = require('lzma').LZMA; var lzmaWorker = new LZMA(); var redis = require('redis'); -var nock = require('nock'); var log4js = require('log4js'); -var pg = require('pg'); const setICUEnvVariable = require('../../lib/cartodb/utils/icu_data_env_setter'); // set environment specific variables @@ -104,16 +104,18 @@ beforeEach(function() { //global afterEach to capture test suites that leave keys in redis afterEach(function(done) { - // restoring nock globally after each suite - nock.cleanAll(); - nock.enableNetConnect(); - var expectedKeys = { 'rails:test_windshaft_cartodb_user_1_db:test_table_private_1': true, 'rails:test_windshaft_cartodb_user_1_db:my_table': true, 'rails:users:localhost:map_key': true, 'rails:users:cartodb250user': true, - 'rails:users:localhost': true + 'rails:users:localhost': true, + 'api_keys:localhost:1234': true, + 'api_keys:localhost:default_public': true, + 'api_keys:cartodb250user:4321': true, + 'api_keys:cartodb250user:default_public': true, + 'api_keys:localhost:regular1': true, + 'api_keys:localhost:regular2': true, }; var databasesTasks = { 0: 'users', 5: 'meta'}; @@ -146,11 +148,6 @@ afterEach(function(done) { }); }); -function cleanPGPoolConnections () { - // TODO: this method will be replaced by psql.end - pg.end(); -} - function deleteRedisKeys(keysToDelete, callback) { if (Object.keys(keysToDelete).length === 0) { @@ -213,7 +210,5 @@ module.exports = { checkSurrogateKey: checkSurrogateKey, checkCache: checkCache, rmdirRecursiveSync: rmdirRecursiveSync, - configureMetadata, - cleanPGPoolConnections + configureMetadata }; - diff --git a/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js index 6e47297d..874fba28 100644 --- a/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require('assert'); var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats'); var MapConfig = require('windshaft').model.MapConfig; diff --git a/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js index c9adfad9..c00260c8 100644 --- a/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js +++ b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require('assert'); var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats'); var MapConfig = require('windshaft').model.MapConfig; diff --git a/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js index 7e774023..861f51d8 100644 --- a/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js +++ b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js @@ -1,3 +1,5 @@ +'use strict'; + var PostgresDatasource = require('../../../../lib/cartodb/backends/turbo-carto-postgres-datasource'); var PSQL = require('cartodb-psql'); var _ = require('underscore'); diff --git a/test/unit/cartodb/cache/model/named_maps_entry.test.js b/test/unit/cartodb/cache/model/named_maps_entry.test.js index abaef82f..335f30c7 100644 --- a/test/unit/cartodb/cache/model/named_maps_entry.test.js +++ b/test/unit/cartodb/cache/model/named_maps_entry.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/cdb_request.test.js b/test/unit/cartodb/cdb_request.test.js index 2d89d25c..0dbf35a3 100644 --- a/test/unit/cartodb/cdb_request.test.js +++ b/test/unit/cartodb/cdb_request.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/error-middleware.test.js b/test/unit/cartodb/error-middleware.test.js index 861bec54..1f17ed9c 100644 --- a/test/unit/cartodb/error-middleware.test.js +++ b/test/unit/cartodb/error-middleware.test.js @@ -1,7 +1,9 @@ +'use strict'; + require('../../support/test_helper.js'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware'); describe('error-middleware', function() { @@ -28,7 +30,7 @@ describe('error-middleware', function() { error.subtype = 'test subtype'; const errors = [error, error]; - + const req = {}; const res = { headers: {}, @@ -43,7 +45,7 @@ describe('error-middleware', function() { send () {} }; - const errorHeader = { + const errorHeader = { mainError: { statusCode: 400, message: error.message, @@ -133,7 +135,7 @@ describe('error-middleware', function() { error.subtype = badString; const errors = [error, error]; - + const req = {}; const res = { headers: {}, @@ -148,7 +150,7 @@ describe('error-middleware', function() { send () {} }; - const errorHeader = { + const errorHeader = { mainError: { statusCode: 400, message: escapedString, diff --git a/test/unit/cartodb/error_messages.test.js b/test/unit/cartodb/error_messages.test.js index bfe0b03a..13ea1661 100644 --- a/test/unit/cartodb/error_messages.test.js +++ b/test/unit/cartodb/error_messages.test.js @@ -1,8 +1,10 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware'); describe('error messages clean up', function() { diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js index 9a41030a..4dc355fb 100644 --- a/test/unit/cartodb/lzmaMiddleware.test.js +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -1,7 +1,9 @@ +'use strict'; + var assert = require('assert'); var testHelper = require('../../support/test_helper'); -var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma'); +var lzmaMiddleware = require('../../../lib/cartodb/api/middlewares/lzma'); describe('lzma-middleware', function() { @@ -12,6 +14,7 @@ describe('lzma-middleware', function() { } }; testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { + const lzma = lzmaMiddleware(); var req = { headers: { host:'localhost' @@ -19,9 +22,13 @@ describe('lzma-middleware', function() { query: { api_key: 'test', lzma: data + }, + profiler: { + done: function () {} } }; - lzmaMiddleware(req, {}, function(err) { + + lzma(req, {}, function(err) { if ( err ) { return done(err); } diff --git a/test/unit/cartodb/mapconfig/adapter.test.js b/test/unit/cartodb/mapconfig/adapter.test.js index eeb71896..d2832c78 100644 --- a/test/unit/cartodb/mapconfig/adapter.test.js +++ b/test/unit/cartodb/mapconfig/adapter.test.js @@ -1,3 +1,5 @@ +'use strict'; + //require('../../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js b/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js index b78d8a85..58eaa709 100644 --- a/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js +++ b/test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js @@ -1,3 +1,5 @@ +'use strict'; + //require('../../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/middlewares/coordinates.test.js b/test/unit/cartodb/middlewares/coordinates.test.js new file mode 100644 index 00000000..f6c39d88 --- /dev/null +++ b/test/unit/cartodb/middlewares/coordinates.test.js @@ -0,0 +1,260 @@ +'use strict'; + +const assert = require('assert'); +const coordinates = require('../../../../lib/cartodb/api/middlewares/coordinates'); + +describe('coordinates middleware', function () { + it('should return error: invalid zoom paramenter (-1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '-1', + x: '0', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + 'Invalid zoom value (-1). It should be an integer number greather than or equal to 0' + ); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should return error: invalid zoom paramenter (1.1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1.1', + x: '0', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + 'Invalid zoom value (1.1). It should be an integer number greather than or equal to 0' + ); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should return error: invalid zoom paramenter (0.1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '0.1', + x: '0', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + 'Invalid zoom value (0.1). It should be an integer number greather than or equal to 0' + ); + assert.equal(err.http_status, 400); + done(); + }); + }); + + + it('should return error: invalid zoom paramenter (wadus)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: 'wadus', + x: '0', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + 'Invalid zoom value (wadus). It should be an integer number greather than or equal to 0' + ); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should NOT return error: \'zoom\' paramenter (1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: '1', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.ifError(err); + done(); + }); + }); + + it('should return error: invalid coordinate \'x\' paramenter (1.1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: '1.1', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal(err.message, `Invalid coodinate 'x' value (1.1). It should be an integer number`); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should return error: invalid coordinate \'x\' paramenter (wadus)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: 'wadus', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal(err.message, `Invalid coodinate 'x' value (wadus). It should be an integer number`); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should NOT return error: \'x\' paramenter (-1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: '-3', + y: '0' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.ifError(err); + done(); + }); + }); + + it('should return error: invalid coordinate \'y\' paramenter (-1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: '0', + y: '-1' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + `Invalid coodinate 'y' value (-1). It should be an integer number greather than or equal to 0` + ); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should return error: invalid coordinate \'y\' paramenter (1.1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: '0', + y: '1.1' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + `Invalid coodinate 'y' value (1.1). It should be an integer number greather than or equal to 0` + ); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should return error: invalid coordinate \'y\' paramenter (wadus)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: '0', + y: 'wadus' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + `Invalid coodinate 'y' value (wadus). It should be an integer number greather than or equal to 0` + ); + assert.equal(err.http_status, 400); + done(); + }); + }); + + it('should NOT return error: \'y\' paramenter (1)', function (done) { + const coords = coordinates(); + const req = { + params: { + z: '1', + x: '1', + y: '1' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.ifError(err); + done(); + }); + }); + + it('should validate zoom and should return error: invalid zoom paramenter (-1.1)', function (done) { + const coords = coordinates({ z: true, x: false, y: false }); + const req = { + params: { + z: '-1.1' + } + }; + const res = {}; + + coords(req, res, function (err) { + assert.equal( + err.message, + 'Invalid zoom value (-1.1). It should be an integer number greather than or equal to 0' + ); + assert.equal(err.http_status, 400); + done(); + }); + }); +}); diff --git a/test/unit/cartodb/model/filter/bbox-filters.test.js b/test/unit/cartodb/model/filter/bbox-filters.test.js index 68327a7a..a0e7a14b 100644 --- a/test/unit/cartodb/model/filter/bbox-filters.test.js +++ b/test/unit/cartodb/model/filter/bbox-filters.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../../support/test_helper'); var util = require('util'); diff --git a/test/unit/cartodb/model/resource-locator.test.js b/test/unit/cartodb/model/resource-locator.test.js index 13ad5203..e9571c3e 100644 --- a/test/unit/cartodb/model/resource-locator.test.js +++ b/test/unit/cartodb/model/resource-locator.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../support/test_helper'); var assert = require('../../../support/assert'); diff --git a/test/unit/cartodb/overviews_query_rewriter.js b/test/unit/cartodb/overviews_query_rewriter.js index c224fed8..a3d07e4c 100644 --- a/test/unit/cartodb/overviews_query_rewriter.js +++ b/test/unit/cartodb/overviews_query_rewriter.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/ported/profiler.test.js b/test/unit/cartodb/ported/profiler.test.js index 29d9b485..d9bc775e 100644 --- a/test/unit/cartodb/ported/profiler.test.js +++ b/test/unit/cartodb/ported/profiler.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/ported/stats_client.test.js b/test/unit/cartodb/ported/stats_client.test.js index 088206a0..230d8b25 100644 --- a/test/unit/cartodb/ported/stats_client.test.js +++ b/test/unit/cartodb/ported/stats_client.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/ported/tile_stats.test.js b/test/unit/cartodb/ported/tile_stats.test.js deleted file mode 100644 index a2d30cd0..00000000 --- a/test/unit/cartodb/ported/tile_stats.test.js +++ /dev/null @@ -1,90 +0,0 @@ -require('../../../support/test_helper.js'); - -var assert = require('assert'); - -var LayergroupController = require('../../../../lib/cartodb/controllers/layergroup'); - -describe('tile stats', function() { - - beforeEach(function () { - this.statsClient = global.statsClient; - }); - - afterEach(function() { - global.statsClient = this.statsClient; - }); - - it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() { - var expectedCalls = 2, // it will call increment once for the general error - invalidFormat = 'png2', - invalidFormatRegexp = new RegExp('invalid'), - formatMatched = false; - mockStatsClientGetInstance({ - increment: function(label) { - formatMatched = formatMatched || !!label.match(invalidFormatRegexp); - expectedCalls--; - } - }); - - var layergroupController = new LayergroupController(); - - var reqMock = { - profiler: { toJSONString:function() {} }, - params: { - format: invalidFormat - } - }; - var resMock = { - status: function() { return this; }, - set: function() {}, - json: function() {}, - jsonp: function() {}, - send: function() {} - }; - - var next = function () {}; - layergroupController.finalizeGetTileOrGrid('Unsupported format png2', reqMock, resMock, null, null, next); - - assert.ok(formatMatched, 'Format was never matched in increment method'); - assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method'); - }); - - it('finalizeGetTileOrGrid calls statsClient when format is supported', function() { - var expectedCalls = 2, // general error + format error - validFormat = 'png', - validFormatRegexp = new RegExp(validFormat), - formatMatched = false; - mockStatsClientGetInstance({ - increment: function(label) { - formatMatched = formatMatched || !!label.match(validFormatRegexp); - expectedCalls--; - } - }); - var reqMock = { - profiler: { toJSONString:function() {} }, - params: { - format: validFormat - } - }; - var resMock = { - status: function() { return this; }, - set: function() {}, - json: function() {}, - jsonp: function() {}, - send: function() {} - }; - - var layergroupController = new LayergroupController(); - - var next = function () {}; - layergroupController.finalizeGetTileOrGrid('Another error happened', reqMock, resMock, null, null, next); - - assert.ok(formatMatched, 'Format was never matched in increment method'); - assert.equal(expectedCalls, 0, 'Unexpected number of calls to increment method'); - }); - - function mockStatsClientGetInstance(instance) { - global.statsClient = Object.assign(global.statsClient, instance); - } - -}); diff --git a/test/unit/cartodb/ported/windshaft_server.test.js b/test/unit/cartodb/ported/windshaft_server.test.js index e6f07ea7..1591dc45 100644 --- a/test/unit/cartodb/ported/windshaft_server.test.js +++ b/test/unit/cartodb/ported/windshaft_server.test.js @@ -1,6 +1,7 @@ +'use strict'; + require('../../../support/test_helper.js'); -var _ = require('underscore'); var assert = require('assert'); var cartodbServer = require('../../../../lib/cartodb/server'); var serverOptions = require('../../../../lib/cartodb/server_options'); @@ -30,15 +31,7 @@ describe('windshaft', function() { function(){ var ws = cartodbServer({unbuffered_logging:true}); ws.listen(); - }, /Cannot read property 'mapnik' of undefined/ + }, /Must initialise server with/ ); }); - - it('options are set on main windshaft object', function(){ - var ws = cartodbServer(serverOptions); - assert.ok(_.isObject(ws.bind)); - assert.ok(_.isObject(ws.grainstore)); - assert.equal(ws.base_url, '/tiles/:table'); - }); - }); diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index dbbfb8bf..5de8a677 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -1,16 +1,18 @@ +'use strict'; + var assert = require('assert'); var _ = require('underscore'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../../lib/cartodb/backends/pg_connection'); -var AuthApi = require('../../../lib/cartodb/api/auth_api'); +var AuthBackend = require('../../../lib/cartodb/backends/auth'); var TemplateMaps = require('../../../lib/cartodb/backends/template_maps'); -const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/context/clean-up-query-params'); -const authorizeMiddleware = require('../../../lib/cartodb/middleware/context/authorize'); -const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/context/db-conn-setup'); -const localsMiddleware = require('../../../lib/cartodb/middleware/context/locals'); +const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/api/middlewares/clean-up-query-params'); +const authorizeMiddleware = require('../../../lib/cartodb/api/middlewares/authorize'); +const dbConnSetupMiddleware = require('../../../lib/cartodb/api/middlewares/db-conn-setup'); +const credentialsMiddleware = require('../../../lib/cartodb/api/middlewares/credentials'); var windshaft = require('windshaft'); @@ -23,6 +25,7 @@ describe('prepare-context', function() { let cleanUpQueryParams; let dbConnSetup; let authorize; + let setCredentials; before(function() { var redisPool = new RedisPool(global.environment.redis); @@ -30,11 +33,12 @@ describe('prepare-context', function() { var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var templateMaps = new TemplateMaps(redisPool); - var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); + var authBackend = new AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps); cleanUpQueryParams = cleanUpQueryParamsMiddleware(); - authorize = authorizeMiddleware(authApi); + authorize = authorizeMiddleware(authBackend); dbConnSetup = dbConnSetupMiddleware(pgConnection); + setCredentials = credentialsMiddleware(); }); @@ -58,20 +62,11 @@ describe('prepare-context', function() { } res.locals.user = 'localhost'; + res.set = function () {}; + return res; } - it('res.locals are created', function(done) { - let req = {}; - let res = {}; - - localsMiddleware(prepareRequest(req), prepareResponse(res), function(err) { - if ( err ) { done(err); return; } - assert.ok(res.hasOwnProperty('locals'), 'response has locals'); - done(); - }); - }); - it('cleans up request', function(done){ var req = {headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}; var res = {}; @@ -103,8 +98,20 @@ describe('prepare-context', function() { }); it('sets also dbuser for authenticated requests', function(done){ - var req = { headers: { host: 'localhost' }, query: { map_key: '1234' }}; - var res = { set: function () {} }; + var req = { + headers: { + host: 'localhost' + }, + query: { + api_key: '1234' + } + }; + var res = { + set: function () {}, + locals: { + api_key: '1234' + } + }; // FIXME: review authorize-pgconnsetup workflow, It might we are doing authorization twice. authorize(prepareRequest(req), prepareResponse(res), function (err) { @@ -154,18 +161,79 @@ describe('prepare-context', function() { } }; var res = {}; - + cleanUpQueryParams(prepareRequest(req), prepareResponse(res), function (err) { if ( err ) { return done(err); } - var query = res.locals; - assert.deepEqual(config, query.config); - assert.equal('test', query.api_key); - assert.equal(undefined, query.non_included); + assert.deepEqual(config, req.query.config); + assert.equal('test', req.query.api_key); + assert.equal(undefined, req.query.non_included); done(); }); }); + describe('Set apikey token', function(){ + it('from query param', function (done) { + var req = { + headers: { + host: 'localhost' + }, + query: { + api_key: '1234', + } + }; + var res = {}; + setCredentials(prepareRequest(req), prepareResponse(res), function (err) { + if (err) { + return done(err); + } + var query = res.locals; + + assert.equal('1234', query.api_key); + done(); + }); + }); + + it('from body param', function (done) { + var req = { + headers: { + host: 'localhost' + }, + body: { + api_key: '1234', + } + }; + var res = {}; + setCredentials(prepareRequest(req), prepareResponse(res), function (err) { + if (err) { + return done(err); + } + var query = res.locals; + + assert.equal('1234', query.api_key); + done(); + }); + }); + + it('from http header', function (done) { + var req = { + headers: { + host: 'localhost', + authorization: 'Basic bG9jYWxob3N0OjEyMzQ=', // user: localhost, password: 1234 + } + }; + var res = {}; + setCredentials(prepareRequest(req), prepareResponse(res), function (err) { + if (err) { + return done(err); + } + var query = res.locals; + + assert.equal('1234', query.api_key); + done(); + }); + }); + }); }); diff --git a/test/unit/cartodb/substitution-tokens.test.js b/test/unit/cartodb/substitution-tokens.test.js index 70c3a0d0..61b78e39 100644 --- a/test/unit/cartodb/substitution-tokens.test.js +++ b/test/unit/cartodb/substitution-tokens.test.js @@ -1,3 +1,5 @@ +'use strict'; + var assert = require('assert'); var SubstitutionTokens = require('../../../lib/cartodb/utils/substitution-tokens'); diff --git a/test/unit/cartodb/table_name_parser.js b/test/unit/cartodb/table_name_parser.js index c08ebd8a..3c7aac48 100644 --- a/test/unit/cartodb/table_name_parser.js +++ b/test/unit/cartodb/table_name_parser.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/template_maps.test.js b/test/unit/cartodb/template_maps.test.js index 78dcfd38..d504dc2d 100644 --- a/test/unit/cartodb/template_maps.test.js +++ b/test/unit/cartodb/template_maps.test.js @@ -1,3 +1,5 @@ +'use strict'; + var testHelper = require('../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/template_maps_auth.test.js b/test/unit/cartodb/template_maps_auth.test.js index 5cf145ae..9497adde 100644 --- a/test/unit/cartodb/template_maps_auth.test.js +++ b/test/unit/cartodb/template_maps_auth.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/template_maps_defaults.test.js b/test/unit/cartodb/template_maps_defaults.test.js index bd61641c..dc612cd5 100644 --- a/test/unit/cartodb/template_maps_defaults.test.js +++ b/test/unit/cartodb/template_maps_defaults.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('assert'); diff --git a/test/unit/cartodb/utils/date-wrapper.test.js b/test/unit/cartodb/utils/date-wrapper.test.js new file mode 100644 index 00000000..0ef7c126 --- /dev/null +++ b/test/unit/cartodb/utils/date-wrapper.test.js @@ -0,0 +1,20 @@ +'use strict'; + +var assert = require('assert'); +var dateWrapper = require('../../../../lib/cartodb/utils/date-wrapper'); + +describe('date-wrapper', function() { + + it('should wrap property fields with spaces', function() { + const actual = dateWrapper.wrapDates( + 'select * from table', + [{name: 'a'}, {name: 'b c'}] + ); + const expected = ` + SELECT + "a","b c" + FROM + (select * from table) _cdb_epoch_transformation `; + assert.equal(actual, expected); + }); +}); diff --git a/test/unit/cartodb/valid_template_maps.test.js b/test/unit/cartodb/valid_template_maps.test.js index 7c3327d5..c9a90b2c 100644 --- a/test/unit/cartodb/valid_template_maps.test.js +++ b/test/unit/cartodb/valid_template_maps.test.js @@ -1,3 +1,5 @@ +'use strict'; + require('../../support/test_helper'); var assert = require('assert'); diff --git a/yarn.lock b/yarn.lock deleted file mode 100644 index 26f8cc5b..00000000 --- a/yarn.lock +++ /dev/null @@ -1,2497 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@carto/mapnik@3.6.2-carto.2", "@carto/mapnik@~3.6.2-carto.0": - version "3.6.2-carto.2" - resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.2.tgz#45a055fd2d39530a873ef9ce5a325baacc81c196" - dependencies: - mapnik-vector-tile "1.5.0" - nan "~2.7.0" - node-pre-gyp "~0.6.30" - protozero "1.5.1" - -"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb1": - version "2.5.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/b0b5559f948e77b337bc9a9ae0bf6ec4249fba21" - dependencies: - "@carto/mapnik" "~3.6.2-carto.0" - "@mapbox/sphericalmercator" "~1.0.1" - mapnik-pool "~0.1.3" - -"@mapbox/sphericalmercator@~1.0.1": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" - -abaculus@cartodb/abaculus#2.0.3-cdb2: - version "2.0.3-cdb2" - resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc" - dependencies: - "@carto/mapnik" "~3.6.2-carto.0" - d3-queue "^2.0.2" - sphericalmercator "1.0.x" - -abbrev@1, abbrev@1.0.x: - version "1.0.9" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" - -accepts@~1.3.4: - version "1.3.4" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f" - dependencies: - mime-types "~2.1.16" - negotiator "0.6.1" - -ajv@^4.9.1: - version "4.11.8" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" - dependencies: - co "^4.6.0" - json-stable-stringify "^1.0.1" - -ajv@^5.1.0: - version "5.5.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.5.2.tgz#73b5eeca3fab653e3d3f9422b341ad42205dc965" - dependencies: - co "^4.6.0" - fast-deep-equal "^1.0.0" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.3.0" - -align-text@^0.1.1, align-text@^0.1.3: - version "0.1.4" - resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" - dependencies: - kind-of "^3.0.2" - longest "^1.0.1" - repeat-string "^1.5.2" - -amdefine@>=0.0.4: - version "1.0.1" - resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5" - -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - -ansi-styles@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" - -aproba@^1.0.3: - version "1.2.0" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" - -are-we-there-yet@~1.1.2: - version "1.1.4" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.4.tgz#bb5dca382bb94f05e15194373d16fd3ba1ca110d" - dependencies: - delegates "^1.0.0" - readable-stream "^2.0.6" - -argparse@^1.0.7: - version "1.0.9" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86" - dependencies: - sprintf-js "~1.0.2" - -array-flatten@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" - -asn1@~0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.3.tgz#dac8787713c9966849fc8180777ebe9c1ddf3b86" - -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - -assert-plus@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234" - -assertion-error@^1.0.1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b" - -async@1.x, async@^1.4.0, async@^1.5.2: - version "1.5.2" - resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" - -async@~0.2.0: - version "0.2.10" - resolved "https://registry.yarnpkg.com/async/-/async-0.2.10.tgz#b6bbe0b0674b9d719708ca38de8c237cb526c3d1" - -asynckit@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - -aws-sign2@~0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" - -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - -aws4@^1.2.1, aws4@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" - -balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - -bcrypt-pbkdf@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz#63bc5dcb61331b92bc05fd528953c33462a06f8d" - dependencies: - tweetnacl "^0.14.3" - -bindings@^1.2.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7" - -block-stream@*: - version "0.0.9" - resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" - dependencies: - inherits "~2.0.0" - -body-parser@1.18.2, body-parser@^1.18.2: - version "1.18.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.2.tgz#87678a19d84b47d859b83199bd59bce222b10454" - dependencies: - bytes "3.0.0" - content-type "~1.0.4" - debug "2.6.9" - depd "~1.1.1" - http-errors "~1.6.2" - iconv-lite "0.4.19" - on-finished "~2.3.0" - qs "6.5.1" - raw-body "2.3.2" - type-is "~1.6.15" - -boom@2.x.x: - version "2.10.1" - resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f" - dependencies: - hoek "2.x.x" - -boom@4.x.x: - version "4.3.1" - resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" - dependencies: - hoek "4.x.x" - -boom@5.x.x: - version "5.2.0" - resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" - dependencies: - hoek "4.x.x" - -brace-expansion@^1.1.7: - version "1.1.8" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" - dependencies: - balanced-match "^1.0.0" - concat-map "0.0.1" - -browser-stdout@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f" - -buffer-writer@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" - -builtin-modules@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" - -bunyan@1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.1.tgz#68c6a4a502d5620bc9f72d6736810c1b1898097f" - optionalDependencies: - dtrace-provider "~0.6" - moment "^2.10.6" - mv "~2" - safe-json-stringify "~1" - -bytes@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" - -camelcase@^1.0.2: - version "1.2.1" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-1.2.1.tgz#9bb5304d2e0b56698b2c758b08a3eaa9daa58a39" - -camelcase@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" - -camshaft@0.61.2: - version "0.61.2" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.2.tgz#5c0d43ca769377c6cfb9808f1023d20ea3df55b3" - dependencies: - async "^1.5.2" - bunyan "1.8.1" - cartodb-psql "^0.10.1" - debug "^3.1.0" - dot "^1.0.3" - request "^2.69.0" - -canvas@cartodb/node-canvas#1.6.2-cdb2: - version "1.6.2-cdb2" - resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" - dependencies: - nan "^2.4.0" - -carto@0.16.3: - version "0.16.3" - resolved "https://registry.yarnpkg.com/carto/-/carto-0.16.3.tgz#f42df84557ff28a9036af716660b1e721cedc8c0" - dependencies: - chroma-js "~1.1.1" - husl "^6.0.1" - js-yaml "^3.4.6" - lodash "^4.5.1" - mapnik-reference "~8.5.3" - semver "^5.1.0" - yargs "^4.2.0" - -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - -carto@cartodb/carto#0.15.1-cdb3: - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "1.8.3" - -cartocolor@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" - dependencies: - colorbrewer "1.0.0" - -cartodb-psql@0.10.2, cartodb-psql@^0.10.1: - version "0.10.2" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.2.tgz#8c505066e4a635cfa0ee4c603769c83f6e2187dd" - dependencies: - debug "^3.1.0" - pg cartodb/node-postgres#6.1.6-cdb1 - underscore "~1.6.0" - -cartodb-query-tables@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756" - -cartodb-redis@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.15.0.tgz#509ab9f62b8cae0838bcb8db1cb9d6355704ace3" - dependencies: - dot "~1.0.2" - redis-mpool "^0.5.0" - underscore "~1.6.0" - -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - -center-align@^0.1.1: - version "0.1.3" - resolved "https://registry.yarnpkg.com/center-align/-/center-align-0.1.3.tgz#aa0d32629b6ee972200411cbd4461c907bc2b7ad" - dependencies: - align-text "^0.1.3" - lazy-cache "^1.0.3" - -"chai@>=1.9.2 <4.0.0": - version "3.5.0" - resolved "https://registry.yarnpkg.com/chai/-/chai-3.5.0.tgz#4d02637b067fe958bdbfdd3a40ec56fef7373247" - dependencies: - assertion-error "^1.0.1" - deep-eql "^0.1.3" - type-detect "^1.0.0" - -chalk@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" - dependencies: - ansi-styles "^2.2.1" - escape-string-regexp "^1.0.2" - has-ansi "^2.0.0" - strip-ansi "^3.0.0" - supports-color "^2.0.0" - -chroma-js@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.1.1.tgz#9bb9434959336ece75700aaadfeedc71806d8c05" - -cli@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/cli/-/cli-1.0.1.tgz#22817534f24bfa4950c34d532d48ecbc621b8c14" - dependencies: - exit "0.1.2" - glob "^7.1.1" - -cliui@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-2.1.0.tgz#4b475760ff80264c762c3a1719032e91c7fea0d1" - dependencies: - center-align "^0.1.1" - right-align "^0.1.1" - wordwrap "0.0.2" - -cliui@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" - dependencies: - string-width "^1.0.1" - strip-ansi "^3.0.1" - wrap-ansi "^2.0.0" - -co@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" - -code-point-at@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" - -colorbrewer@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/colorbrewer/-/colorbrewer-1.0.0.tgz#4f97333b969ba7612382be4bc3394b341fb4c8a2" - -combined-stream@^1.0.5, combined-stream@~1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009" - dependencies: - delayed-stream "~1.0.0" - -commander@2.9.0: - version "2.9.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" - dependencies: - graceful-readlink ">= 1.0.0" - -concat-map@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" - -console-browserify@1.1.x: - version "1.1.0" - resolved "https://registry.yarnpkg.com/console-browserify/-/console-browserify-1.1.0.tgz#f0241c45730a9fc6323b206dbf38edc741d0bb10" - dependencies: - date-now "^0.1.4" - -console-control-strings@^1.0.0, console-control-strings@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" - -content-disposition@0.5.2: - version "0.5.2" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" - -content-type@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" - -cookie-signature@1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" - -cookie@0.3.1: - version "0.3.1" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.3.1.tgz#e7e0a1f9ef43b4c8ba925c5c5a96e806d16873bb" - -core-util-is@1.0.2, core-util-is@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - -cryptiles@2.x.x: - version "2.0.5" - resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8" - dependencies: - boom "2.x.x" - -cryptiles@3.x.x: - version "3.1.2" - resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" - dependencies: - boom "5.x.x" - -d3-queue@^2.0.2: - version "2.0.3" - resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" - -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - dependencies: - assert-plus "^1.0.0" - -date-now@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" - -debug@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b" - dependencies: - ms "0.7.2" - -debug@2.6.9, debug@^2.2.0: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - dependencies: - ms "2.0.0" - -debug@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" - dependencies: - ms "2.0.0" - -debug@^3.1.0, debug@~3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - dependencies: - ms "2.0.0" - -decamelize@^1.0.0, decamelize@^1.1.1: - version "1.2.0" - resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" - -deep-eql@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2" - dependencies: - type-detect "0.1.1" - -deep-equal@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" - -deep-extend@~0.4.0: - version "0.4.2" - resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.2.tgz#48b699c27e334bf89f10892be432f6e4c7d34a7f" - -deep-is@~0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" - -delayed-stream@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - -delegates@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" - -depd@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" - -depd@~1.1.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" - -destroy@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" - -detect-libc@^1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" - -diff@3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9" - -dom-serializer@0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82" - dependencies: - domelementtype "~1.1.1" - entities "~1.1.1" - -domelementtype@1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.0.tgz#b17aed82e8ab59e52dd9c19b1756e0fc187204c2" - -domelementtype@~1.1.1: - version "1.1.3" - resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.1.3.tgz#bd28773e2642881aec51544924299c5cd822185b" - -domhandler@2.3: - version "2.3.0" - resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.3.0.tgz#2de59a0822d5027fabff6f032c2b25a2a8abe738" - dependencies: - domelementtype "1" - -domutils@1.5: - version "1.5.1" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" - dependencies: - dom-serializer "0" - domelementtype "1" - -dot@^1.0.3, dot@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" - -double-ended-queue@^2.1.0-0: - version "2.1.0-0" - resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" - -dtrace-provider@~0.6: - version "0.6.0" - resolved "https://registry.yarnpkg.com/dtrace-provider/-/dtrace-provider-0.6.0.tgz#0b078d5517937d873101452d9146737557b75e51" - dependencies: - nan "^2.0.8" - -ecc-jsbn@~0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz#0fc73a9ed5f0d53c38193398523ef7e543777505" - dependencies: - jsbn "~0.1.0" - -ee-first@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" - -encodeurl@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" - -entities@1.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/entities/-/entities-1.0.0.tgz#b2987aa3821347fcde642b24fdfc9e4fb712bf26" - -entities@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0" - -error-ex@^1.2.0: - version "1.3.1" - resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.1.tgz#f855a86ce61adc4e8621c3cda21e7a7612c3a8dc" - dependencies: - is-arrayish "^0.2.1" - -es6-promise@3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.1.2.tgz#795e25ceb47f7babb263d151afbedd92d18e6a07" - -escape-html@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" - -escape-string-regexp@1.0.5, escape-string-regexp@^1.0.2: - version "1.0.5" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" - -escodegen@1.8.x: - version "1.8.1" - resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.8.1.tgz#5a5b53af4693110bebb0867aa3430dd3b70a1018" - dependencies: - esprima "^2.7.1" - estraverse "^1.9.1" - esutils "^2.0.2" - optionator "^0.8.1" - optionalDependencies: - source-map "~0.2.0" - -esprima@2.7.x, esprima@^2.7.1: - version "2.7.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" - -esprima@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" - -estraverse@^1.9.1: - version "1.9.3" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-1.9.3.tgz#af67f2dc922582415950926091a4005d29c9bb44" - -esutils@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b" - -etag@~1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" - -exit@0.1.2, exit@0.1.x: - version "0.1.2" - resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" - -express@~4.16.0: - version "4.16.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c" - dependencies: - accepts "~1.3.4" - array-flatten "1.1.1" - body-parser "1.18.2" - content-disposition "0.5.2" - content-type "~1.0.4" - cookie "0.3.1" - cookie-signature "1.0.6" - debug "2.6.9" - depd "~1.1.1" - encodeurl "~1.0.1" - escape-html "~1.0.3" - etag "~1.8.1" - finalhandler "1.1.0" - fresh "0.5.2" - merge-descriptors "1.0.1" - methods "~1.1.2" - on-finished "~2.3.0" - parseurl "~1.3.2" - path-to-regexp "0.1.7" - proxy-addr "~2.0.2" - qs "6.5.1" - range-parser "~1.2.0" - safe-buffer "5.1.1" - send "0.16.1" - serve-static "1.13.1" - setprototypeof "1.1.0" - statuses "~1.3.1" - type-is "~1.6.15" - utils-merge "1.0.1" - vary "~1.1.2" - -extend@~3.0.0, extend@~3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" - -extsprintf@1.3.0, extsprintf@^1.2.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - -fast-deep-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" - -fast-json-stable-stringify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" - -fast-levenshtein@~2.0.4: - version "2.0.6" - resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" - -fastly-purge@~1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/fastly-purge/-/fastly-purge-1.0.1.tgz#3bdfe9ea1d0fbf2a65712f2f5fe2eca63fcb5960" - dependencies: - request "^2.55.0" - -finalhandler@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.0.tgz#ce0b6855b45853e791b2fcc680046d88253dd7f5" - dependencies: - debug "2.6.9" - encodeurl "~1.0.1" - escape-html "~1.0.3" - on-finished "~2.3.0" - parseurl "~1.3.2" - statuses "~1.3.1" - unpipe "~1.0.0" - -find-up@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" - dependencies: - path-exists "^2.0.0" - pinkie-promise "^2.0.0" - -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - -form-data@~2.1.1: - version "2.1.4" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1" - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.5" - mime-types "^2.1.12" - -form-data@~2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.5" - mime-types "^2.1.12" - -forwarded@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" - -fresh@0.5.2: - version "0.5.2" - resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" - -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - -fstream-ignore@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/fstream-ignore/-/fstream-ignore-1.0.5.tgz#9c31dae34767018fe1d249b24dada67d092da105" - dependencies: - fstream "^1.0.0" - inherits "2" - minimatch "^3.0.0" - -fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: - version "1.0.11" - resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.11.tgz#5c1fb1f117477114f0632a0eb4b71b3cb0fd3171" - dependencies: - graceful-fs "^4.1.2" - inherits "~2.0.0" - mkdirp ">=0.5 0" - rimraf "2" - -gauge@~2.7.3: - version "2.7.4" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" - dependencies: - aproba "^1.0.3" - console-control-strings "^1.0.0" - has-unicode "^2.0.0" - object-assign "^4.1.0" - signal-exit "^3.0.0" - string-width "^1.0.1" - strip-ansi "^3.0.1" - wide-align "^1.1.0" - -gdal@~0.9.2: - version "0.9.6" - resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.6.tgz#0cf75d830d35847b4274368b10e04a925321a0ba" - dependencies: - nan "~2.6.2" - node-pre-gyp "~0.6.36" - -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.3" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" - -generic-pool@~2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.1.1.tgz#af04dc2c325cfcb975023fa52bfce9617a7435fd" - -generic-pool@~2.2.0, generic-pool@~2.2.1: - version "2.2.2" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" - -get-caller-file@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" - -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - dependencies: - assert-plus "^1.0.0" - -glob@7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.2" - once "^1.3.0" - path-is-absolute "^1.0.0" - -glob@^5.0.15: - version "5.0.15" - resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" - dependencies: - inflight "^1.0.4" - inherits "2" - minimatch "2 || 3" - once "^1.3.0" - path-is-absolute "^1.0.0" - -glob@^6.0.1: - version "6.0.4" - resolved "https://registry.yarnpkg.com/glob/-/glob-6.0.4.tgz#0f08860f6a155127b2fadd4f9ce24b1aab6e4d22" - dependencies: - inflight "^1.0.4" - inherits "2" - minimatch "2 || 3" - once "^1.3.0" - path-is-absolute "^1.0.0" - -glob@^7.0.5, glob@^7.1.1, glob@^7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - -graceful-fs@^4.1.2: - version "4.1.11" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" - -"graceful-readlink@>= 1.0.0": - version "1.0.1" - resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" - -grainstore@1.8.2: - version "1.8.2" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.8.2.tgz#79dd7a91a098bf8b0ea3189961775c8cc7474319" - dependencies: - carto "0.16.3" - debug "~3.1.0" - generic-pool "~2.2.0" - millstone "0.6.17" - postcss "~5.2.8" - postcss-scss "0.4.0" - postcss-strip-inline-comments "0.1.5" - semver "~5.0.3" - underscore "~1.6.0" - -growl@1.9.2: - version "1.9.2" - resolved "https://registry.yarnpkg.com/growl/-/growl-1.9.2.tgz#0ea7743715db8d8de2c5ede1775e1b45ac85c02f" - -handlebars@^4.0.1: - version "4.0.11" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.11.tgz#630a35dfe0294bc281edae6ffc5d329fc7982dcc" - dependencies: - async "^1.4.0" - optimist "^0.6.1" - source-map "^0.4.4" - optionalDependencies: - uglify-js "^2.6" - -har-schema@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" - -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - -har-validator@~4.2.1: - version "4.2.1" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-4.2.1.tgz#33481d0f1bbff600dd203d75812a6a5fba002e2a" - dependencies: - ajv "^4.9.1" - har-schema "^1.0.5" - -har-validator@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" - dependencies: - ajv "^5.1.0" - har-schema "^2.0.0" - -has-ansi@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" - dependencies: - ansi-regex "^2.0.0" - -has-flag@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-1.0.0.tgz#9d9e793165ce017a00f00418c43f942a7b1d11fa" - -has-unicode@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" - -hawk@3.1.3, hawk@~3.1.3: - version "3.1.3" - resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" - dependencies: - boom "2.x.x" - cryptiles "2.x.x" - hoek "2.x.x" - sntp "1.x.x" - -hawk@~6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" - dependencies: - boom "4.x.x" - cryptiles "3.x.x" - hoek "4.x.x" - sntp "2.x.x" - -hiredis@~0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" - dependencies: - bindings "^1.2.1" - nan "^2.3.4" - -hoek@2.x.x: - version "2.16.3" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" - -hoek@4.x.x: - version "4.2.0" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" - -hosted-git-info@^2.1.4: - version "2.5.0" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" - -htmlparser2@3.8.x: - version "3.8.3" - resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.8.3.tgz#996c28b191516a8be86501a7d79757e5c70c1068" - dependencies: - domelementtype "1" - domhandler "2.3" - domutils "1.5" - entities "1.0" - readable-stream "1.1" - -http-errors@1.6.2, http-errors@~1.6.2: - version "1.6.2" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.2.tgz#0a002cc85707192a7e7946ceedc11155f60ec736" - dependencies: - depd "1.1.1" - inherits "2.0.3" - setprototypeof "1.0.3" - statuses ">= 1.3.1 < 2" - -http-signature@~1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf" - dependencies: - assert-plus "^0.2.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - -husl@^6.0.1: - version "6.0.6" - resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" - -iconv-lite@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" - -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2, inherits@2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" - -ini@~1.3.0: - version "1.3.5" - resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" - -invert-kv@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" - -ipaddr.js@1.5.2: - version "1.5.2" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.5.2.tgz#d4b505bde9946987ccf0fc58d9010ff9607e3fa0" - -is-arrayish@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" - -is-buffer@^1.1.5: - version "1.1.6" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" - -is-builtin-module@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-1.0.0.tgz#540572d34f7ac3119f8f76c30cbc1b1e037affbe" - dependencies: - builtin-modules "^1.0.0" - -is-fullwidth-code-point@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" - dependencies: - number-is-nan "^1.0.0" - -is-typedarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - -is-utf8@^0.2.0: - version "0.2.1" - resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" - -isarray@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" - -isarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - -isexe@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" - -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - -istanbul@~0.4.3: - version "0.4.5" - resolved "https://registry.yarnpkg.com/istanbul/-/istanbul-0.4.5.tgz#65c7d73d4c4da84d4f3ac310b918fb0b8033733b" - dependencies: - abbrev "1.0.x" - async "1.x" - escodegen "1.8.x" - esprima "2.7.x" - glob "^5.0.15" - handlebars "^4.0.1" - js-yaml "3.x" - mkdirp "0.5.x" - nopt "3.x" - once "1.x" - resolve "1.1.x" - supports-color "^3.1.0" - which "^1.1.1" - wordwrap "^1.0.0" - -js-base64@^2.1.9: - version "2.4.3" - resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.4.3.tgz#2e545ec2b0f2957f41356510205214e98fad6582" - -js-string-escape@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" - -js-yaml@3.x, js-yaml@^3.4.6: - version "3.10.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" - dependencies: - argparse "^1.0.7" - esprima "^4.0.0" - -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - -jshint@~2.9.4: - version "2.9.5" - resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.5.tgz#1e7252915ce681b40827ee14248c46d34e9aa62c" - dependencies: - cli "~1.0.0" - console-browserify "1.1.x" - exit "0.1.x" - htmlparser2 "3.8.x" - lodash "3.7.x" - minimatch "~3.0.2" - shelljs "0.3.x" - strip-json-comments "1.0.x" - -json-schema-traverse@^0.3.0: - version "0.3.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" - -json-schema@0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" - -json-stable-stringify@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af" - dependencies: - jsonify "~0.0.0" - -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - -json3@3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/json3/-/json3-3.3.2.tgz#3c0434743df93e2f5c42aee7b19bcb483575f4e1" - -jsonify@~0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" - -jsprim@^1.2.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.2.3" - verror "1.10.0" - -kind-of@^3.0.2: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - dependencies: - is-buffer "^1.1.5" - -lazy-cache@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e" - -lcid@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835" - dependencies: - invert-kv "^1.0.0" - -levn@~0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" - dependencies: - prelude-ls "~1.1.2" - type-check "~0.3.2" - -load-json-file@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" - dependencies: - graceful-fs "^4.1.2" - parse-json "^2.2.0" - pify "^2.0.0" - pinkie-promise "^2.0.0" - strip-bom "^2.0.0" - -lodash._baseassign@^3.0.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz#8c38a099500f215ad09e59f1722fd0c52bfe0a4e" - dependencies: - lodash._basecopy "^3.0.0" - lodash.keys "^3.0.0" - -lodash._basecopy@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz#8da0e6a876cf344c0ad8a54882111dd3c5c7ca36" - -lodash._basecreate@^3.0.0: - version "3.0.3" - resolved "https://registry.yarnpkg.com/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz#1bc661614daa7fc311b7d03bf16806a0213cf821" - -lodash._getnative@^3.0.0: - version "3.9.1" - resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" - -lodash._isiterateecall@^3.0.0: - version "3.0.9" - resolved "https://registry.yarnpkg.com/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz#5203ad7ba425fae842460e696db9cf3e6aac057c" - -lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.1.0, lodash.assign@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" - -lodash.create@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/lodash.create/-/lodash.create-3.1.1.tgz#d7f2849f0dbda7e04682bb8cd72ab022461debe7" - dependencies: - lodash._baseassign "^3.0.0" - lodash._basecreate "^3.0.0" - lodash._isiterateecall "^3.0.0" - -lodash.isarguments@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" - -lodash.isarray@^3.0.0: - version "3.0.4" - resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55" - -lodash.keys@^3.0.0: - version "3.1.2" - resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a" - dependencies: - lodash._getnative "^3.0.0" - lodash.isarguments "^3.0.0" - lodash.isarray "^3.0.0" - -lodash@2.4.1: - version "2.4.1" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.1.tgz#5b7723034dda4d262e5a46fb2c58d7cc22f71420" - -lodash@3.7.x: - version "3.7.0" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.7.0.tgz#3678bd8ab995057c07ade836ed2ef087da811d45" - -lodash@^4.5.1: - version "4.17.5" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511" - -log4js@cartodb/log4js-node#cdb: - version "0.6.25" - resolved "https://codeload.github.com/cartodb/log4js-node/tar.gz/145d5f91e35e7fb14a6278cbf7a711ced6603727" - dependencies: - async "~0.2.0" - readable-stream "~1.0.2" - semver "~4.3.3" - underscore "1.8.2" - -longest@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" - -lru-cache@2.6.5: - version "2.6.5" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.6.5.tgz#e56d6354148ede8d7707b58d143220fd08df0fd5" - -lzma@~2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/lzma/-/lzma-2.3.2.tgz#3783b24858b9c0e747a0df3cbf1fb5fcaa92c441" - -mapnik-pool@~0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/mapnik-pool/-/mapnik-pool-0.1.3.tgz#9bdad1cb34d87a19bc78ecb4cbfbaa203b2f3f2b" - dependencies: - generic-pool "~2.2.1" - xtend "~4.0.0" - -mapnik-reference@~6.0.2: - version "6.0.5" - resolved "https://registry.yarnpkg.com/mapnik-reference/-/mapnik-reference-6.0.5.tgz#d52319e1df88e5a07a811e84def2a1a423b6f05f" - -mapnik-reference@~8.5.3: - version "8.5.6" - resolved "https://registry.yarnpkg.com/mapnik-reference/-/mapnik-reference-8.5.6.tgz#5e8079821409322d6c958ce1823c67883ebb7655" - dependencies: - semver "^5.1.0" - -mapnik-vector-tile@1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/mapnik-vector-tile/-/mapnik-vector-tile-1.5.0.tgz#c647bfb8027e9dc40db583505a436f35e2101407" - -media-typer@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" - -merge-descriptors@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" - -methods@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" - -millstone@0.6.17: - version "0.6.17" - resolved "https://registry.yarnpkg.com/millstone/-/millstone-0.6.17.tgz#5859f6ab5d5a1a9135963fc3137c5eac566a1a50" - dependencies: - generic-pool "~2.4.1" - mime "~1.2.11" - mkdirp "~0.5.0" - optimist "~0.6.1" - request "2.x" - sqlite3 "2.x || 3.x" - srs "1.x" - step "~0.0.5" - underscore "~1.6.0" - zipfile "~0.5.5" - -mime-db@~1.30.0: - version "1.30.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" - -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17, mime-types@~2.1.7: - version "2.1.17" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" - dependencies: - mime-db "~1.30.0" - -mime@1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" - -mime@~1.2.11: - version "1.2.11" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" - -mime@~1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" - -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - dependencies: - brace-expansion "^1.1.7" - -minimist@0.0.8, minimist@~0.0.1: - version "0.0.8" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" - -minimist@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" - -minimist@~0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" - -mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: - version "0.5.1" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" - dependencies: - minimist "0.0.8" - -mocha@~3.4.1: - version "3.4.2" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" - dependencies: - browser-stdout "1.3.0" - commander "2.9.0" - debug "2.6.0" - diff "3.2.0" - escape-string-regexp "1.0.5" - glob "7.1.1" - growl "1.9.2" - json3 "3.3.2" - lodash.create "3.1.1" - mkdirp "0.5.1" - supports-color "3.1.2" - -moment@^2.10.6, moment@~2.18.1: - version "2.18.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" - -ms@0.7.2: - version "0.7.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" - -ms@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - -mv@~2: - version "2.1.1" - resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" - dependencies: - mkdirp "~0.5.1" - ncp "~2.0.0" - rimraf "~2.4.0" - -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" - -nan@~2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" - -nan@~2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" - -nan@~2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" - -ncp@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" - -negotiator@0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" - -nock@~2.11.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/nock/-/nock-2.11.0.tgz#2554cc09f0c3077100aae185b88d030bad950fcd" - dependencies: - chai ">=1.9.2 <4.0.0" - debug "^1.0.4" - deep-equal "^1.0.0" - lodash "2.4.1" - mkdirp "^0.5.0" - propagate "0.3.x" - -node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: - version "0.6.39" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649" - dependencies: - detect-libc "^1.0.2" - hawk "3.1.3" - mkdirp "^0.5.1" - nopt "^4.0.1" - npmlog "^4.0.2" - rc "^1.1.7" - request "2.81.0" - rimraf "^2.6.1" - semver "^5.3.0" - tar "^2.2.1" - tar-pack "^3.4.0" - -node-statsd@~0.0.7: - version "0.0.7" - resolved "https://registry.yarnpkg.com/node-statsd/-/node-statsd-0.0.7.tgz#96d4bbd21dff2d798b3374a3e9329cfecce3afa8" - -nopt@3.x: - version "3.0.6" - resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" - dependencies: - abbrev "1" - -nopt@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.1.tgz#d0d4685afd5415193c8c7505602d0d17cd64474d" - dependencies: - abbrev "1" - osenv "^0.1.4" - -normalize-package-data@^2.3.2: - version "2.4.0" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" - dependencies: - hosted-git-info "^2.1.4" - is-builtin-module "^1.0.0" - semver "2 || 3 || 4 || 5" - validate-npm-package-license "^3.0.1" - -npmlog@^4.0.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" - dependencies: - are-we-there-yet "~1.1.2" - console-control-strings "~1.1.0" - gauge "~2.7.3" - set-blocking "~2.0.0" - -number-is-nan@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" - -oauth-sign@~0.8.1, oauth-sign@~0.8.2: - version "0.8.2" - resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" - -object-assign@4.1.0, object-assign@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" - -object-keys@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" - -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" - dependencies: - ee-first "1.1.1" - -on-headers@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.1.tgz#928f5d0f470d49342651ea6794b0857c100693f7" - -once@1.x, once@^1.3.0, once@^1.3.3: - version "1.4.0" - resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - dependencies: - wrappy "1" - -optimist@^0.6.1, optimist@~0.6.0, optimist@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686" - dependencies: - minimist "~0.0.1" - wordwrap "~0.0.2" - -optionator@^0.8.1: - version "0.8.2" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.2.tgz#364c5e409d3f4d6301d6c0b4c05bba50180aeb64" - dependencies: - deep-is "~0.1.3" - fast-levenshtein "~2.0.4" - levn "~0.3.0" - prelude-ls "~1.1.2" - type-check "~0.3.2" - wordwrap "~1.0.0" - -os-homedir@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" - -os-locale@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-1.4.0.tgz#20f9f17ae29ed345e8bde583b13d2009803c14d9" - dependencies: - lcid "^1.0.0" - -os-tmpdir@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" - -osenv@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.4.tgz#42fe6d5953df06c8064be6f176c3d05aaaa34644" - dependencies: - os-homedir "^1.0.0" - os-tmpdir "^1.0.0" - -packet-reader@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.2.0.tgz#819df4d010b82d5ea5671f8a1a3acf039bcd7700" - -parse-json@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" - dependencies: - error-ex "^1.2.0" - -parseurl@~1.3.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" - -path-exists@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b" - dependencies: - pinkie-promise "^2.0.0" - -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - -path-to-regexp@0.1.7: - version "0.1.7" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" - -path-type@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" - dependencies: - graceful-fs "^4.1.2" - pify "^2.0.0" - pinkie-promise "^2.0.0" - -performance-now@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" - -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - -pg-connection-string@0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" - -pg-int8@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" - -pg-pool@1.*: - version "1.8.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37" - dependencies: - generic-pool "2.4.3" - object-assign "4.1.0" - -pg-types@1.*: - version "1.13.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.13.0.tgz#75f490b8a8abf75f1386ef5ec4455ecf6b345c63" - dependencies: - pg-int8 "1.0.1" - postgres-array "~1.0.0" - postgres-bytea "~1.0.0" - postgres-date "~1.0.0" - postgres-interval "^1.1.0" - -pg@cartodb/node-postgres#6.1.6-cdb1: - version "6.1.6" - resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" - dependencies: - buffer-writer "1.0.1" - js-string-escape "1.0.1" - packet-reader "0.2.0" - pg-connection-string "0.1.3" - pg-pool "1.*" - pg-types "1.*" - pgpass "1.x" - semver "4.3.2" - -pgpass@1.x: - version "1.0.2" - resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" - dependencies: - split "^1.0.0" - -pify@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" - -pinkie-promise@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa" - dependencies: - pinkie "^2.0.0" - -pinkie@^2.0.0: - version "2.0.4" - resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" - -postcss-scss@0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/postcss-scss/-/postcss-scss-0.4.0.tgz#087c052c529b9270d9580bd1248a0f93d3b40d57" - dependencies: - postcss "^5.2.5" - -postcss-strip-inline-comments@0.1.5: - version "0.1.5" - resolved "https://registry.yarnpkg.com/postcss-strip-inline-comments/-/postcss-strip-inline-comments-0.1.5.tgz#7ff6bcdc14e633ed4cdfa020bae3eddad4f84b90" - dependencies: - postcss "^5.0.18" - -postcss-value-parser@3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-3.3.0.tgz#87f38f9f18f774a4ab4c8a232f5c5ce8872a9d15" - -postcss@5.0.19: - version "5.0.19" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.0.19.tgz#b6342a01dc75b8cab7e968afda96aefc67f888af" - dependencies: - js-base64 "^2.1.9" - source-map "^0.5.1" - supports-color "^3.1.2" - -postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8: - version "5.2.18" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.18.tgz#badfa1497d46244f6390f58b319830d9107853c5" - dependencies: - chalk "^1.1.3" - js-base64 "^2.1.9" - source-map "^0.5.6" - supports-color "^3.2.3" - -postgres-array@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-1.0.2.tgz#8e0b32eb03bf77a5c0a7851e0441c169a256a238" - -postgres-bytea@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" - -postgres-date@~1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" - -postgres-interval@^1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.1.tgz#acdb0f897b4b1c6e496d9d4e0a853e1c428f06f0" - dependencies: - xtend "^4.0.0" - -prelude-ls@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" - -process-nextick-args@~1.0.6: - version "1.0.7" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" - -progress-stream@~0.5.x: - version "0.5.0" - resolved "https://registry.yarnpkg.com/progress-stream/-/progress-stream-0.5.0.tgz#cc4759167a6ff4f05876179384f0b6ae0d1d7587" - dependencies: - single-line-log "~0.3.1" - speedometer "~0.1.2" - through2 "~0.2.3" - -propagate@0.3.x: - version "0.3.1" - resolved "https://registry.yarnpkg.com/propagate/-/propagate-0.3.1.tgz#e3a84404a7ece820dd6bbea9f6d924e3135ae09c" - -protozero@1.5.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.5.1.tgz#5a27df6fb6e1ed743f510812ae76c082f5b16638" - -proxy-addr@~2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.2.tgz#6571504f47bb988ec8180253f85dd7e14952bdec" - dependencies: - forwarded "~0.1.2" - ipaddr.js "1.5.2" - -punycode@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" - -qs@6.5.1, qs@~6.5.1: - version "6.5.1" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" - -qs@~6.4.0: - version "6.4.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" - -queue-async@~1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" - -range-parser@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e" - -raw-body@2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.2.tgz#bcd60c77d3eb93cde0050295c3f379389bc88f89" - dependencies: - bytes "3.0.0" - http-errors "1.6.2" - iconv-lite "0.4.19" - unpipe "1.0.0" - -rc@^1.1.7: - version "1.2.5" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.5.tgz#275cd687f6e3b36cc756baa26dfee80a790301fd" - dependencies: - deep-extend "~0.4.0" - ini "~1.3.0" - minimist "^1.2.0" - strip-json-comments "~2.0.1" - -read-pkg-up@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02" - dependencies: - find-up "^1.0.0" - read-pkg "^1.0.0" - -read-pkg@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28" - dependencies: - load-json-file "^1.0.0" - normalize-package-data "^2.3.2" - path-type "^1.0.0" - -readable-stream@1.1, readable-stream@~1.1.9: - version "1.1.13" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - -readable-stream@^2.0.6, readable-stream@^2.1.4: - version "2.3.3" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~1.0.6" - safe-buffer "~5.1.1" - string_decoder "~1.0.3" - util-deprecate "~1.0.1" - -readable-stream@~1.0.2: - version "1.0.34" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - -redis-commands@^1.2.0: - version "1.3.1" - resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.3.1.tgz#81d826f45fa9c8b2011f4cd7a0fe597d241d442b" - -redis-mpool@0.5.0, redis-mpool@^0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.5.0.tgz#9a22dcffb4ad796ec88ce3038b991deae9d1fda7" - dependencies: - generic-pool "~2.1.1" - hiredis "~0.5.0" - redis "^2.8.0" - underscore "~1.6.0" - -redis-parser@^2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b" - -redis@2.8.0, redis@^2.8.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/redis/-/redis-2.8.0.tgz#202288e3f58c49f6079d97af7a10e1303ae14b02" - dependencies: - double-ended-queue "^2.1.0-0" - redis-commands "^1.2.0" - redis-parser "^2.6.0" - -repeat-string@^1.5.2: - version "1.6.1" - resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" - -request@2.81.0: - version "2.81.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.12.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~4.2.1" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - performance-now "^0.2.0" - qs "~6.4.0" - safe-buffer "^5.0.1" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "^0.6.0" - uuid "^3.0.0" - -request@2.x, request@^2.55.0, request@^2.69.0, request@^2.83.0: - version "2.83.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.6.0" - caseless "~0.12.0" - combined-stream "~1.0.5" - extend "~3.0.1" - forever-agent "~0.6.1" - form-data "~2.3.1" - har-validator "~5.0.3" - hawk "~6.0.2" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.17" - oauth-sign "~0.8.2" - performance-now "^2.1.0" - qs "~6.5.1" - safe-buffer "^5.1.1" - stringstream "~0.0.5" - tough-cookie "~2.3.3" - tunnel-agent "^0.6.0" - uuid "^3.1.0" - -require-directory@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" - -require-main-filename@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" - -resolve@1.1.x: - version "1.1.7" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" - -right-align@^0.1.1: - version "0.1.3" - resolved "https://registry.yarnpkg.com/right-align/-/right-align-0.1.3.tgz#61339b722fe6a3515689210d24e14c96148613ef" - dependencies: - align-text "^0.1.1" - -rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: - version "2.6.2" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" - dependencies: - glob "^7.0.5" - -rimraf@~2.4.0: - version "2.4.5" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.4.5.tgz#ee710ce5d93a8fdb856fb5ea8ff0e2d75934b2da" - dependencies: - glob "^6.0.1" - -safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" - -safe-json-stringify@~1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" - -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - -semver@4.3.2: - version "4.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" - -semver@~4.3.3: - version "4.3.6" - resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" - -semver@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" - -send@0.16.1: - version "0.16.1" - resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" - dependencies: - debug "2.6.9" - depd "~1.1.1" - destroy "~1.0.4" - encodeurl "~1.0.1" - escape-html "~1.0.3" - etag "~1.8.1" - fresh "0.5.2" - http-errors "~1.6.2" - mime "1.4.1" - ms "2.0.0" - on-finished "~2.3.0" - range-parser "~1.2.0" - statuses "~1.3.1" - -serve-static@1.13.1: - version "1.13.1" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.1.tgz#4c57d53404a761d8f2e7c1e8a18a47dbf278a719" - dependencies: - encodeurl "~1.0.1" - escape-html "~1.0.3" - parseurl "~1.3.2" - send "0.16.1" - -set-blocking@^2.0.0, set-blocking@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" - -setprototypeof@1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.0.3.tgz#66567e37043eeb4f04d91bd658c0cbefb55b8e04" - -setprototypeof@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" - -shelljs@0.3.x: - version "0.3.0" - resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" - -signal-exit@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" - -single-line-log@~0.3.1: - version "0.3.1" - resolved "https://registry.yarnpkg.com/single-line-log/-/single-line-log-0.3.1.tgz#a7ad6507f218ce5dfe16c4bf2d659246419e7a06" - -sntp@1.x.x: - version "1.0.9" - resolved "https://registry.yarnpkg.com/sntp/-/sntp-1.0.9.tgz#6541184cc90aeea6c6e7b35e2659082443c66198" - dependencies: - hoek "2.x.x" - -sntp@2.x.x: - version "2.1.0" - resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.1.0.tgz#2c6cec14fedc2222739caf9b5c3d85d1cc5a2cc8" - dependencies: - hoek "4.x.x" - -source-map@^0.4.4: - version "0.4.4" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" - dependencies: - amdefine ">=0.0.4" - -source-map@^0.5.1, source-map@^0.5.6, source-map@~0.5.1: - version "0.5.7" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" - -source-map@~0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.2.0.tgz#dab73fbcfc2ba819b4de03bd6f6eaa48164b3f9d" - dependencies: - amdefine ">=0.0.4" - -spdx-correct@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40" - dependencies: - spdx-license-ids "^1.0.2" - -spdx-expression-parse@~1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-1.0.4.tgz#9bdf2f20e1f40ed447fbe273266191fced51626c" - -spdx-license-ids@^1.0.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz#c9df7a3424594ade6bd11900d596696dc06bac57" - -speedometer@~0.1.2: - version "0.1.4" - resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" - -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" - -split@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" - dependencies: - through "2" - -sprintf-js@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - -"sqlite3@2.x || 3.x": - version "3.1.13" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.13.tgz#d990a05627392768de6278bafd1a31fdfe907dd9" - dependencies: - nan "~2.7.0" - node-pre-gyp "~0.6.38" - -srs@1.x: - version "1.2.0" - resolved "https://registry.yarnpkg.com/srs/-/srs-1.2.0.tgz#ecb447854df76ceb12b4529032be23e9e18eecdc" - dependencies: - gdal "~0.9.2" - -sshpk@^1.7.0: - version "1.13.1" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - dashdash "^1.12.0" - getpass "^0.1.1" - optionalDependencies: - bcrypt-pbkdf "^1.0.0" - ecc-jsbn "~0.1.1" - jsbn "~0.1.0" - tweetnacl "~0.14.0" - -"statuses@>= 1.3.1 < 2", statuses@~1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" - -step-profiler@~0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/step-profiler/-/step-profiler-0.3.0.tgz#841368ce44f2330c862edd5ec60e7fbec148f0b3" - dependencies: - debug "^2.2.0" - -step@~0.0.5, step@~0.0.6: - version "0.0.6" - resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" - -strftime@~0.8.2: - version "0.8.4" - resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" - -string-width@^1.0.1, string-width@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" - dependencies: - code-point-at "^1.0.0" - is-fullwidth-code-point "^1.0.0" - strip-ansi "^3.0.0" - -string_decoder@~0.10.x: - version "0.10.31" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" - -string_decoder@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" - dependencies: - safe-buffer "~5.1.0" - -stringstream@~0.0.4, stringstream@~0.0.5: - version "0.0.5" - resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" - -strip-ansi@^3.0.0, strip-ansi@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - dependencies: - ansi-regex "^2.0.0" - -strip-bom@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" - dependencies: - is-utf8 "^0.2.0" - -strip-json-comments@1.0.x: - version "1.0.4" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-1.0.4.tgz#1e15fbcac97d3ee99bf2d73b4c656b082bbafb91" - -strip-json-comments@~2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" - -supports-color@3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5" - dependencies: - has-flag "^1.0.0" - -supports-color@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" - -supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: - version "3.2.3" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.2.3.tgz#65ac0504b3954171d8a64946b2ae3cbb8a5f54f6" - dependencies: - has-flag "^1.0.0" - -tar-pack@^3.4.0: - version "3.4.1" - resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.1.tgz#e1dbc03a9b9d3ba07e896ad027317eb679a10a1f" - dependencies: - debug "^2.2.0" - fstream "^1.0.10" - fstream-ignore "^1.0.5" - once "^1.3.3" - readable-stream "^2.1.4" - rimraf "^2.5.1" - tar "^2.2.1" - uid-number "^0.0.6" - -tar@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1" - dependencies: - block-stream "*" - fstream "^1.0.2" - inherits "2" - -through2@~0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/through2/-/through2-0.2.3.tgz#eb3284da4ea311b6cc8ace3653748a52abf25a3f" - dependencies: - readable-stream "~1.1.9" - xtend "~2.1.1" - -through@2: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb5: - version "0.6.18-cdb5" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cec846025e60837c60af193d600d972917ea8d35" - dependencies: - "@carto/mapnik" "~3.6.2-carto.0" - generic-pool "~2.4.0" - mime "~1.6.0" - sphericalmercator "~1.0.4" - step "~0.0.5" - -tilelive@5.12.2: - version "5.12.2" - resolved "https://registry.yarnpkg.com/tilelive/-/tilelive-5.12.2.tgz#9ede3aefded4b04e0be32e0dfc6a1ce01247eef7" - dependencies: - minimist "~0.2.0" - progress-stream "~0.5.x" - queue-async "~1.0.7" - sphericalmercator "~1.0.1" - -torque.js@~2.11.0: - version "2.11.5" - resolved "https://registry.yarnpkg.com/torque.js/-/torque.js-2.11.5.tgz#8fe8dfb53fb76497a3c427fcb7ba18e427c1ef6b" - dependencies: - carto CartoDB/carto#0.15.1-cdb1 - -tough-cookie@~2.3.0, tough-cookie@~2.3.3: - version "2.3.3" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" - dependencies: - punycode "^1.4.1" - -tunnel-agent@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - dependencies: - safe-buffer "^5.0.1" - -turbo-carto@0.20.2: - version "0.20.2" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.2.tgz#2b737597a65c2918432f70ea414f12fbec2b6a6f" - dependencies: - cartocolor "4.0.0" - colorbrewer "1.0.0" - debug "^3.1.0" - es6-promise "3.1.2" - postcss "5.0.19" - postcss-value-parser "3.3.0" - -tweetnacl@^0.14.3, tweetnacl@~0.14.0: - version "0.14.5" - resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - -type-check@~0.3.2: - version "0.3.2" - resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" - dependencies: - prelude-ls "~1.1.2" - -type-detect@0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-0.1.1.tgz#0ba5ec2a885640e470ea4e8505971900dac58822" - -type-detect@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" - -type-is@~1.6.15: - version "1.6.15" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410" - dependencies: - media-typer "0.3.0" - mime-types "~2.1.15" - -uglify-js@^2.6: - version "2.8.29" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" - dependencies: - source-map "~0.5.1" - yargs "~3.10.0" - optionalDependencies: - uglify-to-browserify "~1.0.0" - -uglify-to-browserify@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz#6e0924d6bda6b5afe349e39a6d632850a0f882b7" - -uid-number@^0.0.6: - version "0.0.6" - resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" - -underscore@1.8.2: - version "1.8.2" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.2.tgz#64df2eb590899de950782f3735190ba42ebf311d" - -underscore@1.8.3: - version "1.8.3" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" - -underscore@~1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" - -unpipe@1.0.0, unpipe@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" - -util-deprecate@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - -utils-merge@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" - -uuid@^3.0.0, uuid@^3.1.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14" - -validate-npm-package-license@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc" - dependencies: - spdx-correct "~1.0.0" - spdx-expression-parse "~1.0.0" - -vary@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" - -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - -which-module@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" - -which@^1.1.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" - dependencies: - isexe "^2.0.0" - -wide-align@^1.1.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.2.tgz#571e0f1b0604636ebc0dfc21b0339bbe31341710" - dependencies: - string-width "^1.0.2" - -window-size@0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d" - -window-size@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" - -windshaft@4.5.3: - version "4.5.3" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.3.tgz#45e792af06b224f78f44b6eb3b0ecb9d90dcc943" - dependencies: - "@carto/mapnik" "3.6.2-carto.2" - "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb1 - abaculus cartodb/abaculus#2.0.3-cdb2 - canvas cartodb/node-canvas#1.6.2-cdb2 - carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "^0.10.1" - debug "^3.1.0" - dot "~1.0.2" - grainstore "1.8.2" - queue-async "~1.0.7" - redis-mpool "^0.5.0" - request "^2.83.0" - semver "~5.0.3" - sphericalmercator "1.0.4" - step "~0.0.6" - tilelive "5.12.2" - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb5 - torque.js "~2.11.0" - underscore "~1.6.0" - -wordwrap@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.2.tgz#b79669bb42ecb409f83d583cad52ca17eaa1643f" - -wordwrap@^1.0.0, wordwrap@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" - -wordwrap@~0.0.2: - version "0.0.3" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107" - -wrap-ansi@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85" - dependencies: - string-width "^1.0.1" - strip-ansi "^3.0.1" - -wrappy@1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - -xtend@^4.0.0, xtend@~4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" - -xtend@~2.1.1: - version "2.1.2" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-2.1.2.tgz#6efecc2a4dad8e6962c4901b337ce7ba87b5d28b" - dependencies: - object-keys "~0.4.0" - -y18n@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" - -yargs-parser@^2.4.1: - version "2.4.1" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4" - dependencies: - camelcase "^3.0.0" - lodash.assign "^4.0.6" - -yargs-parser@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-3.2.0.tgz#5081355d19d9d0c8c5d81ada908cb4e6d186664f" - dependencies: - camelcase "^3.0.0" - lodash.assign "^4.1.0" - -yargs@^4.2.0: - version "4.8.1" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-4.8.1.tgz#c0c42924ca4aaa6b0e6da1739dfb216439f9ddc0" - dependencies: - cliui "^3.2.0" - decamelize "^1.1.1" - get-caller-file "^1.0.1" - lodash.assign "^4.0.3" - os-locale "^1.4.0" - read-pkg-up "^1.0.1" - require-directory "^2.1.1" - require-main-filename "^1.0.1" - set-blocking "^2.0.0" - string-width "^1.0.1" - which-module "^1.0.0" - window-size "^0.2.0" - y18n "^3.2.1" - yargs-parser "^2.4.1" - -yargs@~3.10.0: - version "3.10.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-3.10.0.tgz#f7ee7bd857dd7c1d2d38c0e74efbd681d1431fd1" - dependencies: - camelcase "^1.0.2" - cliui "^2.1.0" - decamelize "^1.0.0" - window-size "0.1.0" - -yargs@~5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-5.0.0.tgz#3355144977d05757dbb86d6e38ec056123b3a66e" - dependencies: - cliui "^3.2.0" - decamelize "^1.1.1" - get-caller-file "^1.0.1" - lodash.assign "^4.2.0" - os-locale "^1.4.0" - read-pkg-up "^1.0.1" - require-directory "^2.1.1" - require-main-filename "^1.0.1" - set-blocking "^2.0.0" - string-width "^1.0.2" - which-module "^1.0.0" - window-size "^0.2.0" - y18n "^3.2.1" - yargs-parser "^3.2.0" - -zipfile@~0.5.5: - version "0.5.11" - resolved "https://registry.yarnpkg.com/zipfile/-/zipfile-0.5.11.tgz#a46c1b76706138fb9bcf83890193caec679f685e" - dependencies: - nan "~2.4.0" - node-pre-gyp "~0.6.30"