Compare commits

...

122 Commits

Author SHA1 Message Date
Raul Ochoa
03778c370c Release 2.91.2 2017-04-24 19:35:23 +02:00
Raul Ochoa
7812af8ff9 Bump version and update news 2017-04-24 19:34:52 +02:00
Raul Ochoa
f599346138 Fix release version 2017-04-24 19:34:08 +02:00
Raul Ochoa
eb8f6095b0 Merge pull request #672 from CartoDB/upgrade-deps-010
Upgrade deps, target grainstore 1.6.3
2017-04-24 19:29:41 +02:00
Raul Ochoa
30d5d43d2c Upgrade deps, target grainstore 1.6.3 2017-04-24 19:20:03 +02:00
Raul Ochoa
f354aaf500 Merge pull request #669 from CartoDB/upgrade-camshaft-010
Upgrades camshaft to 0.54.1
2017-04-24 14:36:41 +02:00
Raul Ochoa
167639ddb9 Upgrades camshaft to 0.54.1 2017-04-24 14:22:06 +02:00
Raul Ochoa
57281ad0ce Ignore errors when removing npm-shrinkwrap.json 2017-04-24 12:41:33 +02:00
Raul Ochoa
7e7e14766e Merge pull request #667 from CartoDB/upgrade-camshaft-010
Upgrades camshaft to 0.54.0
2017-04-20 18:40:39 +02:00
Raul Ochoa
7bd7a05f3c Upgrades camshaft to 0.54.0 2017-04-20 18:34:23 +02:00
Daniel García Aubert
c503548db6 Stubs next version 2017-04-11 19:31:20 +02:00
Daniel García Aubert
93537f7473 Merge tag '2.90.1' into v2.x
2.90.1
Released 2017-04-11

Announcements:
 - Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1)
2017-04-11 19:30:13 +02:00
Daniel García Aubert
56acb80fa9 Release 2.90.1 2017-04-11 18:50:29 +02:00
Daniel
f13463d350 Merge pull request #663 from CartoDB/upgrade-camshaft-to-0.53.1-v2.x
Upgrade camshaft to 0.53.1
2017-04-11 18:48:32 +02:00
Daniel García Aubert
88e7d2e583 Update shrinkwrap 2017-04-11 18:46:25 +02:00
Daniel García Aubert
f5c366890f Upgrade camshaft to 0.53.1 2017-04-11 18:18:41 +02:00
Daniel García Aubert
a3c51fad02 Stubs next version 2017-04-10 16:15:33 +02:00
Daniel García Aubert
b8e961ffc0 Release 2.90.0 2017-04-10 12:31:08 +02:00
Daniel
7b767f4e7b Merge pull request #661 from CartoDB/fix_invalidation_for_v2x_version
Fix invalidation cache issues for v2x version
2017-04-10 12:11:15 +02:00
Daniel García Aubert
8cba537ac6 Update shrinkwrap 2017-04-10 12:05:24 +02:00
Daniel García Aubert
8fa7f7d779 Upgrade camshaft to 0.53.0 2017-04-10 11:54:31 +02:00
Mario de Frutos
5c22a7439f Point to camshaft branch to test properly 2017-04-07 16:40:51 +02:00
Mario de Frutos
fff2c33100 Make the cache headers tests idempotent 2017-04-07 16:17:04 +02:00
Mario de Frutos
1fc49b5ad5 Add cdb_invalidate_varnish function fixture to tests 2017-04-07 16:16:58 +02:00
Mario de Frutos
07880928e8 Include check for surrogate-key header and renamed the test file 2017-04-07 16:16:52 +02:00
Mario de Frutos
7631edb3db Affected tables are now included in X-Cache-Channel 2017-04-07 16:16:47 +02:00
Mario de Frutos
1ba42e37e9 Add more tests for x-cache-channel but with analysis 2017-04-07 16:16:40 +02:00
Mario de Frutos
47dd485282 Get affected tables and add it to the layergroup 2017-04-07 16:16:34 +02:00
Raul Ochoa
c1fa8897ce Merge pull request #660 from CartoDB/upgrade-camshaft-010
Upgrade camshaft to 0.52.0
2017-04-07 15:19:33 +02:00
Raul Ochoa
c5d0b99fae Upgrade camshaft to 0.52.0 2017-04-07 15:12:58 +02:00
Raul Ochoa
e57c4df663 Merge pull request #658 from CartoDB/upgrade-camshaft-010
Upgrade camshaft to 0.51.0
2017-04-03 16:15:55 +02:00
Raul Ochoa
7ea3fd060e Upgrade camshaft to 0.51.0 2017-04-03 16:01:19 +02:00
Raul Ochoa
3c110cad6d Merge pull request #656 from CartoDB/static-maps-layers-filter-010
Static maps layers filter
2017-04-03 12:58:59 +02:00
Raul Ochoa
217749c340 Throw on invalid params argument 2017-04-03 12:16:32 +02:00
Raul Ochoa
e2c2e06d65 Configure extra allowed params per endpoint via middleware
Instead of making all params available in all endpoints, we control
what endpoints allow what extra params.

Dataviews endpoints should be migrated to this.
2017-04-03 12:16:24 +02:00
Raul Ochoa
1b510f05ac Add test to go red 2017-04-03 12:16:13 +02:00
Raul Ochoa
7e449f76c8 Merge pull request #651 from CartoDB/upgrade-grainstore-1.6.2-v2.x
Upgrade grainstore 1.6.2
2017-03-29 16:22:35 +02:00
Daniel García Aubert
97428b3f3e Upgrade grainstore 1.6.2 2017-03-29 16:15:20 +02:00
Raul Ochoa
ec33d85f6f Remove unused import 2017-03-23 01:22:44 +01:00
Raul Ochoa
a9b36d69d6 Use crc32 instead of md5 for computing subdomain candidate 2017-03-23 01:22:39 +01:00
Raul Ochoa
fa649881dc Generate URLs for resources based on CDN + template rules 2017-03-22 19:21:26 +01:00
Raul Ochoa
5ed8b78f34 Use v2.x branch 2017-03-21 17:04:23 +01:00
Daniel García Aubert
e0519e7851 Stubs next version 2017-03-17 17:06:39 +01:00
Daniel García Aubert
38294d29f5 Release 2.89.0 2017-03-17 16:45:18 +01:00
Daniel
5b131cc8a7 Merge pull request #639 from CartoDB/upgrade-windshaft-to-2.8.0
Upgrade windshaft to 2.8.0
2017-03-17 16:37:07 +01:00
Daniel García Aubert
5dee654132 Upgrade windshaft to 2.8.0 2017-03-17 16:26:26 +01:00
Daniel García Aubert
d902476780 Fix assertions, now MapCofig.getLayer() return {} if layer not found 2017-03-17 11:06:40 +00:00
Ubuntu
bc5dabef3c Revert "Fix assertions, now MapCofig.getLayer() return {} if layer not found"
This reverts commit c839a0b0a3.
2017-03-17 11:04:50 +00:00
Daniel García Aubert
024f1e4851 Fix assertions, now MapCofig.getLayer() return {} if layer not found 2017-03-17 10:45:59 +00:00
Raul Ochoa
5f87417d9e Merge pull request #637 from CartoDB/histogram-type-discovery
Histogram column type discovery query uses non-filtered query
2017-03-17 11:03:15 +01:00
Raul Ochoa
fa94550261 Include changes for overviews implementation 2017-03-16 19:15:34 +01:00
Raul Ochoa
2656a26272 Merge pull request #622 from strk/typo
Trip epoch is over...
2017-03-16 16:26:53 +01:00
Raul Ochoa
992b2b6ba7 Histogram column type discovery query uses non-filtered query
Pass all queries to the dataview and use the no filters one for
discovering what is the column type associated to the histogram dataview.
2017-03-13 18:40:29 +01:00
Raul Ochoa
924f009390 Test for #606: function avg(timestamp with time zone) does not exist 2017-03-13 18:36:43 +01:00
Raul Ochoa
48a1244fa0 Stubs next version 2017-03-10 11:06:47 +01:00
Raul Ochoa
8789a959e5 Release 2.88.4 2017-03-10 11:06:06 +01:00
Raul Ochoa
5765ac59cc Merge pull request #636 from CartoDB/upgrade-camshaft
Upgrades camshaft to 0.50.3
2017-03-10 11:05:30 +01:00
Raul Ochoa
3b9bf96431 Upgrades camshaft to 0.50.3 2017-03-10 10:58:07 +01:00
Mario de Frutos
8868066445 Stubs next version 2017-03-02 11:00:53 +01:00
Mario de Frutos
b446c31cbc Other category now uses the selected aggregated function (#633)
* Other category in category widget uses selected aggregation function
Fixes https://github.com/CartoDB/Windshaft-cartodb/issues/628
2017-03-02 10:48:20 +01:00
Raul Ochoa
2d6e7070a6 Stubs next version 2017-02-23 18:12:54 +01:00
Raul Ochoa
473e0cb902 Release 2.88.2 2017-02-23 18:12:14 +01:00
Raul Ochoa
1a8fca0534 Formatting 2017-02-23 18:11:39 +01:00
Raul Ochoa
e24bc12fc9 Merge pull request #632 from CartoDB/upgrade-camshaft
Upgrades camshaft to 0.50.2
2017-02-23 18:10:22 +01:00
Raul Ochoa
e77c9141ed Upgrades camshaft to 0.50.2 2017-02-23 18:00:55 +01:00
Raul Ochoa
321157b17b Make target to generate shrinkwrap file applying prune
It removes dev dependencies from the local node_modules
and avoid getting those into shrinkwrap file.
2017-02-23 17:57:52 +01:00
Raul Ochoa
6ac6574b4c clean target to completely delete node_modules dir 2017-02-23 17:57:44 +01:00
Raul Ochoa
933d486a57 Merge pull request #630 from CartoDB/close-response-timer
Close timer for response preparation
2017-02-22 18:46:56 +01:00
Raul Ochoa
8a1c7f5b52 Close timer for response preparation
Timer for affectedTables is taking everything from response timing,
adding a tag to represent all the response preparation.
That way affectedTables only represents the time for retrieving the
affected tables themselves.
2017-02-22 18:38:25 +01:00
Rafa de la Torre
822954be5d Stub next version 2017-02-21 11:06:54 +01:00
Rafa de la Torre
57dc17518c Release 2.88.1 2017-02-21 11:01:15 +01:00
Mario de Frutos
3df8d4844e Stubs next version 2017-02-21 08:17:09 +01:00
Mario de Frutos
5a0443618f Merge pull request #626 from CartoDB/camshaft_v050
Camshaft 0.50.0
2017-02-21 08:08:34 +01:00
Raul Ochoa
8a76cd506f Fix issue with global Promise object 2017-02-20 18:46:48 +01:00
Mario de Frutos
50d05eae47 Version 2.88.0
Update camshaft version to 0.50.0
2017-02-20 18:46:47 +01:00
Raul Ochoa
ca41b3b600 Merge pull request #627 from CartoDB/change_release
We don't announce release deploy in google groups anymore
2017-02-20 10:31:39 +01:00
Mario de Frutos
43a17ddc7d We don't announce release deploy in google groups anymore 2017-02-17 18:01:59 +01:00
Mario de Frutos
dfa347f860 Update camshaft dependency to 0.50 2017-02-17 16:29:34 +01:00
Raul Ochoa
28f1179336 Stubs next version 2017-02-02 16:44:41 +01:00
Raul Ochoa
de4d9e285e Release 2.87.5 2017-02-02 16:43:43 +01:00
Raul Ochoa
e0faaac822 Update news 2017-02-02 16:43:04 +01:00
Raul Ochoa
c84f27dd3f Merge pull request #623 from CartoDB/fix-dataviews-override-params-casting
Cast all dataview overrides values to Number
2017-02-02 16:42:48 +01:00
Raul Ochoa
12279d5c00 Cast dataview override values to Number or throw error
We were letting params expected as Numbers to be passed as any type
when they were not Numbers.
2017-02-02 16:20:16 +01:00
Raul Ochoa
281588abd2 Add test to validate bins param is casted as Number 2017-02-02 16:12:49 +01:00
Sandro Santilli
7e206b84aa Fix typo 2017-01-31 13:16:36 +01:00
Mario de Frutos
f69f999694 Fixed contributing link 2017-01-25 10:43:24 +01:00
Daniel García Aubert
c0c062592f Stubs next version 2017-01-20 11:40:00 +01:00
Daniel García Aubert
06885e2ba3 Release 2.87.4 2017-01-20 11:32:11 +01:00
Daniel
89a268d087 Merge pull request #617 from CartoDB/497-null-category
Be able to not compute NULL categories and null values
2017-01-20 11:26:12 +01:00
csobier
3648b8b0b1 removed "default" from descriptions
As per a customer Support inquiry, I removed "default" from the Static Maps Limits section, as it was confusing to the user that it implied
2017-01-17 11:21:23 -05:00
Daniel García Aubert
83301238d2 Port changes to overviews 2017-01-17 17:10:08 +01:00
Daniel García Aubert
a4a1fb930a Be able to not compute NULL categories and null values wheter aggregation operation is not 'count' 2017-01-17 17:09:17 +01:00
Daniel García Aubert
6555353e0e Improve test to handle NULL values in category and aggregation columns using any operation 2017-01-16 19:23:08 +01:00
Daniel García Aubert
f5f0601e53 Add test to check if NULL category count values properly 2017-01-16 17:00:28 +01:00
Raul Ochoa
2598595e42 Merge pull request #614 from CartoDB/zipfile-cartodb
Use zipfile from cartodb
2016-12-23 14:47:40 +01:00
Raul Ochoa
49b78a85c9 Use zipfile from cartodb
Adding it as dependency in combination with loose definition in
millstone makes it to use the top level defined dependency.
2016-12-22 19:12:30 +01:00
Javier Goizueta
0918c8e68c Stub next version 2016-12-19 17:20:18 +01:00
Javier Goizueta
1603a07de1 Release 2.87.3 2016-12-19 17:13:12 +01:00
Javier Goizueta
0a37aa4ba1 Merge pull request #604 from CartoDB/fix-overviews-dataviews
Fixes for dataviews using overviews
2016-12-19 16:55:13 +01:00
Javier Goizueta
b721a80fcc Merge branch 'master' into fix-overviews-dataviews 2016-12-19 16:45:28 +01:00
Rafa de la Torre
01365d035e Stub version 2.87.3 2016-12-19 16:38:27 +01:00
Rafa de la Torre
a4f059e20f Merge pull request #605 from CartoDB/update-camshaft-0.48.5
Update camshaft to 0.48.5
2016-12-19 16:33:27 +01:00
Rafa de la Torre
79c35118d7 Update camshaft to 0.48.5
Use exception safe Dataservices API functions. See
https://github.com/CartoDB/dataservices-api/issues/314 and
https://github.com/CartoDB/camshaft/issues/242
2016-12-19 15:52:25 +01:00
Javier Goizueta
6a4f5d52ec Don't use overviews for date histograms 2016-12-16 17:51:36 +01:00
Javier Goizueta
ccaae2dd66 Remove spurious parameter from overviews dataviews functions
In the overviews-specialized dataview classes the sql-generating
functions had an unneeded parameter filters.
In some cases, since this parameter was not being paaased from
the base dataviews class it was masking the override parameter.
2016-12-16 17:37:05 +01:00
Raul Ochoa
d335e64f88 Stubs next version 2016-12-13 14:16:49 +01:00
Raul Ochoa
177d7ed07a Release 2.87.1 2016-12-13 14:15:50 +01:00
Raul Ochoa
85a1e15b58 Merge pull request #603 from CartoDB/upgrade-deps
Upgrade windshaft and request deps
2016-12-13 14:15:04 +01:00
Raul Ochoa
432b58a078 Upgrade windshaft and request deps 2016-12-13 14:10:42 +01:00
xavijam
75e3c5daef Ready for next release 2016-12-13 10:43:16 +01:00
Raul Ochoa
deb71c27b0 Merge pull request #601 from CartoDB/update-turbocarto
Update turbocarto package version
2016-12-12 18:56:54 +01:00
Raul Ochoa
8f5e1de6d8 Merge branch 'master' into update-turbocarto 2016-12-12 18:51:04 +01:00
Raul Ochoa
4836d62d7a Merge pull request #602 from CartoDB/travis-improvements
Travis improvements
2016-12-12 18:50:39 +01:00
Raul Ochoa
d27b0617b2 Remove comments 2016-12-12 18:44:25 +01:00
Raul Ochoa
28a2c29a39 Attempt to use postgis 2.3 2016-12-12 18:39:57 +01:00
Raul Ochoa
fcb6478407 Attempt to use postgis 2.2 2016-12-12 18:33:19 +01:00
Raul Ochoa
6d72afe40e Test sudoless travis 2016-12-12 17:30:14 +01:00
Raul Ochoa
e775266c64 Remove notifications 2016-12-12 17:29:32 +01:00
xavijam
12f25b38c0 Updated news 2016-12-12 17:03:56 +01:00
xavijam
c67a1107cb Package version updated with turbo-carto cahnges 2016-12-12 17:02:31 +01:00
xavijam
34bfb0d62c Updated news 2016-12-12 17:02:15 +01:00
Javier Goizueta
ede45cad1f Stub next version 2016-12-02 15:25:52 +01:00
40 changed files with 3608 additions and 2639 deletions

1
.gitignore vendored
View File

@@ -3,6 +3,7 @@ config.status*
config/environments/*.js
.idea
.vscode
.nvmrc
tools/munin/windshaft.conf
logs/
pids/

View File

@@ -1,9 +1,9 @@
dist: trusty # only environment that supports Postgres 9.5 at this time
sudo: required
dist: trusty
addons:
postgresql: "9.5"
apt:
packages:
- postgresql-9.5-postgis-2.3
- postgresql-plpython-9.5
- pkg-config
- libcairo2-dev
@@ -22,9 +22,3 @@ env:
language: node_js
node_js:
- "0.10"
notifications:
irc:
channels:
- "irc.freenode.org#cartodb"
use_notice: true

View File

@@ -8,4 +8,4 @@ We love pull requests from everyone, see [Contributing to Open Source on GitHub]
## Submitting Contributions
* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).
* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

View File

@@ -1,11 +1,10 @@
1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json
3. Ensure NEWS section exists for the new version, review it, add release date
4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap`
4. Recreate npm-shrinkwrap.json with: `make shrinkwrap`
5. Commit package.json, npm-shrinwrap.json, NEWS
6. git tag -a Major.Minor.Patch # use NEWS section as content
7. Announce on cartodb@googlegroups.com
8. Stub NEWS/package for next version
7. Stub NEWS/package for next version
Versions:

View File

@@ -7,7 +7,13 @@ all:
@$(SHELL) ./scripts/install.sh
clean:
rm -rf node_modules/*
rm -rf node_modules/
shrinkwrap: clean
-rm npm-shrinkwrap.json
npm install --no-shrinkwrap --production
npm prune
npm shrinkwrap
distclean: clean
rm config.status*

127
NEWS.md
View File

@@ -1,5 +1,132 @@
# Changelog
## 2.91.2
Released 2017-04-24
Announcements:
- Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3).
## 2.91.1
Released 2017-04-24
Announcements:
- Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1).
## 2.91.0
Released 2017-04-11
Announcements:
- Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0).
## 2.90.2
Released 2017-04-11
Announcements:
- Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1)
## 2.90.0
Released 2017-04-10
Bug fixes:
- Fix invalidation of cache for maps with analyses #661.
Announcements:
- Upgrades camshaft to [0.53.0](https://github.com/CartoDB/camshaft/releases/tag/0.53.0)
## 2.89.0
Released 2017-03-17
**Deprecation warning**: v2.89.0 is the last release that supports Node v0.10.x. Next mayor release will support Node v6.9.x and further versions.
Announcements:
- Upgrades windshaft to [2.8.0](https://github.com/CartoDB/windshaft/releases/tag/2.8.0).
Bug fixes:
- Histogram column type discovery query uses non-filtered query #637
## 2.88.4
Released 2017-03-10
Announcements:
- Upgrades camshaft to [0.50.3](https://github.com/CartoDB/camshaft/releases/tag/0.50.3).
## 2.88.3
Released 2017-03-02
Bug fixes:
- Category dataviews now uses the proper aggregation function for the 'Other' category. See https://github.com/CartoDB/Windshaft-cartodb/issues/628
## 2.88.2
Released 2017-02-23
Announcements:
- Upgrades camshaft to [0.50.2](https://github.com/CartoDB/camshaft/releases/tag/0.50.2).
## 2.88.1
Released 2017-02-21
Announcements:
- Upgrades camshaft to [0.50.1](https://github.com/CartoDB/camshaft/releases/tag/0.50.1)
## 2.88.0
Released 2017-02-21
Announcements:
- Upgrades camshaft to [0.50.0](https://github.com/CartoDB/camshaft/releases/tag/0.50.0).
- Upgrades cartodb-psql to [0.7.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.7.1).
- Upgrades windshaft to [2.7.0](https://github.com/CartoDB/windshaft/releases/tag/2.7.0).
## 2.87.5
Released 2017-02-02
Bug fixes:
- Cast dataview override values to Number or throw error.
## 2.87.4
Released 2017-01-20
Bug fixes:
- Be able to not compute NULL categories and null values in aggregation dataviews #617.
## 2.87.3
Released 2016-12-19
Bug fixes:
- Fix overviews-related dataviews problems. See https://github.com/CartoDB/Windshaft-cartodb/pull/604
## 2.87.2
Released 2016-12-19
- Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242
## 2.87.1
Released 2016-12-13
Announcements:
- Upgrades windshaft to [2.6.4](https://github.com/CartoDB/Windshaft/releases/tag/2.6.4).
- Upgrades request dependency.
- Regenerate npm-shrinkwrap.json: missing dependency updates.
## 2.87.0
Released 2016-12-12
Enhancements:
- Upgrade turbo-carto dependency to version 0.19.0.
## 2.86.1
Released 2016-12-02

View File

@@ -146,9 +146,9 @@ It is important to note that generated images are cached from the live data refe
### Limits
* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
* Image resolution by default is set to 72 DPI
* JPEG quality by default is 85%
* While images can encompass an entirety of a map, the limit for pixel range is 8192 x 8192.
* Image resolution is set to 72 DPI
* JPEG quality is 85%
* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
## Examples

View File

@@ -79,7 +79,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
function castNumbers(overrides, val, k) {
overrides[k] = Number.isFinite(+val) ? +val : val;
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
}
overrides[k] = +val;
return overrides;
},
{ownFilter: ownFilter}

View File

@@ -36,7 +36,7 @@ function BaseController(authApi, pgConnection) {
module.exports = BaseController;
// jshint maxcomplexity:9
// jshint maxcomplexity:10
/**
* Whitelist input and get database name & default geometry type from
* subdomain/user metadata held in CartoDB Redis
@@ -77,7 +77,11 @@ BaseController.prototype.req2params = function(req, callback){
return;
}
req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
if (Array.isArray(req.context.allowedQueryParams)) {
allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams);
}
req.query = _.pick(req.query, allowedQueryParams);
req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
var user = req.context.user;

View File

@@ -6,6 +6,7 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var allowQueryParams = require('../middleware/allow-query-params');
var DataviewBackend = require('../backends/dataview');
var AnalysisStatusBackend = require('../backends/analysis-status');
@@ -67,11 +68,13 @@ LayergroupController.prototype.register = function(app) {
this.attributes.bind(this));
app.get(app.base_url_mapconfig +
'/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware,
'/static/center/:token/:z/:lat/:lng/:width/:height.:format',
cors(), userMiddleware, allowQueryParams(['layer']),
this.center.bind(this));
app.get(app.base_url_mapconfig +
'/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
'/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
cors(), userMiddleware, allowQueryParams(['layer']),
this.bbox.bind(this));
// Undocumented/non-supported API endpoint methods.
@@ -338,6 +341,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
var self = this;
req.profiler.done('res');
res.set('Cache-Control', 'public,max-age=31536000');
// Set Last-Modified header
@@ -386,13 +391,15 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
function getSQL(err, mapConfig) {
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
var queries = [];
mapConfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
queries.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
return queries.length ? queries.join(';') : null;
},

View File

@@ -1,11 +1,11 @@
var _ = require('underscore');
var dot = require('dot');
dot.templateSettings.strip = false;
var assert = require('assert');
var step = require('step');
var windshaft = require('windshaft');
var QueryTables = require('cartodb-query-tables');
var ResourceLocator = require('../models/resource-locator');
var util = require('util');
var BaseController = require('./base');
@@ -46,20 +46,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.layergroupAffectedTables = layergroupAffectedTables;
this.mapConfigAdapter = mapConfigAdapter;
this.resourcesUrlTemplates = null;
if (global.environment.resources_url_templates) {
var templates = global.environment.resources_url_templates;
if (templates.http) {
this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}');
}
if (templates.https) {
this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}');
}
}
this.resourceLocator = new ResourceLocator(global.environment);
}
util.inherits(MapController, BaseController);
@@ -325,9 +312,15 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
done();
});
var sql = mapconfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
var sql = [];
mapconfig.getLayers().forEach(function(layer) {
sql.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
sql.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
var dbName = req.params.dbname;
var layergroupId = layergroup.layergroupid;
@@ -338,7 +331,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
},
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
@@ -399,7 +392,7 @@ MapController.prototype.addAnalysesMetadata = function(username, layergroup, ana
var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
var nodeRepr = {
status: node.getStatus(),
url: this.getUrls(username, nodeResource)
url: this.resourceLocator.getUrls(username, nodeResource)
};
if (includeQuery) {
nodeRepr.query = node.getQuery();
@@ -429,7 +422,7 @@ MapController.prototype.addDataviewsUrls = function(username, layergroup, mapCon
Object.keys(dataviews).forEach(function(dataviewName) {
var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
layergroup.metadata.dataviews[dataviewName] = {
url: this.getUrls(username, resource)
url: this.resourceLocator.getUrls(username, resource)
};
}.bind(this));
};
@@ -444,7 +437,7 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig
var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
layer.widgets[widgetName] = {
type: mapConfigLayer.options.widgets[widgetName].type,
url: this.getUrls(username, resource)
url: this.resourceLocator.getUrls(username, resource)
};
}.bind(this));
}
@@ -452,46 +445,3 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig
}.bind(this));
}
};
MapController.prototype.getUrls = function(username, resource) {
if (this.resourcesUrlTemplates) {
return this.getUrlsFromTemplate(username, resource);
}
var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
if (cdnUrl) {
return {
http: 'http://' + cdnUrl.http + '/' + username + '/api/v1/map/' + resource,
https: 'https://' + cdnUrl.https + '/' + username + '/api/v1/map/' + resource
};
} else {
var port = global.environment.port;
return {
http: 'http://' + username + '.' + 'localhost.lan:' + port + '/api/v1/map/' + resource
};
}
};
MapController.prototype.getUrlsFromTemplate = function(username, resource) {
var urls = {};
var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url || {};
if (this.resourcesUrlTemplates.http) {
urls.http = this.resourcesUrlTemplates.http({
cdn_url: cdnUrl.http,
user: username,
port: global.environment.port,
resource: resource
});
}
if (this.resourcesUrlTemplates.https) {
urls.https = this.resourcesUrlTemplates.https({
cdn_url: cdnUrl.https,
user: username,
port: global.environment.port,
resource: resource
});
}
return urls;
};

View File

@@ -8,6 +8,7 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var allowQueryParams = require('../middleware/allow-query-params');
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
@@ -31,7 +32,7 @@ NamedMapsController.prototype.register = function(app) {
this.tile.bind(this));
app.get(app.base_url_mapconfig +
'/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
'/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']),
this.staticMap.bind(this));
};

View File

@@ -0,0 +1,9 @@
module.exports = function allowQueryParams(params) {
if (!Array.isArray(params)) {
throw new Error('allowQueryParams must receive an Array of params');
}
return function allowQueryParamsMiddleware(req, res, next) {
req.context.allowedQueryParams = params;
next();
};
};

View File

@@ -19,25 +19,38 @@ var rankedCategoriesQueryTpl = dot.template([
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
')'
].join('\n'));
var categoriesSummaryQueryTpl = dot.template([
'categories_summary AS(',
' SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
var categoriesSummaryMinMaxQueryTpl = dot.template([
'categories_summary_min_max AS(',
' SELECT max(value) max_val, min(value) min_val',
' FROM categories',
')'
].join('\n'));
var categoriesSummaryCountQueryTpl = dot.template([
'categories_summary_count AS(',
' SELECT count(1) AS categories_count',
' FROM (',
' SELECT {{=it._column}} AS category',
' FROM ({{=it._query}}) _cdb_categories',
' GROUP BY {{=it._column}}',
' ) _cdb_categories_count',
')'
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
' count, categories_count',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
].join('\n'));
@@ -45,7 +58,7 @@ var rankedAggregationQueryTpl = dot.template([
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
'ORDER BY value DESC'
].join('\n'));
@@ -114,24 +127,10 @@ Aggregation.prototype.sql = function(psql, override, callback) {
var _query = this.query;
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
_query: _query,
_column: this.column
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
}),
categoriesSummaryQueryTpl({
_query: _query,
_column: this.column
})
].join(',\n'),
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
aggregationQueryTpl({
_query: _query,
_column: this.column,
@@ -141,25 +140,11 @@ Aggregation.prototype.sql = function(psql, override, callback) {
].join('\n');
} else {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
_query: _query,
_column: this.column
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
}),
categoriesSummaryQueryTpl({
_query: _query,
_column: this.column
})
].join(',\n'),
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
rankedAggregationQueryTpl({
_query: _query,
_column: this.column,
_aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
_limit: CATEGORIES_LIMIT
})
].join('\n');
@@ -170,6 +155,32 @@ Aggregation.prototype.sql = function(psql, override, callback) {
return callback(null, aggregationSql);
};
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
return [
"WITH",
[
summaryQueryTpl({
_query: query,
_column: column
}),
rankedCategoriesQueryTpl({
_query: query,
_column: column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
categoriesSummaryMinMaxQueryTpl({
_query: query,
_column: column
}),
categoriesSummaryCountQueryTpl({
_query: query,
_column: column
})
].join(',\n')
].join('\n');
};
var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
Aggregation.prototype.getAggregationSql = function() {
return aggregationFnQueryTpl({

View File

@@ -11,7 +11,7 @@ var DataviewFactory = {
if (!this.dataviews[type]) {
throw new Error('Invalid dataview type: "' + type + '"');
}
return new this.dataviews[type](query, dataviewDefinition.options);
return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql);
}
};

View File

@@ -109,12 +109,13 @@ var TYPE = 'histogram';
}
}
*/
function Histogram(query, options) {
function Histogram(query, options, queries) {
if (!_.isString(options.column)) {
throw new Error('Histogram expects `column` in widget options');
}
this.query = query;
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
@@ -143,7 +144,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.query
column: _column, query: this.queries.no_filters
});
if (this._columnType === null) {

View File

@@ -18,25 +18,37 @@ var rankedCategoriesQueryTpl = dot.template([
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
')'
].join('\n'));
var categoriesSummaryQueryTpl = dot.template([
'categories_summary AS(',
' SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
var categoriesSummaryMinMaxQueryTpl = dot.template([
'categories_summary_min_max AS(',
' SELECT max(value) max_val, min(value) min_val',
' FROM categories',
')'
].join('\n'));
var categoriesSummaryCountQueryTpl = dot.template([
'categories_summary_count AS(',
' SELECT count(1) AS categories_count',
' FROM (',
' SELECT {{=it._column}} AS category',
' FROM ({{=it._query}}) _cdb_categories',
' GROUP BY {{=it._column}}',
' ) _cdb_categories_count',
')'
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
].join('\n'));
@@ -44,7 +56,7 @@ var rankedAggregationQueryTpl = dot.template([
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
'ORDER BY value DESC'
].join('\n'));
@@ -65,7 +77,7 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, filters, override, callback) {
Aggregation.prototype.sql = function(psql, override, callback) {
if (!callback) {
callback = override;
override = {};
@@ -85,9 +97,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
_aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
}),
categoriesSummaryQueryTpl({
categoriesSummaryMinMaxQueryTpl({
_query: _query,
_column: this.column
}),
categoriesSummaryCountQueryTpl({
_query: _query,
_column: this.column
})
@@ -110,9 +127,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
_aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
}),
categoriesSummaryQueryTpl({
categoriesSummaryMinMaxQueryTpl({
_query: _query,
_column: this.column
}),
categoriesSummaryCountQueryTpl({
_query: _query,
_column: this.column
})

View File

@@ -55,20 +55,20 @@ BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
};
// Default behaviour
BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) {
BaseOverviewsDataview.prototype.defaultSql = function(psql, override, callback) {
var query = this.query;
var dataview = this.baseDataview;
if ( SETTINGS.defaultOverviews ) {
query = this.rewrittenQuery(query);
dataview = new this.BaseDataview(query, this.queryOptions);
}
return dataview.sql(psql, filters, override, callback);
return dataview.sql(psql, override, callback);
};
// default implementation that can be override in derived classes:
BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) {
return this.defaultSql(psql, filters, override, callback);
BaseOverviewsDataview.prototype.sql = function(psql, override, callback) {
return this.defaultSql(psql, override, callback);
};
BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {

View File

@@ -14,7 +14,8 @@ OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinit
return parentFactory.getDataview(query, dataviewDefinition);
}
return new dataviews[type](
query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options,
dataviewDefinition.sql
);
};

View File

@@ -36,7 +36,7 @@ Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, filters, override, callback) {
Formula.prototype.sql = function(psql, override, callback) {
var formulaQueryTpl = formulaQueryTpls[this.operation];
if ( formulaQueryTpl ) {
@@ -52,5 +52,5 @@ Formula.prototype.sql = function(psql, filters, override, callback) {
}
// default behaviour
return this.defaultSql(psql, filters, override, callback);
return this.defaultSql(psql, override, callback);
};

View File

@@ -8,7 +8,6 @@ dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
@@ -97,10 +96,11 @@ var histogramQueryTpl = dot.template([
'ORDER BY bin'
].join('\n'));
function Histogram(query, options, queryRewriter, queryRewriteData, params) {
function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
this.query = query;
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
@@ -130,7 +130,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.rewrittenQuery(this.query)
column: _column, query: this.rewrittenQuery(this.queries.no_filters)
});
if (this._columnType === null) {
@@ -149,7 +149,9 @@ Histogram.prototype.sql = function(psql, override, callback) {
}
if (this._columnType === 'date') {
_column = columnCastTpl({column: _column});
// overviews currently aggregate dates to NULL
// to avoid problem we don't use overviews for histograms of date columns
return this.defaultSql(psql, override, callback);
}
var _query = this.rewrittenQuery(this.query);

View File

@@ -115,6 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
}
layer.options.sql = analysisSql;
layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode);
} else {
missingNodesErrors.push(
new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
@@ -330,4 +331,13 @@ function AnalysisError(message) {
this.message = message;
}
function getAllAffectedTablesFromSourceNodes(node) {
var affectedTables = node.getAllInputNodes(function (node) {
return node.getType() === 'source';
}).reduce(function(list, node) {
return list.concat(node.getAffectedTables());
},[]);
return affectedTables;
}
require('util').inherits(AnalysisError, Error);

View File

@@ -4,6 +4,13 @@ var dot = require('dot');
dot.templateSettings.strip = false;
var queue = require('queue-async');
var PSQL = require('cartodb-psql');
/**
* cartodb-psql creates `global.Promise` as an empty constructor.
* However, `turbo-carto` relies on a polyfil that fails to create the polyfil
* as it finds `global.Promise` but it doesn't find `Promise.resolve`.
*/
global.Promise = global.Promise || function() {};
global.Promise.resolve = global.Promise.resolve || function() {};
var turboCarto = require('turbo-carto');
var SubstitutionTokens = require('../../../utils/substitution-tokens');

View File

@@ -0,0 +1,119 @@
var dot = require('dot');
dot.templateSettings.strip = false;
function ResourceLocator(environment) {
this.environment = environment;
this.resourcesUrlTemplates = null;
if (this.environment.resources_url_templates) {
var templates = environment.resources_url_templates;
if (templates.http) {
this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}');
}
if (templates.https) {
this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}');
}
}
}
module.exports = ResourceLocator;
ResourceLocator.prototype.getUrls = function(username, resource) {
if (this.resourcesUrlTemplates) {
return this.getUrlsFromTemplate(username, resource);
}
var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource);
if (cdnDomain) {
return {
http: 'http://' + cdnDomain.http + '/' + username + '/api/v1/map/' + resource,
https: 'https://' + cdnDomain.https + '/' + username + '/api/v1/map/' + resource
};
} else {
var port = this.environment.port;
return {
http: 'http://' + username + '.' + 'localhost.lan:' + port + '/api/v1/map/' + resource
};
}
};
ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource) {
var urls = {};
var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
if (this.resourcesUrlTemplates.http) {
urls.http = this.resourcesUrlTemplates.http({
cdn_url: cdnDomain.http,
user: username,
port: this.environment.port,
resource: resource
});
}
if (this.resourcesUrlTemplates.https) {
urls.https = this.resourcesUrlTemplates.https({
cdn_url: cdnDomain.https,
user: username,
port: this.environment.port,
resource: resource
});
}
return urls;
};
function getCdnDomain(serverMetadata, resource) {
if (serverMetadata && serverMetadata.cdn_url) {
var cdnUrl = serverMetadata.cdn_url;
var http = cdnUrl.http;
var https = cdnUrl.https;
if (cdnUrl.templates) {
var templates = cdnUrl.templates;
var httpUrlTemplate = templates.http.url;
var httpsUrlTemplate = templates.https.url;
http = httpUrlTemplate
.replace(/^(http[s]*:\/\/)/, '')
.replace('{s}', subdomain(templates.http.subdomains, resource));
https = httpsUrlTemplate
.replace(/^(http[s]*:\/\/)/, '')
.replace('{s}', subdomain(templates.https.subdomains, resource));
}
return {
http: http,
https: https,
};
}
return null;
}
// ref https://jsperf.com/js-crc32
function crcTable() {
var c;
var table = [];
for (var n = 0; n < 256; n++) {
c = n;
for (var k = 0; k < 8; k++) {
c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
}
table[n] = c;
}
return table;
}
var CRC_TABLE = crcTable();
function crc32(str) {
var crc = 0 ^ (-1);
for (var i = 0; i < str.length; i++) {
crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xFF];
}
return (crc ^ (-1)) >>> 0;
}
function subdomain(subdomains, resource) {
var index = crc32(resource) % subdomains.length;
return subdomains[index];
}
module.exports.subdomain = subdomain;

4463
npm-shrinkwrap.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "2.86.1",
"version": "2.91.2",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -20,8 +20,8 @@
],
"dependencies": {
"body-parser": "~1.14.0",
"camshaft": "0.48.4",
"cartodb-psql": "~0.6.1",
"camshaft": "0.54.1",
"cartodb-psql": "~0.7.1",
"cartodb-query-tables": "~0.1.0",
"cartodb-redis": "0.13.1",
"debug": "~2.2.0",
@@ -34,13 +34,14 @@
"node-statsd": "~0.0.7",
"queue-async": "~1.0.7",
"redis-mpool": "~0.4.0",
"request": "~2.62.0",
"request": "~2.79.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"turbo-carto": "0.18.0",
"turbo-carto": "0.19.0",
"underscore": "~1.6.0",
"windshaft": "2.6.3",
"yargs": "~5.0.0"
"windshaft": "cartodb/windshaft#v2.x",
"yargs": "~5.0.0",
"zipfile": "cartodb/node-zipfile#0.5.11-cdb1"
},
"devDependencies": {
"istanbul": "~0.4.3",

393
test/acceptance/cache/cache_headers.js vendored Normal file
View File

@@ -0,0 +1,393 @@
var testHelper = require('../../support/test_helper');
var assert = require('../../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../../lib/cartodb/server');
var serverOptions = require('../../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var LayergroupToken = require('../../support/layergroup-token');
describe('get requests with cache headers', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var statusOkResponse = {
status: 200
};
var mapConfigs = [
{
"description": "cache headers should be present",
"cache_headers": {
"x_cache_channel": {
"db_name": "test_windshaft_cartodb_user_1_db",
"tables": ["public.test_table"]
},
"surrogate_keys": "t:77pJnX"
},
"data":
{
version: '1.5.0',
layers: [
{
options: {
source: {
id: "2570e105-7b37-40d2-bdf4-1af889598745"
},
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
],
analyses: [
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "source",
"params": {
"query": "select * from test_table limit 2"
}
}
]
},
},
{
"description": "cache headers should be present and be composed with source table name",
"cache_headers": {
"x_cache_channel": {
"db_name": "test_windshaft_cartodb_user_1_db",
"tables": ["public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d",
"public.test_table"]
},
"surrogate_keys": "t:77pJnX t:iL4eth"
},
"data":
{
version: '1.5.0',
layers: [
{
options: {
source: {
id: "2570e105-7b37-40d2-bdf4-1af889598745"
},
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
],
analyses: [
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from test_table limit 2"
}
},
"radius": 50000
}
}
]
}
}];
var layergroupRequest = function(mapConfig) {
return {
url: '/api/v1/map?api_key=1234&config=' + encodeURIComponent(JSON.stringify(mapConfig)),
method: 'GET',
headers: {
host: 'localhost'
}
};
};
function getRequest(url, addApiKey, callbackName) {
var params = {};
if (!!addApiKey) {
params.api_key = '1234';
}
if (!!callbackName) {
params.callback = callbackName;
}
return {
url: url + '?' + qs.stringify(params),
method: 'GET',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
};
}
function validateCacheHeaders(done, expectedCacheHeaders) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(res.headers['x-cache-channel']);
assert.ok(res.headers['surrogate-key']);
if (expectedCacheHeaders) {
validateXChannelHeaders(res.headers, expectedCacheHeaders);
assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys);
}
done();
};
}
function validateXChannelHeaders(headers, expectedCacheHeaders) {
var dbName = headers['x-cache-channel'].split(':')[0];
var tables = headers['x-cache-channel'].split(':')[1].split(',').sort();
assert.equal(dbName, expectedCacheHeaders.x_cache_channel.db_name);
assert.deepEqual(tables, expectedCacheHeaders.x_cache_channel.tables.sort());
}
function noCacheHeaders(done) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(
!res.headers['x-cache-channel'],
'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
);
assert.ok(
!res.headers['surrogate-key'],
'did not expect surrogate-key header, got: `' + res.headers['surrogate-key'] + '`'
);
done();
};
}
function withLayergroupId(mapConfig, callback) {
assert.response(
server,
layergroupRequest(mapConfig),
statusOkResponse,
function(res, err) {
if (err) {
return callback(err);
}
var layergroupId = JSON.parse(res.body).layergroupid;
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
callback(null, layergroupId, res);
}
);
}
mapConfigs.forEach(function(mapConfigData) {
describe(mapConfigData.description, function() {
var mapConfig = mapConfigData.data;
var expectedCacheHeaders = mapConfigData.cache_headers;
it('/api/v1/map Map instantiation', function(done) {
var testFn = validateCacheHeaders(done, expectedCacheHeaders);
withLayergroupId(mapConfig, function(err, layergroupId, res) {
testFn(res);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
});
});
describe('cache headers should NOT be present', function() {
it('/', function(done) {
assert.response(
server,
getRequest('/'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/version', function(done) {
assert.response(
server,
getRequest('/version'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/health', function(done) {
assert.response(
server,
getRequest('/health'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/api/v1/map/named list named maps', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named', true),
statusOkResponse,
noCacheHeaders(done)
);
});
describe('with named maps', function() {
var templateName = 'x_cache';
beforeEach(function(done) {
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
layergroup: mapConfigs[0].data
};
var namedMapRequest = {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(template)
};
assert.response(
server,
namedMapRequest,
statusOkResponse,
function(res, err) {
done(err);
}
);
});
afterEach(function(done) {
assert.response(
server,
{
url: '/api/v1/map/named/' + templateName + '?api_key=1234',
method: 'DELETE',
headers: {
host: 'localhost'
}
},
{
status: 204
},
function(res, err) {
done(err);
}
);
});
it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
statusOkResponse,
noCacheHeaders(done)
);
});
});
});
});

View File

@@ -1,9 +1,8 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('aggregations', function() {
describe('aggregations happy cases', function() {
afterEach(function(done) {
if (this.testClient) {
@@ -13,30 +12,36 @@ describe('aggregations', function() {
}
});
function aggregationOperationMapConfig(operation) {
return {
function aggregationOperationMapConfig(operation, query, column, aggregationColumn) {
column = column || 'adm0name';
aggregationColumn = aggregationColumn || 'pop_max';
query = query || 'select * from populated_places_simple_reduced';
var mapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
sql: query,
cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
cartocss_version: '2.0.1',
widgets: {
adm0name: {
type: 'aggregation',
options: {
column: 'adm0name',
aggregation: operation,
aggregationColumn: 'pop_max'
}
}
}
widgets: {}
}
}
]
};
mapConfig.layers[0].options.widgets[column] = {
type: 'aggregation',
options: {
column: column,
aggregation: operation,
aggregationColumn: aggregationColumn
}
};
return mapConfig;
}
var operations = ['count', 'sum', 'avg', 'max', 'min'];
@@ -56,4 +61,88 @@ describe('aggregations', function() {
});
});
});
var query = [
'select 1 as val, \'a\' as cat, ST_Transform(ST_SetSRID(ST_MakePoint(0,0),4326),3857) as the_geom_webmercator',
'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(0,1),4326),3857)',
'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(1,0),4326),3857)',
'select null, null, ST_Transform(ST_SetSRID(ST_MakePoint(1,1),4326),3857)'
].join(' UNION ALL ');
operations.forEach(function (operation) {
var not = operation === 'count' ? ' not ' : ' ';
var description = 'should' +
not +
'handle NULL values in category and aggregation columns using "' +
operation +
'" as aggregation operation';
it(description, function (done) {
this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val'));
this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
assert.ifError(err);
assert.ok(aggregation);
assert.equal(aggregation.type, 'aggregation');
assert.ok(aggregation.categories);
assert.equal(aggregation.categoriesCount, 3);
assert.equal(aggregation.count, 4);
assert.equal(aggregation.nulls, 1);
var hasNullCategory = false;
aggregation.categories.forEach(function (category) {
if (category.category === null) {
hasNullCategory = true;
}
});
if (operation === 'count') {
assert.ok(hasNullCategory, 'aggregation has not a category NULL');
} else {
assert.ok(!hasNullCategory, 'aggregation has category NULL');
}
done();
});
});
});
var operations_and_values = {'count': 9, 'sum': 45, 'avg': 5, 'max': 9, 'min': 1};
var query_other = [
'select generate_series(1,3) as val, \'other_a\' as cat, NULL as the_geom_webmercator',
'select generate_series(4,6) as val, \'other_b\' as cat, NULL as the_geom_webmercator',
'select generate_series(7,9) as val, \'other_c\' as cat, NULL as the_geom_webmercator',
'select generate_series(10,12) as val, \'category_1\' as cat, NULL as the_geom_webmercator',
'select generate_series(10,12) as val, \'category_2\' as cat, NULL as the_geom_webmercator',
'select generate_series(10,12) as val, \'category_3\' as cat, NULL as the_geom_webmercator',
'select generate_series(10,12) as val, \'category_4\' as cat, NULL as the_geom_webmercator',
'select generate_series(10,12) as val, \'category_5\' as cat, NULL as the_geom_webmercator'
].join(' UNION ALL ');
Object.keys(operations_and_values).forEach(function (operation) {
var description = 'should aggregate OTHER category using "' + operation + '"';
it(description, function (done) {
this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
assert.ifError(err);
assert.ok(aggregation);
assert.equal(aggregation.type, 'aggregation');
assert.ok(aggregation.categories);
assert.equal(aggregation.categoriesCount, 8);
assert.equal(aggregation.count, 24);
assert.equal(aggregation.nulls, 0);
var aggregated_categories = aggregation.categories.filter( function(category) {
return category.agg === true;
});
assert.equal(aggregated_categories.length, 1);
assert.equal(aggregated_categories[0].value, operations_and_values[operation]);
done();
});
});
});
});

View File

@@ -77,4 +77,24 @@ describe('histogram-dataview', function() {
done();
});
});
it('should cast all overridable params to numbers', function(done) {
var params = {
bins: '256 AS other, (select 256 * 2) AS bins_number--',
start: 1e3,
end: 0,
response: TestClient.RESPONSE.ERROR
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('pop_max_histogram', params, function(err, res) {
assert.ok(!err, err);
assert.ok(res.errors);
assert.equal(res.errors.length, 1);
assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/));
done();
});
});
});

View File

@@ -144,6 +144,22 @@ describe('dataviews using tables with overviews', function() {
aggregationColumn: 'name',
}
},
test_histogram: {
type: 'histogram',
source: {id: 'data-source'},
options: {
column: 'value',
bins: 2
}
},
test_histogram_date: {
type: 'histogram',
source: {id: 'data-source'},
options: {
column: 'updated_at',
bins: 2
}
},
test_avg: {
type: 'formula',
source: {id: 'data-source'},
@@ -265,8 +281,83 @@ describe('dataviews using tables with overviews', function() {
});
});
it("should expose a histogram", function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_histogram', function (err, histogram) {
if (err) {
return done(err);
}
assert.ok(histogram);
assert.equal(histogram.type, 'histogram');
assert.ok(Array.isArray(histogram.bins));
testClient.drain(done);
});
});
describe('filters', function() {
describe('histogram', function () {
it("should expose a filtered histogram", function (done) {
var params = {
filters: {
dataviews: { test_histogram: { min: 2 } }
}
};
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_histogram', params, function (err, histogram) {
if (err) {
return done(err);
}
assert.ok(histogram);
assert.equal(histogram.type, 'histogram');
assert.ok(Array.isArray(histogram.bins));
assert.equal(histogram.bins.length, 4);
testClient.drain(done);
});
});
it("should expose a filtered histogram with no results", function (done) {
var params = {
filters: {
dataviews: { test_histogram: { max: -1 } }
}
};
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_histogram', params, function (err, histogram) {
if (err) {
return done(err);
}
assert.ok(histogram);
assert.equal(histogram.type, 'histogram');
assert.ok(Array.isArray(histogram.bins));
assert.equal(histogram.bins.length, 0);
testClient.drain(done);
});
});
it("should expose a filtered date histogram with no results", function (done) {
// This most likely works because the overviews will pass
// the responsibility to the normal dataviews.
var params = {
filters: {
dataviews: { test_histogram_date: { max: -1 } }
}
};
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_histogram_date', params, function (err, histogram) {
if (err) {
return done(err);
}
assert.ok(histogram);
assert.equal(histogram.type, 'histogram');
assert.ok(Array.isArray(histogram.bins));
assert.equal(histogram.bins.length, 0);
testClient.drain(done);
});
});
});
describe('category', function () {
var params = {

View File

@@ -574,7 +574,7 @@ describe(suiteName, function() {
if ( err ) {
return done(err);
}
// trip epoch
// strip epoch
expected_token = expected_token.split(':')[0];
keysToDelete['map_cfg|' + expected_token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;

View File

@@ -21,7 +21,7 @@ describe('named maps static view', function() {
var IMAGE_TOLERANCE = 20;
function createTemplate(view) {
function createTemplate(view, layers) {
return {
version: '0.0.1',
name: templateName,
@@ -36,7 +36,7 @@ describe('named maps static view', function() {
},
view: view,
layergroup: {
layers: [
layers: layers || [
{
type: 'mapnik',
options: {
@@ -198,4 +198,43 @@ describe('named maps static view', function() {
});
});
it('should allow to select the layers to render', function (done) {
var view = {
bounds: {
west: 0,
south: 0,
east: 45,
north: 45
}
};
var layers = [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: <%= color %>; }',
cartocss_version: '2.3.0'
}
},
{
type: 'mapnik',
options: {
sql: 'select ST_Transform(ST_MakeEnvelope(-45, -45, 45, 45, 4326), 3857) the_geom_webmercator',
cartocss: '#layer { polygon-fill: <%= color %>; }',
cartocss_version: '2.3.0'
}
}
];
templateMaps.addTemplate(username, createTemplate(view, layers), function (err) {
if (err) {
return done(err);
}
getStaticMap({ layer: 0 }, function(err, img) {
assert.ok(!err);
assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
});
});
});
});

View File

@@ -1052,8 +1052,9 @@ describe('template_api', function() {
'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
var cc = res.headers['x-cache-channel'];
var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
assert.ok(cc);
assert.ok(cc.match, /ciao/, cc);
assert.equal(cc, expectedCC);
// hack simulating restart...
server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache
var get_request = {
@@ -1072,8 +1073,9 @@ describe('template_api', function() {
'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
var cc = res.headers['x-cache-channel'];
var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart");
assert.ok(cc.match, /ciao/, cc);
assert.equal(cc, expectedCC);
return null;
},
function deleteTemplate(err)

View File

@@ -304,6 +304,37 @@ describe('widgets', function() {
});
});
it('can use a datetime filtered column with no results', function(done) {
this.testClient = new TestClient(histogramsMapConfig({
updated_at: {
type: 'histogram',
options: {
column: 'updated_at'
}
}
}));
var params = {
own_filter: 1,
filters: {
layers: [{
updated_at: {
// this will remove all results
max: -1
}
}]
}
};
this.testClient.getWidget('updated_at', params, function (err, res, histogram) {
assert.ok(!err, err);
assert.ok(histogram);
assert.equal(histogram.type, 'histogram');
assert.equal(histogram.bins.length, 0);
done();
});
});
it('can getTile with datetime filtered column', function(done) {
this.testClient = new TestClient(histogramsMapConfig({
updated_at: {

View File

@@ -1,307 +0,0 @@
var testHelper = require('../support/test_helper');
var assert = require('../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var LayergroupToken = require('../support/layergroup-token');
describe('get requests x-cache-channel', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var statusOkResponse = {
status: 200
};
var mapConfig = {
version: '1.3.0',
layers: [
{
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
]
};
var layergroupRequest = {
url: '/api/v1/map?config=' + encodeURIComponent(JSON.stringify(mapConfig)),
method: 'GET',
headers: {
host: 'localhost'
}
};
function getRequest(url, addApiKey, callbackName) {
var params = {};
if (!!addApiKey) {
params.api_key = '1234';
}
if (!!callbackName) {
params.callback = callbackName;
}
return {
url: url + '?' + qs.stringify(params),
method: 'GET',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
};
}
function validateXCacheChannel(done, expectedCacheChannel) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(res.headers['x-cache-channel']);
if (expectedCacheChannel) {
assert.equal(res.headers['x-cache-channel'], expectedCacheChannel);
}
done();
};
}
function noXCacheChannelHeader(done) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(
!res.headers['x-cache-channel'],
'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
);
done();
};
}
function withLayergroupId(callback) {
assert.response(
server,
layergroupRequest,
statusOkResponse,
function(res, err) {
if (err) {
return callback(err);
}
var layergroupId = JSON.parse(res.body).layergroupid;
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
callback(null, layergroupId, res);
}
);
}
describe('header should be present', function() {
it('/api/v1/map Map instantiation', function(done) {
var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table');
withLayergroupId(function(err, layergroupId, res) {
testFn(res);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
});
describe('header should NOT be present', function() {
it('/', function(done) {
assert.response(
server,
getRequest('/'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/version', function(done) {
assert.response(
server,
getRequest('/version'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/health', function(done) {
assert.response(
server,
getRequest('/health'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/api/v1/map/named list named maps', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named', true),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
describe('with named maps', function() {
var templateName = 'x_cache';
beforeEach(function(done) {
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
layergroup: mapConfig
};
var namedMapRequest = {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(template)
};
assert.response(
server,
namedMapRequest,
statusOkResponse,
function(res, err) {
done(err);
}
);
});
afterEach(function(done) {
assert.response(
server,
{
url: '/api/v1/map/named/' + templateName + '?api_key=1234',
method: 'DELETE',
headers: {
host: 'localhost'
}
},
{
status: 204
},
function(res, err) {
done(err);
}
);
});
it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
});
});
});

View File

@@ -145,7 +145,7 @@ describe('named_layers datasources', function() {
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(0), undefined);
assert.deepEqual(datasource.getLayerDatasource(0), {});
done();
}
@@ -160,7 +160,7 @@ describe('named_layers datasources', function() {
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(0), undefined);
assert.deepEqual(datasource.getLayerDatasource(0), {});
assert.equal(layers[1].type, 'cartodb');
assert.equal(layers[1].options.sql, wadusTemplateSql);
@@ -181,7 +181,7 @@ describe('named_layers datasources', function() {
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(0), undefined);
assert.deepEqual(datasource.getLayerDatasource(0), {});
assert.equal(layers[1].type, 'mapnik');
assert.equal(layers[1].options.sql, wadusMapnikSql);
@@ -263,7 +263,7 @@ describe('named_layers datasources', function() {
assert.equal(layers[3].type, 'cartodb');
assert.equal(layers[3].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(3), undefined);
assert.deepEqual(datasource.getLayerDatasource(3), {});
assert.equal(layers[4].type, 'cartodb');
assert.equal(layers[4].options.sql, wadusTemplateSql);
@@ -273,7 +273,7 @@ describe('named_layers datasources', function() {
assert.equal(layers[5].type, 'cartodb');
assert.equal(layers[5].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(5), undefined);
assert.deepEqual(datasource.getLayerDatasource(5), {});
assert.equal(layers[6].type, 'mapnik');
assert.equal(layers[6].options.sql, wadusMapnikSql);
@@ -298,6 +298,7 @@ describe('named_layers datasources', function() {
var context = {};
mapConfigNamedLayersAdapter.getMapConfig(username, testScenario.config, params, context,
function(err, mapConfig) {
assert.ifError(err);
testScenario.test(err, mapConfig.layers, context.datasource, done);
}
);

View File

@@ -75,7 +75,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
dropdb "${TEST_DB}"
createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check'
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
CURL_ARGS=""

View File

@@ -0,0 +1,6 @@
CREATE OR REPLACE FUNCTION CDB_Invalidate_Varnish(table_name TEXT)
RETURNS void AS
$$
BEGIN
END;
$$ LANGUAGE PLPGSQL;

View File

@@ -307,6 +307,13 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
url += '?' + qs.stringify(extraParams);
}
var expectedResponse = params.response || {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
var layergroupId;
step(
function createLayergroup() {
@@ -372,12 +379,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
host: 'localhost'
}
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
expectedResponse,
function(res, err) {
if (err) {
return next(err);

View File

@@ -0,0 +1,132 @@
require('../../../support/test_helper');
var assert = require('../../../support/assert');
var ResourceLocator = require('../../../../lib/cartodb/models/resource-locator');
describe('ResourceLocator.getUrls', function() {
var USERNAME = 'username';
var RESOURCE = 'wadus';
var HTTP_SUBDOMAINS = ['1', '2', '3', '4'];
var HTTPS_SUBDOMAINS = ['a', 'b', 'c', 'd'];
it('should return default urls when no serverMetadata is in environment', function() {
var resourceLocator = new ResourceLocator({});
var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
assert.ok(urls);
});
var BASIC_ENVIRONMENT = {
serverMetadata: {
cdn_url: {
http: 'cdn.carto.com',
https: 'cdn.ssl.carto.com'
}
}
};
it('should return default urls when basic http and https domains are provided', function() {
var resourceLocator = new ResourceLocator(BASIC_ENVIRONMENT);
var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
assert.ok(urls);
assert.equal(urls.http, ['http://cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
assert.equal(urls.https, ['https://cdn.ssl.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
});
var RESOURCE_TEMPLATES_ENVIRONMENT = {
serverMetadata: {
cdn_url: {
http: 'cdn.carto.com',
https: 'cdn.ssl.carto.com'
}
},
resources_url_templates: {
http: 'http://{{=it.user}}.localhost.lan/api/v1/map',
https: 'https://{{=it.user}}.ssl.localhost.lan/api/v1/map'
}
};
it('resources_url_templates should take precedence over http and https domains', function() {
var resourceLocator = new ResourceLocator(RESOURCE_TEMPLATES_ENVIRONMENT);
var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
assert.ok(urls);
assert.equal(urls.http, ['http://' + USERNAME + '.localhost.lan', 'api/v1/map', RESOURCE].join('/'));
assert.equal(urls.https, ['https://' + USERNAME + '.ssl.localhost.lan', 'api/v1/map', RESOURCE].join('/'));
});
var CDN_TEMPLATES_ENVIRONMENT = {
serverMetadata: {
cdn_url: {
http: 'cdn.carto.com',
https: 'cdn.ssl.carto.com',
templates: {
http: {
url: "http://{s}.cdn.carto.com",
subdomains: HTTP_SUBDOMAINS
},
https: {
url: "https://cdn_{s}.ssl.cdn.carto.com",
subdomains: HTTPS_SUBDOMAINS
}
}
}
}
};
it('cdn_url templates should take precedence over http and https domains', function() {
var resourceLocator = new ResourceLocator(CDN_TEMPLATES_ENVIRONMENT);
var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
assert.ok(urls);
var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
assert.equal(
urls.http,
['http://' + httpSubdomain + '.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
);
assert.equal(
urls.https,
['https://cdn_' + httpsSubdomain + '.ssl.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
);
});
var CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT = {
serverMetadata: {
cdn_url: {
http: 'cdn.carto.com',
https: 'cdn.ssl.carto.com',
templates: {
http: {
url: "http://{s}.cdn.carto.com",
subdomains: HTTP_SUBDOMAINS
},
https: {
url: "https://cdn_{s}.ssl.cdn.carto.com",
subdomains: HTTPS_SUBDOMAINS
}
}
}
},
resources_url_templates: {
http: 'http://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map',
https: 'https://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map'
}
};
it('should mix cdn_url templates and resources_url_templates', function() {
var resourceLocator = new ResourceLocator(CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT);
var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
assert.ok(urls);
var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
assert.equal(
urls.http,
['http://' + httpSubdomain + '.cdn.carto.com', 'u', USERNAME, 'api/v1/map', RESOURCE].join('/')
);
assert.equal(
urls.https,
['https://cdn_' + httpsSubdomain + '.ssl.cdn.carto.com', 'u', USERNAME, 'api/v1/map', RESOURCE].join('/')
);
});
});