Compare commits

...

423 Commits

Author SHA1 Message Date
Simon
7aefca3f82 updating NEWS.md 2017-08-22 16:26:36 +02:00
Simon Martín
3d409274e0 Merge pull request #730 from CartoDB/upgrade-camshaft
Upgrade camshaft to 0.55.8

All working fine in local and staging
2017-08-22 16:13:06 +02:00
Raul Ochoa
4491fa2faf Upgrade camshaft to 0.55.8 2017-08-22 14:17:17 +02:00
Raul Ochoa
0ed46930dd Stubs next version 2017-08-16 15:37:29 +02:00
Raul Ochoa
f3b7a857f2 Release 3.12.2 2017-08-16 15:36:16 +02:00
Raul Ochoa
3a22adf966 Update news 2017-08-16 15:35:34 +02:00
Raul Ochoa
1c6a76af72 Merge pull request #726 from CartoDB/725-polygon-count
725 fix polygon count
2017-08-16 15:33:19 +02:00
Raul Ochoa
175d3ac317 Merge pull request #728 from CartoDB/fix-test-gauge-error
Restore statsClient after performing test
2017-08-16 15:32:14 +02:00
Simon
175d070f09 using const instead of let and var and adding assert.ifError 2017-08-16 10:07:27 +02:00
Raul Ochoa
339f1aafa9 Stubs next version 2017-08-13 17:56:23 +02:00
Raul Ochoa
fef0dc302a Release 3.12.1 2017-08-13 17:55:45 +02:00
Raul Ochoa
58fec46117 Merge pull request #729 from CartoDB/upgrade-deps
Upgrades cartodb-psql, windshaft, and camshaft
2017-08-13 17:54:53 +02:00
Raul Ochoa
7be74d6ce1 Upgrades cartodb-psql, windshaft, and camshaft 2017-08-13 17:40:06 +02:00
Daniel García Aubert
d0f5ebd7ab Restore statsClient after performing test 2017-08-11 17:55:55 +02:00
Simon
92d33bf7fd linter details for polygons count test 2017-08-10 18:20:15 +02:00
Daniel García Aubert
490adbce4b Stubs next version 2017-08-10 18:19:20 +02:00
Simon
fab7832dee added ascii art for polygons count test 2017-08-10 18:16:53 +02:00
Simon
e678957a8f move polygon count test to widgets regression testfile, and check the only returned polygon is the expected one 2017-08-10 18:09:18 +02:00
Daniel García Aubert
e1e22de65f Release 3.12.0 2017-08-10 17:54:09 +02:00
Daniel
43312922fc Merge pull request #717 from CartoDB/response-time-limit
[limits] Response time limits
2017-08-10 17:35:47 +02:00
Daniel García Aubert
01a22a45bb Move setUserDatabaseTimeoutLimit from class method to a instance method 2017-08-10 17:09:05 +02:00
Raul Ochoa
9524433437 Use instance server 2017-08-10 16:24:40 +02:00
Raul Ochoa
14d5ee4178 Remove user param 2017-08-10 16:22:45 +02:00
Raul Ochoa
e7c206762d String comparison and regex to match errors instead of indexOf 2017-08-10 16:09:26 +02:00
Raul Ochoa
69eaa72819 String comparison and regex to match errors instead of indexOf 2017-08-10 16:06:10 +02:00
Raul Ochoa
23edf78a67 Remove unnecessary step 2017-08-10 15:58:25 +02:00
Simon
44c5eb051d formatting the query of polygon count test 2017-08-10 11:05:36 +02:00
Simon
814b123b2b fix 725 using the ST_Intersects function instead of && 2017-08-09 18:55:14 +02:00
Simon
ff560ffde7 add test boundingBox-polygon-counter 2017-08-09 18:49:59 +02:00
Daniel García Aubert
14f85abd39 Merge branch 'master' into response-time-limit 2017-08-09 18:48:38 +02:00
Daniel García Aubert
ce97844f37 Upgrade windshaft version to 3.3.0 2017-08-09 18:42:44 +02:00
Daniel García Aubert
d27a281067 Upgrade cartodb-redis to 0.14.0 2017-08-09 17:55:39 +02:00
Daniel García Aubert
3611752677 Stubs next version 2017-08-08 11:25:18 +02:00
Daniel García Aubert
b24858a17c Release 3.11.0 2017-08-08 11:21:44 +02:00
Daniel
26a967d0a7 Merge pull request #723 from CartoDB/fix-time-series-issues
Fix time series issues
2017-08-08 10:51:13 +02:00
Daniel García Aubert
c643160671 Prefix date interval query (to calculate automatic aggregation) to avoid name collision 2017-08-08 10:32:53 +02:00
Daniel García Aubert
e7a0b246a3 Prefix with double underscore 2017-08-08 10:20:36 +02:00
Daniel García Aubert
3c061769c6 Prefix basics columns to avoid name collision 2017-08-08 10:11:35 +02:00
Daniel García Aubert
7e159c565b Prefix iqr query calculation to avoid name collision 2017-08-08 10:03:40 +02:00
Daniel García Aubert
ff3d7ed7b2 Fix jshint typo 2017-08-08 09:57:12 +02:00
Daniel García Aubert
cf71489a7f Prefix nans and infinities counters to avoid name collision 2017-08-08 09:54:49 +02:00
Daniel García Aubert
c7e5dbf158 Fix query aliases 2017-08-08 09:46:34 +02:00
Daniel García Aubert
34cf45bc9d Prefix infinities query to avoid name collision 2017-08-08 09:43:31 +02:00
Daniel García Aubert
7e058955ea Use final naming to group by 2017-08-08 09:39:51 +02:00
Daniel García Aubert
a9e3bc3cda Prefix filtered source to avoid name collision 2017-08-08 09:36:23 +02:00
Daniel García Aubert
3ee064a59f Prefix nulls query to avoid name collision 2017-08-08 09:32:13 +02:00
Daniel García Aubert
f3ababffc1 Prefix bins query to avoid name collision 2017-08-08 09:12:52 +02:00
Daniel García Aubert
0f8de9e74b Add prefix to basics query to avoid name collision 2017-08-08 09:06:36 +02:00
Daniel García Aubert
91d5a0e4e4 Remove column avg_val in group_by 2017-08-08 08:46:26 +02:00
Daniel García Aubert
e446160151 Use final columns to group by 2017-08-08 08:45:05 +02:00
Daniel García Aubert
823925d091 Add prefix to bins_array to avoid name collision 2017-08-08 08:41:22 +02:00
Daniel García Aubert
994e58bef7 Add prefix to bins_number and nulls_cout to avoid name collision 2017-08-08 08:33:26 +02:00
Daniel García Aubert
5c80ff8191 Extract query: move condition to a method 2017-08-07 19:24:15 +02:00
Daniel García Aubert
0f45675652 Update NEWS 2017-08-07 19:06:31 +02:00
Daniel García Aubert
b2bbc329ea Apply prefix for intermediate query variables to avoid name colision 2017-08-07 19:03:49 +02:00
Daniel García Aubert
2024e89c6a Update news 2017-08-07 18:30:17 +02:00
Daniel García Aubert
1f8da14c2a Cast to timestamp start_date and end_date to calculate bins when date column is timestamptz 2017-08-07 18:27:24 +02:00
Daniel García Aubert
660078f284 Fix minor issues with timezones 2017-08-07 16:53:08 +02:00
Daniel García Aubert
7eae2a0618 Stubs next version 2017-08-04 09:51:16 +02:00
Daniel García Aubert
e52cf0960c Release 3.10.1 2017-08-04 09:46:00 +02:00
Daniel García Aubert
c39a6a6806 Update news 2017-08-04 09:42:21 +02:00
Javier Goizueta
82cab3ccc7 Merge pull request #719 from CartoDB/ramp-infinities-nans
Exclude Infinities & NaNs from ramps
2017-08-04 09:38:44 +02:00
Javier Goizueta
e01730e8e4 Syntax fixes 2017-08-03 19:16:08 +02:00
Javier Goizueta
eed33fc76d Add tests for excluding NaNs, Ininities from ramps 2017-08-03 19:07:02 +02:00
Daniel
2a531024d7 Merge pull request #720 from CartoDB/fix-aggregation-falsy-values
Fix bad condition when timestampt_start has falsy values
2017-08-03 18:36:37 +02:00
Daniel García Aubert
48ad7059e1 Going green: do not rely on falsy conditional 2017-08-03 18:23:55 +02:00
Daniel García Aubert
6c063095a3 Going red: aggregation is undefined when automattic mode is enabled and timestamp start is 1970-01-01 (epoch) 2017-08-03 18:18:35 +02:00
Daniel García Aubert
1d1a046439 Stubs next version 2017-08-03 15:33:32 +02:00
Daniel García Aubert
fe7a2451ef Release 3.10.0 2017-08-03 15:25:47 +02:00
Daniel
a696bdc723 Merge pull request #706 from CartoDB/705-special-numeric-values
Support special numeric values for json responses
2017-08-03 15:21:29 +02:00
Daniel
a98c884e1a Merge pull request #698 from CartoDB/691-date-histogram
Support histograms with date/time expressions for aggregations
2017-08-03 15:16:13 +02:00
Daniel García Aubert
431ca9c56f Update NEWS 2017-08-03 15:15:37 +02:00
Daniel García Aubert
b56d2ec30b Validate aggregation value 2017-08-03 12:24:05 +02:00
Daniel García Aubert
90ded34af7 Do not fail if layergroup is undefined 2017-08-03 12:22:30 +02:00
Daniel García Aubert
7fed91900d Handle error 2017-08-03 12:19:34 +02:00
Javier Goizueta
b4799124e6 Exclude non-finite values when computing ramps 2017-08-02 17:28:46 +02:00
Daniel García Aubert
1bc5c04489 Remove unused method 2017-08-02 13:15:40 +02:00
Daniel García Aubert
0a57e86cb8 Do not build data histogram infinitely when overriding aggregation with auto mode 2017-08-02 12:06:10 +02:00
Daniel García Aubert
3574700c2d Remove tiler render limit 2017-08-02 11:07:44 +02:00
Daniel García Aubert
ab879e2634 Use new version of getUserTimeoutRenderLimits 2017-08-01 19:13:55 +02:00
Daniel García Aubert
9034508244 Support automattic aggregation only when aggregation para is set to 'auto' 2017-08-01 17:15:45 +02:00
Daniel García Aubert
b2b68ffd5c Merge branch 'master' into 691-date-histogram 2017-08-01 16:07:27 +02:00
Daniel García Aubert
0594407b38 Change error message 2017-08-01 15:03:09 +02:00
Daniel García Aubert
262f854e68 Remove error wrapping 2017-08-01 13:10:55 +02:00
Daniel García Aubert
9258ad7ecc Fix style typo 2017-08-01 12:56:03 +02:00
Daniel García Aubert
46fee774bd Fix misconfiguration in test's hook 2017-08-01 12:54:41 +02:00
Daniel García Aubert
05ddf1d505 Add test to check if asset fallback is working when enabled and database timeout erro happens 2017-08-01 12:53:29 +02:00
Daniel García Aubert
4c3e3005aa Apply asset fallback to database timeout errors 2017-08-01 12:52:34 +02:00
Daniel García Aubert
7d13603163 Implement test to validate database timeout error for static api 2017-08-01 11:58:43 +02:00
Daniel García Aubert
40af73d524 Implement test to check render timeout error for static api 2017-08-01 11:47:50 +02:00
Daniel García Aubert
91b3e373b7 Add helper method to fetch static images 2017-08-01 11:46:48 +02:00
Daniel García Aubert
aa4bb62f38 Fix test 2017-08-01 10:29:46 +02:00
Daniel García Aubert
9af372381c Fix content-type assertion 2017-08-01 10:29:29 +02:00
Daniel García Aubert
0c4e67d6a8 Implemented database timeout test while requesting tiles 2017-08-01 10:21:39 +02:00
Daniel García Aubert
dd5209b9a7 Add torque.png timeout error test 2017-08-01 09:39:37 +02:00
Daniel García Aubert
44fc34b1ce Improve timeout error message 2017-07-31 20:07:31 +02:00
Daniel García Aubert
1fdc0621e7 Categorize timeout errors 2017-07-31 19:36:07 +02:00
Daniel García Aubert
5974413d5c Use 429 to indicate timeout errors 2017-07-31 19:08:29 +02:00
Daniel García Aubert
bb59902535 Refactoring tests hooks 2017-07-31 18:52:09 +02:00
Daniel García Aubert
49d2f513c6 Fix typo 2017-07-31 18:51:23 +02:00
Daniel García Aubert
b1114fc606 Add timeout test for toque.json tiles 2017-07-31 18:26:45 +02:00
Daniel García Aubert
227c2b336b Uncomment database timeout configuration 2017-07-31 18:25:34 +02:00
Daniel García Aubert
ac7509b01a Expose function to clean up database connections 2017-07-31 18:24:42 +02:00
Daniel García Aubert
9b5482489e Fix content-type defaults 2017-07-31 18:23:17 +02:00
Daniel García Aubert
f079c24554 Use parsed body variable 2017-07-31 18:22:13 +02:00
Daniel García Aubert
04da57fe0c Do not create layergroup if it is already provided 2017-07-31 18:14:32 +02:00
Daniel García Aubert
aa6d01f151 Add timeout test for attributes 2017-07-31 18:12:33 +02:00
Daniel García Aubert
435d902e45 Expose function to clean all database connections in the pool 2017-07-31 18:10:14 +02:00
Daniel García Aubert
664db4b5cf Pass proper param to check content-type and status 2017-07-31 18:07:44 +02:00
Daniel García Aubert
64f19b65ec Remove attributes param 2017-07-31 18:01:19 +02:00
Daniel García Aubert
398369a5c7 Do not rely on expected defaults headers 2017-07-31 17:58:33 +02:00
Daniel García Aubert
f2e043b063 Do not expose database error info 2017-07-31 17:56:58 +02:00
Daniel García Aubert
6936107b68 Adjust pg_sleep to timeout 2017-07-28 16:04:11 +02:00
Daniel García Aubert
c3e137bb00 Update dependency 2017-07-28 16:02:54 +02:00
Daniel García Aubert
cca570e832 Uncomment DB and role configuration 2017-07-28 13:23:13 +02:00
Daniel García Aubert
815eac5a48 Add hook to refresh all connections in the pool 2017-07-28 13:22:16 +02:00
Daniel García Aubert
b023a155b7 Be more accurate with timeouts and pg_sleep 2017-07-28 13:21:17 +02:00
Daniel García Aubert
33e77a42f2 Separate user timeout suites between render and database 2017-07-27 18:50:27 +02:00
Daniel García Aubert
664a4e673a Add missing after-each hooks and merged duplicates 2017-07-27 17:08:29 +02:00
Daniel García Aubert
eba97a41e5 Going green, solves issue with role timeout in database 2017-07-27 16:30:57 +02:00
Daniel García Aubert
9e491e7e9a Fix tests names 2017-07-27 16:29:36 +02:00
Daniel García Aubert
522fc79d71 Do not remove redis' keys if layergroup was not created 2017-07-27 12:46:38 +02:00
Daniel García Aubert
768d06c582 Assert layer params is defined but mapnik layers 2017-07-26 18:35:40 +02:00
Daniel García Aubert
058f19ab36 Fix skipped test 2017-07-26 13:27:58 +02:00
Daniel García Aubert
788b2f0683 Implement test to validate response limits work as expected 2017-07-25 19:16:37 +02:00
Daniel García Aubert
526e850f26 Add method to set statements timeout for user's role and database 2017-07-25 19:15:43 +02:00
Daniel García Aubert
444595d49d Rely on windshaft's repo branch 2017-07-25 19:14:04 +02:00
Daniel García Aubert
eee4fc815e Do not expose database error details 2017-07-25 19:11:56 +02:00
Daniel García Aubert
edacd85d5c Merge branch 'master' into response-time-limit 2017-07-24 11:47:51 +02:00
Daniel García Aubert
52da3bfa55 Stub next version 2017-07-21 11:05:51 +02:00
Daniel García Aubert
35b9448e9a Release 3.9.8 2017-07-21 10:58:37 +02:00
Daniel
9959e009eb Merge pull request #714 from CartoDB/bump-windshaft-3.2.2
Bump windshaft version to 3.2.2
2017-07-21 10:44:35 +02:00
Daniel García Aubert
106b9a64b2 Update NEWS 2017-07-21 10:39:33 +02:00
Daniel García Aubert
42d05f29ee Bump windshaft version to 3.2.2 2017-07-20 19:47:01 +02:00
Daniel García Aubert
fc3a959da1 Stub next version 2017-07-20 15:00:39 +02:00
Daniel García Aubert
20003c49ce Release 3.9.7 2017-07-20 14:53:20 +02:00
Daniel
d1d9401539 Merge pull request #713 from CartoDB/upgrade-turbo-carto-to-0.19.2
Bump turbo-carto version to 0.19.2
2017-07-20 14:50:02 +02:00
Daniel García Aubert
cc8a1df388 Bump turbo-carto version to 0.19.2 2017-07-20 12:11:09 +02:00
Daniel García Aubert
e9bc0732c0 Use switch statement instead of if ... else if ... else ... 2017-07-19 12:33:57 +02:00
Daniel García Aubert
8907082a85 Parse body relying on content-type header 2017-07-19 12:24:37 +02:00
Daniel García Aubert
87eb5407a8 WIP: implement timeout limit for raster 2017-07-18 20:50:31 +02:00
Daniel García Aubert
a17916488b Update config to get the right key to enable stats metadate after layergroup creation 2017-07-18 17:40:36 +02:00
Daniel García Aubert
669707b26c Fix typo 2017-07-18 11:56:54 +02:00
Daniel García Aubert
40dc94e010 Merge branch 'master' into response-time-limit 2017-07-18 11:25:23 +02:00
Daniel
868930de46 Merge pull request #712 from CartoDB/mvt-no-content
Respond with 204 when mvt is empty
2017-07-18 11:23:12 +02:00
Daniel García Aubert
f306c26da6 Update News 2017-07-18 11:08:39 +02:00
Daniel García Aubert
446e2d0802 Add empty line at the end of file 2017-07-18 11:05:45 +02:00
Daniel García Aubert
0aab434f13 Remove duplicated assertion 2017-07-18 10:52:24 +02:00
Daniel García Aubert
ff13996255 Add test to check that mvt returns 200 when tile has data 2017-07-18 10:44:27 +02:00
Daniel García Aubert
eccc3597aa Respond with 204 when vector tile is empty 2017-07-17 19:44:18 +02:00
Carlos Matallín
6766b76545 Merge pull request #710 from CartoDB/691-date-histogram-offset
replace timezone => offset
2017-07-14 18:47:23 +02:00
Carlos Matallín
e30b883906 Merge branch '691-date-histogram' into 691-date-histogram-offset 2017-07-14 18:38:13 +02:00
Carlos Matallín
70b4d5b7fd replace timezone => offset 2017-07-14 18:30:36 +02:00
Daniel García Aubert
0fffafa1db Add timestamp_start in histogram summary to help to build the histogram in frontend side 2017-07-14 18:22:05 +02:00
Daniel García Aubert
21b8655f85 Return timezone applied or 0 if not present 2017-07-13 19:42:22 +02:00
Daniel García Aubert
c8286233be Do not apply timezone for minute an hour aggregations 2017-07-12 17:08:55 +02:00
Daniel García Aubert
b67f6053e8 Should respect first timestamp as bin_start 2017-07-12 15:19:28 +02:00
Daniel García Aubert
967dca9578 Improve readability 2017-07-12 15:10:39 +02:00
Daniel García Aubert
a35b1e3e86 Stubs next version 2017-07-11 09:59:30 +02:00
Daniel García Aubert
5b8ecd3df0 Release 3.9.6 2017-07-11 09:53:12 +02:00
Daniel
5ea5c1b2dc Merge pull request #708 from CartoDB/11245-aggregation-search
widgets: support for aggregation in search results
2017-07-11 09:44:33 +02:00
Daniel García Aubert
e36266a80f Added test to check all aggregation operations work as expected when searching dataviews 2017-07-10 12:29:25 +02:00
Carlos Matallín
b1c9dd537e Update Makefile 2017-07-07 17:47:28 +02:00
Carlos Matallín
dd934a3913 linter 2017-07-07 17:44:32 +02:00
Carlos Matallín
7fa154c062 widgets: add aggregation for search results, specs 2017-07-07 17:38:15 +02:00
Carlos Matallín
f7a763b637 widgets: add aggregation for search results 2017-07-07 17:09:17 +02:00
Daniel García Aubert
ad1506ae97 Remove empty lines 2017-07-06 16:24:50 +02:00
Daniel García Aubert
32bcf9ca89 Fix jshint typo 2017-07-06 16:24:18 +02:00
Daniel García Aubert
23aab7a09f User feature branch of cartodb-redis 2017-07-06 16:23:42 +02:00
Daniel García Aubert
37c970903e Avoid uncaught exception when layergroup is not present 2017-07-05 19:09:14 +02:00
Daniel García Aubert
0684c1b9d3 Work in progress: get timeout from redis 2017-07-05 19:08:19 +02:00
Daniel García Aubert
468f641af8 Going green: automatic mode works with dates 2017-06-29 16:57:27 +02:00
Daniel García Aubert
6d2934b30b Going red: add test to check automatic mode works with dates 2017-06-29 16:53:52 +02:00
Daniel García Aubert
7018af18b6 Support automatic aggregation for time-series histogram 2017-06-28 19:58:45 +02:00
Daniel García Aubert
f507f7a74b Stub next version 2017-06-27 17:18:08 +02:00
Daniel García Aubert
2f1cacdfc7 Release 3.9.5 2017-06-27 17:07:52 +02:00
Daniel García Aubert
3a442bea44 Update NEWS 2017-06-27 17:06:24 +02:00
Daniel
49f5b0b480 Merge pull request #700 from CartoDB/dataviews-special-float-values
Handle Infinity and NaN values for dataviews
2017-06-27 16:58:44 +02:00
Daniel García Aubert
01027b73da Fix jshint typo 2017-06-27 14:36:18 +02:00
Daniel García Aubert
af42fba53b Check that quarter aggreagtion uses filters properly in date histogram dataview 2017-06-27 14:28:23 +02:00
Daniel García Aubert
3e12bfe27a Going green: support special numeric values for json responses 2017-06-27 11:53:22 +02:00
Daniel García Aubert
13764e18ce Going red: attributes service do not support special numeric values (Infinity, -Infinity, NaN) 2017-06-27 11:21:05 +02:00
Javier Goizueta
b2f3735e95 The formula widget wasn't using the no_filters query for checking column types 2017-06-23 18:59:51 +02:00
Javier Goizueta
166e29e8ce Forward queries parameter from overview dataviews to base dataviews 2017-06-23 16:53:16 +02:00
Daniel García Aubert
32274e66fd Dataview formula: count infinities and nans as we do with nulls 2017-06-23 12:24:22 +02:00
Rafa de la Torre
15b88c6a67 Stub next version 2017-06-22 18:26:33 +02:00
Rafa de la Torre
2dae09c35b Fix yarn.lock and update HOWTO_RELEASE
In previous commit I updated all the dependencies with the newest
versions when I really intended to update just one. After asking the
node gurus we agreed on changing the docs to reflect a less aggressive
strategy for dependency upgrades.
2017-06-22 18:11:58 +02:00
Daniel García Aubert
a6daca9628 Support date histograms using timestamp with and without timezones 2017-06-22 18:04:23 +02:00
Rafa de la Torre
14e71b929a Release version 3.9.4 2017-06-22 17:48:55 +02:00
Daniel García Aubert
6f7cb75256 Fix bad datetime conversion 2017-06-21 20:19:02 +02:00
Daniel García Aubert
5555b8ad8e Going green: support numeric NaN values for dataviews 2017-06-21 18:59:36 +02:00
Daniel García Aubert
e44d418db3 Going red: Add test to check that aggregation dataview supports numeric special value 2017-06-21 18:44:21 +02:00
Ivan Malagon
6bfedef7eb Cast histogram width bucket to timestamp 2017-06-19 12:47:08 +02:00
Daniel García Aubert
77cb3dbbdc Merge branch 'master' into 691-date-histogram 2017-06-19 10:59:28 +02:00
Daniel García Aubert
17aebf53e2 Merge branch 'master' into dataviews-special-float-values 2017-06-19 10:58:13 +02:00
Daniel García Aubert
02f117af1b Merge branch 'dataviews-special-float-values' of github.com:CartoDB/Windshaft-cartodb into dataviews-special-float-values 2017-06-16 13:01:39 +02:00
Daniel García Aubert
b1ac5b8ca9 Handle special float values only if column is float 2017-06-16 12:57:46 +02:00
Mario de Frutos
e2b976d9d0 Fix version 2017-06-16 11:16:17 +02:00
Mario de Frutos
8e95cf20c0 Upgrade camshaft to 0.55.5 (#703) 2017-06-16 11:13:40 +02:00
Daniel García Aubert
20d7f1a7c5 Handle special float values only when aggregation columns is float (overviews) 2017-06-15 19:22:26 +02:00
Daniel García Aubert
115d8fe685 Handle special float values only when aggregation columns is float 2017-06-15 19:07:31 +02:00
Mario de Frutos
2a366ec16f Stubs next version 2017-06-15 18:32:28 +02:00
Mario de Frutos
849caf9b58 Upgrade to cammshaft 0.55.4 (#702) 2017-06-15 18:30:13 +02:00
Daniel García Aubert
ad570ab6f2 Use dataview base to get column type in formula dataviews 2017-06-15 18:04:35 +02:00
Daniel García Aubert
443c1100d7 Formula dataview: support special values only if column is a float column 2017-06-15 16:31:31 +02:00
Daniel García Aubert
7d0af4e259 Going green: handle special float values for formula when overviews are involved 2017-06-14 19:44:48 +02:00
Daniel García Aubert
81f60959e5 Going green: handle special float values for formula when overviews are involved 2017-06-14 19:20:39 +02:00
Daniel García Aubert
ef849aec34 Going red: add test to check that special float values are not being filtered out in formula dataview when the layer uses overviews 2017-06-14 19:19:08 +02:00
Daniel García Aubert
dee00e6abd Going green: handle special float values for histogram when overviews are involved 2017-06-14 19:00:37 +02:00
Daniel García Aubert
06d40e8b1e Going red: add test to check that special float values are not being filtered out when the layer uses overviews 2017-06-14 18:57:32 +02:00
Daniel García Aubert
3f17c8b15a Filter out special float values before categorizing them 2017-06-14 15:05:46 +02:00
Daniel García Aubert
668b22628c Going green: support special float values in aggergation dataview with overviews 2017-06-13 19:01:59 +02:00
Daniel García Aubert
c08db78a0b Going red: implement test to check aggregation with overviews support special float values 2017-06-13 19:01:28 +02:00
Daniel García Aubert
551b6d409a Remove bad condition 2017-06-13 09:44:53 +02:00
Daniel García Aubert
3ae66e4143 Do not filter special values out if aggregation column is not defined 2017-06-13 09:30:43 +02:00
Daniel García Aubert
227937bf4c Remove test filter 2017-06-12 19:50:24 +02:00
Daniel García Aubert
cb7ec5d556 Fix jshint typos 2017-06-12 19:49:58 +02:00
Daniel García Aubert
8b2fa27ba7 Calculate aggregation filtering out special float values 2017-06-12 19:45:06 +02:00
Daniel García Aubert
962fa05574 Remove console 2017-06-12 19:28:49 +02:00
Daniel García Aubert
75d07745e6 Improve readability 2017-06-12 19:22:37 +02:00
Daniel García Aubert
7b5111614c Summarize special float values for ranked aggregation 2017-06-12 19:21:41 +02:00
Daniel García Aubert
e60bb770db Summarizes infinity and NaN values 2017-06-12 18:55:33 +02:00
Daniel García Aubert
ba6dc62a38 Going red: implementet test to check special float values support 2017-06-12 18:15:39 +02:00
Daniel García Aubert
e6aededf08 Fix typo 2017-06-12 17:19:05 +02:00
Daniel García Aubert
0aae29fb4b Fix jshint typo 2017-06-09 15:28:18 +02:00
Daniel García Aubert
9ba65bd5a4 Going green: Fix test for formula overviews flavour 2017-06-09 15:18:52 +02:00
Daniel García Aubert
7a3498e8ec Going red: formula does not work with Infinity or NaN values 2017-06-09 12:17:16 +02:00
Daniel García Aubert
fe5c76d65b Remove jshint hook 2017-06-08 19:25:05 +02:00
Daniel García Aubert
29a6658e3d Migrate dataviews endpoints to use the allow-query-params 2017-06-08 19:22:33 +02:00
Daniel García Aubert
2772fc62d2 Use a set/dict for checking the existence 2017-06-08 18:38:44 +02:00
Daniel García Aubert
0d4ac64f00 Merge branch 'master' into 691-date-histogram 2017-06-08 18:34:25 +02:00
Daniel
271887eb46 Merge pull request #696 from CartoDB/dataview-backend-small-refactor
Dataview backend small refactor
2017-06-08 18:24:59 +02:00
Daniel García Aubert
cd53eda0a5 Handle Infinity and NaN values for histograms 2017-06-08 16:01:41 +02:00
Daniel García Aubert
6c301403e3 Histogram going red: fails while quering Infinity and NanN values 2017-06-08 15:59:33 +02:00
Daniel García Aubert
47af013157 Merge branch 'master' into 691-date-histogram 2017-06-07 16:11:50 +02:00
Daniel García Aubert
35d4fb4d27 Improve readability, using extract method pattern 2017-06-07 16:11:09 +02:00
Daniel García Aubert
42e2f9e4b1 Update commented use cases 2017-06-07 15:54:28 +02:00
Daniel García Aubert
02bf1dd2d7 Release 3.9.1 2017-06-06 17:34:38 +02:00
Daniel
d365e092b9 Merge pull request #699 from CartoDB/update_camshaft_0553
Bump camshaft version to  0.55.3
2017-06-06 17:32:18 +02:00
Daniel García Aubert
45d1d07ea2 Use released version of comshaft (0.55.3) 2017-06-06 17:21:33 +02:00
Mario de Frutos
f14a61528a Upgrade to camshaft 0.55.3 2017-06-06 10:52:37 +02:00
Daniel García Aubert
d3bcf6f80d Do not omit nonexistent property 2017-06-06 09:15:49 +02:00
Daniel García Aubert
eeea51e10d Removed unused column in group by statement 2017-06-05 17:26:37 +02:00
Daniel García Aubert
9337cd948c Fix typo 2017-06-05 16:26:29 +02:00
Daniel García Aubert
527e005952 Merge branch '691-date-histogram' of github.com:CartoDB/Windshaft-cartodb into 691-date-histogram 2017-06-05 16:26:09 +02:00
Daniel García Aubert
1ff0954390 Fix typo 2017-06-05 16:25:47 +02:00
Daniel García Aubert
e82d688a18 Fix typo 2017-06-05 16:09:47 +02:00
Daniel García Aubert
95a6ad3b86 Support quarter aggregation in histograms over date columns 2017-06-05 16:04:42 +02:00
Daniel García Aubert
d01787842f Support UTC timezone override 2017-06-05 15:23:04 +02:00
Daniel García Aubert
c86f92f8eb Improve test description 2017-06-05 15:05:23 +02:00
Daniel García Aubert
003227fb29 Fix assertion 2017-06-05 14:59:35 +02:00
Daniel García Aubert
869408b7b7 Use Eastern Daylight Time while testing 2017-06-05 14:50:49 +02:00
Daniel García Aubert
dc844f8131 Remove console.log 2017-06-05 14:23:53 +02:00
Daniel García Aubert
71e9e62db0 Improved histogram assertion with moment.js 2017-06-05 14:18:24 +02:00
Daniel García Aubert
6ff3b33cde Removed bins_start as query output 2017-06-05 14:17:50 +02:00
Daniel García Aubert
32eeb57fce Reduce complexity in function 2017-06-02 19:00:26 +02:00
Daniel García Aubert
8bc38a375a Support timezone aggregation 2017-06-02 18:37:49 +02:00
Daniel García Aubert
c1fac13d6b Be able to accept timezone parameter 2017-06-02 12:45:34 +02:00
Daniel García Aubert
6374d2e4b6 Fix typo 2017-06-02 12:17:55 +02:00
Daniel García Aubert
9c34428984 Allow override start and end params 2017-06-02 12:15:43 +02:00
Daniel García Aubert
1d66e49910 WIP implemented date histogram 2017-06-01 20:07:46 +02:00
Raul Ochoa
4b562e6768 Merge conditions 2017-05-31 13:00:37 -04:00
Raul Ochoa
b4fbe0b8cf No in advance var definition 2017-05-31 12:58:35 -04:00
Raul Ochoa
62514fc563 Extract query rewrite data to function 2017-05-31 12:58:20 -04:00
Raul Ochoa
ef3cad6599 Extract to function 2017-05-31 12:52:41 -04:00
Raul Ochoa
4e53803b3b Create BBoxFilter independently from rewrite data 2017-05-31 12:51:07 -04:00
Raul Ochoa
40a73f2eaf Merge pull request #695 from CartoDB/fix-bboxes-filters
Add test to detect and fix incorrect bbox filter splitting
2017-05-31 11:23:42 -04:00
Javier Goizueta
31557b06be Add test to detect and fix incorrect bbox filter splitting
When bbox crosses date line and is split in two, the eastern box wasn't correct
2017-05-31 11:09:51 -04:00
Daniel García Aubert
c8ea595f47 Stubs next version 2017-05-31 12:14:58 +02:00
Daniel García Aubert
dbd0398d9b Release 3.9.0 2017-05-31 12:13:25 +02:00
Daniel
4f7ec0dd4d Merge pull request #683 from CartoDB/664-layergroup-stats
Use windshaft-stats to extract info about layer metadata
2017-05-31 11:58:00 +02:00
Daniel García Aubert
5a1623b667 Merge branch 'master' into 664-layergroup-stats 2017-05-31 11:50:32 +02:00
Daniel García Aubert
2c509d97b1 Point windshaft to the last released version 2017-05-31 11:46:46 +02:00
Daniel García Aubert
da3f30dd9f Update NEWS 2017-05-31 11:27:41 +02:00
Mario de Frutos
440953b1cd If we have the stats FF disbabled return empty array instead of null
But we keep checking for elements in the returned object because
we don't want to include the stats property if the FF is disabled
2017-05-30 10:47:47 +02:00
Raul Ochoa
45471597bb Merge pull request #693 from CartoDB/test-improvements
Do not assert inside assert.response
2017-05-26 13:20:44 +02:00
Raul Ochoa
882aeacac2 Rewrite test to take advantage of changes in assert.response/TestClient
This should avoid the issue of preventing the whole suite to halt, as
in https://travis-ci.org/CartoDB/Windshaft-cartodb/builds/236337027.
2017-05-26 13:13:19 +02:00
Raul Ochoa
248adab05b Catch response body if any to capture Redis keys 2017-05-26 13:06:04 +02:00
Raul Ochoa
f4e99629f6 Do not assert inside response, but pass error into callback
Preferably we should put response outside of assert and change its
callback signature. However, I don't think it is worth the effort
right now.
2017-05-26 13:02:02 +02:00
Daniel
df08463dcf Merge pull request #690 from CartoDB/allow-es6-syntax
Allow es6 syntax
2017-05-25 14:28:02 +02:00
Daniel García Aubert
b84bb469e5 Allow es6 syntax 2017-05-25 11:26:04 +02:00
Daniel
26e4d19635 Merge pull request #689 from CartoDB/upgrade-camshaft-to-0.55.2
Upgrade camshaft to 0.55.2
2017-05-24 12:01:39 +02:00
Daniel García Aubert
69984ed895 Upgrade camshaft to 0.55.2 2017-05-24 11:54:30 +02:00
Daniel García Aubert
d471905358 Merge branch 'master' into 664-layergroup-stats 2017-05-23 13:47:27 +02:00
Daniel García Aubert
e5223980cf Update NEWS 2017-05-23 13:46:56 +02:00
Daniel
1f65968de3 Merge pull request #688 from CartoDB/upgrade-camshaft-to-0.55.1
Upgrade camshaft to 0.55.1
2017-05-23 13:45:57 +02:00
Daniel García Aubert
51c11aff03 Upgrade camshaft to 0.55.1 2017-05-23 13:37:34 +02:00
Daniel García Aubert
87e6e64d42 Merge branch 'master' into 664-layergroup-stats 2017-05-23 12:41:14 +02:00
Raul Ochoa
7dfb7c605e Merge pull request #687 from CartoDB/remove-turbo-carto-promise-global
Remove Promise hack from turbo-carto adapter
2017-05-22 18:52:27 +02:00
Raul Ochoa
8b0964ad7e Remove Promise hack from turbo-carto adapter 2017-05-22 18:45:24 +02:00
Daniel García Aubert
5135b6e14a Stubs next version 2017-05-22 14:57:03 +02:00
Daniel García Aubert
dcb26560e3 Release 3.8.0 2017-05-22 14:50:41 +02:00
Daniel
921468668b Merge pull request #686 from CartoDB/upgrade-camshaft-to-0.55.0
Upgrade camshaft to 0.55.0 and turbo-carto to 0.19.1
2017-05-22 14:39:10 +02:00
Mario de Frutos
76c2f8dc0e Includes release changes of turbo-carto too 2017-05-22 14:33:14 +02:00
Daniel García Aubert
7efb196247 Upgrade camshaft to 0.55.0 2017-05-22 13:09:42 +02:00
Raul Ochoa
95c324ddd1 Merge pull request #685 from CartoDB/update-dev-deps
Update dev deps
2017-05-18 18:25:32 +02:00
Raul Ochoa
0a3d1fbdf9 Upgrade jshint and fix test 2017-05-18 17:39:56 +02:00
Raul Ochoa
45c1ccab9e Upgrade mocha dep 2017-05-18 17:39:26 +02:00
Daniel García Aubert
cc2a96579b Merge branch 'master' into 664-layergroup-stats 2017-05-18 16:39:52 +02:00
Daniel García Aubert
61b19856e9 Stubs next version 2017-05-18 16:36:50 +02:00
Daniel García Aubert
67aa2d1a00 Release 3.7.1 2017-05-18 16:34:56 +02:00
Daniel
c6ee2eac62 Merge pull request #684 from CartoDB/fix-buffersize-undefined-in-mapconfig
Fix buffersize undefined in mapconfig instantiation
2017-05-18 16:33:31 +02:00
Daniel García Aubert
3978d58d66 Remove empty line 2017-05-18 16:12:28 +02:00
Daniel García Aubert
cd86387fa7 Merge branch 'fix-buffersize-undefined-in-mapconfig' into 664-layergroup-stats 2017-05-18 15:58:12 +02:00
Daniel García Aubert
3ce38d7081 Going green: fix type error when no buffersize is defined in mapconfig 2017-05-18 15:49:57 +02:00
Daniel García Aubert
e9112da305 Going red: add test to check undefined buffersize in requested mapconfig throws error 2017-05-18 15:41:41 +02:00
Mario de Frutos
c9e6e921cb Merge branch 'master' into 664-layergroup-stats 2017-05-18 13:45:38 +02:00
Daniel García Aubert
4e715f6ba4 Stubs next version 2017-05-18 13:24:48 +02:00
Daniel García Aubert
8f156b9f13 Release 3.7.0 2017-05-18 13:23:22 +02:00
Daniel
954876f738 Merge pull request #673 from CartoDB/665-buffer-size
Manage multiple values of buffer-size for different formats
2017-05-18 13:20:25 +02:00
Daniel García Aubert
fd178bcf71 Upgrade windshaft to 3.2.0 2017-05-18 13:14:51 +02:00
Daniel García Aubert
acaff98da5 Merge branch 'master' into 665-buffer-size 2017-05-18 13:08:46 +02:00
Mario de Frutos
ed56094be2 PR changes 2017-05-18 11:51:12 +02:00
Daniel García Aubert
c65518cf41 Get back layer-stats from windshaft-stats 2017-05-17 20:16:43 +02:00
Daniel García Aubert
fb4ee61b83 Use setInmmediate vs process.nextTick 2017-05-17 12:55:05 +02:00
Daniel García Aubert
808c729a0e Now supported formats for buffer-size customization are bound to the adapter 2017-05-17 12:33:41 +02:00
Daniel García Aubert
4602fb3ecf Send stats for png32 tiles 2017-05-17 12:16:16 +02:00
Daniel García Aubert
c59996303d Send stats for mvt tiles 2017-05-17 12:04:11 +02:00
Daniel García Aubert
13b1978d49 Include layer param to reach the right tile for grid.json 2017-05-17 11:40:53 +02:00
Daniel García Aubert
e13ae8d5af Do not make optional layer param in URL template 2017-05-17 11:40:18 +02:00
Daniel García Aubert
0f16c0e396 Merge branch '664-layergroup-stats' of github.com:CartoDB/Windshaft-cartodb into 664-layergroup-stats 2017-05-16 12:15:40 +02:00
Daniel García Aubert
29361f5392 Use windhshaft with latest changes for layer stats 2017-05-16 12:07:17 +02:00
Mario de Frutos
422867762b package.json for staging 2017-05-12 17:14:07 +02:00
Mario de Frutos
5969c99e8a Removed not used parameters for layer stats 2017-05-12 16:42:10 +02:00
Mario de Frutos
5417933ecc Change to tests for layer stats because now uses CDB_EstimateRowCount fuction 2017-05-12 16:42:10 +02:00
Mario de Frutos
59585b5cd9 We leave only one feature flag for stats 2017-05-12 16:42:10 +02:00
Mario de Frutos
522c86e6f2 Change from branch to commits for staging dependencies 2017-05-12 16:42:04 +02:00
Mario de Frutos
1a7fd9bf31 Changed stats name from featureCount to estimatedFeatureCount 2017-05-12 16:40:34 +02:00
Mario de Frutos
7596df96ed package.json for staging 2017-05-12 12:58:13 +02:00
Mario de Frutos
44cca38538 Change to tests for layer stats because now uses CDB_EstimateRowCount fuction 2017-05-12 12:51:45 +02:00
Mario de Frutos
f6fff6953e We leave only one feature flag for stats 2017-05-12 12:50:57 +02:00
Mario de Frutos
35df0c3a68 Change from branch to commits for staging dependencies 2017-05-11 14:26:00 +02:00
Daniel García Aubert
f9c8178d99 Stubs next version 2017-05-11 13:46:34 +02:00
Daniel García Aubert
787ca1607a Release 3.6.6 2017-05-11 13:37:43 +02:00
Daniel
7179c0a5f1 Merge pull request #682 from CartoDB/camshaft-error-node-id
Upgrade camshaft to 0.54.4
2017-05-11 13:29:29 +02:00
Daniel García Aubert
b739db1023 Use released version of camshaft 2017-05-11 13:19:14 +02:00
Daniel García Aubert
66a898cdc2 Upgrade camshaft to get error node-id 2017-05-11 12:55:53 +02:00
Mario de Frutos
61f9ea6e86 Changed stats name from featureCount to estimatedFeatureCount 2017-05-11 12:47:35 +02:00
Daniel García Aubert
5a44d6c547 Drop geojson support for buffersize customization 2017-05-10 18:35:30 +02:00
Daniel García Aubert
53d1b2fbbf Rename mapconfig-named-map-adapter by mapconfig-buffer-size-adapter 2017-05-10 18:16:22 +02:00
Daniel García Aubert
2c9d30e042 Be more flexible validating buffer-size customization 2017-05-10 17:49:28 +02:00
Mario de Frutos
968677e275 package.json poiting to staging branches 2017-05-10 17:26:12 +02:00
Mario de Frutos
daf19c5e27 Stats backend only provides stats not metadata 2017-05-10 17:17:01 +02:00
Raul Ochoa
ac94118798 Merge pull request #678 from CartoDB/print-attributions
added print attributions from Docs FAQs to Static Maps API content
2017-05-10 09:26:07 +02:00
Mario de Frutos
7d5b6b0820 Lint changes and yarn.lock 2017-05-09 18:24:24 +02:00
Mario de Frutos
b87e442801 Remove vertex count from stats tests 2017-05-09 18:24:11 +02:00
Raul Ochoa
1a197bb9cf Stubs next version 2017-05-09 15:06:49 +02:00
Raul Ochoa
5b96db2ba2 Release 3.6.5 2017-05-09 15:06:00 +02:00
Raul Ochoa
3b687ce09a Merge pull request #681 from CartoDB/upgrade-camshaft
Upgrades camshaft to 0.54.3
2017-05-09 15:05:15 +02:00
Raul Ochoa
7bb039b13c Upgrades camshaft to 0.54.3 2017-05-09 14:58:25 +02:00
Mario de Frutos
474d68687c Include vertexCount in the stats tests 2017-05-09 14:47:39 +02:00
Mario de Frutos
b25540720c Added acceptance tests for stats functionallity 2017-05-09 14:36:35 +02:00
Mario de Frutos
759d28f12f Include feature flag to enable/disable stats 2017-05-09 12:51:22 +02:00
Mario de Frutos
15c68711aa Create stats backend to decouple stats logic from map controller 2017-05-09 12:31:16 +02:00
Mario de Frutos
568d6b5458 Include the layers metadata into the layergroup object 2017-05-09 11:59:11 +02:00
Mario de Frutos
525c0f2afa Added rendererCache as dependency for map controller 2017-05-09 11:58:48 +02:00
Mario de Frutos
3f6c8fa51c Use windshaft-stats to get layer stats 2017-05-08 18:42:40 +02:00
Raul Ochoa
0ac53db73a Stubs next version 2017-05-05 16:04:25 +02:00
Raul Ochoa
36e9239056 Release 3.6.4 2017-05-05 16:03:53 +02:00
Raul Ochoa
4e6e267f10 Fix news 2017-05-05 16:03:38 +02:00
Raul Ochoa
2c235b6629 Merge pull request #680 from CartoDB/upgrade-deps
Upgrade deps
2017-05-05 16:02:24 +02:00
Raul Ochoa
6bd7537467 Upgrade deps 2017-05-05 15:57:02 +02:00
Daniel García Aubert
55a351d751 Point windshaft to a specific buffer-size commit 2017-05-03 17:35:55 +02:00
Daniel García Aubert
05d3b3bf66 Point windshaft to specific commit for buffer-size configuration 2017-05-03 17:21:08 +02:00
Daniel García Aubert
e97466378e Add test for different formats to anonymous maps and named maps with placeholders 2017-05-03 11:17:51 +02:00
csobier
8426dd00f1 added print attributions from Docs FAQs to Static Maps API content 2017-05-02 12:48:55 -04:00
Daniel García Aubert
b2b6cf1f02 Merge branch '665-buffer-size' of github.com:CartoDB/Windshaft-cartodb into 665-buffer-size 2017-04-28 19:23:49 +02:00
Daniel García Aubert
c9af38ecd0 Fix issue when 'grid.json' format is not captured properly due to a weird behaviour in regex 2017-04-28 19:21:51 +02:00
Mario de Frutos
be58adb1b9 Be able to override buffer-size configuration without placeholders in named maps 2017-04-28 19:20:00 +02:00
Mario de Frutos
bfb283c5ba wip 2017-04-28 14:46:36 +02:00
Mario de Frutos
332a56b736 Mapconfig only support object for the buffer-size property 2017-04-28 14:22:16 +02:00
Daniel García Aubert
2f4e4246a4 Refactor test-client in order to use same interface for named and anonymous maps 2017-04-26 18:27:18 +02:00
Daniel García Aubert
c481d6473c Use parseInt instead of number constructor 2017-04-26 17:01:21 +02:00
Daniel García Aubert
40c0e306af Remove invalid assertions 2017-04-25 20:40:17 +02:00
Daniel García Aubert
0d840e6daf Javascript style typo 2017-04-25 19:41:30 +02:00
Daniel García Aubert
07e507e1aa Remove dictionary as placeholder type for named maps 2017-04-25 19:40:12 +02:00
Mario de Frutos
7ea7a991aa Buffersize customizable through named maps' placeholders 2017-04-25 19:27:31 +02:00
Daniel García Aubert
0577fa5308 Add test 2017-04-25 17:54:31 +02:00
Daniel García Aubert
f29ee1b4ac Add test to use placeholder buffer-size value 2017-04-25 15:48:23 +02:00
Daniel García Aubert
0c08713521 First attempt: support buffer-size configuration for named maps 2017-04-25 14:34:17 +02:00
Raul Ochoa
567928a7f5 Stubs next version 2017-04-25 12:36:29 +02:00
Raul Ochoa
ae9e211f30 Release 3.6.3 2017-04-25 12:35:34 +02:00
Raul Ochoa
b5b75df91a Merge pull request #674 from CartoDB/upgrade-windshaft
Upgrades windshaft to 3.1.1
2017-04-25 12:33:22 +02:00
Raul Ochoa
8ddccc0b0c Upgrades windshaft to 3.1.1 2017-04-25 12:19:49 +02:00
Daniel García Aubert
383a1a330a Test with buffer-size 0 2017-04-25 10:43:07 +02:00
Raul Ochoa
95195fff6f Stubs next version 2017-04-24 19:33:03 +02:00
Raul Ochoa
93b77dc4c1 Release 3.6.2 2017-04-24 19:32:10 +02:00
Raul Ochoa
4aee7fb1b8 Merge pull request #671 from CartoDB/upgrade-deps
Upgrades grainstore to 1.6.3
2017-04-24 19:31:20 +02:00
Raul Ochoa
a6d68dba5e Upgrades grainstore to 1.6.3 2017-04-24 19:11:53 +02:00
Daniel García Aubert
109c550187 Remove filter 2017-04-24 18:57:20 +02:00
Daniel García Aubert
06353941e6 Implement test to exercise buffer-size configuration by format 2017-04-24 18:56:15 +02:00
Daniel García Aubert
fed953d195 Support mvt tiles 2017-04-24 18:55:08 +02:00
Raul Ochoa
883f87c7c8 Document gc_interval configuration entry 2017-04-24 18:28:51 +02:00
Raul Ochoa
14d37268d6 Stubs next version 2017-04-24 14:44:38 +02:00
Raul Ochoa
4b6181039d Release 3.6.1 2017-04-24 14:43:46 +02:00
Raul Ochoa
47944671c6 fix release date 2017-04-24 14:43:23 +02:00
Raul Ochoa
f33a7dd665 Merge pull request #668 from CartoDB/upgrade-camshaft
Upgrade camshaft to 0.54.1
2017-04-24 14:41:06 +02:00
Raul Ochoa
781e5a71bf Upgrade camshaft to 0.54.1 2017-04-24 12:29:08 +02:00
Raul Ochoa
c4ff884ad0 Merge pull request #666 from CartoDB/upgrade-camshaft
Upgrades camshaft to 0.54.0
2017-04-20 18:33:07 +02:00
Raul Ochoa
02b9f85b16 Upgrades camshaft to 0.54.0 2017-04-20 18:25:57 +02:00
Daniel García Aubert
2756252368 Stubs next version 2017-04-11 19:23:04 +02:00
Daniel García Aubert
a386abf5a5 Release 3.5.1 2017-04-11 18:34:18 +02:00
Daniel
e5c2c35a81 Merge pull request #662 from CartoDB/upgrade-camshaft-to-0.53.1
Upgrade camshaft to 0.53.1
2017-04-11 18:31:49 +02:00
Daniel García Aubert
227112c7aa Update yarn.lock 2017-04-11 18:27:54 +02:00
Daniel García Aubert
a4ed37bdfc Upgrade camshaft to 0.53.1 2017-04-11 18:19:39 +02:00
Daniel García Aubert
c6a62cee61 Stubs next version 2017-04-10 16:13:30 +02:00
Daniel García Aubert
891bc818b2 Release 3.5.0 2017-04-10 12:20:44 +02:00
Daniel
ebe25d6f20 Merge pull request #638 from CartoDB/629_fix_cache_invalidation
Fix cache invalidation problem with affected tables
2017-04-10 11:55:57 +02:00
Daniel García Aubert
92ec17218b Upgrade camshaft to 0.53.0 2017-04-10 11:36:17 +02:00
Mario de Frutos
e8a0f6b7b6 Point to camshaft branch to test properly 2017-04-07 16:00:29 +02:00
Mario de Frutos
125c39967c Make the cache headers tests idempotent 2017-04-07 15:59:58 +02:00
Mario de Frutos
4132bc755d Add cdb_invalidate_varnish function fixture to tests 2017-04-07 15:59:58 +02:00
Mario de Frutos
9707881bf9 Include check for surrogate-key header and renamed the test file 2017-04-07 15:59:58 +02:00
Mario de Frutos
fa6493ae44 Affected tables are now included in X-Cache-Channel 2017-04-07 15:59:58 +02:00
Mario de Frutos
0c387cf6d9 Add more tests for x-cache-channel but with analysis 2017-04-07 15:59:58 +02:00
Mario de Frutos
5e4d1d5c1c Get affected tables and add it to the layergroup 2017-04-07 15:59:58 +02:00
Raul Ochoa
4d82fd65f6 Merge pull request #659 from CartoDB/upgrade-camshaft
Upgrades camshaft to 0.52.0
2017-04-07 14:41:14 +02:00
Raul Ochoa
6d3644f13b Upgrades camshaft to 0.52.0 2017-04-07 13:29:30 +02:00
Raul Ochoa
7a5aa7ba35 Stubs next version 2017-04-03 16:22:15 +02:00
Raul Ochoa
9c9609eb2b Release 3.4.0 2017-04-03 16:21:29 +02:00
Raul Ochoa
418d3c074f Merge pull request #657 from CartoDB/upgrade-camshaft
Upgrade camshaft to 0.51.0
2017-04-03 16:15:48 +02:00
Raul Ochoa
6bbda3d41e Upgrade camshaft to 0.51.0 2017-04-03 15:53:39 +02:00
Raul Ochoa
25669bb3f2 Stubs next version 2017-04-03 13:07:53 +02:00
83 changed files with 7424 additions and 1360 deletions

4
.gitignore vendored
View File

@@ -8,6 +8,6 @@ tools/munin/windshaft.conf
logs/
pids/
redis.pid
test.log
npm-debug.log
*.log
coverage/
.DS_Store

View File

@@ -40,7 +40,7 @@
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
// "eqnull" : false, // true: Tolerate use of `== null`
// "es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
// "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`)
"esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`)
// "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// // (ex: `for each`, multiple try/catch, function expression…)
// "evil" : false, // true: Tolerate use of `eval` and `new Function()`

View File

@@ -1,7 +1,7 @@
1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json
3. Ensure NEWS section exists for the new version, review it, add release date
4. Recreate yarn.lock with: `yarn upgrade`
4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
5. Commit package.json, yarn.lock, NEWS
6. git tag -a Major.Minor.Patch # use NEWS section as content
7. Stub NEWS/package for next version

218
NEWS.md
View File

@@ -1,5 +1,223 @@
# Changelog
## 3.12.3
Released 2017-08-22
Announcements:
- Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8).
## 3.12.2
Released 2017-08-16
Bug fixes:
- Polygon count problems #725.
## 3.12.1
Released 2017-08-13
- Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1).
- Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1).
- Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7).
## 3.12.0
Released 2017-08-10
Announcements:
- Apply max tile response time for requests to layergoup, tiles, static maps, attributes and dataviews services #717.
- Upgrades windshaft to [3.3.0](https://github.com/CartoDB/windshaft/releases/tag/3.3.0).
- Upgrades cartodb-redis to [0.14.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/0.14.0).
## 3.11.0
Released 2017-08-08
Announcements:
- Allow to override with any aggregation for histograms instantiated w/o aggregation.
Bug fixes:
- Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series.
- Support timestamp with timezones to calculate the number of bins in time-series.
- Fixed issue related to name collision while building time-series query.
## 3.10.1
Released 2017-08-04
Bug fixes:
- Exclude Infinities & NaNs from ramps #719.
- Fixed issue in time-series when aggregation starts at 1970-01-01 (epoch) #720.
## 3.10.0
Released 2017-08-03
Announcements:
- Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698.
- Support special numeric values (±Infinity, NaN) for json responses #706
## 3.9.8
Released 2017-07-21
- Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2).
## 3.9.7
Released 2017-07-20
Bug fixes:
- Respond with 204 (No content) when vector tile has no data #712
Announcements:
- Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2)
## 3.9.6
Released 2017-07-11
- Dataviews: support for aggregation in search results #708
## 3.9.5
Released 2017-06-27
- Dataviews: support special numeric values (±Infinity, NaN) #700
## 3.9.4
Released 2017-06-22
Announcements:
- Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6).
## 3.9.3
Released 2017-06-16
Announcements:
- Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5).
## 3.9.2
Released 2017-06-16
Announcements:
- Upgrades camshaft to [0.55.4](https://github.com/CartoDB/camshaft/releases/tag/0.55.4).
## 3.9.1
Released 2017-06-06
Announcements:
- Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3).
## 3.9.0
Released 2017-05-31
Announcements:
- Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1).
- Add support to retrieve info about layer stats in map instantiation.
- Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2).
- Remove promise polyfill from turbo-carto adapter
## 3.8.0
Released 2017-05-22
Announcements:
- Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0).
- Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1)
## 3.7.1
Released 2017-05-18
Bug fixes:
- Fix buffersize assignment when is not defined in requested mapconfig.
## 3.7.0
Released 2017-05-18
Announcements:
- Manage multiple values of buffer-size for different formats
- Upgrades windshaft to [3.2.0](https://github.com/CartoDB/windshaft/releases/tag/3.2.0).
## 3.6.6
Released 2017-05-11
Announcements:
- Upgrades camshaft to [0.54.4](https://github.com/CartoDB/camshaft/releases/tag/0.54.4).
## 3.6.5
Released 2017-05-09
Announcements:
- Upgrades camshaft to [0.54.3](https://github.com/CartoDB/camshaft/releases/tag/0.54.3).
## 3.6.4
Released 2017-05-05
Announcements:
- Upgrade cartodb-psql to [0.8.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.8.0).
- Upgrades camshaft to [0.54.2](https://github.com/CartoDB/camshaft/releases/tag/0.54.2).
- Upgrades windshaft to [3.1.2](https://github.com/CartoDB/windshaft/releases/tag/3.1.2).
## 3.6.3
Released 2017-04-25
Announcements:
- Upgrades windshaft to [3.1.1](https://github.com/CartoDB/windshaft/releases/tag/3.1.1).
## 3.6.2
Released 2017-04-24
Announcements:
- Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3).
## 3.6.1
Released 2017-04-24
Announcements:
- Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1).
## 3.6.0
Released 2017-04-20
Announcements:
- Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0).
## 3.5.1
Released 2017-04-11
Announcements:
- Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1).
## 3.5.0
Released 2017-04-10
Bug fixes:
- Fix invalidation of cache for maps with analyses #638.
Announcements:
- Upgrades camshaft to [0.53.0](https://github.com/CartoDB/camshaft/releases/tag/0.53.0).
## 3.4.0
Released 2017-04-03
Announcements:
- Upgrades camshaft to [0.51.0](https://github.com/CartoDB/camshaft/releases/tag/0.51.0).
## 3.3.0
Released 2017-04-03

View File

@@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '^(.*)\\.localhost'
@@ -321,8 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: true
layerStats: true
}
};

View File

@@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -321,7 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: false
layerStats: false
}
};

View File

@@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -321,7 +324,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: true
layerStats: true
}
};

View File

@@ -6,6 +6,9 @@ var config = {
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
,uv_threadpool_size: undefined
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
,gc_interval: 10000
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
,user_from_host: '(.*)'
@@ -315,7 +318,7 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerMetadata: true
layerStats: true
}
};

View File

@@ -150,6 +150,10 @@ It is important to note that generated images are cached from the live data refe
* Image resolution is set to 72 DPI
* JPEG quality is 85%
* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
* If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code:
{% highlight javascript %}attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors, &copy; <a href="https://carto.com/attributions">CARTO</a>
{% endhighlight %}
## Examples

View File

@@ -1,3 +1,5 @@
var step = require('step');
/**
*
* @param metadataBackend
@@ -13,16 +15,65 @@ function UserLimitsApi(metadataBackend, options) {
module.exports = UserLimitsApi;
UserLimitsApi.prototype.getRenderLimits = function (username, callback) {
UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) {
var self = this;
this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) {
var limits = {
cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
render: self.options.limits.render || 0
};
self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) {
if (err) {
return callback(err);
}
return callback(null, {
cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
render: renderLimit || self.options.limits.render || 0
});
if (timeoutRenderLimit && timeoutRenderLimit.render) {
if (Number.isFinite(timeoutRenderLimit.render)) {
limits.render = timeoutRenderLimit.render;
}
}
return callback(null, limits);
});
};
UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
var self = this;
step(
function isAuthorized() {
var next = this;
if (!apiKey) {
return next(null, false);
}
self.metadataBackend.getUserMapKey(username, function (err, userApiKey) {
if (err) {
return next(err);
}
return next(null, userApiKey === apiKey);
});
},
function getUserTimeoutRenderLimits(err, authorized) {
var next = this;
if (err) {
return next(err);
}
self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) {
if (err) {
return next(err);
}
next(null, {
render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
});
});
},
callback
);
};

View File

@@ -43,53 +43,19 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
ownFilter = !!ownFilter;
var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(mapConfig.obj().layers, function(l) {
return l.options.source && (l.options.source.id === sourceId);
});
var queryRewriteData = layer && layer.options.query_rewrite_data;
if (queryRewriteData && dataviewDefinition.node.type === 'source') {
queryRewriteData = _.extend({}, queryRewriteData, {
filters: dataviewDefinition.node.filters,
unfiltered_query: dataviewDefinition.sql.own_filter_on
});
}
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
query = bboxFilter.sql(query);
if ( queryRewriteData ) {
var bbox_filter_definition = {
type: 'bbox',
options: {
column: 'the_geom_webmercator',
srid: 3857
},
params: {
bbox: params.bbox
}
};
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
}
}
var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
);
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
function castNumbers(overrides, val, k) {
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
}
overrides[k] = +val;
return overrides;
},
{ownFilter: ownFilter}
);
var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
dataview.getResult(pg, overrideParams, this);
dataview.getResult(pg, getOverrideParams(params, ownFilter), this);
},
function returnCallback(err, result) {
return callback(err, result);
@@ -97,6 +63,56 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
);
};
function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(mapConfig.obj().layers, function(l) {
return l.options.source && (l.options.source.id === sourceId);
});
var queryRewriteData = layer && layer.options.query_rewrite_data;
if (queryRewriteData && dataviewDefinition.node.type === 'source') {
queryRewriteData = _.extend({}, queryRewriteData, {
filters: dataviewDefinition.node.filters,
unfiltered_query: dataviewDefinition.sql.own_filter_on
});
}
if (params.bbox && queryRewriteData) {
var bbox_filter_definition = {
type: 'bbox',
options: {
column: 'the_geom_webmercator',
srid: 3857
},
params: {
bbox: params.bbox
}
};
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
}
return queryRewriteData;
}
function getOverrideParams(params, ownFilter) {
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'),
function castNumbers(overrides, val, k) {
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
}
overrides[k] = +val;
return overrides;
},
{ownFilter: ownFilter}
);
// validation will be delegated to the proper dataview
if (params.aggregation !== undefined) {
overrideParams.aggregation = params.aggregation;
}
return overrideParams;
}
DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
var dataviewName = params.dataviewName;

View File

@@ -0,0 +1,16 @@
function EmptyLayerStats(types) {
this._types = types || {};
}
EmptyLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
EmptyLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
setImmediate(function() {
callback(null, {});
});
};
module.exports = EmptyLayerStats;

View File

@@ -0,0 +1,23 @@
var LayerStats = require('./layer-stats');
var EmptyLayerStats = require('./empty-layer-stats');
var MapnikLayerStats = require('./mapnik-layer-stats');
var TorqueLayerStats = require('./torque-layer-stats');
module.exports = function LayerStatsFactory(type) {
var layerStatsIterator = [];
var selectedType = type || 'ALL';
if (selectedType === 'ALL') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true }));
layerStatsIterator.push(new MapnikLayerStats());
layerStatsIterator.push(new TorqueLayerStats());
} else if (selectedType === 'mapnik') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true }));
layerStatsIterator.push(new MapnikLayerStats());
} else if (selectedType === 'torque') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true }));
layerStatsIterator.push(new TorqueLayerStats());
}
return new LayerStats(layerStatsIterator);
};

View File

@@ -0,0 +1,45 @@
var queue = require('queue-async');
function LayerStats(layerStatsIterator) {
this.layerStatsIterator = layerStatsIterator;
}
LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) {
var self = this;
var stats = [];
if (!mapConfig.getLayers().length) {
return callback(null, stats);
}
var metaQueue = queue(mapConfig.getLayers().length);
mapConfig.getLayers().forEach(function (layer, layerId) {
var layerType = mapConfig.layerType(layerId);
for (var i = 0; i < self.layerStatsIterator.length; i++) {
if (self.layerStatsIterator[i].is(layerType)) {
var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]);
metaQueue.defer(getStats, layer, dbConnection);
break;
}
}
});
metaQueue.awaitAll(function (err, results) {
if (err) {
return callback(err);
}
if (!results) {
return callback(null, null);
}
mapConfig.getLayers().forEach(function (layer, layerIndex) {
stats[layerIndex] = results[layerIndex];
});
return callback(err, stats);
});
};
module.exports = LayerStats;

View File

@@ -0,0 +1,28 @@
var queryUtils = require('../../utils/query-utils');
function MapnikLayerStats () {
this._types = {
mapnik: true,
cartodb: true
};
}
MapnikLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
MapnikLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
// This query would gather stats for postgresql table if not exists
dbConnection.query(queryRowCountSql, function (err, res) {
if (err) {
return callback(null, {estimatedFeatureCount: -1});
} else {
// We decided that the relation is 1 row == 1 feature
return callback(null, {estimatedFeatureCount: res.rows[0].rows});
}
});
};
module.exports = MapnikLayerStats;

View File

@@ -0,0 +1,16 @@
function TorqueLayerStats() {
this._types = {
torque: true
};
}
TorqueLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
TorqueLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
return callback(null, {});
};
module.exports = TorqueLayerStats;

View File

@@ -0,0 +1,16 @@
var layerStats = require('./layer-stats/factory');
function StatsBackend() {
}
module.exports = StatsBackend;
StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) {
var enabledFeatures = global.environment.enabledFeatures;
var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false;
if (layerStatsEnabled) {
layerStats().getStats(mapConfig, dbConnection, callback);
} else {
return callback(null, []);
}
};

View File

@@ -296,7 +296,7 @@ TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
// @param callback function(err)
//
TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
var self = this;
template = templateDefaults(template);
@@ -430,13 +430,17 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
_reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
function _replaceVars (str, params) {
//return _.template(str, params); // lazy way, possibly dangerous
// Construct regular expressions for each param
// Construct regular expressions for each param
Object.keys(params).forEach(function(k) {
str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
});
return str;
}
function isObject(val) {
return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val));
}
TemplateMaps.prototype.instance = function(template, params) {
var all_params = {};
var phold = template.placeholders || {};
@@ -474,6 +478,13 @@ TemplateMaps.prototype.instance = function(template, params) {
// NOTE: we're deep-cloning the layergroup here
var layergroup = JSON.parse(JSON.stringify(template.layergroup));
if (layergroup.buffersize && isObject(layergroup.buffersize)) {
Object.keys(layergroup.buffersize).forEach(function(k) {
layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10);
});
}
for (var i=0; i<layergroup.layers.length; ++i) {
var lyropt = layergroup.layers[i].options;

View File

@@ -10,7 +10,13 @@ function createTemplate(method) {
'max({{=it._column}}) max_val,',
'avg({{=it._column}}) avg_val,',
method,
'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL',
'AND',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float'
].join('\n'));
}

View File

@@ -17,16 +17,8 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
'zoom',
'lon',
'lat',
// widgets & filters
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'bins', // number
'start', // number
'end', // number
'column_type', // string
// widgets search
'q'
// analysis
'filters' // json
];
function BaseController(authApi, pgConnection) {
@@ -195,12 +187,19 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
BaseController.prototype.sendError = function(req, res, err, label) {
var allErrors = Array.isArray(err) ? err : [err];
allErrors = populateTimeoutErrors(allErrors);
label = label || 'UNKNOWN';
err = allErrors[0] || new Error(label);
allErrors[0] = err;
var statusCode = findStatusCode(err);
if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
statusCode = 204;
}
debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
// If a callback was requested, force status to 200
@@ -225,6 +224,18 @@ function stripConnectionInfo(message) {
.replace(/is the server.*encountered/im, 'encountered');
}
var ERROR_INFO_TO_EXPOSE = {
message: true,
layer: true,
type: true,
analysis: true,
subtype: true
};
function shouldBeExposed (prop) {
return !!ERROR_INFO_TO_EXPOSE[prop];
}
function errorMessage(err) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var message = (_.isString(err) ? err : err.message) || 'Unknown error';
@@ -243,7 +254,7 @@ function errorMessageWithContext(err) {
for (var prop in err) {
// type & message are properties from Error's prototype and will be skipped
if (err.hasOwnProperty(prop)) {
if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
error[prop] = err[prop];
}
}
@@ -285,5 +296,38 @@ function statusFromErrorMessage(errMsg) {
statusCode = 404;
}
}
return statusCode;
}
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function populateTimeoutErrors (errors) {
return errors.map(function (error) {
if (isRenderTimeoutError(error)) {
error.subtype = 'render';
}
if (isDatasourceTimeoutError(error)) {
error.subtype = 'datasource';
}
if (isTimeoutError(error)) {
error.message = 'You are over platform\'s limits. Please contact us to know more details';
error.type = 'limit';
error.http_status = 429;
}
return error;
});
}

View File

@@ -79,19 +79,51 @@ LayergroupController.prototype.register = function(app) {
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
app.get(app.base_url_mapconfig +
'/:token/dataview/:dataviewName', cors(), userMiddleware,
this.dataview.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:dataviewName', cors(), userMiddleware,
this.dataview.bind(this));
app.get(app.base_url_mapconfig +
'/:token/dataview/:dataviewName/search', cors(), userMiddleware,
this.dataviewSearch.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware,
this.dataviewSearch.bind(this));
var allowedDataviewQueryParams = [
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'start', // number
'end', // number
'column_type', // string
'bins', // number
'aggregation', //string
'offset', // number
'q' // widgets search
];
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataview.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataview.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataviewSearch.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataviewSearch.bind(this)
);
app.get(app.base_url_mapconfig +
'/:token/analysis/node/:nodeId', cors(), userMiddleware,
@@ -253,7 +285,9 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
grid_json: true,
json_torque: true,
torque_json: true,
png: true
png: true,
png32: true,
mvt: true
};
var formatStat = 'invalid';
@@ -391,13 +425,15 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
function getSQL(err, mapConfig) {
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
var queries = [];
mapConfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
queries.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
return queries.length ? queries.join(';') : null;
},

View File

@@ -20,6 +20,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
/**
* @param {AuthApi} authApi
* @param {PgConnection} pgConnection
@@ -30,10 +31,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {MapConfigAdapter} mapConfigAdapter
* @param {StatsBackend} statsBackend
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) {
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
statsBackend) {
BaseController.call(this, authApi, pgConnection);
@@ -47,6 +50,8 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.mapConfigAdapter = mapConfigAdapter;
this.resourceLocator = new ResourceLocator(global.environment);
this.statsBackend = statsBackend;
}
util.inherits(MapController, BaseController);
@@ -214,7 +219,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
var mapConfigProvider;
var mapConfig;
step(
function setupParams(){
self.req2params(req, this);
@@ -249,7 +253,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
},
function afterLayergroupCreate(err, layergroup) {
assert.ifError(err);
self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this);
self.afterLayergroupCreate(req, res, mapConfig, layergroup,
mapConfigProvider.analysesResults,
this);
},
function finishTemplateInstantiation(err, layergroup) {
if (err) {
@@ -272,7 +278,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
);
};
MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) {
MapController.prototype.afterLayergroupCreate =
function(req, res, mapconfig, layergroup, analysesResults, callback) {
var self = this;
var username = req.context.user;
@@ -306,12 +313,19 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
done();
});
var sql = mapconfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
var sql = [];
mapconfig.getLayers().forEach(function(layer) {
sql.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
sql.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
var dbName = req.params.dbname;
var layergroupId = layergroup.layergroupid;
var dbConnection;
step(
function getPgConnection() {
@@ -319,7 +333,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
},
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
dbConnection = connection;
QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
req.profiler.done('queryTablesAndLastUpdated');
@@ -346,6 +361,21 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
return null;
},
function fetchLayersStats(err) {
assert.ifError(err);
var next = this;
self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) {
if (err) {
return next(err);
}
if (layersStats.length > 0) {
layergroup.metadata.layers.forEach(function (layer, index) {
layer.meta.stats = layersStats[index];
});
}
return next();
});
},
function finish(err) {
done(err);
}

View File

@@ -5,11 +5,32 @@ var debug = require('debug')('windshaft:widget:aggregation');
var dot = require('dot');
dot.templateSettings.strip = false;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var summaryQueryTpl = dot.template([
'summary AS (',
' SELECT',
' count(1) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' {{?it._isFloatColumn}},sum(',
' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1',
' ELSE 0',
' END',
' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')'
].join('\n'));
@@ -18,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' FROM filtered_source',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
@@ -44,22 +65,25 @@ var categoriesSummaryCountQueryTpl = dot.template([
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
' count, categories_count',
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,',
' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n'));
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC'
].join('\n'));
@@ -84,7 +108,7 @@ var TYPE = 'aggregation';
}
}
*/
function Aggregation(query, options) {
function Aggregation(query, options, queries) {
if (!_.isString(options.column)) {
throw new Error('Aggregation expects `column` in widget options');
}
@@ -108,9 +132,11 @@ function Aggregation(query, options) {
BaseWidget.apply(this);
this.query = query;
this.queries = queries;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
}
Aggregation.prototype = new BaseWidget();
@@ -119,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var _query = this.query;
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
@@ -140,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) {
].join('\n');
} else {
aggregationSql = [
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
this.getCategoriesCTESql(
_query,
this.column,
this.aggregation,
this.aggregationColumn,
this._isFloatColumn
),
rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
@@ -155,30 +208,38 @@ Aggregation.prototype.sql = function(psql, override, callback) {
return callback(null, aggregationSql);
};
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) {
return [
"WITH",
[
summaryQueryTpl({
_query: query,
_column: column
}),
rankedCategoriesQueryTpl({
_query: query,
_column: column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
categoriesSummaryMinMaxQueryTpl({
_query: query,
_column: column
}),
categoriesSummaryCountQueryTpl({
_query: query,
_column: column
})
].join(',\n')
].join('\n');
"WITH",
[
filteredQueryTpl({
_isFloatColumn: isFloatColumn,
_query: this.query,
_column: this.column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
summaryQueryTpl({
_isFloatColumn: isFloatColumn,
_query: query,
_column: column,
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
rankedCategoriesQueryTpl({
_query: query,
_column: column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
}),
categoriesSummaryMinMaxQueryTpl({
_query: query,
_column: column
}),
categoriesSummaryCountQueryTpl({
_query: query,
_column: column
})
].join(',\n')
].join('\n');
};
var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
@@ -193,6 +254,8 @@ Aggregation.prototype.format = function(result) {
var categories = [];
var count = 0;
var nulls = 0;
var nans = 0;
var infinities = 0;
var minValue = 0;
var maxValue = 0;
var categoriesCount = 0;
@@ -202,12 +265,15 @@ Aggregation.prototype.format = function(result) {
var firstRow = result.rows[0];
count = firstRow.count;
nulls = firstRow.nulls_count;
nans = firstRow.nans_count;
infinities = firstRow.infinities_count;
minValue = firstRow.min_val;
maxValue = firstRow.max_val;
categoriesCount = firstRow.categories_count;
result.rows.forEach(function(row) {
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val',
'max_val', 'categories_count', 'nans_count', 'infinities_count'));
});
}
@@ -215,6 +281,8 @@ Aggregation.prototype.format = function(result) {
aggregation: this.aggregation,
count: count,
nulls: nulls,
nans: nans,
infinities: infinities,
min: minValue,
max: maxValue,
categoriesCount: categoriesCount,
@@ -253,6 +321,8 @@ Aggregation.prototype.search = function(psql, userQuery, callback) {
var self = this;
var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
var _value = this.aggregation !== 'count' && this.aggregationColumn ?
this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)';
// TODO unfiltered will be wrong as filters are already applied at this point
var query = searchQueryTpl({
@@ -265,7 +335,7 @@ Aggregation.prototype.search = function(psql, userQuery, callback) {
_searchFiltered: filterCategoriesQueryTpl({
_query: this.query,
_column: this.column,
_value: 'count(1)',
_value: _value,
_userQuery: _userQuery
})
});

View File

@@ -1,3 +1,6 @@
var dot = require('dot');
dot.templateSettings.strip = false;
function BaseDataview() {}
module.exports = BaseDataview;
@@ -5,8 +8,11 @@ module.exports = BaseDataview;
BaseDataview.prototype.getResult = function(psql, override, callback) {
var self = this;
this.sql(psql, override, function(err, query) {
psql.query(query, function(err, result) {
if (err) {
return callback(err);
}
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
}
@@ -24,3 +30,42 @@ BaseDataview.prototype.getResult = function(psql, override, callback) {
BaseDataview.prototype.search = function(psql, userQuery, callback) {
return callback(null, this.format({ rows: [] }));
};
var FLOAT_OIDS = {
700: true,
701: true,
1700: true
};
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1'
);
BaseDataview.prototype.getColumnType = function (psql, column, query, callback) {
var readOnlyTransaction = true;
var columnTypeQuery = columnTypeQueryTpl({
column: column, query: query
});
psql.query(columnTypeQuery, function(err, result) {
if (err) {
return callback(err);
}
var pgType = result.rows[0].pg_typeof;
callback(null, getPGTypeName(pgType));
}, readOnlyTransaction);
};
function getPGTypeName (pgType) {
return {
float: FLOAT_OIDS.hasOwnProperty(pgType),
date: DATE_OIDS.hasOwnProperty(pgType)
};
}

View File

@@ -7,9 +7,19 @@ dot.templateSettings.strip = false;
var formulaQueryTpl = dot.template([
'SELECT',
'{{=it._operation}}({{=it._column}}) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
' {{=it._operation}}({{=it._column}}) AS result,',
' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n'));
var VALID_OPERATIONS = {
@@ -31,7 +41,7 @@ var TYPE = 'formula';
}
}
*/
function Formula(query, options) {
function Formula(query, options, queries) {
if (!_.isString(options.operation)) {
throw new Error('Formula expects `operation` in widget options');
}
@@ -47,8 +57,10 @@ function Formula(query, options) {
BaseWidget.apply(this);
this.query = query;
this.queries = queries;
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
}
Formula.prototype = new BaseWidget();
@@ -57,14 +69,27 @@ Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var _query = this.query;
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var formulaSql = formulaQueryTpl({
_query: _query,
_isFloatColumn: this._isFloatColumn,
_query: this.query,
_operation: this.operation,
_column: this.column
});
@@ -78,13 +103,17 @@ Formula.prototype.format = function(result) {
var formattedResult = {
operation: this.operation,
result: 0,
nulls: 0
nulls: 0,
nans: 0,
infinities: 0
};
if (result.rows.length) {
formattedResult.operation = this.operation;
formattedResult.result = result.rows[0].result;
formattedResult.nulls = result.rows[0].nulls_count;
formattedResult.nans = result.rows[0].nans_count;
formattedResult.infinities = result.rows[0].infinities_count;
}
return formattedResult;

View File

@@ -5,108 +5,289 @@ var debug = require('debug')('windshaft:dataview:histogram');
var dot = require('dot');
dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
var dateIntervalQueryTpl = dot.template([
'WITH',
'__cdb_dates AS (',
' SELECT',
' MAX({{=it.column}}::timestamp) AS __cdb_end,',
' MIN({{=it.column}}::timestamp) AS __cdb_start',
' FROM ({{=it.query}}) __cdb_source',
'),',
'__cdb_interval_in_days AS (',
' SELECT' ,
' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days',
' FROM __cdb_dates',
'),',
'__cdb_interval_in_hours AS (',
' SELECT',
' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours',
' FROM __cdb_interval_in_days, __cdb_dates',
'),',
'__cdb_interval_in_minutes AS (',
' SELECT',
' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes',
' FROM __cdb_interval_in_hours, __cdb_dates',
'),',
'__cdb_interval_in_seconds AS (',
' SELECT',
' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds',
' FROM __cdb_interval_in_minutes, __cdb_dates',
')',
'SELECT',
' ROUND(__cdb_days / 365) AS year,',
' ROUND(__cdb_days / 90) AS quarter,',
' ROUND(__cdb_days / 30) AS month,',
' ROUND(__cdb_days / 7) AS week,',
' __cdb_days AS day,',
' __cdb_hours AS hour,',
' __cdb_minutes AS minute,',
' __cdb_seconds AS second',
'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds'
].join('\n'));
var MAX_INTERVAL_VALUE = 366;
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
var filteredQueryTpl = dot.template([
'__cdb_filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) __cdb_filtered_source_query',
' WHERE',
' {{=it._column}} IS NOT NULL',
' {{?it._isFloatColumn}}AND',
' {{=it._column}} != \'infinity\'::float',
' AND',
' {{=it._column}} != \'-infinity\'::float',
' AND',
' {{=it._column}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var basicsQueryTpl = dot.template([
'basics AS (',
'__cdb_basics AS (',
' SELECT',
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,',
' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
' FROM __cdb_filtered_source',
')'
].join(' \n'));
var overrideBasicsQueryTpl = dot.template([
'basics AS (',
'__cdb_basics AS (',
' SELECT',
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,',
' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
' FROM __cdb_filtered_source',
')'
].join('\n'));
var iqrQueryTpl = dot.template([
'iqrange AS (',
' SELECT max(quartile_max) - min(quartile_max) AS iqr',
'__cdb_iqrange AS (',
' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr',
' FROM (',
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile',
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
' FROM __cdb_filtered_source) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile',
' ) _cdb_iqr',
' ) __cdb_iqr',
')'
].join('\n'));
var binsQueryTpl = dot.template([
'bins AS (',
' SELECT CASE WHEN total_rows = 0 OR iqr = 0',
'__cdb_bins AS (',
' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0',
' THEN 1',
' ELSE GREATEST(',
' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),',
' LEAST(',
' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),',
' {{=it._maxBins}}',
' )',
' )',
' END AS bins_number',
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
' END AS __cdb_bins_number',
' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source',
' LIMIT 1',
')'
].join('\n'));
var overrideBinsQueryTpl = dot.template([
'bins AS (',
' SELECT {{=it._bins}} AS bins_number',
'__cdb_bins AS (',
' SELECT {{=it._bins}} AS __cdb_bins_number',
')'
].join('\n'));
var nullsQueryTpl = dot.template([
'nulls AS (',
'__cdb_nulls AS (',
' SELECT',
' count(*) AS nulls_count',
' FROM ({{=it._query}}) _cdb_histogram_nulls',
' count(*) AS __cdb_nulls_count',
' FROM ({{=it._query}}) __cdb_histogram_nulls',
' WHERE {{=it._column}} IS NULL',
')'
].join('\n'));
var infinitiesQueryTpl = dot.template([
'__cdb_infinities AS (',
' SELECT',
' count(*) AS __cdb_infinities_count',
' FROM ({{=it._query}}) __cdb_infinities_query',
' WHERE',
' {{=it._column}} = \'infinity\'::float',
' OR',
' {{=it._column}} = \'-infinity\'::float',
')'
].join('\n'));
var nansQueryTpl = dot.template([
'__cdb_nans AS (',
' SELECT',
' count(*) AS __cdb_nans_count',
' FROM ({{=it._query}}) __cdb_nans_query',
' WHERE {{=it._column}} = \'NaN\'::float',
')'
].join('\n'));
var histogramQueryTpl = dot.template([
'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,',
' nulls_count,',
' avg_val,',
' CASE WHEN min_val = max_val',
' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
' __cdb_bins_number AS bins_number,',
' __cdb_nulls_count AS nulls_count,',
' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,',
' __cdb_nans_count AS nans_count,{{?}}',
' __cdb_avg_val AS avg_val,',
' CASE WHEN __cdb_min_val = __cdb_max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
' ELSE GREATEST(',
' 1,',
' LEAST(',
' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),',
' __cdb_bins_number',
' )',
' ) - 1',
' END AS bin,',
' min({{=it._column}})::numeric AS min,',
' max({{=it._column}})::numeric AS max,',
' avg({{=it._column}})::numeric AS avg,',
' count(*) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
'WHERE {{=it._column}} IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,',
' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}',
'GROUP BY bin, bins_number, bin_width, nulls_count,',
' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
'ORDER BY bin'
].join('\n'));
var dateBasicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,',
' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
' min(date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' )) AS __cdb_start_date,',
' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,',
' count(1) AS __cdb_total_rows',
' FROM ({{=it._query}}) __cdb_basics_query',
')'
].join(' \n'));
var dateOverrideBasicsQueryTpl = dot.template([
'__cdb_basics AS (',
' SELECT',
' max({{=it._end}}) AS __cdb_max_val,',
' min({{=it._start}}) AS __cdb_min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
' min(',
' date_trunc(',
' \'{{=it._aggregation}}\',',
' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' )',
' ) AS __cdb_start_date,',
' max(',
' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' ) AS __cdb_end_date,',
' count(1) AS __cdb_total_rows',
' FROM ({{=it._query}}) __cdb_basics_query',
')'
].join(' \n'));
var dateBinsQueryTpl = dot.template([
'__cdb_bins AS (',
' SELECT',
' __cdb_bins_array,',
' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number',
' FROM (',
' SELECT',
' ARRAY(',
' SELECT GENERATE_SERIES(',
' __cdb_start_date::timestamptz,',
' __cdb_end_date::timestamptz,',
' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval',
' )',
' ) AS __cdb_bins_array',
' FROM __cdb_basics',
' ) __cdb_bins_array_query',
')'
].join('\n'));
var dateHistogramQueryTpl = dot.template([
'SELECT',
' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
' __cdb_bins_number AS bins_number,',
' __cdb_nulls_count AS nulls_count,',
' CASE WHEN __cdb_min_val = __cdb_max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(',
' WIDTH_BUCKET(',
' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',',
' __cdb_bins_array',
' ),',
' __cdb_bins_number',
' )) - 1',
' END AS bin,',
' min(',
' date_part(',
' \'epoch\', ',
' date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' ) AT TIME ZONE \'{{=it._offset}}\'',
' )',
' )::numeric AS timestamp,',
' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,',
' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
' count(*) AS freq',
'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls',
'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start',
'ORDER BY bin'
].join('\n'));
var TYPE = 'histogram';
/**
{
type: 'histogram',
options: {
column: 'name',
bins: 10 // OPTIONAL
}
Numeric histogram:
{
type: 'histogram',
options: {
column: 'name', // column data type: numeric
bins: 10 // OPTIONAL
}
}
Time series:
{
type: 'histogram',
options: {
column: 'date', // column data type: date
aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric)
offset: -7200 // OPTIONAL (UTC offset in seconds)
}
}
*/
function Histogram(query, options, queries) {
@@ -118,6 +299,8 @@ function Histogram(query, options, queries) {
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
this.aggregation = options.aggregation;
this.offset = options.offset;
this._columnType = null;
}
@@ -127,50 +310,55 @@ Histogram.prototype.constructor = Histogram;
module.exports = Histogram;
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
Histogram.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var self = this;
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.queries.no_filters
});
if (this._columnType === null) {
psql.query(columnTypeQuery, function(err, result) {
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
// assume numeric, will fail later
self._columnType = 'numeric';
if (!err && !!result.rows[0]) {
var pgType = result.rows[0].pg_typeof;
if (DATE_OIDS.hasOwnProperty(pgType)) {
self._columnType = 'date';
}
if (!err && !!type) {
self._columnType = Object.keys(type).find(function (key) {
return type[key];
});
}
self.sql(psql, override, callback);
}, true); // use read-only transaction
return null;
}
this._buildQuery(psql, override, callback);
};
Histogram.prototype.isDateHistogram = function (override) {
return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined);
};
Histogram.prototype._buildQuery = function (psql, override, callback) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.query;
if (this.isDateHistogram(override)) {
return this._buildDateHistogramQuery(psql, override, callback);
}
if (this._columnType === 'date') {
_column = columnCastTpl({column: _column});
}
var _query = this.query;
filteredQuery = filteredQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
});
var basicsQuery, binsQuery;
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
if (this._shouldOverride(override)) {
debug('overriding with %j', override);
basicsQuery = overrideBasicsQueryTpl({
_query: _query,
@@ -190,7 +378,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
_column: _column
});
if (override && _.has(override, 'bins')) {
if (this._shouldOverrideBins(override)) {
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
@@ -211,18 +399,34 @@ Histogram.prototype.sql = function(psql, override, callback) {
}
}
var cteSql = [
filteredQuery,
basicsQuery,
binsQuery,
nullsQueryTpl({
_query: _query,
_column: _column
})
];
var histogramSql = [
"WITH",
[
basicsQuery,
binsQuery,
nullsQueryTpl({
if (this._columnType === 'float') {
cteSql.push(
infinitiesQueryTpl({
_query: _query,
_column: _column
}),
nansQueryTpl({
_query: _query,
_column: _column
})
].join(',\n'),
);
}
var histogramSql = [
"WITH",
cteSql.join(',\n'),
histogramQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
})
@@ -233,6 +437,143 @@ Histogram.prototype.sql = function(psql, override, callback) {
return callback(null, histogramSql);
};
Histogram.prototype._shouldOverride = function (override) {
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
};
Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};
var DATE_AGGREGATIONS = {
'auto': true,
'minute': true,
'hour': true,
'day': true,
'week': true,
'month': true,
'quarter': true,
'year': true
};
Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) {
var _column = this.column;
var _query = this.query;
var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset;
if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) {
return callback(new Error('Invalid aggregation value. Valid ones: ' +
Object.keys(DATE_AGGREGATIONS).join(', ')
));
}
if (_aggregation === 'auto') {
this.getAutomaticAggregation(psql, function (err, aggregation) {
if (err || aggregation === 'none') {
this.aggregation = 'day';
} else {
this.aggregation = aggregation;
}
override.aggregation = this.aggregation;
this._buildDateHistogramQuery(psql, override, callback);
}.bind(this));
return null;
}
var dateBasicsQuery;
if (override && _.has(override, 'start') && _.has(override, 'end')) {
dateBasicsQuery = dateOverrideBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_start: getBinStart(override),
_end: getBinEnd(override),
_offset: parseOffset(_offset, _aggregation)
});
} else {
dateBasicsQuery = dateBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
}
var dateBinsQuery = [
dateBinsQueryTpl({
_aggregation: _aggregation
})
].join(',\n');
var nullsQuery = nullsQueryTpl({
_query: _query,
_column: _column
});
var dateHistogramQuery = dateHistogramQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
var histogramSql = [
"WITH",
[
dateBasicsQuery,
dateBinsQuery,
nullsQuery
].join(',\n'),
dateHistogramQuery
].join('\n');
debug(histogramSql);
return callback(null, histogramSql);
};
Histogram.prototype.getAutomaticAggregation = function (psql, callback) {
var dateIntervalQuery = dateIntervalQueryTpl({
query: this.query,
column: this.column
});
debug(dateIntervalQuery);
psql.query(dateIntervalQuery, function (err, result) {
if (err) {
return callback(err);
}
var aggegations = result.rows[0];
var aggregation = Object.keys(aggegations)
.map(function (key) {
return {
name: key,
value: aggegations[key]
};
})
.reduce(function (closer, current) {
if (current.value > MAX_INTERVAL_VALUE) {
return closer;
}
var closerDiff = MAX_INTERVAL_VALUE - closer.value;
var currentDiff = MAX_INTERVAL_VALUE - current.value;
if (Number.isFinite(current.value) && closerDiff > currentDiff) {
return current;
}
return closer;
}, { name: 'none', value: -1 });
callback(null, aggregation.name);
});
};
Histogram.prototype.format = function(result, override) {
override = override || {};
var buckets = [];
@@ -241,7 +582,12 @@ Histogram.prototype.format = function(result, override) {
var width = getWidth(override);
var binsStart = getBinStart(override);
var nulls = 0;
var infinities = 0;
var nans = 0;
var avg;
var timestampStart;
var aggregation;
var offset;
if (result.rows.length) {
var firstRow = result.rows[0];
@@ -249,23 +595,60 @@ Histogram.prototype.format = function(result, override) {
width = firstRow.bin_width || width;
avg = firstRow.avg_val;
nulls = firstRow.nulls_count;
binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min;
timestampStart = firstRow.timestamp_start;
infinities = firstRow.infinities_count;
nans = firstRow.nans_count;
binsStart = populateBinStart(override, firstRow);
if (Number.isFinite(timestampStart)) {
aggregation = getAggregation(override, this.aggregation);
offset = getOffset(override, this.offset);
}
buckets = result.rows.map(function(row) {
return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
return _.omit(
row,
'bins_number',
'bin_width',
'nulls_count',
'infinities_count',
'nans_count',
'avg_val',
'timestamp_start'
);
});
}
return {
aggregation: aggregation,
offset: offset,
timestamp_start: timestampStart,
bin_width: width,
bins_count: binsCount,
bins_start: binsStart,
nulls: nulls,
infinities: infinities,
nans: nans,
avg: avg,
bins: buckets
};
};
function getAggregation(override, aggregation) {
return override && override.aggregation ? override.aggregation : aggregation;
}
function getOffset(override, offset) {
if (override && override.offset) {
return override.offset;
}
if (offset) {
return offset;
}
return 0;
}
function getBinStart(override) {
if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
return Math.min(override.start, override.end);
@@ -295,6 +678,32 @@ function getWidth(override) {
return width;
}
function parseOffset(offset, aggregation) {
if (!offset) {
return '0';
}
if (aggregation === 'hour' || aggregation === 'minute') {
return '0';
}
var offsetInHours = Math.ceil(offset / 3600);
return '' + offsetInHours;
}
function populateBinStart(override, firstRow) {
var binStart;
if (firstRow.hasOwnProperty('timestamp')) {
binStart = firstRow.timestamp;
} else if (override.hasOwnProperty('start')) {
binStart = getBinStart(override);
} else {
binStart = firstRow.min;
}
return binStart;
}
Histogram.prototype.getType = function() {
return TYPE;
};

View File

@@ -1,14 +1,36 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../aggregation');
var debug = require('debug')('windshaft:widget:aggregation:overview');
var dot = require('dot');
dot.templateSettings.strip = false;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
' {{=it._aggregationColumn}} != \'infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'-infinity\'::float',
' AND',
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var summaryQueryTpl = dot.template([
'summary AS (',
' SELECT',
' sum(_feature_count) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' {{?it._isFloatColumn}},sum(',
' CASE',
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
' THEN 1',
' ELSE 0',
' END',
' ) AS infinities_count,',
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')'
].join('\n'));
@@ -17,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' FROM filtered_source',
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
@@ -36,40 +58,46 @@ var categoriesSummaryCountQueryTpl = dot.template([
' SELECT count(1) AS categories_count',
' FROM (',
' SELECT {{=it._column}} AS category',
' FROM ({{=it._query}}) _cdb_categories',
' FROM filtered_source',
' GROUP BY {{=it._column}}',
' ) _cdb_categories_count',
')'
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
'GROUP BY nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
].join('\n'));
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
'GROUP BY category, nulls_count, min_val, max_val, count,',
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
'ORDER BY value DESC'
].join('\n'));
var CATEGORIES_LIMIT = 6;
function Aggregation(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.query = query;
this.queries = queries;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
this._isFloatColumn = null;
}
Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
@@ -78,27 +106,49 @@ Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var _query = this.rewrittenQuery(this.query);
var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
if (this.aggregationColumn && this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
_aggregationColumn: _aggregationColumn
}),
categoriesSummaryMinMaxQueryTpl({
_query: _query,
@@ -110,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
})
].join(',\n'),
aggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
@@ -120,15 +171,23 @@ Aggregation.prototype.sql = function(psql, override, callback) {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
filteredQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
summaryQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_aggregationColumn: _aggregationColumn
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
_aggregationColumn: _aggregationColumn
}),
categoriesSummaryMinMaxQueryTpl({
_query: _query,
@@ -140,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
})
].join(',\n'),
rankedAggregationQueryTpl({
_isFloatColumn: this._isFloatColumn,
_query: _query,
_column: this.column,
_limit: CATEGORIES_LIMIT
@@ -147,6 +207,8 @@ Aggregation.prototype.sql = function(psql, override, callback) {
].join('\n');
}
debug(aggregationSql);
return callback(null, aggregationSql);
};

View File

@@ -1,14 +1,15 @@
var _ = require('underscore');
var BaseDataview = require('../base');
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) {
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) {
this.BaseDataview = BaseDataview;
this.query = query;
this.queryOptions = queryOptions;
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
this.options = options;
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions);
this.queries = queries;
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries);
}
module.exports = BaseOverviewsDataview;

View File

@@ -1,34 +1,61 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overview');
var dot = require('dot');
dot.templateSettings.strip = false;
var formulaQueryTpls = {
'count': dot.template([
'SELECT',
'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'count': dot.template([
'SELECT',
'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
'FROM ({{=it._query}}) _cdb_formula',
'{{?it._isFloatColumn}}WHERE',
' {{=it._column}} != \'infinity\'::float',
'AND',
' {{=it._column}} != \'-infinity\'::float',
'AND',
' {{=it._column}} != \'NaN\'::float{{?}}'
].join('\n')),
};
function Formula(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.column = options.column || '1';
this.operation = options.operation;
this._isFloatColumn = null;
this.queries = queries;
}
Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
@@ -36,21 +63,38 @@ Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, override, callback) {
Formula.prototype.sql = function (psql, override, callback) {
var self = this;
var formulaQueryTpl = formulaQueryTpls[this.operation];
if ( formulaQueryTpl ) {
if (formulaQueryTpl) {
// supported formula for use with overviews
if (this._isFloatColumn === null) {
this._isFloatColumn = false;
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
if (!err && !!type) {
self._isFloatColumn = type.float;
}
self.sql(psql, override, callback);
});
return null;
}
var formulaSql = formulaQueryTpl({
_query: this.rewrittenQuery(this.query),
_isFloatColumn: this._isFloatColumn,
_query: this.rewrittenQuery(this.query),
_operation: this.operation,
_column: this.column
_column: this.column
});
callback = callback || override;
debug(formulaSql);
return callback(null, formulaSql);
}
// default behaviour
return this.defaultSql(psql, override, callback);
};

View File

@@ -1,23 +1,35 @@
var _ = require('underscore');
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../histogram');
var debug = require('debug')('windshaft:dataview:histogram:overview');
var dot = require('dot');
dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
var filteredQueryTpl = dot.template([
'filtered_source AS (',
' SELECT *',
' FROM ({{=it._query}}) _cdb_filtered_source',
' WHERE',
' {{=it._column}} IS NOT NULL',
' {{?it._isFloatColumn}}AND',
' {{=it._column}} != \'infinity\'::float',
' AND',
' {{=it._column}} != \'-infinity\'::float',
' AND',
' {{=it._column}} != \'NaN\'::float{{?}}',
')'
].join(' \n'));
var basicsQueryTpl = dot.template([
'basics AS (',
' SELECT',
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' FROM filtered_source',
')'
].join(' \n'));
@@ -26,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([
' SELECT',
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
' FROM filtered_source',
')'
].join('\n'));
@@ -37,7 +49,7 @@ var iqrQueryTpl = dot.template([
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile',
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
' FROM filtered_source) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile',
' ) _cdb_iqr',
@@ -56,7 +68,7 @@ var binsQueryTpl = dot.template([
' )',
' )',
' END AS bins_number',
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
' FROM basics, iqrange, filtered_source',
' LIMIT 1',
')'
].join('\n'));
@@ -76,11 +88,34 @@ var nullsQueryTpl = dot.template([
')'
].join('\n'));
var infinitiesQueryTpl = dot.template([
'infinities AS (',
' SELECT',
' count(*) AS infinities_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE',
' {{=it._column}} = \'infinity\'::float',
' OR',
' {{=it._column}} = \'-infinity\'::float',
')'
].join('\n'));
var nansQueryTpl = dot.template([
'nans AS (',
' SELECT',
' count(*) AS nans_count',
' FROM ({{=it._query}}) _cdb_histogram_infinities',
' WHERE {{=it._column}} = \'NaN\'::float',
')'
].join('\n'));
var histogramQueryTpl = dot.template([
'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,',
' nulls_count,',
' {{?it._isFloatColumn}}infinities_count,',
' nans_count,{{?}}',
' avg_val,',
' CASE WHEN min_val = max_val',
' THEN 0',
@@ -90,14 +125,14 @@ var histogramQueryTpl = dot.template([
' max({{=it._column}})::numeric AS max,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
' sum(_feature_count) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
'WHERE {{=it._column}} IS NOT NULL',
'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
'ORDER BY bin'
].join('\n'));
function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
this.query = query;
this.queries = queries;
@@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram;
module.exports = Histogram;
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
Histogram.prototype.sql = function(psql, override, callback) {
var self = this;
if (!callback) {
callback = override;
override = {};
}
var self = this;
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.rewrittenQuery(this.queries.no_filters)
});
if (this._columnType === null) {
psql.query(columnTypeQuery, function(err, result) {
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
// assume numeric, will fail later
self._columnType = 'numeric';
if (!err && !!result.rows[0]) {
var pgType = result.rows[0].pg_typeof;
if (DATE_OIDS.hasOwnProperty(pgType)) {
self._columnType = 'date';
}
if (!err && !!type) {
self._columnType = Object.keys(type).find(function (key) {
return type[key];
});
}
self.sql(psql, override, callback);
}, true); // use read-only transaction
@@ -154,11 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) {
return this.defaultSql(psql, override, callback);
}
var histogramSql = this._buildQuery(override);
return callback(null, histogramSql);
};
Histogram.prototype._buildQuery = function (override) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.rewrittenQuery(this.query);
var basicsQuery, binsQuery;
filteredQuery = filteredQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
});
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
if (this._shouldOverride(override)) {
debug('overriding with %j', override);
basicsQuery = overrideBasicsQueryTpl({
_query: _query,
_column: _column,
@@ -177,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
_column: _column
});
if (override && _.has(override, 'bins')) {
if (this._shouldOverrideBins(override)) {
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
@@ -198,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) {
}
}
var cteSql = [
filteredQuery,
basicsQuery,
binsQuery,
nullsQueryTpl({
_query: _query,
_column: _column
})
];
var histogramSql = [
"WITH",
[
basicsQuery,
binsQuery,
nullsQueryTpl({
if (this._columnType === 'float') {
cteSql.push(
infinitiesQueryTpl({
_query: _query,
_column: _column
}),
nansQueryTpl({
_query: _query,
_column: _column
})
].join(',\n'),
);
}
var histogramSql = [
"WITH",
cteSql.join(',\n'),
histogramQueryTpl({
_isFloatColumn: this._columnType === 'float',
_query: _query,
_column: _column
})
].join('\n');
return callback(null, histogramSql);
debug(histogramSql);
return histogramSql;
};
Histogram.prototype._shouldOverride = function (override) {
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
};
Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};

View File

@@ -1,8 +1,8 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../list');
function List(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
function List(query, options, queryRewriter, queryRewriteData, params, queries) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
}
List.prototype = Object.create(BaseOverviewsDataview.prototype);

View File

@@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([
].join('\n'));
var bboxFilterTpl = dot.template(
'{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'
'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))'
);
var LATITUDE_MAX_VALUE = 85.0511287798066;
@@ -66,7 +66,8 @@ function getBoundingBoxes(west, south, east, north) {
bboxes.push([west, south, east, north]);
} else {
bboxes.push([west, south, 180, north]);
bboxes.push([-180, south, east % 180, north]);
// here we assume west,east have been adjusted => west >= -180 => east > 180
bboxes.push([-180, south, east - 360, north]);
}
return bboxes;

View File

@@ -115,6 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
}
layer.options.sql = analysisSql;
layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode);
} else {
missingNodesErrors.push(
new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
@@ -330,4 +331,13 @@ function AnalysisError(message) {
this.message = message;
}
function getAllAffectedTablesFromSourceNodes(node) {
var affectedTables = node.getAllInputNodes(function (node) {
return node.getType() === 'source';
}).reduce(function(list, node) {
return list.concat(node.getAffectedTables());
},[]);
return affectedTables;
}
require('util').inherits(AnalysisError, Error);

View File

@@ -0,0 +1,25 @@
function MapConfigBufferSizeAdapter() {
this.formats = ['png', 'png32', 'mvt', 'grid.json'];
}
module.exports = MapConfigBufferSizeAdapter;
MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
if (!context.templateParams || !context.templateParams.buffersize) {
return callback(null, requestMapConfig);
}
this.formats.forEach(function (format) {
if (Number.isFinite(context.templateParams.buffersize[format])) {
if (requestMapConfig.buffersize === undefined) {
requestMapConfig.buffersize = {};
}
requestMapConfig.buffersize[format] = context.templateParams.buffersize[format];
}
});
setImmediate(function () {
callback(null, requestMapConfig);
});
};

View File

@@ -43,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC
if (nestedNamedLayers.length > 0) {
var nestedNamedMapsError = new Error('Nested named layers are not allowed');
// nestedNamedMapsError.http_status = 400;
return done(nestedNamedMapsError);
}

View File

@@ -4,13 +4,6 @@ var dot = require('dot');
dot.templateSettings.strip = false;
var queue = require('queue-async');
var PSQL = require('cartodb-psql');
/**
* cartodb-psql creates `global.Promise` as an empty constructor.
* However, `turbo-carto` relies on a polyfil that fails to create the polyfil
* as it finds `global.Promise` but it doesn't find `Promise.resolve`.
*/
global.Promise = global.Promise || function() {};
global.Promise.resolve = global.Promise.resolve || function() {};
var turboCarto = require('turbo-carto');
var SubstitutionTokens = require('../../../utils/substitution-tokens');

View File

@@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, this);
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

View File

@@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, this);
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

View File

@@ -90,6 +90,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
},
function instantiateTemplate(err, templateParams) {
assert.ifError(err);
context.templateParams = templateParams;
return self.templateMaps.instance(self.template, templateParams);
},
function prepareAdapterMapConfig(err, requestMapConfig) {
@@ -113,7 +114,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
function prepareContextLimits(err, _mapConfig) {
assert.ifError(err);
mapConfig = _mapConfig;
self.userLimitsApi.getRenderLimits(self.owner, this);
self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this);
},
function cacheAndReturnMapConfig(err, renderLimits) {
self.err = err;

View File

@@ -35,12 +35,15 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin
var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
var MapConfigAdapter = require('./models/mapconfig/adapter');
var StatsBackend = require('./backends/stats');
module.exports = function(serverOptions) {
// Make stats client globally accessible
global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@@ -115,8 +118,27 @@ module.exports = function(serverOptions) {
var onTileErrorStrategy;
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
if (err && err.message === 'Render timed out' && format === 'png') {
return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRasterFormat (format) {
return format === 'png' || format === 'jpg';
}
if (isTimeoutError(err) && isRasterFormat(format)) {
return callback(null, timeoutErrorTile, {
'Content-Type': 'image/png',
}, {});
} else {
return callback(err, tile, headers, stats);
}
@@ -150,11 +172,14 @@ module.exports = function(serverOptions) {
var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
var statsBackend = new StatsBackend();
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var mapConfigAdapter = new MapConfigAdapter(
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
new MapConfigBufferSizeAdapter(),
new SqlWrapMapConfigAdapter(),
new DataviewsWidgetsAdapter(),
new AnalysisMapConfigAdapter(analysisBackend),
@@ -207,7 +232,8 @@ module.exports = function(serverOptions) {
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
mapConfigAdapter
mapConfigAdapter,
statsBackend
).register(app);
new controller.NamedMaps(
@@ -303,6 +329,25 @@ function bootstrap(opts) {
app.enable('jsonp callback');
app.disable('x-powered-by');
app.disable('etag');
// Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705
// See: http://expressjs.com/en/4x/api.html#app.set
app.set('json replacer', function (key, value) {
if (value !== value) {
return 'NaN';
}
if (value === Infinity) {
return 'Infinity';
}
if (value === -Infinity) {
return '-Infinity';
}
return value;
});
app.use(bodyParser.json());
app.use(function bootstrap$prepareRequestResponse(req, res, next) {

View File

@@ -0,0 +1,26 @@
function prepareQuery(sql) {
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
}
module.exports.extractTableNames = function extractTableNames(query) {
return [
'SELECT * FROM CDB_QueryTablesText($windshaft$',
prepareQuery(query),
'$windshaft$) as tablenames'
].join('');
};
module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
};

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "3.3.0",
"version": "3.12.3",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -16,14 +16,15 @@
"contributors": [
"Simon Tokumine <simon@vizzuality.com>",
"Javi Santana <jsantana@vizzuality.com>",
"Sandro Santilli <strk@vizzuality.com>"
"Sandro Santilli <strk@vizzuality.com>",
"Carlos Matallín <matallo@carto.com>"
],
"dependencies": {
"body-parser": "~1.14.0",
"camshaft": "0.50.3",
"cartodb-psql": "~0.7.1",
"camshaft": "0.55.8",
"cartodb-psql": "0.10.1",
"cartodb-query-tables": "0.2.0",
"cartodb-redis": "0.13.2",
"cartodb-redis": "0.14.0",
"debug": "~2.2.0",
"dot": "~1.0.2",
"express": "~4.13.3",
@@ -37,21 +38,23 @@
"request": "~2.79.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"turbo-carto": "0.19.0",
"turbo-carto": "0.19.2",
"underscore": "~1.6.0",
"windshaft": "3.1.0",
"windshaft": "3.3.1",
"yargs": "~5.0.0"
},
"devDependencies": {
"istanbul": "~0.4.3",
"jshint": "~2.6.0",
"mocha": "~1.21.4",
"jshint": "~2.9.4",
"mocha": "~3.4.1",
"moment": "~2.18.1",
"nock": "~2.11.0",
"redis": "~0.12.1",
"semver": "~1.1.4",
"strftime": "~0.8.2"
},
"scripts": {
"lint": "jshint lib test",
"preinstall": "make pre-install",
"test": "make test-all"
},

View File

@@ -373,5 +373,70 @@ describe('analysis-layers error cases', function() {
});
});
it('should return "function does not exist" indicating the node_id and context', function(done) {
var mapConfig = createMapConfig([{
"type": "cartodb",
"options": {
"source": {
"id": "HEAD"
},
"cartocss": '#polygons { polygon-fill: red; }',
"cartocss_version": "2.3.0"
}
}], {}, [{
"id": "HEAD",
"type": "buffer",
"params": {
"source": {
"id": "HEAD2",
"type": "buffer",
"params": {
"source": {
"id": "HEAD3",
"type": 'deprecated-sql-function',
"params": {
"id": "HEAD4",
"function_name": 'DEP_EXT_does_not_exist_fn',
"primary_source": {
"type": 'source',
"params": {
"query": "select * from populated_places_simple_reduced"
}
},
"function_args": ['wadus']
}
},
"radius": 10
}
},
"radius": 10
}
}]);
var testClient = new TestClient(mapConfig, 1234);
testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
assert.ok(!err, err);
assert.equal(layergroupResult.errors.length, 1);
assert.equal(
layergroupResult.errors[0],
'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist'
);
assert.equal(layergroupResult.errors_with_context[0].type, 'analysis');
assert.equal(
layergroupResult.errors_with_context[0].message,
'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist'
);
assert.equal(layergroupResult.errors_with_context[0].analysis.id, 'HEAD');
assert.equal(layergroupResult.errors_with_context[0].analysis.type, 'buffer');
assert.equal(layergroupResult.errors_with_context[0].analysis.node_id, 'HEAD3');
testClient.drain(done);
});
});
});

View File

@@ -0,0 +1,441 @@
require('../support/test_helper');
var fs = require('fs');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
var mapnik = require('windshaft').mapnik;
var IMAGE_TOLERANCE_PER_MIL = 5;
var CARTOCSS_LABELS = [
'#layer {',
' polygon-fill: #374C70;',
' polygon-opacity: 0.9;',
' line-width: 1;',
' line-color: #FFF;',
' line-opacity: 0.5;',
'}',
'#layer::labels {',
' text-name: [name];',
' text-face-name: \'DejaVu Sans Book\';',
' text-size: 20;',
' text-fill: #FFFFFF;',
' text-label-position-tolerance: 0;',
' text-halo-radius: 1;',
' text-halo-fill: #6F808D;',
' text-dy: -10;',
' text-allow-overlap: true;',
' text-placement: point;',
' text-placement-type: dummy;',
'}'
].join('\n');
function createMapConfig (bufferSize, cartocss) {
cartocss = cartocss || CARTOCSS_LABELS;
return {
version: '1.6.0',
buffersize: bufferSize,
layers: [{
type: "cartodb",
options: {
sql: [
'select',
' *',
'from',
' populated_places_simple_reduced',
].join('\n'),
cartocss: cartocss,
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
}]
};
}
describe('buffer size per format', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
mapConfig: createMapConfig({ png: 0, 'grid.json': 0 }),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
mapConfig: createMapConfig({ png: 128, 'grid.json': 128 }),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get mvt tile using buffer-size 0',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.mvt',
mapConfig: createMapConfig({ mvt: 0 }),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
assert.equal(features.length, 1);
callback();
}
},
{
desc: 'should get mvt tile using buffer-size 128',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.mvt',
mapConfig: createMapConfig({ mvt: 128 }),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
assert.equal(features.length, 9);
callback();
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
mapConfig: createMapConfig({ 'grid.json': 0 }),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
mapConfig: createMapConfig({ 'grid.json': 128 }),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
}
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.mapConfig, 1234);
var coords = test.coords;
var options = {
format: test.format,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
// tile.save(test.fixturePath);
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});
function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) {
cartocss = cartocss || CARTOCSS_LABELS;
return {
"version": "0.0.1",
"name": name,
"placeholders": placeholders || {
"buffersize": {
"type": "number",
"default": 0
}
},
"layergroup": createMapConfig(buffersize)
};
}
describe('buffer size per format for named maps', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0 (default value in template)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 (placehoder value)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-custom-buffer-size', { png: '<%= buffersize %>'}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 0 (default value in template by format)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize_png: 0 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-default-buffer-size-by-format', {
png: '<%= buffersize_png %>'
}, {
"buffersize_png": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize_png: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-custom-buffer-size-by-format', {
png: '<%= buffersize_png %>'
}, {
"buffersize_png": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: { buffersize_gridjson: 0 },
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', {
'grid.json': '<%= buffersize_gridjson %>'
}, {
"buffersize_gridjson": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: { buffersize_gridjson: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', {
'grid.json': '<%= buffersize_gridjson %>'
}, {
"buffersize_gridjson": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
}
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png');
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});
describe('buffer size per format for named maps w/o placeholders', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-no-buffer-size-png-0', {}, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-no-buffer-size-png-128', {}, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get mvt tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
placeholders: {
buffersize: {
mvt: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt',
template: createBufferSizeTemplate('named-no-buffer-size-mvt', {}, {}),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
var dataFixture = fs.readFileSync(this.fixturePath);
var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
vtile.setDataSync(dataFixture);
var vtileJSON = vtile.toJSON();
var vtileFeatures = vtileJSON[0].features;
assert.equal(features.length, vtileFeatures.length);
callback();
}
},
{
desc: 'should get mvt tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
placeholders: {
buffersize: {
mvt: 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt',
template: createBufferSizeTemplate('named-no-buffer-size-mvt-128', {}, {}),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
var dataFixture = fs.readFileSync(this.fixturePath);
var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
vtile.setDataSync(dataFixture);
var vtileJSON = vtile.toJSON();
var vtileFeatures = vtileJSON[0].features;
assert.equal(features.length, vtileFeatures.length);
callback();
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: {
buffersize: {
'grid.json': 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
template: createBufferSizeTemplate('named-no-buffer-size-grid-json-0', {}, {}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: {
buffersize: {
'grid.json': 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
template: createBufferSizeTemplate('named-no-buffer-size-grid-json-128', {}, {}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
},
{
desc: 'should get png tile using buffer-size 0' +
' overriden by template params with no buffersize in mapconfig',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-no-buffer-size-mapconfig-png-0', undefined, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save(test.fixturePath);
// require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
// require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});

393
test/acceptance/cache/cache_headers.js vendored Normal file
View File

@@ -0,0 +1,393 @@
var testHelper = require('../../support/test_helper');
var assert = require('../../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../../lib/cartodb/server');
var serverOptions = require('../../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var LayergroupToken = require('../../support/layergroup-token');
describe('get requests with cache headers', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var statusOkResponse = {
status: 200
};
var mapConfigs = [
{
"description": "cache headers should be present",
"cache_headers": {
"x_cache_channel": {
"db_name": "test_windshaft_cartodb_user_1_db",
"tables": ["public.test_table"]
},
"surrogate_keys": "t:77pJnX"
},
"data":
{
version: '1.5.0',
layers: [
{
options: {
source: {
id: "2570e105-7b37-40d2-bdf4-1af889598745"
},
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
],
analyses: [
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "source",
"params": {
"query": "select * from test_table limit 2"
}
}
]
},
},
{
"description": "cache headers should be present and be composed with source table name",
"cache_headers": {
"x_cache_channel": {
"db_name": "test_windshaft_cartodb_user_1_db",
"tables": ["public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d",
"public.test_table"]
},
"surrogate_keys": "t:77pJnX t:iL4eth"
},
"data":
{
version: '1.5.0',
layers: [
{
options: {
source: {
id: "2570e105-7b37-40d2-bdf4-1af889598745"
},
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
],
analyses: [
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "buffer",
"params": {
"source": {
"type": "source",
"params": {
"query": "select * from test_table limit 2"
}
},
"radius": 50000
}
}
]
}
}];
var layergroupRequest = function(mapConfig) {
return {
url: '/api/v1/map?api_key=1234&config=' + encodeURIComponent(JSON.stringify(mapConfig)),
method: 'GET',
headers: {
host: 'localhost'
}
};
};
function getRequest(url, addApiKey, callbackName) {
var params = {};
if (!!addApiKey) {
params.api_key = '1234';
}
if (!!callbackName) {
params.callback = callbackName;
}
return {
url: url + '?' + qs.stringify(params),
method: 'GET',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
};
}
function validateCacheHeaders(done, expectedCacheHeaders) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(res.headers['x-cache-channel']);
assert.ok(res.headers['surrogate-key']);
if (expectedCacheHeaders) {
validateXChannelHeaders(res.headers, expectedCacheHeaders);
assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys);
}
done();
};
}
function validateXChannelHeaders(headers, expectedCacheHeaders) {
var dbName = headers['x-cache-channel'].split(':')[0];
var tables = headers['x-cache-channel'].split(':')[1].split(',').sort();
assert.equal(dbName, expectedCacheHeaders.x_cache_channel.db_name);
assert.deepEqual(tables, expectedCacheHeaders.x_cache_channel.tables.sort());
}
function noCacheHeaders(done) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(
!res.headers['x-cache-channel'],
'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
);
assert.ok(
!res.headers['surrogate-key'],
'did not expect surrogate-key header, got: `' + res.headers['surrogate-key'] + '`'
);
done();
};
}
function withLayergroupId(mapConfig, callback) {
assert.response(
server,
layergroupRequest(mapConfig),
statusOkResponse,
function(res, err) {
if (err) {
return callback(err);
}
var layergroupId = JSON.parse(res.body).layergroupid;
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
callback(null, layergroupId, res);
}
);
}
mapConfigs.forEach(function(mapConfigData) {
describe(mapConfigData.description, function() {
var mapConfig = mapConfigData.data;
var expectedCacheHeaders = mapConfigData.cache_headers;
it('/api/v1/map Map instantiation', function(done) {
var testFn = validateCacheHeaders(done, expectedCacheHeaders);
withLayergroupId(mapConfig, function(err, layergroupId, res) {
testFn(res);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
withLayergroupId(mapConfig, function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true),
validateCacheHeaders(done, expectedCacheHeaders)
);
});
});
});
});
describe('cache headers should NOT be present', function() {
it('/', function(done) {
assert.response(
server,
getRequest('/'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/version', function(done) {
assert.response(
server,
getRequest('/version'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/health', function(done) {
assert.response(
server,
getRequest('/health'),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/api/v1/map/named list named maps', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named', true),
statusOkResponse,
noCacheHeaders(done)
);
});
describe('with named maps', function() {
var templateName = 'x_cache';
beforeEach(function(done) {
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
layergroup: mapConfigs[0].data
};
var namedMapRequest = {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(template)
};
assert.response(
server,
namedMapRequest,
statusOkResponse,
function(res, err) {
done(err);
}
);
});
afterEach(function(done) {
assert.response(
server,
{
url: '/api/v1/map/named/' + templateName + '?api_key=1234',
method: 'DELETE',
headers: {
host: 'localhost'
}
},
{
status: 204
},
function(res, err) {
done(err);
}
);
});
it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true),
statusOkResponse,
noCacheHeaders(done)
);
});
it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
statusOkResponse,
noCacheHeaders(done)
);
});
});
});
});

View File

@@ -145,4 +145,182 @@ describe('aggregations happy cases', function() {
});
});
});
var widgetSearchExpects = {
'count': [ { category: 'other_a', value: 3 } ],
'sum': [ { category: 'other_a', value: 6 } ],
'avg': [ { category: 'other_a', value: 2 } ],
'max': [ { category: 'other_a', value: 3 } ],
'min': [ { category: 'other_a', value: 1 } ]
};
Object.keys(operations_and_values).forEach(function (operation) {
var description = 'should search OTHER category using "' + operation + '"';
it(description, function (done) {
this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) {
assert.ifError(err);
assert.ok(searchResult);
assert.equal(searchResult.type, 'aggregation');
assert.equal(searchResult.categories.length, 1);
assert.deepEqual(
searchResult.categories,
widgetSearchExpects[operation]
);
done();
});
});
});
});
describe('aggregation-dataview: special float values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
val_aggregation: {
source: {
id: 'a0'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'avg',
aggregationColumn: 'val'
}
},
sum_aggregation_numeric: {
source: {
id: 'a1'
},
type: 'aggregation',
options: {
column: 'cat',
aggregation: 'sum',
aggregationColumn: 'val'
}
}
},
[
{
"id": "a0",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS val,',
' CASE',
' WHEN x % 2 = 0 THEN \'category_1\'',
' ELSE \'category_2\'',
' END AS cat',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}, {
"id": "a1",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 3 = 0 THEN \'NaN\'::numeric',
' WHEN x % 3 = 1 THEN x',
' ELSE x',
' END AS val,',
' CASE',
' WHEN x % 2 = 0 THEN \'category_1\'',
' ELSE \'category_2\'',
' END AS cat',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}
]
);
// Source a0
// -----------------------------------------------
// the_geom_webmercator | val | cat
// ----------------------+-----------+------------
// | -Infinity | category_2
// | NaN | category_1
// | 3 | category_2
// | Infinity | category_1
// | -Infinity | category_2
// | NaN | category_1
// | 7 | category_2
// | Infinity | category_1
// | -Infinity | category_2
// | NaN | category_1
// | 11 | category_2
// | " | "
var filters = [{ own_filter: 0 }, {}];
filters.forEach(function (filter) {
it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) {
assert.ifError(err);
assert.ok(dataview.infinities === (250 + 250));
assert.ok(dataview.nans === 250);
assert.ok(dataview.categories.length === 1);
dataview.categories.forEach(function (category) {
assert.ok(category.category === 'category_2');
assert.ok(category.value === 501);
});
done();
});
});
it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) {
assert.ifError(err);
assert.ok(dataview.nans === 333);
assert.ok(dataview.categories.length === 2);
dataview.categories.forEach(function (category) {
assert.ok(category.value !== null);
});
done();
});
});
});
});

View File

@@ -0,0 +1,80 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
describe('formula-dataview: special float values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
val_formula: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: 'val',
operation: 'avg'
}
}
},
[
{
"id": "a0",
"type": "source",
"params": {
"query": [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS val',
'FROM generate_series(1, 1000) x'
].join('\n')
}
}
]
);
it('should filter infinities out and count them in the summary', function(done) {
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('val_formula', {}, function(err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.result, 501);
assert.ok(dataview.infinities === (250 + 250));
assert.ok(dataview.nans === 250);
done();
});
});
});

File diff suppressed because it is too large Load Diff

View File

@@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() {
params: {
query: 'select * from test_table_overviews'
}
},
{
id: 'data-source-special-float-values',
type: 'source',
params: {
query: 'select * from test_special_float_values_table_overviews'
}
}
],
dataviews: {
@@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() {
aggregationColumn: 'name',
}
},
test_categories_special_values: {
type: 'aggregation',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'name',
aggregation: 'sum',
aggregationColumn: 'value',
}
},
test_histogram: {
type: 'histogram',
source: {id: 'data-source'},
@@ -160,6 +178,16 @@ describe('dataviews using tables with overviews', function() {
bins: 2
}
},
test_histogram_special_values: {
type: 'histogram',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'value',
bins: 2
}
},
test_avg: {
type: 'formula',
source: {id: 'data-source'},
@@ -168,6 +196,16 @@ describe('dataviews using tables with overviews', function() {
operation: 'avg'
}
},
test_formula_sum_special_values: {
type: 'formula',
source: {
id: 'data-source-special-float-values'
},
options: {
column: 'value',
operation: 'sum'
}
},
test_count: {
type: 'formula',
source: {id: 'data-source'},
@@ -202,6 +240,17 @@ describe('dataviews using tables with overviews', function() {
cartocss_version: '2.3.0',
source: { id: 'data-source' }
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_special_float_values_table_overviews',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
source: {
id: 'data-source-special-float-values'
}
}
}
]
};
@@ -212,7 +261,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":15,
"infinities": 0,
"nans": 0,
"nulls":0,
"type":"formula"
});
testClient.drain(done);
});
@@ -224,7 +280,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"avg",
"result":3,
"nulls":0,
"type":"formula",
"infinities": 0,
"nans": 0
});
testClient.drain(done);
});
@@ -236,7 +299,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"count",
"result":5,
"nulls":0,
"type":"formula",
"infinities": 0,
"nans": 0
});
testClient.drain(done);
});
@@ -248,7 +318,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "max",
"result": 5,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@@ -260,7 +337,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "min",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@@ -275,7 +359,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":15,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
@@ -372,7 +463,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
});
@@ -383,7 +481,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"avg",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
@@ -395,7 +500,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"count",
"result":1,
"infinities": 0,
"nans": 0,
"nulls":0,
"type":"formula"
});
testClient.drain(done);
});
@@ -407,7 +519,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "max",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@@ -419,7 +538,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation": "min",
"result": 1,
"nulls": 0,
"infinities": 0,
"nans": 0,
"type": "formula"
});
testClient.drain(done);
});
@@ -437,7 +563,14 @@ describe('dataviews using tables with overviews', function() {
if (err) {
return done(err);
}
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
assert.deepEqual(formula_result, {
"operation":"sum",
"result":1,
"nulls":0,
"infinities": 0,
"nans": 0,
"type":"formula"
});
testClient.drain(done);
});
});
@@ -445,5 +578,69 @@ describe('dataviews using tables with overviews', function() {
});
describe('aggregation special float values', function () {
var params = {};
it("should expose an aggregation dataview filtering special float values out", function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_categories_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
aggregation: 'sum',
count: 5,
nulls: 0,
nans: 1,
infinities: 1,
min: 6,
max: 6,
categoriesCount: 1,
categories: [ { category: 'Hawai', value: 6, agg: false } ],
type: 'aggregation'
});
testClient.drain(done);
});
});
it('should expose a histogram dataview filtering special float values out', function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_histogram_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
bin_width: 0,
bins_count: 1,
bins_start: 3,
nulls: 0,
infinities: 1,
nans: 1,
avg: 3,
bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ],
type: 'histogram'
});
testClient.drain(done);
});
});
it('should expose a formula (sum) dataview filtering special float values out', function (done) {
var testClient = new TestClient(overviewsMapConfig);
testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) {
if (err) {
return done(err);
}
assert.deepEqual(dataview, {
operation: 'sum',
result: 6,
nulls: 0,
nans: 1,
infinities: 1,
type: 'formula'
});
testClient.drain(done);
});
});
});
});
});

View File

@@ -1,311 +0,0 @@
var testHelper = require('../support/test_helper');
var assert = require('../support/assert');
var _ = require('underscore');
var redis = require('redis');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var LayergroupToken = require('../support/layergroup-token');
describe('render limits', function() {
var layergroupUrl = '/api/v1/map';
var redisClient = redis.createClient(global.environment.redis.port);
var server;
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var user = 'localhost';
var pointSleepSql = "SELECT pg_sleep(0.5)," +
" 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id";
var pointCartoCss = '#layer { marker-fill:red; }';
var polygonSleepSql = "SELECT pg_sleep(0.5)," +
" ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id";
var polygonCartoCss = '#layer { polygon-fill:red; }';
function singleLayergroupConfig(sql, cartocss) {
return {
version: '1.0.0',
layers: [
{
type: 'mapnik',
options: {
sql: sql,
cartocss: cartocss,
cartocss_version: '2.0.1'
}
}
]
};
}
function createRequest(layergroup, userHost) {
return {
url: layergroupUrl,
method: 'POST',
headers: {
host: userHost,
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
};
}
function withRenderLimit(user, renderLimit, callback) {
redisClient.SELECT(5, function(err) {
if (err) {
return callback(err);
}
var userLimitsKey = 'limits:tiler:' + user;
redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) {
if (err) {
return callback(err);
}
keysToDelete[userLimitsKey] = 5;
return callback();
});
});
}
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategyEnabled;
before(function() {
onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
});
after(function() {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled;
});
it("layergroup creation fails if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed.errors, [ 'Render timed out' ]);
done();
}
);
});
});
it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done();
}
);
});
});
it("layergroup creation works if test tile is fast but tile request fails if they are slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 400
},
function(res) {
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed.errors, ['Render timed out']);
done();
}
);
}
);
});
});
it("tile request does not fail if user limit is high enough", function(done) {
withRenderLimit(user, 5000, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
done(err);
}
);
}
);
});
});
});
describe('with onTileErrorStrategy', function() {
it("layergroup creation works even if test tile is slow", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
var parsed = JSON.parse(res.body);
assert.ok(parsed.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done();
}
);
});
});
it("layergroup creation and tile requests works even if they are slow but returns fallback", function(done) {
withRenderLimit(user, 50, function(err) {
if (err) {
return done(err);
}
var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
assert.response(server,
createRequest(layergroup, user),
{
status: 200
},
function(res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
assert.response(server,
{
url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
layergroupId: JSON.parse(res.body).layergroupid,
z: 0,
x: 0,
y: 0
}),
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'Content-Type': 'image/png'
}
},
function(res, err) {
if (err) {
done(err);
}
var referenceImagePath = './test/fixtures/render-timeout-fallback.png';
assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25,
function(imgErr/*, similarity*/) {
done(imgErr);
}
);
}
);
}
);
});
});
});
});

View File

@@ -1041,7 +1041,7 @@ describe(suiteName, function() {
);
});
}
// See https://github.com/CartoDB/Windshaft-cartodb/issues/91
// and https://github.com/CartoDB/Windshaft-cartodb/issues/38
it("tiles for private tables can be fetched with api_key", function(done) {

63
test/acceptance/mvt.js Normal file
View File

@@ -0,0 +1,63 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
function createMapConfig (sql = TestClient.SQL.ONE_POINT) {
return {
version: '1.6.0',
layers: [{
type: "cartodb",
options: {
sql: sql,
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
}]
};
}
describe('mvt', function () {
const testCases = [
{
desc: 'should get empty mvt with code 204 (no content)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
response: {
status: 204,
headers: {
'Content-Type': undefined
}
},
mapConfig: createMapConfig(TestClient.SQL.EMPTY)
},
{
desc: 'should get mvt tile with code 200 (ok)',
coords: { z: 0, x: 0, y: 0 },
format: 'mvt',
response: {
status: 200,
headers: {
'Content-Type': 'application/x-protobuf'
}
},
mapConfig: createMapConfig()
}
];
testCases.forEach(function (test) {
it(test.desc, done => {
const testClient = new TestClient(test.mapConfig, 1234);
const { z, x, y } = test.coords;
const { format, response } = test;
testClient.getTile(z, x, y, { format, response }, (err, res) => {
assert.ifError(err);
assert.equal(res.statusCode, test.response.status);
testClient.drain(done);
});
});
});
});

View File

@@ -5,6 +5,7 @@ var step = require('step');
var cartodbServer = require('../../../lib/cartodb/server');
var ServerOptions = require('./support/ported_server_options');
var testClient = require('./support/test_client');
var TestClient = require('../../support/test-client');
var BaseController = require('../../../lib/cartodb/controllers/base');
@@ -23,6 +24,14 @@ describe('multilayer error cases', function() {
BaseController.prototype.req2params = req2paramsFn;
});
// var client = null;
afterEach(function(done) {
if (this.client) {
return this.client.drain(done);
}
return done();
});
it("post layergroup with wrong Content-Type", function(done) {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
@@ -153,24 +162,16 @@ describe('multilayer error cases', function() {
]
};
ServerOptions.afterLayergroupCreateCalls = 0;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
var parsed = JSON.parse(res.body);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
var error = parsed.errors[0];
assert.ok(error.match(/column "missing" does not exist/m), error);
// TODO: check which layer introduced the problem ?
done();
} catch (err) { done(err); }
this.client = new TestClient(layergroup);
this.client.getLayergroup({status: 400}, function(err, parsed) {
assert.ok(!err, err);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
var error = parsed.errors[0];
assert.ok(error.match(/column "missing" does not exist/m), error);
done();
});
});

View File

@@ -16,13 +16,13 @@ describe('server_png8_format', function() {
var serverOptionsPng32 = ServerOptions;
serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore);
serverOptionsPng32.grainstore.mapnik_tile_format = 'png32';
var serverPng32 = new cartodbServer(serverOptionsPng32);
var serverPng32 = cartodbServer(serverOptionsPng32);
serverPng32.setMaxListeners(0);
var serverOptionsPng8 = ServerOptions;
serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore);
serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h';
var serverPng8 = new cartodbServer(serverOptionsPng8);
var serverPng8 = cartodbServer(serverOptionsPng8);
serverPng8.setMaxListeners(0);

View File

@@ -0,0 +1,71 @@
require('../support/test_helper');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
describe('special numeric values', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var ATTRIBUTES_LAYER = 1;
function createMapConfig(sql, id, columns) {
return {
version: '1.6.0',
layers: [
{
type: 'mapnik',
options: {
sql: "select 1 as id, 'SRID=4326;POINT(0 0)'::geometry as the_geom",
cartocss: '#style { }',
cartocss_version: '2.0.1'
}
},
{
type: 'mapnik',
options: {
sql: sql || "select 1 as i, 6 as n, 'SRID=4326;POINT(0 0)'::geometry as the_geom",
attributes: {
id: id || 'i',
columns: columns || ['n']
},
cartocss: '#style { }',
cartocss_version: '2.0.1'
}
}
]
};
}
it('should retrieve special numeric values', function (done) {
var featureId = 1;
var sql = [
'SELECT',
' 1 as cartodb_id,',
' null::geometry the_geom_webmercator,',
' \'infinity\'::float as infinity,',
' \'-infinity\'::float as _infinity,',
' \'NaN\'::float as nan'
].join('\n');
var id = 'cartodb_id';
var columns = ['infinity', '_infinity', 'nan'];
var mapConfig = createMapConfig(sql, id, columns);
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getFeatureAttributes(featureId, ATTRIBUTES_LAYER, {}, function (err, attributes) {
assert.ifError(err);
assert.equal(attributes.infinity, 'Infinity');
assert.equal(attributes._infinity, '-Infinity');
assert.equal(attributes.nan, 'NaN');
done();
});
});
});

View File

@@ -0,0 +1,279 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('Create mapnik layergroup', function() {
before(function() {
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerStats = true;
});
after(function() {
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var mapnikLayer1 = {
type: 'mapnik',
options: {
sql: 'select * from test_table limit 1',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer2 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_2 limit 2',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer3 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_3 limit 3',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer4 = {
type: 'mapnik',
options: {
sql: [
'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
' from test_table t1, test_table_2 t2',
' where t1.cartodb_id = t2.cartodb_id;'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var mapnikLayerGeomColumn = {
type: 'mapnik',
options: {
sql: 'select *, the_geom as my_geom from test_table_3 limit 2',
geom_column: 'my_geom',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId(index) {
return 'layer' + index;
}
function typeLayerId(type, index) {
return type + '-' + mapnikBasicLayerId(index);
}
it('with one mapnik layer should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
testClient.drain(done);
});
});
it('with two mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
testClient.drain(done);
});
});
it('with three mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2,
mapnikLayer3
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2));
assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3);
testClient.drain(done);
});
});
it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer3,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
assert.ok(!layergroup.metadata.layers[2]);
testClient.drain(done);
});
});
it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].type, 'mapnik');
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[1].type, 'http');
testClient.drain(done);
});
});
it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss);
testClient.drain(done);
});
});
it('should work with different geom_column', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayerGeomColumn
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
// we don't care about stats here as is an aliased column
assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount'));
testClient.drain(done);
});
});
it('should not include the stats part if the FF is disabled', function(done) {
global.environment.enabledFeatures.layerStats = false;
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats'));
assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
assert.equal(layergroup.metadata.layers[2].type, 'http');
testClient.drain(done);
});
});
});

View File

@@ -0,0 +1,225 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('multilayer stats disabled', function() {
before(function () {
this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata;
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerMetadata = true;
global.environment.enabledFeatures.layerStats = false;
});
after(function () {
global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig;
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
function testLayerMetadataStats(testScenario) {
it(testScenario.desc, function(done) {
var mapConfig = {
version: '1.3.0',
layers: testScenario.layers
};
var testClient = new TestClient(mapConfig);
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
layergroup.metadata.layers.forEach(function (layer) {
if (layer.type !== 'torque' && layer.type !== 'mapnik') {
assert.ok(!('stats' in layer.meta));
} else if (layer.type !== 'torque') {
assert.ok(!('stats' in layer.meta));
assert.ok('cartocss' in layer.meta);
} else {
assert.ok('cartocss' in layer.meta);
// check torque metadata at least match in number
var torqueLayers = mapConfig.layers.filter(function(layer) { return layer.type === 'torque'; });
if (torqueLayers.length) {
assert.equal(Object.keys(layergroup.metadata.torque).length, torqueLayers.length);
}
}
});
testClient.drain(done);
});
});
}
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var torqueLayer = {
type: 'torque',
options: {
sql: "select 1 id, '1970-01-02'::date d, 'POINT(0 0)'::geometry the_geom_webmercator",
cartocss: [
"Map {",
"-torque-frame-count:2;",
"-torque-resolution:3;",
"-torque-time-attribute:d;",
"-torque-aggregation-function:'count(id)';",
"}"
].join(' '),
cartocss_version: '2.0.1'
}
};
var mapnikLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikInteractivityLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var cartodbInteractivityLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbWadusInteractivityLayer = {
type: 'cartodb',
options: {
sql: sqlWadus,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'wadus'
}
};
var noTypeLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var noTypeInteractivityLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var testScenarios = [
{
desc: 'one layer, no interactivity',
layers: [cartodbLayer]
},
{
desc: 'two layers, different interactivity columns',
layers: [
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'torque + interactivity layers',
layers: [
torqueLayer,
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'interactivity + torque + interactivity',
layers: [
cartodbInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'http + interactivity + torque + no interactivity + torque + interactivity',
layers: [
httpLayer,
cartodbInteractivityLayer,
torqueLayer,
cartodbLayer,
torqueLayer,
cartodbWadusInteractivityLayer
]
},
{
desc: 'mapnik type two layers, interactivity mix',
layers: [
mapnikLayer,
mapnikInteractivityLayer
]
},
{
desc: 'mapnik type http + interactivity + torque + interactivity',
layers: [
httpLayer,
mapnikInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'no type two layers, interactivity mix',
layers: [
noTypeLayer,
noTypeInteractivityLayer
]
},
{
desc: 'no type http + interactivity + torque + interactivity',
layers: [
httpLayer,
noTypeInteractivityLayer,
torqueLayer,
noTypeInteractivityLayer
]
}
];
testScenarios.forEach(testLayerMetadataStats);
});

View File

@@ -1052,8 +1052,9 @@ describe('template_api', function() {
'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
var cc = res.headers['x-cache-channel'];
var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
assert.ok(cc);
assert.ok(cc.match, /ciao/, cc);
assert.equal(cc, expectedCC);
// hack simulating restart...
server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache
var get_request = {
@@ -1072,8 +1073,9 @@ describe('template_api', function() {
'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
var cc = res.headers['x-cache-channel'];
var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart");
assert.ok(cc.match, /ciao/, cc);
assert.equal(cc, expectedCC);
return null;
},
function deleteTemplate(err)

View File

@@ -0,0 +1,786 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
const pointSleepSql = `
SELECT
pg_sleep(0.3),
'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator,
1 cartodb_id,
2 val
`;
const validationPointSleepSql = `
SELECT
pg_sleep(0.3),
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = pointSleepSql,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id',
attributes
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
attributes,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
const DATASOURCE_TIMEOUT_ERROR = {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
};
describe('user database timeout limit', function () {
describe('dataview', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but dataview request fails due to statement timeout', function (done) {
const params = {
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getDataview('count', params, (err, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
describe('raster', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching raster tiles', function () {
describe('with user\'s timeout of 200 ms', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.setUserDatabaseTimeoutLimit(200, (err) => {
if (err) {
return done(err);
}
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('"png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'png',
layers: [ 0 ]
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
it('"static png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png'
};
this.testClient.getStaticCenter(params, function (err, res, tile) {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.setUserDatabaseTimeoutLimit(200, (err) => {
if (err) {
return done(err);
}
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('"png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'png',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
it('"static png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getStaticCenter(params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
});
describe('vector', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching vector tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"mvt" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'mvt',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('interactivity', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql, interactivity: 'val' });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'layer0', index: 0, type: 'mapnik' }
}]
});
done();
});
});
});
describe('fetching interactivity tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ interactivity: 'val' });
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"grid.json" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'grid.json',
layers: 'mapnik',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('torque', function () {
describe('while validating in layergroup creation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
type: 'torque',
cartocss: TestClient.CARTOCSS.TORQUE
});
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: { id: 'torque-layer0', index: 0, type: 'torque' }
}]
});
done();
});
});
});
describe('fetching torque tiles', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
type: 'torque',
cartocss: TestClient.CARTOCSS.TORQUE
});
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"torque.json" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'torque.json',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
it('".png" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'torque.png',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, attributes) => {
assert.ifError(err);
assert.deepEqual(attributes, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
describe('attributes:', function () {
describe('while validating in map instatiation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
attributes: {
id: 'cartodb_id',
columns: [ 'val' ]
}
});
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.deepEqual(timeoutError, {
errors: [ 'You are over platform\'s limits. Please contact us to know more details' ],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details',
layer: {
id: 'layer0',
index: 0,
type: 'mapnik'
}
}]
});
done();
});
});
});
describe('fetching by feature id', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({
attributes: {
id: 'cartodb_id',
columns: [ 'val' ]
}
});
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function (done) {
this.testClient.drain(done);
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
featureId: 1,
layer: 0,
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getAttributes(params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR);
done();
});
});
});
});
});
});

View File

@@ -0,0 +1,394 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
const pointSleepSql = `
SELECT
pg_sleep(0.5),
'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator,
1 cartodb_id,
2 val
`;
// during instatiation we validate tile 30/0/0, creating a point in that tile `pg_sleep` will throw a timeout
const validationPointSleepSql = `
SELECT
pg_sleep(0.5),
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = pointSleepSql,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id'
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
describe('user render timeout limit', function () {
describe('map instantiation => validation', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig({ sql: validationPointSleepSql });
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation fails due to statement timeout', function (done) {
const expectedResponse = {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details",
layer: {
id: "layer0",
index: 0,
type: "mapnik"
}
}]
});
done();
});
});
});
describe('raster', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but tile request fails due to render timeout', function (done) {
this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works and render tile fails', function (done) {
var params = {
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details"
}]
});
done();
});
});
});
});
describe('vector', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but vector tile request fails due to render timeout', function (done) {
const params = {
format: 'mvt',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
});
done();
});
});
});
describe('interativity', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but "grid.json" tile request fails due to render timeout', function (done) {
const params = {
layers: 'mapnik',
format: 'grid.json',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
});
done();
});
});
});
describe('static images', function () {
describe('with onTileErrorStrategy ENABLED', function () {
let onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = true;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but static image fails due to render timeout', function (done) {
const params = {
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png'
};
this.testClient.getStaticCenter(params, function (err, res, tile) {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
assert.ifError(err);
done();
});
});
});
});
describe('with onTileErrorStrategy DISABLED', function() {
var onTileErrorStrategy;
beforeEach(function (done) {
onTileErrorStrategy = global.environment.enabledFeatures.onTileErrorStrategy;
global.environment.enabledFeatures.onTileErrorStrategy = false;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategy;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works and render tile fails', function (done) {
const params = {
zoom: 0,
lat: 0,
lng: 0,
width: 256,
height: 256,
format: 'png',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getStaticCenter(params, function (err, res, timeoutError) {
assert.ifError(err);
assert.deepEqual(timeoutError, {
errors: ["You are over platform\'s limits. Please contact us to know more details"],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: "You are over platform\'s limits. Please contact us to know more details"
}]
});
done();
});
});
});
});
});

View File

@@ -322,6 +322,25 @@ describe('widgets', function() {
});
});
});
[adm0name].forEach(function(userQuery) {
it('should search with sum aggregation: ' + userQuery, function(done) {
this.testClient = new TestClient(aggregationSumMapConfig);
this.testClient.widgetSearch('adm0name', userQuery, function (err, res, searchResult) {
assert.ok(!err, err);
assert.ok(searchResult);
assert.equal(searchResult.type, 'aggregation');
assert.equal(searchResult.categories.length, 1);
assert.deepEqual(
searchResult.categories,
[{ category:"Argentina", value:28015640 }]
);
done();
});
});
});
});
});

View File

@@ -218,6 +218,114 @@ describe('widgets-regressions', function() {
});
});
it('should not count the polygons outside the bounding box', function(done) {
// $ % $ = not intersecting left triangle
// $$ **VVVVV** %% % = not intersecting right triangle
// $$$ *VVVVV* %%% * = intersecting triangle
// $$$$ ***** %%%% V = bounding box
// $$$$$ *** %%%%%
// $$$$$$ * %%%%%%
// $$$$$$$ %%%%%%%
// $$$$$$$$ %%%%%%%%
const notIntersectingLeftTriangle = {
type: "Polygon",
coordinates:[[
[-161.015625,69.28725695167886],
[-162.7734375,-7.710991655433217],
[-40.78125,-8.059229627200192],
[-161.015625,69.28725695167886]
]]
};
const notIntersectingRightTriangle = {
type: "Polygon",
coordinates: [[
[-29.179687499999996,-7.01366792756663],
[103.71093749999999,-6.664607562172573],
[105.46875,69.16255790810501],
[-29.179687499999996,-7.01366792756663]
]]
};
const intersectingTriangle = {
type: "Polygon",
coordinates:[[
[-117.42187500000001,68.13885164925573],
[-35.859375,20.96143961409684],
[59.4140625,68.52823492039876],
[-117.42187500000001,68.13885164925573]
]]
};
const query = `
SELECT
ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON(
'${JSON.stringify(notIntersectingLeftTriangle)}'
), 4326), 3857) AS the_geom_webmercator, 1 AS cartodb_id, 'notIntersectingLeftTriangle' AS name
UNION
SELECT
ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON(
'${JSON.stringify(notIntersectingRightTriangle)}'
), 4326), 3857), 2, 'notIntersectingRightTriangle'
UNION
SELECT
ST_TRANSFORM(ST_SETSRID(ST_GeomFromGeoJSON(
'${JSON.stringify(intersectingTriangle)}'
), 4326), 3857), 3, 'intersectingTriangle'
`;
const mapConfig = {
version: '1.5.0',
layers: [
{
"type": "cartodb",
"options": {
"source": {
"id": "a0"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
dataviews: {
val_formula: {
source: {
id: 'a0'
},
type: 'aggregation',
options: {
column: "name",
aggregation: "count",
}
}
},
analyses: [
{
"id": "a0",
"type": "source",
"params": {
"query": query
}
}
]
};
this.testClient = new TestClient(mapConfig, 1234);
const params = {
bbox: '-77.34374999999999,45.82879925192134,17.578125,55.97379820507658'
};
this.testClient.getDataview('val_formula', params, function(err, dataview) {
assert.ifError(err);
assert.equal(dataview.categories.length, 1);
assert.equal(dataview.categories[0].category, 'intersectingTriangle');
done();
});
});
});
});

View File

@@ -1,307 +0,0 @@
var testHelper = require('../support/test_helper');
var assert = require('../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var LayergroupToken = require('../support/layergroup-token');
describe('get requests x-cache-channel', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
testHelper.deleteRedisKeys(keysToDelete, done);
});
var statusOkResponse = {
status: 200
};
var mapConfig = {
version: '1.3.0',
layers: [
{
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
attributes: {
id:'cartodb_id',
columns: [
'name',
'address'
]
}
}
}
]
};
var layergroupRequest = {
url: '/api/v1/map?config=' + encodeURIComponent(JSON.stringify(mapConfig)),
method: 'GET',
headers: {
host: 'localhost'
}
};
function getRequest(url, addApiKey, callbackName) {
var params = {};
if (!!addApiKey) {
params.api_key = '1234';
}
if (!!callbackName) {
params.callback = callbackName;
}
return {
url: url + '?' + qs.stringify(params),
method: 'GET',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
};
}
function validateXCacheChannel(done, expectedCacheChannel) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(res.headers['x-cache-channel']);
if (expectedCacheChannel) {
assert.equal(res.headers['x-cache-channel'], expectedCacheChannel);
}
done();
};
}
function noXCacheChannelHeader(done) {
return function(res, err) {
if (err) {
return done(err);
}
assert.ok(
!res.headers['x-cache-channel'],
'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
);
done();
};
}
function withLayergroupId(callback) {
assert.response(
server,
layergroupRequest,
statusOkResponse,
function(res, err) {
if (err) {
return callback(err);
}
var layergroupId = JSON.parse(res.body).layergroupid;
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
callback(null, layergroupId, res);
}
);
}
describe('header should be present', function() {
it('/api/v1/map Map instantiation', function(done) {
var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table');
withLayergroupId(function(err, layergroupId, res) {
testFn(res);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
withLayergroupId(function(err, layergroupId) {
assert.response(
server,
getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'),
validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
);
});
});
});
describe('header should NOT be present', function() {
it('/', function(done) {
assert.response(
server,
getRequest('/'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/version', function(done) {
assert.response(
server,
getRequest('/version'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/health', function(done) {
assert.response(
server,
getRequest('/health'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/api/v1/map/named list named maps', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named', true),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
describe('with named maps', function() {
var templateName = 'x_cache';
beforeEach(function(done) {
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
layergroup: mapConfig
};
var namedMapRequest = {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(template)
};
assert.response(
server,
namedMapRequest,
statusOkResponse,
function(res, err) {
done(err);
}
);
});
afterEach(function(done) {
assert.response(
server,
{
url: '/api/v1/map/named/' + templateName + '?api_key=1234',
method: 'DELETE',
headers: {
host: 'localhost'
}
},
{
status: 204
},
function(res, err) {
done(err);
}
);
});
it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
assert.response(
server,
getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
statusOkResponse,
noXCacheChannelHeader(done)
);
});
});
});
});

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

View File

@@ -0,0 +1 @@
{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[-53839,4629161]},"properties":{"name":"Alicante","cartodb_id":1200}},{"type":"Feature","geometry":{"type":"Point","coordinates":[242835,5069332]},"properties":{"name":"Barcelona","cartodb_id":5330}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-5567,4861644]},"properties":{"name":"Castello","cartodb_id":1201}},{"type":"Feature","geometry":{"type":"Point","coordinates":[272735,5092314]},"properties":{"name":"Mataro","cartodb_id":615}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125787,4576600]},"properties":{"name":"Murcia","cartodb_id":952}},{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139148,5030112]},"properties":{"name":"Tarragona","cartodb_id":616}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-44746,4791667]},"properties":{"name":"Valencia","cartodb_id":5942}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-99072,5108695]},"properties":{"name":"Zaragoza","cartodb_id":5932}}]}

View File

@@ -0,0 +1 @@
{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !! ","!!! !!!!! ","!!!!!!! ! ","!!! !!!!! "," !! ! "," "," "," "," "," "," "," "," ### # "," ####### ###"," ####### ## ","$ ## #### ## ","$$ ","$$ ","$$ "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","9","2","1"],"data":{"1":{"cartodb_id":5942},"2":{"cartodb_id":5500},"9":{"cartodb_id":1201}}}

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 KiB

View File

@@ -0,0 +1 @@
{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !!! ! "," !!!!!!! !!!"," !!!!!!! !! "," !! !!!! !! "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","1"],"data":{"1":{"cartodb_id":5500}}}

View File

@@ -0,0 +1 @@
{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}}]}

Binary file not shown.

Binary file not shown.

View File

@@ -126,22 +126,25 @@ assert.response = function(server, req, res, callback) {
// Assert response body
if (res.body) {
var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
assert.ok(
eql,
colorize('[red]{Invalid response body.}\n' +
if (!eql) {
return callback(response, new Error(colorize(
'[red]{Invalid response body.}\n' +
' Expected: [green]{' + res.body + '}\n' +
' Got: [red]{' + response.body + '}')
);
' Got: [red]{' + response.body + '}'))
);
}
}
// Assert response status
if (typeof status === 'number') {
assert.equal(response.statusCode, status,
colorize('[red]{Invalid response status code.}\n' +
if (response.statusCode != status) {
return callback(response, new Error(colorize(
'[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body)
);
' Body: ' + response.body))
);
}
}
// Assert response headers
@@ -152,11 +155,13 @@ assert.response = function(server, req, res, callback) {
actual = response.headers[name.toLowerCase()],
expected = res.headers[name],
headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
assert.ok(headerEql,
colorize('Invalid response header [bold]{' + name + '}.\n' +
if (!headerEql) {
return callback(response, new Error(colorize(
'Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expected + '}\n' +
' Got: [red]{' + actual + '}')
);
' Got: [red]{' + actual + '}'))
);
}
}
}

View File

@@ -75,8 +75,8 @@ if test x"$PREPARE_PGSQL" = xyes; then
dropdb "${TEST_DB}"
createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount'
CURL_ARGS=""
for i in ${REMOTE_SQL_SCRIPTS}

View File

@@ -0,0 +1,6 @@
CREATE OR REPLACE FUNCTION CDB_Invalidate_Varnish(table_name TEXT)
RETURNS void AS
$$
BEGIN
END;
$$ LANGUAGE PLPGSQL;

View File

@@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES
INSERT INTO _vovw_1_test_table_overviews VALUES
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5);
-- table with overviews whit special float values
CREATE TABLE test_special_float_values_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id;
SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true);
ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass);
INSERT INTO test_special_float_values_table_overviews VALUES
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1),
(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1),
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1),
(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1);
ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id);
CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom);
CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator);
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
CREATE TABLE _vovw_1_test_special_float_values_table_overviews (
cartodb_id integer NOT NULL,
name character varying,
address character varying,
value float8,
the_geom geometry,
the_geom_webmercator geometry,
_feature_count integer,
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
);
GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER;
GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER;
INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2),
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2);
-- analysis tables -----------------------------------------------
@@ -649,3 +721,5 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters int
END;
$$ LANGUAGE plpgsql;
GRANT ALL ON FUNCTION cdb_crankshaft.CDB_KMeans(text, integer, integer) TO :TESTUSER;
ANALYZE;

View File

@@ -3,7 +3,8 @@
var qs = require('querystring');
var step = require('step');
var urlParser = require('url');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
var mapnik = require('windshaft').mapnik;
var LayergroupToken = require('./layergroup-token');
@@ -14,16 +15,33 @@ var helper = require('./test_helper');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
serverOptions.analysis.batch.inlineExecution = true;
var server = new CartodbWindshaft(serverOptions);
function TestClient(mapConfig, apiKey) {
this.mapConfig = mapConfig;
const MAPNIK_SUPPORTED_FORMATS = {
'png': true,
'png32': true,
'grid.json': true,
'geojson': true,
'mvt': true
}
function TestClient(config, apiKey) {
this.mapConfig = isMapConfig(config) ? config : null;
this.template = isTemplate(config) ? config : null;
this.apiKey = apiKey;
this.keysToDelete = {};
this.server = new CartodbWindshaft(serverOptions);
}
module.exports = TestClient;
function isMapConfig(config) {
return config && config.layers;
}
function isTemplate(config) {
return config && config.layergroup;
}
module.exports.RESPONSE = {
ERROR: {
status: 400,
@@ -63,9 +81,42 @@ module.exports.CARTOCSS = {
' line-width: 0.5;',
' line-opacity: 1;',
'}'
].join('\n'),
TORQUE: [
'Map {',
' -torque-frame-count: 256;',
' -torque-animation-duration: 30;',
' -torque-time-attribute: "cartodb_id";',
' -torque-aggregation-function: "count(1)";',
' -torque-resolution: 4;',
' -torque-data-aggregation: linear;',
'}',
'#layer {',
' marker-width: 7;',
' marker-fill: #FFB927;',
' marker-fill-opacity: 0.9;',
' marker-line-width: 1;',
' marker-line-color: #FFF;',
' marker-line-opacity: 1;',
' comp-op: lighter;',
'}',
'#layer[frame-offset=1] {',
' marker-width: 9;',
' marker-fill-opacity: 0.45;',
'}',
'#layer[frame-offset=2] {',
' marker-width: 11;',
' marker-fill-opacity: 0.225;',
'}'
].join('\n')
};
module.exports.SQL = {
EMPTY: 'select 1 as cartodb_id, null::geometry as the_geom_webmercator',
ONE_POINT: 'select 1 as cartodb_id, \'SRID=3857;POINT(0 0)\'::geometry the_geom_webmercator'
}
TestClient.prototype.getWidget = function(widgetName, params, callback) {
var self = this;
@@ -83,7 +134,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@@ -142,7 +193,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@@ -194,7 +245,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@@ -251,7 +302,7 @@ TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, call
}
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@@ -318,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@@ -360,7 +411,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
};
['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) {
['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) {
if (params.hasOwnProperty(extraParam)) {
urlParams[extraParam] = params[extraParam];
}
@@ -371,7 +422,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
}
url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams);
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'GET',
@@ -390,9 +441,115 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
);
},
function finish(err, dataview) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
return callback(err, dataview);
if (err) {
return callback(err);
}
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(null, dataview);
}
);
};
TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, callback) {
var self = this;
if (!callback) {
callback = params;
params = {};
}
var extraParams = {};
if (this.apiKey) {
extraParams.api_key = this.apiKey;
}
if (params && params.filters) {
extraParams.filters = JSON.stringify(params.filters);
}
var url = '/api/v1/map';
if (Object.keys(extraParams).length > 0) {
url += '?' + qs.stringify(extraParams);
}
var expectedResponse = params.response || {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
var layergroupId;
step(
function createLayergroup() {
var next = this;
assert.response(self.server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return next(null, parsedBody.layergroupid);
}
);
},
function getFeatureAttributes(err, layergroupId) {
assert.ifError(err);
var next = this;
url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId;
assert.response(self.server,
{
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
},
expectedResponse,
function(res, err) {
if (err) {
return next(err);
}
next(null, JSON.parse(res.body));
}
);
},
function finish(err, attributes) {
if (err) {
return callback(err);
}
return callback(null, attributes);
}
);
};
@@ -406,24 +563,77 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
var url = '/api/v1/map';
var urlNamed = url + '/named';
if (this.apiKey) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
var layergroupId;
if (params.layergroupid) {
layergroupId = params.layergroupid
}
step(
function createLayergroup() {
function createTemplate () {
var next = this;
assert.response(server,
if (!self.template) {
return next();
}
if (!self.apiKey) {
return next(new Error('apiKey param is mandatory to create a new template'));
}
params.placeholders = params.placeholders || {};
assert.response(self.server,
{
url: url,
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
data: JSON.stringify(self.template)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function (res, err) {
if (err) {
return next(err);
}
return next(null, JSON.parse(res.body).template_id);
}
);
},
function createLayergroup(err, templateId) {
var next = this;
if (layergroupId) {
return next(null, layergroupId);
}
var data = templateId ? params.placeholders : self.mapConfig
var path = templateId ?
urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) :
url;
assert.response(self.server,
{
url: path,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(data)
},
{
status: 200,
@@ -456,6 +666,10 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
var format = params.format || 'png';
if (layers === undefined && !MAPNIK_SUPPORTED_FORMATS[format]) {
throw new Error(`Missing layer filter while fetching ${format} tile, review params argument`);
}
url += [z,x,y].join('/');
url += '.' + format;
@@ -471,37 +685,76 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
};
var expectedResponse = {
var expectedResponse = Object.assign({}, {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
'Content-Type': 'image/png'
}
};
}, params.response);
var isPng = format.match(/png$/);
if (isPng) {
request.encoding = 'binary';
expectedResponse.headers['Content-Type'] = 'image/png';
}
assert.response(server, request, expectedResponse, function(res, err) {
var isMvt = format.match(/mvt$/);
if (isMvt) {
request.encoding = 'binary';
if (expectedResponse.status === 200) {
expectedResponse.headers['Content-Type'] = 'application/x-protobuf';
}
}
var isGeojson = format.match(/geojson$/);
if (isGeojson) {
request.encoding = 'utf-8';
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
var isGridJSON = format.match(/grid.json$/);
if (isGridJSON) {
request.encoding = 'utf-8';
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
if (params.contentType) {
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var obj;
if (isPng) {
obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
} else {
obj = JSON.parse(res.body);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/x-protobuf':
body = new mapnik.VectorTile(z, x, y);
body.setDataSync(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, obj);
next(null, res, body);
});
},
function finish(err, res, image) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, image);
}
);
@@ -526,7 +779,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@@ -538,18 +791,124 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
},
expectedResponse,
function(res, err) {
// If there is a response, we are still interested in catching the created keys
// to be able to delete them on the .drain() method.
if (res) {
var parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
}
if (err) {
return callback(err);
}
var parsedBody = JSON.parse(res.body);
return callback(null, parsedBody);
}
);
};
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
TestClient.prototype.getStaticCenter = function (params, callback) {
var self = this;
let { layergroupid, z, lat, lng, width, height, format } = params
var url = `/api/v1/map/`;
if (this.apiKey) {
url += '?' + qs.stringify({api_key: this.apiKey});
}
step(
function createLayergroup() {
var next = this;
if (layergroupid) {
return next(null, layergroupid);
}
return callback(null, parsedBody);
var data = self.mapConfig
var path = url;
assert.response(self.server,
{
url: path,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(data)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
return next(null, JSON.parse(res.body).layergroupid);
}
);
},
function getStaticResult(err, _layergroupid) {
assert.ifError(err);
var next = this;
layergroupid = _layergroupid;
url = `/api/v1/map/static/center/${layergroupid}/${z}/${lat}/${lng}/${width}/${height}.${format}`
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
}
var request = {
url: url,
encoding: 'binary',
method: 'GET',
headers: {
host: 'localhost'
}
};
var expectedResponse = Object.assign({}, {
status: 200,
headers: {
'Content-Type': 'image/png'
}
}, params.response);
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
var body;
switch (res.headers['content-type']) {
case 'image/png':
body = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body
break;
}
next(null, res, body);
});
},
function finish(err, res, image) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, image);
}
);
};
@@ -568,7 +927,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
step(
function createLayergroup() {
var next = this;
assert.response(server,
assert.response(self.server,
{
url: url,
method: 'POST',
@@ -629,7 +988,7 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
}
};
assert.response(server, request, expectedResponse, function(res, err) {
assert.response(self.server, request, expectedResponse, function(res, err) {
assert.ifError(err);
next(null, res, JSON.parse(res.body));
});
@@ -642,11 +1001,119 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
);
};
TestClient.prototype.getAttributes = function(params, callback) {
var self = this;
if (!Number.isFinite(params.featureId)) {
throw new Error('featureId param must be a number')
}
if (!Number.isFinite(params.layer)) {
throw new Error('layer param must be a number')
}
var url = '/api/v1/map';
if (this.apiKey) {
url += '?' + qs.stringify({ api_key: this.apiKey });
}
var layergroupid;
if (params.layergroupid) {
layergroupid = params.layergroupid
}
step(
function createLayergroup() {
var next = this;
if (layergroupid) {
return next(null, layergroupid);
}
assert.response(self.server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
return next(null, parsedBody.layergroupid);
}
);
},
function getAttributes(err, _layergroupid) {
assert.ifError(err);
var next = this;
layergroupid = _layergroupid;
url = `/api/v1/map/${layergroupid}/${params.layer}/attributes/${params.featureId}`;
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});
}
var request = {
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
};
var expectedResponse = params.response || {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
assert.response(self.server, request, expectedResponse, function (res, err) {
assert.ifError(err);
var attributes = JSON.parse(res.body);
next(null, res, attributes);
});
},
function finish(err, res, attributes) {
if (layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, res, attributes);
}
);
};
TestClient.prototype.drain = function(callback) {
helper.deleteRedisKeys(this.keysToDelete, callback);
};
module.exports.getStaticMap = function getStaticMap(templateName, params, callback) {
var self = this;
self.server = new CartodbWindshaft(serverOptions);
if (!callback) {
callback = params;
params = null;
@@ -677,9 +1144,56 @@ module.exports.getStaticMap = function getStaticMap(templateName, params, callba
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
assert.response(self.server, requestOptions, expectedResponse, function (res, err) {
helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() {
return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary')));
});
});
};
TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) {
const userTimeoutLimitsKey = `limits:timeout:${user}`;
const params = [
userTimeoutLimitsKey,
'render', userTimeoutLimit,
'render_public', userTimeoutLimit
];
this.keysToDelete[userTimeoutLimitsKey] = 5;
helper.configureMetadata('hmset', params, callback);
};
TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callback) {
const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db';
const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 })
const pass = _.template(global.environment.postgres_auth_pass, { user_id: 1 })
const publicuser = global.environment.postgres.user;
// we need to guarantee all new connections have the new settings
helper.cleanPGPoolConnections()
const psql = new PSQL({
user: 'postgres',
dbname: dbname,
host: global.environment.postgres.host,
port: global.environment.postgres.port
});
step(
function configureTimeouts () {
const timeoutSQLs = [
`ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`
];
const group = this.group();
timeoutSQLs.forEach(sql => psql.query(sql, group()));
},
callback
);
};

View File

@@ -14,6 +14,7 @@ var lzmaWorker = new LZMA();
var redis = require('redis');
var nock = require('nock');
var log4js = require('log4js');
var pg = require('pg');
// set environment specific variables
global.environment = require(__dirname + '/../../config/environments/test');
@@ -127,6 +128,11 @@ afterEach(function(done) {
});
});
function cleanPGPoolConnections () {
// TODO: this method will be replaced by psql.end
pg.end();
}
function deleteRedisKeys(keysToDelete, callback) {
if (Object.keys(keysToDelete).length === 0) {
@@ -166,12 +172,30 @@ function rmdirRecursiveSync(dirname) {
}
}
function configureMetadata(action, params, callback) {
redisClient.SELECT(5, function (err) {
if (err) {
return callback(err);
}
redisClient[action](params, function (err) {
if (err) {
return callback(err);
}
return callback();
});
});
}
module.exports = {
deleteRedisKeys: deleteRedisKeys,
lzma_compress_to_base64: lzma_compress_to_base64,
checkNoCache: checkNoCache,
checkSurrogateKey: checkSurrogateKey,
checkCache: checkCache,
rmdirRecursiveSync: rmdirRecursiveSync
rmdirRecursiveSync: rmdirRecursiveSync,
configureMetadata,
cleanPGPoolConnections
};

View File

@@ -0,0 +1,153 @@
var assert = require('assert');
var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
function getDbConnectionMock () {
return {
query: function(sql, callback) {
return callback(null, {
rows: [{rows: 1}]
});
}
};
}
describe('mapnik-layer-stats', function() {
beforeEach(function () {
this.dbConnectionMock = getDbConnectionMock();
this.rendererCacheMock = {};
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
}
]
};
var testMapConfigTwoLayers = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
]
};
var testMapConfigOneLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
]
};
var testMapConfigTwoLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_3', 'test_table_4']
}
},
]
};
it('should return 1 feature for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayers);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
it('should return 1 feature for one layers with two tables', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers and two tables', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
});

View File

@@ -0,0 +1,36 @@
var assert = require('assert');
var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
describe('torque-layer-stats', function () {
beforeEach(function () {
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'torque',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
}
},
]
};
it('should return torque stats for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layerId = 0;
var layer = mapConfig.getLayer(layerId);
var testSubject = new TorqueLayerStats();
testSubject.getStats(layer, {}, function (err, result) {
assert.ifError(err);
assert.deepEqual({}, result);
done();
});
});
});

View File

@@ -0,0 +1,44 @@
var PostgresDatasource = require('../../../../lib/cartodb/backends/turbo-carto-postgres-datasource');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
var assert = require('assert');
describe('turbo-carto-postgres-datasource', function() {
beforeEach(function () {
const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db';
const psql = new PSQL({
user: 'postgres',
dbname: dbname,
host: global.environment.postgres.host,
port: global.environment.postgres.port
});
const sql = [
'SELECT',
' null::geometry the_geom_webmercator,',
' CASE',
' WHEN x % 4 = 0 THEN \'infinity\'::float',
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
' WHEN x % 4 = 2 THEN \'NaN\'::float',
' ELSE x',
' END AS values',
'FROM generate_series(1, 1000) x'
].join('\n');
this.datasource = new PostgresDatasource(psql, sql);
});
it('should ignore NaNs and Infinities when computing ramps', function(done) {
var column = 'values';
var buckets = 4;
var method = 'equal';
this.datasource.getRamp(column, buckets, method, function(err, result) {
var expected_result = {
ramp: [ 252, 501, 750, 999 ],
stats: { min_val: 3, max_val: 999, avg_val: 501 },
strategy: undefined
};
assert.deepEqual(result, expected_result);
done();
});
});
});

View File

@@ -111,6 +111,23 @@ describe('Bounding box filter', function() {
createRef([-180, -45, 0, 45])
);
});
it('generating multiple bbox', function() {
var bbox = [90, -45, 190, 45];
var bboxFilter = createFilter(bbox);
assert.equal(bboxFilter.bboxes.length, 2);
assert.deepEqual(
bboxFilter.bboxes[0],
createRef([90, -45, 180, 45])
);
assert.deepEqual(
bboxFilter.bboxes[1],
createRef([-180, -45, -170, 45])
);
});
});
describe('out of bounds', function() {

View File

@@ -6,10 +6,13 @@ var LayergroupController = require('../../../../lib/cartodb/controllers/layergro
describe('tile stats', function() {
after(function() {
global.statsClient = null;
beforeEach(function () {
this.statsClient = global.statsClient;
});
afterEach(function() {
global.statsClient = this.statsClient;
});
it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() {
var expectedCalls = 2, // it will call increment once for the general error

654
yarn.lock

File diff suppressed because it is too large Load Diff