Compare commits

...

92 Commits

Author SHA1 Message Date
Raul Ochoa
2b7bd58fd5 Release 1.28.1 2015-02-17 12:20:07 +01:00
Raul Ochoa
055932d38e Upgrades windshaft to 0.37.1 2015-02-17 12:19:16 +01:00
Raul Ochoa
c65a29acf4 Stubs next version 2015-02-17 11:04:41 +01:00
Raul Ochoa
27cda49fd8 Release 1.28.0 2015-02-17 11:02:29 +01:00
Raul Ochoa
2d7b706507 Merge pull request #217 from strk/master-tolerant-configure
Have ./configure tolerate unknown options
2015-02-16 18:55:45 +01:00
Raul Ochoa
3ea3638fe9 Merge pull request #218 from strk/master-own-testuser
Use a windshaft-specific name for the test user
2015-02-16 18:55:34 +01:00
Raul Ochoa
1f9387bb68 Rely on published versions 2015-02-16 15:02:39 +01:00
Raul Ochoa
00542bbc57 Upgrades windshaft to 0.37.0 2015-02-16 14:57:03 +01:00
Raul Ochoa
5be8afbbeb Merge pull request #261 from CartoDB/private-cdb_tablemetadata
Authenticated query to retrieve last update
2015-02-16 14:29:10 +01:00
Raul Ochoa
9f7dcc2354 Update news 2015-02-16 14:21:56 +01:00
Raul Ochoa
21b3f44441 Merge branch 'master' into private-cdb_tablemetadata 2015-02-16 14:18:56 +01:00
Raul Ochoa
f295f847d1 Stubs next version 2015-02-16 12:15:08 +01:00
Raul Ochoa
0478905689 Release 1.27.0 2015-02-16 12:14:24 +01:00
Raul Ochoa
d311dd4245 Use PgConnection to set db auth
No need to use request context anymore
X-Cache-Channel will be set now even for private tables: fixes #253
2015-02-16 11:57:53 +01:00
Raul Ochoa
b25bb03cdf Merge branch 'master' into private-cdb_tablemetadata
Conflicts:
	lib/cartodb/server_options.js
	test/support/sql/windshaft.test.sql
2015-02-16 11:41:45 +01:00
Raul Ochoa
afa625e3d2 Merge pull request #260 from CartoDB/medusa-improvements
Medusa improvements
2015-02-16 11:34:23 +01:00
Raul Ochoa
e08b1ea1a0 Update news with changes in branch 2015-02-16 11:21:22 +01:00
Raul Ochoa
c6d328ee07 Upgrades to windshaft 0.36.0 2015-02-16 10:55:38 +01:00
Raul Ochoa
8d10d0f760 Remove draft 2015-02-16 10:54:53 +01:00
Raul Ochoa
0f668aabf1 Rely on windshaft's master branch as everything for medusa is already there 2015-02-11 14:48:06 +01:00
Raul Ochoa
b1b57d6f24 Regenerate npm-shrinkwrap.json 2015-02-10 11:41:22 +01:00
Raul Ochoa
75c4153f9b No need to retrieve api key externally, QueryTablesApi takes care 2015-02-09 19:41:38 +01:00
Raul Ochoa
8e038b0323 Merge pull request #259 from CartoDB/empty-flush_cache-endpoint
Empty flush cache endpoint
2015-02-09 19:40:50 +01:00
Raul Ochoa
0a994c731c Merge branch 'master' into empty-flush_cache-endpoint 2015-02-09 19:05:25 +01:00
Raul Ochoa
13ae1b4067 Remove flush_cache tool as it doesn't make sense anymore 2015-02-09 19:04:38 +01:00
Raul Ochoa
742a9744ea Remove select permission for publicuser 2015-02-09 18:56:01 +01:00
Raul Ochoa
8ed864ad18 apt-get update before apt-get install 2015-02-09 18:49:58 +01:00
Raul Ochoa
87638168ff apt-get update before apt-get install 2015-02-09 18:39:24 +01:00
Raul Ochoa
8364da683a Merge branch 'master' into medusa-improvements 2015-02-09 18:39:14 +01:00
Raul Ochoa
6eec5822f0 Create CREATE EXTENSION plpythonu for tests 2015-02-09 18:36:47 +01:00
Raul Ochoa
d667dbcc2f Merge branch 'master' into private-cdb_tablemetadata 2015-02-09 18:36:19 +01:00
Raul Ochoa
40de1a8f86 Create CREATE EXTENSION plpythonu for tests 2015-02-09 18:34:28 +01:00
Raul Ochoa
b53c25e514 Merge pull request #258 from CartoDB/stop-testing-node-0.8
Don't test against node 0.8
2015-02-09 15:09:57 +01:00
Raul Ochoa
81919706ea Adds default image placeholder for http renderer to use as fallback 2015-02-09 15:08:36 +01:00
Raul Ochoa
d38fc16b57 Don't test against node 0.8 2015-02-09 14:49:01 +01:00
Raul Ochoa
90b22b2718 QueryTables and last updated_at retrieved with user
Move setDBAuth and setDBConn to PgConnection entity
 - It uses cartodb-redis to retrieve datasource configuration
Start using it in ServerOptions, TemplateMaps and QueryTablesApi
QueryTablesApi don't receive anymore the connection/credentials
 - It will always use an authenticated query to retrieve last update
 - That will allow to query affected private tables last update
2015-02-09 14:46:52 +01:00
Raul Ochoa
04af57cab9 Add some entries to cdb_tablemetadata for tables being used in tests 2015-02-09 14:38:59 +01:00
Raul Ochoa
d40b15454b Run some tests only if they are using the SQL API 2015-02-09 14:38:29 +01:00
Raul Ochoa
e1e925bd9e Run postgresql/sql-api dependant tests against two implementations
This time for real.
2015-02-09 14:33:17 +01:00
Raul Ochoa
151968ae13 Merge branch 'master' into private-cdb_tablemetadata 2015-02-06 18:42:18 +01:00
Raul Ochoa
547782eea5 Use windshaft's plain-renderer-plus-http-default-plus-per-layer-datasource
branch to pick latest developments for medusa
2015-02-06 12:03:17 +01:00
Raul Ochoa
6bd967e9fb Merge branch 'master' into medusa-improvements
Conflicts:
	lib/cartodb/server_options.js
2015-02-06 12:01:28 +01:00
Raul Ochoa
13f5fda1b8 Merge pull request #255 from CartoDB/empty-flush_cache-endpoint
Remove per table flush cache endpoint
2015-02-05 17:37:56 +01:00
Raul Ochoa
673bd4f3f2 Add querytables and cdb_tablemetadata for proper testing its integration 2015-02-05 17:21:38 +01:00
Raul Ochoa
d56affae2d Merge pull request #256 from CartoDB/private-cdb_tablemetadata
Remove no longer needed method from query_tables_api
2015-02-05 17:21:21 +01:00
Raul Ochoa
09527b6808 Remove no longer needed method from query_tables_api 2015-02-05 17:08:20 +01:00
Raul Ochoa
d065ace036 Remove per table flush cache endpoint 2015-02-05 17:05:50 +01:00
Raul Ochoa
2736b93c69 test to validate it's not possible to override authorization
with a crafted layergroup
2015-02-05 16:47:37 +01:00
Raul Ochoa
10b7ab307e Merge pull request #254 from CartoDB/add_cdbjs_layer
Added layer source info for cartodb.js
2015-02-05 15:23:53 +01:00
Andy Eschbacher
56c24a738a Added layer source info for cartodb.js 2015-02-04 16:23:34 -05:00
Raul Ochoa
fa8b27231c Removed unused/old template lock functionality 2015-02-04 19:36:16 +01:00
Raul Ochoa
c17af23a40 A non empty datasource from MapConfigNamedLayersAdapter.getLayers
means the affected tables can have private tables involved.
That implies QueryTablesApi will need the proper user to use
CDB_QueryTables. So we store it in a request context to use it in
the afterLayergroupCreate call.

Tiles for these layergroups will fail to add a X-Cache-Channel
header because it won't be possible to use the proper user within
those tiles. Ok, they will fail if they are not requested through
the same tiler instance because if they are they most likely will
reuse the in memory cache.

See https://github.com/CartoDB/Windshaft-cartodb/issues/253
2015-02-04 19:31:20 +01:00
Raul Ochoa
fbecc11aa5 Do not use the SQL API Emulator for testing named layers as it hides
an integration issue with cdb_querytables
2015-02-04 19:01:14 +01:00
Raul Ochoa
8cacc3bb9e Merge branch 'master' into 239-mapconfig-named-maps-extension 2015-02-04 18:59:07 +01:00
Raul Ochoa
a82af16347 Adds a template test with http layer 2015-02-04 18:57:46 +01:00
Raul Ochoa
5018d32af6 Add querytables and cdb_tablemetadata for proper testing its integration 2015-02-04 18:52:37 +01:00
Raul Ochoa
2c7bc6adde Datasource to give per-layer authentication in named layers
Make beforeLayergroupCreate to return a datasource with different
 authentication for the different layers.
 - Named layers will get access to private tables in case it's needed

Changes in MapConfigNamedLayersAdapter:
  - It will retrieve the dbAuth params only if named layers are present so
  there is no extra overhead for normal layers
  - Rename queue function signature from `callback` to `done` so it is easier
  to follow the code

Add several tests to validate `named` layers authentication
2015-02-04 11:30:36 +01:00
Raul Ochoa
58f9f5f7a8 Remove unused object, rename suite 2015-02-03 14:16:55 +01:00
Raul Ochoa
e4e633cf86 Fix next reference 2015-02-02 17:44:15 +01:00
Raul Ochoa
4ca5c5fa3c Merge pull request #252 from namessanti/api-test-doc
Static Maps API
2015-02-02 15:50:44 +01:00
Raul Ochoa
1bb0d8738e Add test case for layers with private tables 2015-02-02 14:38:26 +01:00
Raul Ochoa
4949616c4e Some acceptance tests, http_status = 403 should not happen in adapter,
needs improvement
2015-01-30 19:29:45 +01:00
Raul Ochoa
12c5d835c5 Fix integration tests as I messed with the filename 2015-01-30 19:11:54 +01:00
Raul Ochoa
87eaeb0074 Some integration tests for different cases in named layers type 2015-01-30 18:57:01 +01:00
Raul Ochoa
8b07156a2d Make templateMaps available in cartodb windshaft 2015-01-30 16:51:09 +01:00
Raul Ochoa
358b296750 Remove beforeEach and afterEach, in combination with suite they are
triggered for every single test even outside of the suite they were
invoked in.
2015-01-30 16:50:06 +01:00
Raul Ochoa
d0ef87b0cf Add a before layergroup creation action to allow first level named
maps layer type to be extended as other layers
2015-01-30 15:31:49 +01:00
Raul Ochoa
e28fe1fdc0 Initialize template maps in server options 2015-01-30 15:30:13 +01:00
Raul Ochoa
aecb07b008 Create redis pool in server options when not supplied 2015-01-30 15:28:55 +01:00
Raul Ochoa
5573dfda84 Add queue-async dependency 2015-01-30 15:26:27 +01:00
Raul Ochoa
7a22973258 Use windshaft's before-layergroup-step branch 2015-01-30 12:59:26 +01:00
Raul Ochoa
f099a69df3 Add limitations for named maps of the user account 2015-01-30 11:27:16 +01:00
Raul Ochoa
938b6579c0 MapConfig named maps extension specification (draft) 2015-01-30 11:17:29 +01:00
Raul Ochoa
e445d0de01 Stubs next version 2015-01-28 17:37:38 +01:00
Raul Ochoa
7a35b9695f Release 1.26.2 2015-01-28 17:36:44 +01:00
Raul Ochoa
9523d40937 Merge pull request #250 from CartoDB/accept-open-string-as-valid-auth
Accept open string as valid auth
2015-01-28 17:35:38 +01:00
Raul Ochoa
697323dbbc Fix typo 2015-01-28 17:31:22 +01:00
Raul Ochoa
efe090f5b0 Accept 'open' string in templated auth as authorized 2015-01-28 17:29:50 +01:00
Raul Ochoa
ee1454d91c Stubs next version 2015-01-28 13:16:57 +01:00
Raul Ochoa
38242813be Release 1.26.1 2015-01-28 13:15:42 +01:00
Raul Ochoa
f9373dd8d0 Fix typos 2015-01-28 13:15:18 +01:00
Raul Ochoa
0e4e56f333 Fix version number 2015-01-28 13:14:32 +01:00
Raul Ochoa
c1d4da870f Upgrades windshaft to 0.35.1, see https://github.com/CartoDB/Windshaft/pull/254 2015-01-28 13:09:53 +01:00
Raul Ochoa
3c26f1f986 Stubs next version 2015-01-27 17:51:04 +01:00
Raul Ochoa
e9195967a4 Release 1.26.0 2015-01-27 17:50:13 +01:00
Raul Ochoa
30c6a390ac Upgrades windshaft to 0.35.0, supports mapconfig version 1.3.0 2015-01-27 17:49:08 +01:00
Raul Ochoa
3a97af767f Stubs next version 2015-01-26 17:33:22 +01:00
Santiago Giraldo
6a39893e20 Brittany reviewed text and markdown, added basemap and limit information 2015-01-23 16:15:30 -05:00
Santiago Giraldo
2e84d18c3c Updated Maps API v2 2015-01-22 13:44:13 -05:00
Santiago Giraldo
b26fe87430 added-static-api-doc 2015-01-22 12:07:34 -05:00
Sandro Santilli
e9f5c60719 Use a windshaft-specific name for the test user
Avoids problems with running cartodb-sql-api tests after
windshaft-cartodb tests (as the test database is not cleaned up)
2014-10-08 10:31:32 +02:00
Sandro Santilli
9b06ac833a Have ./configure tolerate unknown options 2014-10-08 10:10:58 +02:00
36 changed files with 2313 additions and 456 deletions

View File

@@ -2,7 +2,9 @@ addons:
postgresql: "9.3"
before_install:
- sudo apt-get update
- sudo apt-get install -y pkg-config libcairo2-dev libjpeg8-dev libgif-dev
- sudo apt-get install postgresql-plpython-9.3
- createdb template_postgis
- psql -c "CREATE EXTENSION postgis" template_postgis
@@ -11,7 +13,6 @@ env:
language: node_js
node_js:
- "0.8"
- "0.10"
notifications:

View File

@@ -17,8 +17,11 @@ config/environments/test.js: config.status--test
check-local: config/environments/test.js
./run_tests.sh ${RUNTESTFLAGS} \
test/unit/cartodb/*.js \
test/acceptance/*.js
test/unit/cartodb/*.js \
test/unit/cartodb/cache/model/*.js \
test/integration/*.js \
test/acceptance/*.js \
test/acceptance/cache/*.js
check-submodules:
PATH="$$PATH:$(srcdir)/node_modules/.bin/"; \

57
NEWS.md
View File

@@ -1,3 +1,56 @@
1.28.1 -- 2015-02-17
--------------------
Announcements:
- Upgrades windshaft to [0.37.1](https://github.com/CartoDB/Windshaft/releases/tag/0.37.1)
1.28.0 -- 2015-02-17
--------------------
Announcements:
- Upgrades windshaft to [0.37.0](https://github.com/CartoDB/Windshaft/releases/tag/0.37.0)
New features:
- QueryTablesApi will always use an authenticated query to retrieve last update, this allows to query affected private
tables last update (#253)
1.27.0 -- 2015-02-16
--------------------
Announcements:
- Adds default image placeholder for http renderer to use as fallback
New features:
- `named` layers type, see [MapConfig-NamedMaps-extension](docs/MapConfig-NamedMaps-extension.md)
- Starts using datasource per layer feature from Windshaft ([2c7bc6a](https://github.com/CartoDB/Windshaft-cartodb/commit/2c7bc6adde561b20ed955b905e3c7bcd6795d128))
Bugfixes:
- Fixes tests with beforeEach and afterEach triggers
1.26.2 -- 2015-01-28
--------------------
Bugfixes:
- Accept 'open' string in templates' `auth` as authorized.
1.26.1 -- 2015-01-28
--------------------
Announcements:
- Upgrades windshaft to 0.35.1, see https://github.com/CartoDB/Windshaft/pull/254
1.26.0 -- 2015-01-27
--------------------
Announcements:
- Upgrades windshaft to 0.35.0, supports mapconfig version `1.3.0`
1.25.0 -- 2015-01-26
--------------------
@@ -27,9 +80,9 @@ New features:
- Emits for create, update and delete templates
- VarnishHttpCacheBackend will invalidate a varnish instance via HTTP PURGE method
- In the future there could be more backends, for instance to invalidate a CDN.
- NamedMapsEntry has the responibility to generate a cache key for a named map
- NamedMapsEntry has the responsibility to generate a cache key for a named map
- It probably should receive a template/named map instead of owner and template name
- SurrogateKeysCache is resposible to tag responses with a header
- SurrogateKeysCache is responsible to tag responses with a header
- It also is responsible for invalidations given an Invalidation Backend
- In the future it could have several backends so it can invalidates different caches
- SurrogateKeysCache is subscribed to TemplateMaps events to do the invalidations

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

View File

@@ -94,7 +94,13 @@ var config = {
proxy: undefined, // the url for a proxy server
whitelist: [ // the whitelist of urlTemplates that can be used
'http://{s}.example.com/{z}/{x}/{y}.png'
]
],
// image to use as placeholder when urlTemplate is not in the whitelist
// if provided the http renderer will use it instead of throw an error
fallbackImage: {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
}
}
,millstone: {

View File

@@ -88,7 +88,13 @@ var config = {
proxy: undefined, // the url for a proxy server
whitelist: [ // the whitelist of urlTemplates that can be used
'http://{s}.example.com/{z}/{x}/{y}.png'
]
],
// image to use as placeholder when urlTemplate is not in the whitelist
// if provided the http renderer will use it instead of throw an error
fallbackImage: {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
}
}
,millstone: {

View File

@@ -88,7 +88,13 @@ var config = {
proxy: undefined, // the url for a proxy server
whitelist: [ // the whitelist of urlTemplates that can be used
'http://{s}.example.com/{z}/{x}/{y}.png'
]
],
// image to use as placeholder when urlTemplate is not in the whitelist
// if provided the http renderer will use it instead of throw an error
fallbackImage: {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
}
}
,millstone: {

View File

@@ -50,7 +50,7 @@ var config = {
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "testpublicuser",
user: "test_windshaft_publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
@@ -87,8 +87,16 @@ var config = {
timeout: 2000, // the timeout in ms for a http tile request
proxy: undefined, // the url for a proxy server
whitelist: [ // the whitelist of urlTemplates that can be used
'http://{s}.example.com/{z}/{x}/{y}.png'
]
'http://{s}.example.com/{z}/{x}/{y}.png',
// for testing purposes
'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png'
],
// image to use as placeholder when urlTemplate is not in the whitelist
// if provided the http renderer will use it instead of throw an error
fallbackImage: {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
}
}
,millstone: {

5
configure vendored
View File

@@ -56,9 +56,8 @@ while test -n "$1"; do
ENVIRONMENT=`echo "$1" | cut -d= -f2`
;;
*)
echo "Unknown option '$1'" >&2
usage >&2
exit 1
echo "Unused option '$1'" >&2
;;
esac
shift
done

View File

@@ -1,14 +1,14 @@
## Maps API
The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account, and you can apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and you can apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
You can create two types of maps with the Maps API:
- **Anonymous maps**
Maps that can be created using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example]({{ '/cartodb-platform/cartodb-js.html' | prepend: site.baseurl }}).
You can create maps using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example]({{ '/cartodb-platform/cartodb-js.html' | prepend: site.baseurl }}).
- **Named maps**
Maps that have access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls.
There are also maps that have access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls.
## Quickstart
@@ -82,16 +82,12 @@ To get the `URL` to fetch the tiles you need to instantiate the map, where `temp
curl -X POST 'http://{account}.cartodb.com/api/v1/map/named/:template_id' -H 'Content-Type: application/json'
```
The response will return JSON with properties for the `layergroupid`, `cdn_url`, and the timestamp (`last_updated`) of the last data modification.
The response will return JSON with properties for the `layergroupid` and the timestamp (`last_updated`) of the last data modification.
Here is an example response:
```javascript
{
"cdn_url": {
"http": "ashbu.cartocdn.com",
"https": "cartocdn-ashbu.global.ssl.fastly.net"
},
"layergroupid": "c01a54877c62831bb51720263f91fb33:0",
"last_updated": "1970-01-01T00:00:00.000Z"
}
@@ -109,7 +105,7 @@ The following concepts are the same for every endpoint in the API except when it
### Auth
By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data, an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a named map).
By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a named map).
To execute an authorized request, api_key=YOURAPIKEY should be added to the request URL. The param can be also passed as POST param. We **strongly advise** using HTTPS when you are performing requests that include your `api_key`.
@@ -129,7 +125,7 @@ If you use JSONP, the 200 HTTP code is always returned so the JavaScript client
### CORS support
All the endpoints which might be accessed using a web browser add CORS headers and allow OPTIONS method.
All the endpoints, which might be accessed using a web browser, add CORS headers and allow OPTIONS method.
## Anonymous Maps
@@ -226,7 +222,7 @@ Which returns JSON with the attributes defined, like:
{ c: 1, d: 2 }
```
Notice UTF Grid and attributes endpoints need an integer parameter, ``layer``. That number is the 0-based index of the layer inside the mapconfig. So in this case 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example mapconfig. If a second layer was available it could be returned with 1, a third layer with 2, etc.
Notice UTF Grid and attributes endpoints need an integer parameter, ``layer``. That number is the 0-based index of the layer inside the mapconfig. In this case, 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example mapconfig. If a second layer was available it could be returned with 1, a third layer with 2, etc.
### Create JSONP
@@ -271,12 +267,12 @@ callback({
### Remove
Anonymous maps cannot be removed by an API call. They will expire after about five minutes but sometimes longer. If an anonymous map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time returns to the map an attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.
Anonymous maps cannot be removed by an API call. They will expire after about five minutes but sometimes longer. If an anonymous map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.
## Named Maps
Named maps are essentially the same as anonymous maps except the mapconfig is stored on the server and the map is given a unique name. Two other big differences are that you can create named maps from private data, and that users without an API Key can see them even though they are from that private data.
Named maps are essentially the same as anonymous maps except the mapconfig is stored on the server and the map is given a unique name. Two other big differences are: you can create named maps from private data and that users without an API Key can see them even though they are from that private data.
The main two differences compared to anonymous maps are:
@@ -284,9 +280,9 @@ The main two differences compared to anonymous maps are:
This allows you to control who is able to see the map based on a token auth
- **templates**
Since the mapconfig is static it can contain some variables so the client can modify the map's appearance using those variables
Since the mapconfig is static it can contain some variables so the client can modify the map's appearance using those variables.
Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see [auth section](http://docs.cartodb.com/cartodb-platform/maps-api.html#auth)).
Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see auth section).
### Create
@@ -345,7 +341,7 @@ POST /api/v1/map/named
- **auth**:
- **method** `"token"` or `"open"` (the default if no `"method"` is given).
- **valid_tokens** when `"method"` is set to `"token"`, the values listed here allow you to instantiate the named map.
- **placeholders**: Variables not listed here are not substituted. Variable not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
- **placeholders**: Variables not listed here are not substituted. Variables not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
- **layergroup**: the layer list definition. This is the MapConfig explained in anonymous maps. See [MapConfig documentation](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.1.0.md) for more info.
#### Template Format
@@ -371,7 +367,7 @@ The placeholder type will determine the kind of escaping for the associated valu
- **number** can only contain numerical representation
- **css_color** can only contain color names or hex-values
Placeholder default values will be used whenever new values are not provided as options at the time of creation on the client. They can also be used to test the template by creating a default version with now options provided.
Placeholder default values will be used whenever new values are not provided as options at the time of creation on the client. They can also be used to test the template by creating a default version with new options provided.
When using templates, be very careful about your selections as they can give broad access to your data if they are defined losely.
@@ -413,7 +409,7 @@ POST /api/v1/map/named/:template_name
}
```
The fields you pass as `params.json` depend on the variables allowed by the named map. If there are variables missing it will raise an error (HTTP 400).
The fields you pass as `params.json` depend on the variables allowed by the named map. If there are variables missing it will raise an error (HTTP 400)
- **auth_token** *optional* if the named map needs auth
@@ -446,7 +442,7 @@ curl -X POST \
}
```
You can then use the `layergroupid` for fetching tiles and grids as you would normally (see anonymous map section). However, you'll need to show the `auth_token`, if required by the template.
You can then use the `layergroupid` for fetching tiles and grids as you would normally (see anonymous map section). However you'll need to show the `auth_token`, if required by the template.
### Using JSONP
@@ -549,7 +545,7 @@ If a template with the same name does NOT exist, a 400 HTTP response is generate
### Delete
Delete the specified template map from the server and disables any previously initialized versions of the map.
Delete the specified template map from the server and it disables any previously initialized versions of the map.
#### Definition
@@ -576,7 +572,7 @@ curl -X DELETE 'https://documentation.cartodb.com/api/v1/map/named/:template_nam
}
```
On success, a 204 (No Content) response would be issued. Otherwise a 4xx response with with an error will be returned.
On success, a 204 (No Content) response will be issued. Otherwise a 4xx response with an error will be returned.
### Listing Available Templates
@@ -671,4 +667,195 @@ cartodb.createLayer('map_dom_id',layerSource)
```
See [CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js.html) methods [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js.html#layersetparamskey-value) and [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js.html#layersetauthtokenauthtoken).
[CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js.html) has methods for accessing your named maps.
1. [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js.html#layersetparamskey-value) allows you to change the template variables (in the placeholders object) via JavaScript
2. [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js.html#layersetauthtokenauthtoken) allows you to set the auth tokens to create the layer
##Static Maps API
The Static Maps API can be initiated using both named and anonymous maps using the 'layergroupid' token. The API can be used to create static images of parts of maps and thumbnails for use in web design, graphic design, print, field work, and many other applications that require standard image formats.
### Maps API endpoints
Begin by instantiating either a named or anonymous map using the `layergroupid token` as demonstrated in the Maps API documentation above. The `layergroupsid token` calls to the map and allows for parameters in the definition to generate static images.
##### Definition
<div class="code-title notitle code-request"></div>
```bash
GET /api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format
```
##### Params
* **:token**: the layergroupid token from the map instantiation
* **:z**: the zoom level of the map
* **:lat**: the latitude for the center of the map
* **:lng**: the longitude for the center of the map
* **:width**: the width in pixels for the output image
* **:height**: the height in pixels for the output image
* **:format**: the format for the image, supported types: `png`, `jpg`
* **jpg** will have a default quality of 85.
#### Bounding Box
##### Definition
<div class="code-title notitle code-request"></div>
```bash
GET /api/v1/map/static/bbox/:token/:bbox/:width/:height.:format`
```
##### Params
* **:token**: the layergroupid token from the map instantiation
* **:bbox**: the bounding box in WGS 84 (EPSG:4326), comma separated values for:
- LowerCorner longitude, in decimal degrees (aka most western)
- LowerCorner latitude, in decimal degrees (aka most southern)
- UpperCorner longitude, in decimal degrees (aka most eastern)
- UpperCorner latitude, in decimal degrees (aka most northern)
* **:width**: the width in pixels for the output image
* **:height**: the height in pixels for the output image
* **:format**: the format for the image, supported types: `png`, `jpg`
* **jpg** will have a default quality of 85.
Note: you can see this endpoint as:
```bash
GET /api/v1/map/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`
```
####Layers
The Static Maps API allows for multiple layers of incorporation into the `MapConfig` to allow for maximum versatility in creating a static map. The examples below were used to generate the static image example in the next section, and appear in the specific order designated.
**Basemaps**
```javascript
{
"type": "http",
"options": {
"urlTemplate": "http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png",
"subdomains": [
"a",
"b",
"c"
]
}
},
```
By manipulating the `"urlTemplate"` custom basemaps can be used in generating static images. Supported map types for the Static Maps API are:
'http://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png',
'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
'http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png',
'http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png',
**Mapnik**
```javascript
{
"type": "mapnik",
"options": {
"sql": "select null::geometry the_geom_webmercator",
"cartocss": "#layer {\n\tpolygon-fill: #FF3300;\n\tpolygon-opacity: 0;\n\tline-color: #333;\n\tline-width: 0;\n\tline-opacity: 0;\n}",
"cartocss_version": "2.2.0"
}
},
```
**CartoDB**
```javascript
{
"type": "cartodb",
"options": {
"sql": "select * from park",
"cartocss": "/** simple visualization */\n\n#park{\n polygon-fill: #229A00;\n polygon-opacity: 0.7;\n line-color: #FFF;\n line-width: 0;\n line-opacity: 1;\n}",
"cartocss_version": "2.1.1"
}
},
```
Additoinally, static images from Torque maps and other map layers can be used together to generate highly customizable and versatile static maps.
####Caching
It is important to note that generated images are cached from the live data referenced with the `layergroupid token` on the specified CartoDB account. This means that if the data changes, the cached image will also change. When linking dynamically, it is important to take into consideration the state of the data and longevity of the static image to avoid broken images or changes in how the image is displayed. To obtain a static snapshot of the map as it is today and preserve the image long-term regardless of changes in data, the image must be saved and stored locally.
####Limits
* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
* Resolution limitations for jpegs are at default a quality of 85. Limitations for quality are restricted to a maximum of XXX dpi.
* Timeout limits for generating static maps are the same across the CartoDB Editor and Platform. It is important to ensure timely processing of queries.
### Examples
After instantiating a map from a CartoDB account:
<div class="code-title code-request with-result">REQUEST</div>
```bash
GET /api/v1/map/static/center/4b615ff367e498e770e7d05e99181873:1420231989550.8699/14/40.71502926732618/-73.96039009094238/600/400.png
```
####Response
<div clas="wrap"><p class="wrap-border"><img src="https://raw.githubusercontent.com/namessanti/Pictures/master/static_api.png" alt="static-api"/></p>,</div>
####MapConfig
For this map, the multiple layers, order, and stylings are defined by the MapConfig.
```javascript
{
"version": "1.3.0-alpha",
"layers": [
{
"type": "http",
"options": {
"urlTemplate": "http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png",
"subdomains": [
"a",
"b",
"c"
]
}
},
{
"type": "mapnik",
"options": {
"sql": "select null::geometry the_geom_webmercator",
"cartocss": "#layer {\n\tpolygon-fill: #FF3300;\n\tpolygon-opacity: 0;\n\tline-color: #333;\n\tline-width: 0;\n\tline-opacity: 0;\n}",
"cartocss_version": "2.2.0"
}
},
{
"type": "cartodb",
"options": {
"sql": "select * from park",
"cartocss": "/** simple visualization */\n\n#park{\n polygon-fill: #229A00;\n polygon-opacity: 0.7;\n line-color: #FFF;\n line-width: 0;\n line-opacity: 1;\n}",
"cartocss_version": "2.1.1"
}
},
{
"type": "cartodb",
"options": {
"sql": "select * from residential_zoning_2009",
"cartocss": "/** simple visualization */\n\n#residential_zoning_2009{\n polygon-fill: #c7eae5;\n polygon-opacity: 1;\n line-color: #FFF;\n line-width: 0.2;\n line-opacity: 0.5;\n}",
"cartocss_version": "2.1.1"
}
},
{
"type": "cartodb",
"options": {
"sql": "select * from nycha_developments_july2011",
"cartocss": "/** simple visualization */\n\n#nycha_developments_july2011{\n polygon-fill: #ef3b2c;\n polygon-opacity: 0.7;\n line-color: #FFF;\n line-width: 0;\n line-opacity: 1;\n}",
"cartocss_version": "2.1.1"
}
}
]
}
```

View File

@@ -0,0 +1,56 @@
# 1. Purpose
This specification describes an extension for
[MapConfig 1.3.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.3.0.md) version.
# 2. Changes over specification
This extension introduces a new layer type so it's possible to use a named map by its name as a layer.
## 2.1 Named layers definition
```javascript
{
// REQUIRED
// string, `named` is the only supported value
type: "named",
// REQUIRED
// object, set `named` map layers configuration
options: {
// REQUIRED
// string, the name for the named map to use
name: "world_borders",
// OPTIONAL
// object, the replacement values for the named map's template placeholders
// See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#instantiate-1 for more details
config: {
"color": "#000"
},
// OPTIONAL
// string array, the authorized tokens in case the named map has auth method set to `token`
// See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#named-maps-1 for more details
auth_tokens: [
"token1",
"token2"
]
}
}
```
## 2.2 Limitations
1. A Named Map will not allow to have `named` type layers inside their templates layergroup's layers definition.
2. A `named` layer does not allow Named Maps form other accounts, it's only possible to use Named Maps from the very
same user account.
# History
## 1.0.0
- Initial version

View File

@@ -1,7 +1,10 @@
var sqlApi = require('../sql/sql_api'),
PSQL = require('cartodb-psql');
var sqlApi = require('../sql/sql_api');
var PSQL = require('cartodb-psql');
var Step = require('step');
function QueryTablesApi() {
function QueryTablesApi(pgConnection, metadataBackend) {
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
}
var affectedTableRegexCache = {
@@ -13,33 +16,12 @@ var affectedTableRegexCache = {
module.exports = QueryTablesApi;
QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, tableNames, callback) {
var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY['+
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(',') +
'])';
// call sql api
sqlApi.query(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not find last updated timestamp: ' + msg));
return;
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var last_updated = 0;
if(rows.length !== 0) {
last_updated = rows[0].max || 0;
}
callback(null, last_updated*1000);
});
};
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
runQuery(username, options, query, handleAffectedTablesInQueryRows, callback);
this.runQuery(username, query, handleAffectedTablesInQueryRows, callback);
};
function handleAffectedTablesInQueryRows(err, rows, callback) {
@@ -54,7 +36,7 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
callback(null, tableNames);
}
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
var query = [
'WITH querytables AS (',
@@ -65,7 +47,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
this.runQuery(username, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
};
function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
@@ -89,20 +71,60 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
}
function runQuery(username, options, query, queryHandler, callback) {
QueryTablesApi.prototype.runQuery = function(username, query, queryHandler, callback) {
var self = this;
if (shouldQueryPostgresDirectly()) {
var psql = new PSQL(options);
psql.query(query, function(err, resultSet) {
resultSet = resultSet || {};
var rows = resultSet.rows || [];
queryHandler(err, rows, callback);
});
var params = {};
Step(
function setAuth() {
self.pgConnection.setDBAuth(username, params, this);
},
function setConn(err) {
if (err) {
throw err;
}
self.pgConnection.setDBConn(username, params, this);
},
function executeQuery(err) {
if (err) {
throw err;
}
var psql = new PSQL({
user: params.dbuser,
pass: params.dbpass,
host: params.dbhost,
port: params.dbport,
dbname: params.dbname
});
psql.query(query, function(err, resultSet) {
resultSet = resultSet || {};
var rows = resultSet.rows || [];
queryHandler(err, rows, callback);
});
}
);
} else {
sqlApi.query(username, options.api_key, query, function(err, rows) {
queryHandler(err, rows, callback);
});
Step(
function getApiKey() {
self.metadataBackend.getUserMapKey(username, this);
},
function executeQuery(err, apiKey) {
if (err) {
throw err;
}
sqlApi.query(username, apiKey, query, function(err, rows) {
queryHandler(err, rows, callback);
});
}
);
}
}
};
function prepareSql(sql) {

View File

@@ -0,0 +1,96 @@
var Step = require('step');
var _ = require('underscore');
function PgConnection(metadataBackend) {
this.metadataBackend = metadataBackend;
}
module.exports = PgConnection;
// Set db authentication parameters to those of the given username
//
// @param username the cartodb username, mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set auth options into
// added params are: "dbuser" and "dbpassword"
//
// @param callback function(err)
//
PgConnection.prototype.setDBAuth = function(username, params, callback) {
var self = this;
var user_params = {};
var auth_user = global.environment.postgres_auth_user;
var auth_pass = global.environment.postgres_auth_pass;
Step(
function getId() {
self.metadataBackend.getUserId(username, this);
},
function(err, user_id) {
if (err) throw err;
user_params['user_id'] = user_id;
var dbuser = _.template(auth_user, user_params);
_.extend(params, {dbuser:dbuser});
// skip looking up user_password if postgres_auth_pass
// doesn't contain the "user_password" label
if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
self.metadataBackend.getUserDBPass(username, this);
},
function(err, user_password) {
if (err) throw err;
user_params['user_password'] = user_password;
if ( auth_pass ) {
var dbpass = _.template(auth_pass, user_params);
_.extend(params, {dbpassword:dbpass});
}
return true;
},
function finish(err) {
callback(err);
}
);
};
// Set db connection parameters to those for the given username
//
// @param dbowner cartodb username of database owner,
// mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set connection options into
// added params are: "dbname", "dbhost"
//
// @param callback function(err)
//
PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
var self = this;
// Add default database connection parameters
// if none given
_.defaults(params, {
dbuser: global.environment.postgres.user,
dbpassword: global.environment.postgres.password,
dbhost: global.environment.postgres.host,
dbport: global.environment.postgres.port
});
Step(
function getConnectionParams() {
self.metadataBackend.getUserDBConnectionParams(dbowner, this);
},
function extendParams(err, dbParams){
if (err) throw err;
// we don't want null values or overwrite a non public user
if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
delete dbParams.dbuser;
}
if ( dbParams ) _.extend(params, dbParams);
return null;
},
function finish(err) {
callback(err);
}
);
};

View File

@@ -1,11 +1,9 @@
var _ = require('underscore')
, Step = require('step')
, Windshaft = require('windshaft')
, TemplateMaps = require('./template_maps.js')
, Cache = require('./cache_validator')
, os = require('os')
, HealthCheck = require('./monitoring/health_check')
;
var _ = require('underscore');
var Step = require('step');
var Windshaft = require('windshaft');
var Cache = require('./cache_validator');
var os = require('os');
var HealthCheck = require('./monitoring/health_check');
if ( ! process.env['PGAPPNAME'] )
process.env['PGAPPNAME']='cartodb_tiler';
@@ -25,6 +23,8 @@ var CartodbWindshaft = function(serverOptions) {
var cartoData = require('cartodb-redis')({pool: redisPool});
var templateMaps = serverOptions.templateMaps;
if(serverOptions.cache_enabled) {
console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
@@ -48,12 +48,6 @@ var CartodbWindshaft = function(serverOptions) {
//
var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
var templateMapsOpts = {
max_user_templates: global.environment.maxUserTemplates
};
var templateMaps = new TemplateMaps(redisPool, templateMapsOpts);
serverOptions.templateMaps = templateMaps;
var SurrogateKeysCache = require('./cache/surrogate_keys_cache'),
NamedMapsCacheEntry = require('./cache/model/named_maps_entry'),
VarnishHttpCacheBackend = require('./cache/backend/varnish_http'),
@@ -158,7 +152,14 @@ var CartodbWindshaft = function(serverOptions) {
var TemplateMapsController = require('./controllers/template_maps'),
templateMapsController = new TemplateMapsController(
ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry
ws,
serverOptions,
templateMaps,
cartoData,
template_baseurl,
surrogateKeysCache,
NamedMapsCacheEntry,
serverOptions.pgConnection
);
templateMapsController.register(ws);
@@ -209,26 +210,11 @@ var CartodbWindshaft = function(serverOptions) {
/**
* Helper API to allow per table tile cache (and sql cache) to be invalidated remotely.
* TODO: Move?
* Keep endpoint for backwards compatibility
*/
ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
if ( req.profiler && req.profiler.statsd_client ) {
req.profiler.start('windshaft-cartodb.flush_cache');
}
ws.doCORS(res);
Step(
function flushCache(){
serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
},
function sendResponse(err, data){
if (err){
ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE', err);
//ws.sendResponse(res, [500]);
} else {
ws.sendResponse(res, [{status: 'ok'}, 200]);
}
}
);
ws.sendResponse(res, [{status: 'ok'}, 200]);
});
var healthCheck = new HealthCheck(cartoData, Windshaft.tilelive);

View File

@@ -2,7 +2,7 @@ var Step = require('step');
var _ = require('underscore');
function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache,
NamedMapsCacheEntry) {
NamedMapsCacheEntry, pgConnection) {
this.app = app;
this.serverOptions = serverOptions;
this.templateMaps = templateMaps;
@@ -10,6 +10,7 @@ function TemplateMapsController(app, serverOptions, templateMaps, metadataBacken
this.templateBaseUrl = templateBaseUrl;
this.surrogateKeysCache = surrogateKeysCache;
this.NamedMapsCacheEntry = NamedMapsCacheEntry;
this.pgConnection = pgConnection;
}
module.exports = TemplateMapsController;
@@ -465,11 +466,11 @@ TemplateMapsController.prototype.setDBParams = function(cdbuser, params, callbac
var self = this;
Step(
function setAuth() {
self.serverOptions.setDBAuth(cdbuser, params, this);
self.pgConnection.setDBAuth(cdbuser, params, this);
},
function setConn(err) {
if ( err ) throw err;
self.serverOptions.setDBConn(cdbuser, params, this);
self.pgConnection.setDBConn(cdbuser, params, this);
},
function finish(err) {
callback(err);

View File

@@ -0,0 +1,120 @@
var queue = require('queue-async');
var _ = require('underscore');
var Datasource = require('windshaft').Datasource;
function MapConfigNamedLayersAdapter(templateMaps) {
this.templateMaps = templateMaps;
}
module.exports = MapConfigNamedLayersAdapter;
MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbMetadata, callback) {
var self = this;
var adaptLayersQueue = queue(layers.length);
function adaptLayer(layer, done) {
if (isNamedTypeLayer(layer)) {
if (!layer.options.name) {
return done(new Error('Missing Named Map `name` in layer options'));
}
var templateName = layer.options.name;
var templateConfigParams = layer.options.config || {};
var templateAuthTokens = layer.options.auth_tokens;
self.templateMaps.getTemplate(username, templateName, function(err, template) {
if (err || !template) {
return done(new Error("Template '" + templateName + "' of user '" + username + "' not found"));
}
if (self.templateMaps.isAuthorized(template, templateAuthTokens)) {
var nestedNamedLayers = template.layergroup.layers.filter(function(layer) {
return layer.type === 'named';
});
if (nestedNamedLayers.length > 0) {
var nestedNamedMapsError = new Error('Nested named layers are not allowed');
// nestedNamedMapsError.http_status = 400;
return done(nestedNamedMapsError);
}
try {
var templateLayergroupConfig = self.templateMaps.instance(template, templateConfigParams);
return done(null, {
datasource: true,
layers: templateLayergroupConfig.layers
});
} catch (err) {
return done(err);
}
} else {
var unauthorizedError = new Error("Unauthorized '" + templateName + "' template instantiation");
unauthorizedError.http_status = 403;
return done(unauthorizedError);
}
});
} else {
return done(null, {
datasource: false,
layers: [layer]
});
}
}
var datasourceBuilder = new Datasource.Builder();
function layersAdaptQueueFinish(err, layersResults) {
if (err) {
return callback(err);
}
if (!layersResults || layersResults.length === 0) {
return callback(new Error('Missing layers array from layergroup config'));
}
var layers = [],
currentLayerIndex = 0;
layersResults.forEach(function(layersResult) {
layersResult.layers.forEach(function(layer) {
layers.push(layer);
if (layersResult.datasource) {
datasourceBuilder.withLayerDatasource(currentLayerIndex, {
user: dbAuth.dbuser
});
}
currentLayerIndex++;
});
});
return callback(null, layers, datasourceBuilder.build());
}
var dbAuth = {};
if (_.some(layers, isNamedTypeLayer)) {
// Lazy load dbAuth
dbMetadata.setDBAuth(username, dbAuth, function(err) {
if (err) {
return callback(err);
}
layers.forEach(function(layer) {
adaptLayersQueue.defer(adaptLayer, layer);
});
adaptLayersQueue.awaitAll(layersAdaptQueueFinish);
});
} else {
return callback(null, layers, datasourceBuilder.build());
}
};
function isNamedTypeLayer(layer) {
return layer.type === 'named';
}

View File

@@ -1,10 +1,11 @@
var _ = require('underscore')
, Step = require('step')
, Cache = require('./cache_validator')
, QueryTablesApi = require('./api/query_tables_api')
, crypto = require('crypto')
, LZMA = require('lzma').LZMA
;
var _ = require('underscore');
var Step = require('step');
var QueryTablesApi = require('./api/query_tables_api');
var PgConnection = require('./backends/pg_connection');
var crypto = require('crypto');
var LZMA = require('lzma').LZMA;
var TemplateMaps = require('./template_maps.js');
var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter');
// This is for backward compatibility with 1.3.3
if ( _.isUndefined(global.environment.sqlapi.domain) ) {
@@ -34,10 +35,13 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
];
module.exports = function(redisPool) {
var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis;
var cartoData = require('cartodb-redis')(redisOpts),
redisPool = redisPool
|| require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:server_options'}));
var cartoData = require('cartodb-redis')({ pool: redisPool }),
lzmaWorker = new LZMA(),
queryTablesApi = new QueryTablesApi();
pgConnection = new PgConnection(cartoData),
queryTablesApi = new QueryTablesApi(pgConnection, cartoData);
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
@@ -104,6 +108,16 @@ module.exports = function(redisPool) {
// Re-use redisPool
me.redis.pool = redisPool;
// Re-use pgConnection
me.pgConnection = pgConnection;
var templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
me.templateMaps = templateMaps;
var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
/* This whole block is about generating X-Cache-Channel { */
// TODO: review lifetime of elements of this cache
@@ -194,38 +208,8 @@ module.exports = function(redisPool) {
}
return [req.params.table];
}
var user, key;
var next = this;
Step (
function findUserKey() {
if ( req.params.hasOwnProperty('_authorizedBySigner') ) {
user = req.params._authorizedBySigner;
cartoData.getUserMapKey(user, this);
} else {
user = that.userByReq(req);
key = req.params.map_key || req.params.api_key;
return null;
}
},
function getAffected(err, data) {
if ( err ) throw err;
if ( data ) {
if ( req.profiler ) req.profiler.done('getSignerMapKey');
key = data;
}
queryTablesApi.getAffectedTablesInQuery(user, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this); // in addCacheChannel
},
function finish(err, data) {
next(err,data);
}
);
queryTablesApi.getAffectedTablesInQuery(that.userByReq(req), sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
if ( err ) throw err;
@@ -296,6 +280,17 @@ module.exports = function(redisPool) {
});
};
me.beforeLayergroupCreate = function(req, requestMapConfig, callback) {
mapConfigNamedLayersAdapter.getLayers(this.userByReq(req), requestMapConfig.layers, pgConnection, function(err, layers, datasource) {
if (err) {
return callback(err);
}
requestMapConfig.layers = layers;
return callback(null, requestMapConfig, datasource)
});
};
me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
var token = response.layergroupid;
@@ -331,28 +326,16 @@ module.exports = function(redisPool) {
done();
});
var sql = [];
_.each(mapconfig.layers, function(lyr) {
sql.push(lyr.options.sql);
});
sql = sql.join(';');
var sql = mapconfig.layers.map(function(layer) {
return layer.options.sql;
}).join(';');
var dbName = req.params.dbname;
var usr = this.userByReq(req);
var key = req.params.map_key || req.params.api_key;
var cacheKey = dbName + ':' + token;
Step(
function getAffectedTablesAndLastUpdatedTime() {
queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this);
queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
@@ -407,92 +390,6 @@ module.exports = function(redisPool) {
return mat[1];
};
// Set db authentication parameters to those of the given username
//
// @param username the cartodb username, mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set auth options into
// added params are: "dbuser" and "dbpassword"
//
// @param callback function(err)
//
me.setDBAuth = function(username, params, callback) {
var user_params = {};
var auth_user = global.environment.postgres_auth_user;
var auth_pass = global.environment.postgres_auth_pass;
Step(
function getId() {
cartoData.getUserId(username, this);
},
function(err, user_id) {
if (err) throw err;
user_params['user_id'] = user_id;
var dbuser = _.template(auth_user, user_params);
_.extend(params, {dbuser:dbuser});
// skip looking up user_password if postgres_auth_pass
// doesn't contain the "user_password" label
if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
cartoData.getUserDBPass(username, this);
},
function(err, user_password) {
if (err) throw err;
user_params['user_password'] = user_password;
if ( auth_pass ) {
var dbpass = _.template(auth_pass, user_params);
_.extend(params, {dbpassword:dbpass});
}
return true;
},
function finish(err) {
callback(err);
}
);
};
// Set db connection parameters to those for the given username
//
// @param dbowner cartodb username of database owner,
// mapped to a database username
// via CartodbRedis metadata records
//
// @param params the parameters to set connection options into
// added params are: "dbname", "dbhost"
//
// @param callback function(err)
//
me.setDBConn = function(dbowner, params, callback) {
// Add default database connection parameters
// if none given
_.defaults(params, {
dbuser: global.environment.postgres.user,
dbpassword: global.environment.postgres.password,
dbhost: global.environment.postgres.host,
dbport: global.environment.postgres.port
});
Step(
function getConnectionParams() {
cartoData.getUserDBConnectionParams(dbowner, this);
},
function extendParams(err, dbParams){
if (err) throw err;
// we don't want null values or overwrite a non public user
if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
delete dbParams.dbuser;
}
if ( dbParams ) _.extend(params, dbParams);
return null;
},
function finish(err) {
callback(err);
}
);
};
// Check if a request is authorized by a signer
//
// @param req express request object
@@ -591,7 +488,7 @@ module.exports = function(redisPool) {
_.extend(req.params, { _authorizedByApiKey: true });
// authorized by api key, login as the given username and stop
that.setDBAuth(user, req.params, function(err) {
pgConnection.setDBAuth(user, req.params, function(err) {
callback(err, true); // authorized (or error)
});
},
@@ -626,7 +523,7 @@ module.exports = function(redisPool) {
// Authorized by "signed_by" !
_.extend(req.params, { _authorizedBySigner: signed_by });
that.setDBAuth(signed_by, req.params, function(err) {
pgConnection.setDBAuth(signed_by, req.params, function(err) {
if (req.profiler) req.profiler.done('setDBAuth');
callback(err, true); // authorized (or error)
});
@@ -740,7 +637,7 @@ module.exports = function(redisPool) {
},
function getDatabase(err){
if(err) throw err;
that.setDBConn(user, req.params, this);
pgConnection.setDBConn(user, req.params, this);
},
function getGeometryType(err){
if (req.profiler) req.profiler.done('setDBConn');
@@ -824,33 +721,5 @@ module.exports = function(redisPool) {
);
};
/**
* Helper to clear out tile cache on request
* @param req
* @param callback
*/
me.flushCache = function(req, Cache, callback){
var that = this;
Step(
function getParams(){
// this is mostly to compute req.params.dbname
that.req2params(req, this);
},
function flushInternalCache(err){
// TODO: implement this, see
// http://github.com/Vizzuality/Windshaft-cartodb/issues/73
return true;
},
function flushVarnishCache(err){
if (err) { callback(err); return; }
if(Cache) {
Cache.invalidate_db(req.params.dbname, req.params.table);
}
callback(null, true);
}
);
};
return me;
};

View File

@@ -44,11 +44,6 @@ function TemplateMaps(redis_pool, opts) {
// User templates (HASH:tpl_id->tpl_val)
this.key_usr_tpl = dot.template("map_tpl|{{=it.owner}}");
// User template locks (HASH:tpl_id->ctime)
this.key_usr_tpl_lck = dot.template("map_tpl|{{=it.owner}}|locks");
this.lock_ttl = this.opts['lock_ttl'] || 5000;
}
util.inherits(TemplateMaps, EventEmitter);
@@ -398,6 +393,10 @@ o.isAuthorized = function(template, authTokens) {
return false;
}
if (_.isString(templateAuth) && templateAuth === 'open') {
return true;
}
if (templateAuth.method === 'open') {
return true;
}

47
npm-shrinkwrap.json generated
View File

@@ -1,10 +1,10 @@
{
"name": "windshaft-cartodb",
"version": "1.25.0",
"version": "1.28.1",
"dependencies": {
"cartodb-psql": {
"version": "0.4.0",
"from": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
"from": "cartodb-psql@~0.4.0",
"resolved": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
"dependencies": {
"pg": {
@@ -18,8 +18,7 @@
},
"buffer-writer": {
"version": "1.0.0",
"from": "buffer-writer@1.0.0",
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz"
"from": "buffer-writer@1.0.0"
}
}
}
@@ -27,7 +26,7 @@
},
"cartodb-redis": {
"version": "0.11.0",
"from": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
"from": "cartodb-redis@~0.11.0",
"resolved": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
"dependencies": {
"redis-mpool": {
@@ -88,8 +87,7 @@
},
"isarray": {
"version": "0.0.1",
"from": "isarray@0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
"from": "isarray@0.0.1"
},
"string_decoder": {
"version": "0.10.31",
@@ -97,7 +95,7 @@
},
"inherits": {
"version": "2.0.1",
"from": "inherits@2"
"from": "inherits@~2.0.1"
}
}
}
@@ -113,9 +111,14 @@
"from": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
"resolved": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0"
},
"queue-async": {
"version": "1.0.7",
"from": "queue-async@~1.0.7",
"resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz"
},
"redis-mpool": {
"version": "0.3.0",
"from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0",
"from": "redis-mpool@~0.3.0",
"resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0",
"dependencies": {
"generic-pool": {
@@ -178,24 +181,17 @@
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
},
"windshaft": {
"version": "0.34.0",
"from": "https://github.com/CartoDB/Windshaft/tarball/0.34.0",
"resolved": "https://github.com/CartoDB/Windshaft/tarball/0.34.0",
"version": "0.37.1",
"from": "windshaft@~0.37.1",
"dependencies": {
"chronograph": {
"version": "0.1.0",
"from": "chronograph@git://github.com/CartoDB/chronographjs.git#0.1.0",
"resolved": "git://github.com/CartoDB/chronographjs.git#0b8c35eee510cfa14a16be24d70533b38ecc1d2d"
},
"queue-async": {
"version": "1.0.7",
"from": "queue-async@~1.0.7",
"resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz"
},
"grainstore": {
"version": "0.22.1",
"from": "https://github.com/CartoDB/grainstore/tarball/0.22.1",
"resolved": "https://github.com/CartoDB/grainstore/tarball/0.22.1",
"version": "0.23.0",
"from": "grainstore@~0.23.0",
"dependencies": {
"carto": {
"version": "0.9.5-cdb2",
@@ -1637,9 +1633,9 @@
"resolved": "https://registry.npmjs.org/connect/-/connect-1.9.2.tgz",
"dependencies": {
"formidable": {
"version": "1.0.16",
"version": "1.0.17",
"from": "formidable@1.0.x",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.16.tgz"
"resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz"
}
}
},
@@ -2159,8 +2155,8 @@
},
"torque.js": {
"version": "2.7.1",
"from": "https://github.com/CartoDB/torque/tarball/master",
"resolved": "https://github.com/CartoDB/torque/tarball/master",
"from": "https://github.com/CartoDB/torque/tarball/2.8",
"resolved": "https://github.com/CartoDB/torque/tarball/2.8",
"dependencies": {
"carto": {
"version": "0.15.1",
@@ -2213,8 +2209,7 @@
},
"isarray": {
"version": "0.0.1",
"from": "isarray@0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
"from": "isarray@0.0.1"
},
"string_decoder": {
"version": "0.10.31",

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.25.0",
"version": "1.28.1",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -25,12 +25,13 @@
"node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
"underscore" : "~1.6.0",
"dot": "~1.0.2",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.34.0",
"windshaft": "~0.37.1",
"step": "~0.0.5",
"queue-async": "~1.0.7",
"request": "~2.9.203",
"cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
"cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
"redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0",
"cartodb-redis": "~0.11.0",
"cartodb-psql": "~0.4.0",
"redis-mpool": "~0.3.0",
"lzma": "~1.3.7",
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
"rollbar": "~0.3.13"

View File

@@ -16,9 +16,13 @@ var serverOptions = ServerOptions();
suite('templates surrogate keys', function() {
var redisClient,
sqlApiServer,
server;
var sqlApiServer;
var redisClient = redis.createClient(global.environment.redis.port);
// Enable Varnish purge for tests
serverOptions.varnish_purge_enabled = true;
var server = new CartodbWindshaft(serverOptions);
var templateOwner = 'localhost',
templateName = 'acceptance',
@@ -45,19 +49,12 @@ suite('templates surrogate keys', function() {
expectedBody = { template_id: expectedTemplateId };
suiteSetup(function(done) {
// Enable Varnish purge for tests
serverOptions.varnish_purge_enabled = true;
server = new CartodbWindshaft(serverOptions);
sqlApiServer = new SqlApiEmulator(global.environment.sqlapi.port, done);
redisClient = redis.createClient(global.environment.redis.port);
});
var surrogateKeysCacheInvalidateFn = SurrogateKeysCache.prototype.invalidate;
beforeEach(function(done) {
function createTemplate(callback) {
var postTemplateRequest = {
url: '/tiles/template?api_key=1234',
method: 'POST',
@@ -90,10 +87,10 @@ suite('templates surrogate keys', function() {
return true;
},
function finish(err) {
done(err);
callback(err);
}
);
});
}
test("update template calls surrogate keys invalidation", function(done) {
var cacheEntryKey;
@@ -104,7 +101,13 @@ suite('templates surrogate keys', function() {
};
Step(
function putValidTemplate() {
function createTemplateToUpdate() {
createTemplate(this);
},
function putValidTemplate(err) {
if (err) {
throw err;
}
var updateTemplateRequest = {
url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234',
method: 'PUT',
@@ -163,7 +166,13 @@ suite('templates surrogate keys', function() {
};
Step(
function putValidTemplate() {
function createTemplateToDelete() {
createTemplate(this);
},
function deleteValidTemplate(err) {
if (err) {
throw err;
}
var deleteTemplateRequest = {
url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234',
method: 'DELETE',
@@ -199,12 +208,10 @@ suite('templates surrogate keys', function() {
);
});
afterEach(function(done) {
SurrogateKeysCache.prototype.invalidate = surrogateKeysCacheInvalidateFn;
done();
});
suiteTeardown(function(done) {
SurrogateKeysCache.prototype.invalidate = surrogateKeysCacheInvalidateFn;
// Enable Varnish purge for tests
serverOptions.varnish_purge_enabled = false;
sqlApiServer.close(done);
});

View File

@@ -7,7 +7,7 @@ var server = new CartodbWindshaft(serverOptions);
suite('health checks', function () {
beforeEach(function (done) {
function resetHealthConfig() {
global.environment.health = {
enabled: true,
username: 'localhost',
@@ -15,8 +15,7 @@ suite('health checks', function () {
x: 0,
y: 0
};
done();
});
}
var healthCheckRequest = {
url: '/health',
@@ -27,13 +26,14 @@ suite('health checks', function () {
};
test('returns 200 and ok=true with enabled configuration', function (done) {
resetHealthConfig();
assert.response(server,
healthCheckRequest,
{
status: 200
},
function (res, err) {
console.log(res.body);
assert.ok(!err);
var parsed = JSON.parse(res.body);
@@ -47,6 +47,8 @@ suite('health checks', function () {
});
test('fails for invalid user because it is not in redis', function (done) {
resetHealthConfig();
global.environment.health.username = 'invalid';
assert.response(server,

View File

@@ -24,8 +24,6 @@ server.setMaxListeners(0);
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
@@ -38,6 +36,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
var test_database = test_user + '_db';
suiteSetup(function(done){
global.environment.enabledFeatures = { cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue };
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
});
@@ -112,15 +111,17 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
assert.ok(cc);
var dbname = test_database;
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join('');
assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames )'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
var jsonquery = cc.substring(dbname.length + 1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join('');
assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames )'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
}
assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL,
function(err, similarity) {
@@ -337,7 +338,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
layers: [
{ options: {
sql: 'select 1 as cartodb_id, '
+ 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+ 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator from test_table limit 1',
cartocss: '#layer { polygon-fill:red; }',
cartocss_version: '2.0.1',
interactivity: 'cartodb_id'
@@ -388,21 +389,23 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
// Check X-Cache-Channel
var cc = res.headers['x-cache-channel'];
assert.ok(cc);
assert.ok(cc);
var dbname = test_database;
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = layergroup.layers[0].options.sql
.replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)')
.replace(/!pixel_width!/g, '1')
.replace(/!pixel_height!/g, '1');
assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames )'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
var jsonquery = cc.substring(dbname.length + 1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = layergroup.layers[0].options.sql
.replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)')
.replace(/!pixel_width!/g, '1')
.replace(/!pixel_height!/g, '1');
assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames )'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
}
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
@@ -428,18 +431,20 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
assert.ok(cc);
var dbname = test_database;
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = layergroup.layers[0].options.sql
.replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
.replace('!pixel_width!', '1')
.replace('!pixel_height!', '1');
assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames )'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
var jsonquery = cc.substring(dbname.length + 1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = layergroup.layers[0].options.sql
.replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
.replace('!pixel_width!', '1')
.replace('!pixel_height!', '1');
assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames )'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
}
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
@@ -1218,8 +1223,10 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
var parsedBody = JSON.parse(res.body);
var token_components = parsedBody.layergroupid.split(':');
expected_token = token_components[0];
var last_request = sqlapi_server.getLastRequest();
assert.equal(last_request.method, 'POST');
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
var last_request = sqlapi_server.getLastRequest();
assert.equal(last_request.method, 'POST');
}
return null;
},
function cleanup(err) {
@@ -1283,6 +1290,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
);
});
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
// See https://github.com/CartoDB/Windshaft-cartodb/issues/167
test("lack of response from sql-api will result in a timeout", function(done) {
@@ -1323,6 +1331,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
}
);
});
}
var layergroupTtlRequest = {
url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify({
@@ -1368,6 +1377,50 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
});
test("it's not possible to override authorization with a crafted layergroup", function(done) {
var layergroup = {
version: '1.0.0',
layers: [
{
options: {
sql: 'select * from test_table_private_1',
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
}
],
template: {
auth: {
method: "open"
},
name: "open"
}
};
assert.response(
server,
{
url: '/api/v1/map?signer=localhost',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 403
},
function(res, err) {
assert.ok(res.body.match(/permission denied for relation test_table_private_1/));
done();
}
);
});
suiteTeardown(function(done) {
// This test will add map_style records, like

View File

@@ -0,0 +1,527 @@
var test_helper = require('../support/test_helper');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var server = new CartodbWindshaft(serverOptions);
var RedisPool = require('redis-mpool');
var TemplateMaps = require('../../lib/cartodb/template_maps.js');
var Step = require('step');
var _ = require('underscore');
suite('named_layers', function() {
// configure redis pool instance to use in tests
var redisPool = RedisPool(global.environment.redis);
var templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
var wadusLayer = {
type: 'cartodb',
options: {
sql: 'select 1 cartodb_id, null::geometry the_geom_webmercator',
cartocss: '#layer { marker-fill: <%= color %>; }',
cartocss_version: '2.3.0'
}
};
var username = 'localhost';
var templateName = 'valid_template';
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
"placeholders": {
"color": {
"type": "css_color",
"default": "#cc3300"
}
},
layergroup: {
layers: [
wadusLayer
]
}
};
var tokenAuthTemplateName = 'auth_valid_template';
var tokenAuthTemplate = {
version: '0.0.1',
name: tokenAuthTemplateName,
auth: {
method: 'token',
valid_tokens: ['valid1', 'valid2']
},
placeholders: {
color: {
"type": "css_color",
"default": "#cc3300"
}
},
layergroup: {
layers: [
wadusLayer
]
}
};
var namedMapLayer = {
type: 'named',
options: {
name: templateName,
config: {},
auth_tokens: []
}
};
var nestedNamedMapTemplateName = 'nested_template';
var nestedNamedMapTemplate = {
version: '0.0.1',
name: nestedNamedMapTemplateName,
auth: {
method: 'open'
},
layergroup: {
layers: [
namedMapLayer
]
}
};
suiteSetup(function(done) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true};
templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) {
if (err) {
return done(err);
}
templateMaps.addTemplate(username, tokenAuthTemplate, function(err) {
if (err) {
return done(err);
}
templateMaps.addTemplate(username, template, function(err) {
return done(err);
});
});
});
});
test('should fail for non-existing template name', function(done) {
var layergroup = {
version: '1.3.0',
layers: [
{
type: 'named',
options: {
name: 'nonexistent'
}
}
]
};
Step(
function createLayergroup() {
var next = this;
assert.response(server,
{
url: '/tiles/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 400
},
function(res, err) {
next(err, res);
}
);
},
function checkLayergroup(err, response) {
if (err) {
throw err;
}
var parsedBody = JSON.parse(response.body);
assert.deepEqual(parsedBody, { errors: ["Template 'nonexistent' of user 'localhost' not found"] });
return null;
},
function finish(err) {
done(err);
}
);
});
test('should return 403 if not properly authorized', function(done) {
var layergroup = {
version: '1.3.0',
layers: [
{
type: 'named',
options: {
name: tokenAuthTemplateName,
config: {},
auth_tokens: ['token1']
}
}
]
};
Step(
function createLayergroup() {
var next = this;
assert.response(server,
{
url: '/tiles/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 403
},
function(res, err) {
next(err, res);
}
);
},
function checkLayergroup(err, response) {
if (err) {
throw err;
}
var parsedBody = JSON.parse(response.body);
assert.deepEqual(
parsedBody,
{ errors: [ "Unauthorized 'auth_valid_template' template instantiation" ] }
);
return null;
},
function finish(err) {
done(err);
}
);
});
test('should return 200 and layergroup if properly authorized', function(done) {
var layergroup = {
version: '1.3.0',
layers: [
{
type: 'named',
options: {
name: tokenAuthTemplateName,
config: {},
auth_tokens: ['valid1']
}
}
]
};
Step(
function createLayergroup() {
var next = this;
assert.response(server,
{
url: '/tiles/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 200
},
function(res, err) {
next(err, res);
}
);
},
function checkLayergroup(err, response) {
if (err) {
throw err;
}
var parsedBody = JSON.parse(response.body);
assert.ok(parsedBody.layergroupid);
assert.ok(parsedBody.last_updated);
return null;
},
function finish(err) {
done(err);
}
);
});
test('should return 400 for nested named map layers', function(done) {
var layergroup = {
version: '1.3.0',
layers: [
{
type: 'named',
options: {
name: nestedNamedMapTemplateName
}
}
]
};
Step(
function createLayergroup() {
var next = this;
assert.response(server,
{
url: '/tiles/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 400
},
function(res, err) {
next(err, res);
}
);
},
function checkLayergroup(err, response) {
if (err) {
throw err;
}
var parsedBody = JSON.parse(response.body);
assert.deepEqual(parsedBody, { errors: [ 'Nested named layers are not allowed' ] });
return null;
},
function finish(err) {
done(err);
}
);
});
test('should return 200 and layergroup with private tables', function(done) {
var privateTableTemplateName = 'private_table_template';
var privateTableTemplate = {
version: '0.0.1',
name: privateTableTemplateName,
auth: {
method: 'open'
},
layergroup: {
layers: [
{
type: 'cartodb',
options: {
sql: 'select * from test_table_private_1',
cartocss: '#layer { marker-fill: #cc3300; }',
cartocss_version: '2.3.0'
}
}
]
}
};
var layergroup = {
version: '1.3.0',
layers: [
{
type: 'named',
options: {
name: privateTableTemplateName
}
}
]
};
Step(
function createTemplate() {
templateMaps.addTemplate(username, privateTableTemplate, this);
},
function createLayergroup(err) {
if (err) {
throw err;
}
var next = this;
assert.response(server,
{
url: '/tiles/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 200
},
function(res, err) {
next(err, res);
}
);
},
function checkLayergroup(err, response) {
if (err) {
throw err;
}
var parsedBody = JSON.parse(response.body);
assert.ok(parsedBody.layergroupid);
assert.ok(parsedBody.last_updated);
return parsedBody.layergroupid;
},
function requestTile(err, layergroupId) {
if (err) {
throw err;
}
var next = this;
assert.response(server,
{
url: '/tiles/layergroup/' + layergroupId + '/0/0/0.png',
method: 'GET',
headers: {
host: 'localhost'
},
encoding: 'binary'
},
{
status: 200,
headers: {
'content-type': 'image/png'
}
},
function(res, err) {
next(err, res);
}
);
},
function handleTileResponse(err, res) {
if (err) {
throw err;
}
test_helper.checkCache(res);
return true;
},
function deleteTemplate(err) {
var next = this;
templateMaps.delTemplate(username, privateTableTemplate, function(/*delErr*/) {
// ignore deletion error
next(err);
});
},
function finish(err) {
done(err);
}
);
});
test('should return 403 when private table is accessed from non named layer', function(done) {
var layergroup = {
version: '1.3.0',
layers: [
{
type: 'cartodb',
options: {
sql: 'select * from test_table_private_1',
cartocss: '#layer { marker-fill: #cc3300; }',
cartocss_version: '2.3.0'
}
},
{
type: 'named',
options: {
name: templateName
}
}
]
};
Step(
function createLayergroup() {
var next = this;
assert.response(server,
{
url: '/tiles/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
{
status: 403
},
function(res, err) {
next(err, res);
}
);
},
function checkLayergroup(err, response) {
if (err) {
throw err;
}
var parsedBody = JSON.parse(response.body);
assert.ok(parsedBody.errors[0].match(/permission denied for relation test_table_private_1/));
return null;
},
function finish(err) {
done(err);
}
);
});
suiteTeardown(function(done) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false};
templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) {
if (err) {
return done(err);
}
templateMaps.delTemplate(username, tokenAuthTemplateName, function(err) {
if (err) {
return done(err);
}
templateMaps.delTemplate(username, templateName, function(err) {
return done(err);
});
});
});
});
});

View File

@@ -20,10 +20,7 @@ server.setMaxListeners(0);
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
suite('server:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@@ -47,8 +44,9 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
// A couple of styles to use during testing
var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}";
var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
suiteSetup(function(done){
global.environment.enabledFeatures = { cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue };
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
});
@@ -1152,10 +1150,12 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
assert(cc, 'Missing X-Cache-Channel');
var dbname = test_database;
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.api_key, qo.map_key);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)');
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
var jsonquery = cc.substring(dbname.length + 1);
var sentquery = JSON.parse(jsonquery);
assert.equal(sentquery.api_key, qo.map_key);
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)');
}
return null;
},
function finish(err) {
@@ -1164,6 +1164,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
);
});
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was a post if it's using the SQL API
test("passes hostname header to sqlapi", function(done){
var qo = {
sql: "SELECT * from gadm4",
@@ -1195,7 +1196,9 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
}
);
});
}
if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
test("requests to skip cache on sqlapi error", function(done){
var qo = {
sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator "
@@ -1228,6 +1231,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
}
);
});
}
// Zoom is a special variable
test("Specifying zoom level in CartoCSS does not need a 'zoom' variable in SQL output", function(done){

View File

@@ -27,9 +27,8 @@ server.setMaxListeners(0);
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
serverOptions.channelCache = {};
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@@ -37,6 +36,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
suiteSetup(function(done){
global.environment.enabledFeatures = { cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue };
sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
// TODO: check redis is clean ?
});
@@ -1923,6 +1923,182 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
);
});
test("can use an http layer", function(done) {
var username = 'localhost';
var httpTemplateName = 'acceptance_http';
var httpTemplate = {
version: '0.0.1',
name: httpTemplateName,
layergroup: {
version: '1.3.0',
layers: [
{
type: "http",
options: {
urlTemplate: "http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png",
subdomains: [
"a",
"b",
"c"
]
}
},
{
type: 'cartodb',
options: {
sql: "select * from test_table_private_1",
cartocss: '#layer { marker-fill:blue; }',
cartocss_version: '2.0.2',
interactivity: 'cartodb_id'
}
}
]
}
};
var template_params = {};
var errors = [];
var expectedTemplateId = username + '@' + httpTemplateName;
var layergroupid;
Step(
function createTemplate()
{
var next = this;
assert.response(
server,
{
url: '/tiles/template?api_key=1234',
method: 'POST',
headers: {
host: username,
'Content-Type': 'application/json'
},
data: JSON.stringify(httpTemplate)
},
{
status: 200
},
function(res, err) {
next(err, res);
}
);
},
function instantiateTemplate(err, res) {
if (err) {
throw err;
}
assert.deepEqual(JSON.parse(res.body), { template_id: expectedTemplateId });
var next = this;
assert.response(
server,
{
url: '/tiles/template/' + expectedTemplateId,
method: 'POST',
headers: {
host: username,
'Content-Type': 'application/json'
},
data: JSON.stringify(template_params)
},
{
status: 200
},
function(res) {
next(null, res);
}
);
},
function fetchTile(err, res) {
if (err) {
throw err;
}
var parsed = JSON.parse(res.body);
assert.ok(parsed.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from response body: " + res.body);
layergroupid = parsed.layergroupid;
var next = this;
assert.response(
server,
{
url: '/tiles/layergroup/' + layergroupid + '/all/0/0/0.png',
method: 'GET',
headers: {
host: username
},
encoding: 'binary'
},
{
status: 200
},
function(res) {
next(null, res);
}
);
},
function checkTile(err, res) {
if (err) {
throw err;
}
assert.equal(res.headers['content-type'], "image/png");
return null;
},
function deleteTemplate(err) {
if (err) {
throw err;
}
var next = this;
assert.response(
server,
{
url: '/tiles/template/' + expectedTemplateId + '?api_key=1234',
method: 'DELETE',
headers: {
host: username
}
},
{
status: 204
},
function(res, err) {
next(err, res);
}
);
},
function finish(err) {
if (err) {
errors.push(err);
}
redis_client.keys("map_*|localhost", function(err, keys) {
if ( err ) errors.push(err.message);
var todrop = _.map(keys, function(m) {
if ( m.match(/^map_(tpl|crt)|/) )
return m;
});
if ( todrop.length ) {
errors.push(new Error("Unexpected keys in redis: " + todrop));
redis_client.del(todrop, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) {
done(new Error(errors));
}
else done(null);
});
} else {
if ( errors.length ) {
done(new Error(errors));
}
else done(null);
}
});
}
);
});
suiteTeardown(function(done) {
// This test will add map_style records, like

View File

@@ -0,0 +1,298 @@
var test_helper = require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var TemplateMaps = require('../../lib/cartodb/template_maps.js');
var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter');
var Step = require('step');
var _ = require('underscore');
// configure redis pool instance to use in tests
var redisPool = RedisPool(global.environment.redis);
var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool }));
var templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
var wadusSql = 'select 1 wadusLayer, null::geometry the_geom_webmercator';
var wadusLayer = {
type: 'cartodb',
options: {
sql: wadusSql,
cartocss: '#layer { marker-fill: black; }',
cartocss_version: '2.3.0'
}
};
var wadusTemplateSql = 'select 1 wadusTemplateLayer, null::geometry the_geom_webmercator';
var wadusTemplateLayer = {
type: 'cartodb',
options: {
sql: wadusTemplateSql,
cartocss: '#layer { marker-fill: <%= color %>; }',
cartocss_version: '2.3.0'
}
};
var wadusMapnikSql = 'select 1 wadusMapnikLayer, null::geometry the_geom_webmercator';
var wadusMapnikLayer = {
type: 'mapnik',
options: {
sql: wadusMapnikSql,
cartocss: '#layer { polygon-fill: <%= polygon_color %>; }',
cartocss_version: '2.3.0'
}
};
var username = 'localhost';
var templateName = 'valid_template';
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
"placeholders": {
"color": {
"type": "css_color",
"default": "#cc3300"
}
},
layergroup: {
layers: [
wadusTemplateLayer
]
}
};
var multipleLayersTemplateName = 'multiple_valid_template';
var multipleLayersTemplate = {
version: '0.0.1',
name: multipleLayersTemplateName,
auth: {
method: 'token',
valid_tokens: ['valid1', 'valid2']
},
"placeholders": {
"polygon_color": {
"type": "css_color",
"default": "green"
},
"color": {
"type": "css_color",
"default": "red"
}
},
layergroup: {
layers: [
wadusMapnikLayer,
wadusTemplateLayer
]
}
};
suite('named_layers datasources', function() {
suiteSetup(function(done) {
templateMaps.addTemplate(username, template, function(err) {
if (err) {
return done(err);
}
templateMaps.addTemplate(username, multipleLayersTemplate, done);
});
});
function makeNamedMapLayerConfig(layers) {
return {
version: '1.3.0',
layers: layers
};
}
var simpleNamedLayer = {
type: 'named',
options: {
name: templateName
}
};
var multipleLayersNamedLayer = {
type: 'named',
options: {
name: multipleLayersTemplateName,
auth_tokens: ['valid2']
}
};
var testScenarios = [
{
desc: 'without datasource for non-named layers',
config: makeNamedMapLayerConfig([wadusLayer]),
test: function(err, layers, datasource, done) {
assert.ok(!err);
assert.equal(layers.length, 1);
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(0), undefined);
done();
}
},
{
desc: 'with datasource for the named layer but not for the normal',
config: makeNamedMapLayerConfig([wadusLayer, simpleNamedLayer]),
test: function(err, layers, datasource, done) {
assert.ok(!err);
assert.equal(layers.length, 2);
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(0), undefined);
assert.equal(layers[1].type, 'cartodb');
assert.equal(layers[1].options.sql, wadusTemplateSql);
var layerDatasource = datasource.getLayerDatasource(1);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
done();
}
},
{
desc: 'with datasource for the multiple layers in the named but not for the normal',
config: makeNamedMapLayerConfig([wadusLayer, multipleLayersNamedLayer]),
test: function(err, layers, datasource, done) {
assert.ok(!err);
assert.equal(layers.length, 3);
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(0), undefined);
assert.equal(layers[1].type, 'mapnik');
assert.equal(layers[1].options.sql, wadusMapnikSql);
var layerDatasource = datasource.getLayerDatasource(1);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[2].type, 'cartodb');
assert.equal(layers[2].options.sql, wadusTemplateSql);
layerDatasource = datasource.getLayerDatasource(2);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
done();
}
},
{
desc: 'all with datasource because all are named',
config: makeNamedMapLayerConfig([multipleLayersNamedLayer, simpleNamedLayer]),
test: function(err, layers, datasource, done) {
assert.ok(!err);
assert.equal(layers.length, 3);
assert.equal(layers[0].type, 'mapnik');
assert.equal(layers[0].options.sql, wadusMapnikSql);
var layerDatasource = datasource.getLayerDatasource(0);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[1].type, 'cartodb');
assert.equal(layers[1].options.sql, wadusTemplateSql);
layerDatasource = datasource.getLayerDatasource(1);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[2].type, 'cartodb');
assert.equal(layers[2].options.sql, wadusTemplateSql);
layerDatasource = datasource.getLayerDatasource(2);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
done();
}
},
{
desc: 'with a mix of datasource and no datasource depending if layers are named or not',
config: makeNamedMapLayerConfig([simpleNamedLayer, multipleLayersNamedLayer, wadusLayer, simpleNamedLayer, wadusLayer, multipleLayersNamedLayer]),
test: function(err, layers, datasource, done) {
assert.ok(!err);
assert.equal(layers.length, 8);
assert.equal(layers[0].type, 'cartodb');
assert.equal(layers[0].options.sql, wadusTemplateSql);
var layerDatasource = datasource.getLayerDatasource(0);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[1].type, 'mapnik');
assert.equal(layers[1].options.sql, wadusMapnikSql);
layerDatasource = datasource.getLayerDatasource(1);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[2].type, 'cartodb');
assert.equal(layers[2].options.sql, wadusTemplateSql);
layerDatasource = datasource.getLayerDatasource(2);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[3].type, 'cartodb');
assert.equal(layers[3].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(3), undefined);
assert.equal(layers[4].type, 'cartodb');
assert.equal(layers[4].options.sql, wadusTemplateSql);
layerDatasource = datasource.getLayerDatasource(4);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[5].type, 'cartodb');
assert.equal(layers[5].options.sql, wadusSql);
assert.equal(datasource.getLayerDatasource(5), undefined);
assert.equal(layers[6].type, 'mapnik');
assert.equal(layers[6].options.sql, wadusMapnikSql);
layerDatasource = datasource.getLayerDatasource(6);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
assert.equal(layers[7].type, 'cartodb');
assert.equal(layers[7].options.sql, wadusTemplateSql);
layerDatasource = datasource.getLayerDatasource(7);
assert.notEqual(layerDatasource, undefined);
assert.ok(layerDatasource.user);
done();
}
}
];
testScenarios.forEach(function(testScenario) {
test('should return a list of layers ' + testScenario.desc, function(done) {
mapConfigNamedLayersAdapter.getLayers(username, testScenario.config.layers, pgConnection, function(err, layers, datasource) {
testScenario.test(err, layers, datasource, done);
});
});
});
suiteTeardown(function(done) {
templateMaps.delTemplate(username, templateName, function(err) {
if (err) {
return done(err);
}
templateMaps.delTemplate(username, multipleLayersTemplateName, done);
});
});
});

View File

@@ -0,0 +1,312 @@
var testHelper = require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var TemplateMaps = require('../../lib/cartodb/template_maps.js');
var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter');
var Step = require('step');
var _ = require('underscore');
suite('mapconfig_named_layers_adapter', function() {
// configure redis pool instance to use in tests
var redisPool = RedisPool(global.environment.redis);
var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool }));
var templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
var wadusLayer = {
type: 'cartodb',
options: {
sql: 'select 1 cartodb_id, null::geometry the_geom_webmercator',
cartocss: '#layer { marker-fill: <%= color %>; }',
cartocss_version: '2.3.0'
}
};
var wadusMapnikLayer = {
type: 'mapnik',
options: {
sql: 'select 1 cartodb_id, null::geometry the_geom_webmercator',
cartocss: '#layer { polygon-fill: <%= polygon_color %>; }',
cartocss_version: '2.3.0'
}
};
var username = 'localhost';
var templateName = 'valid_template';
var template = {
version: '0.0.1',
name: templateName,
auth: {
method: 'open'
},
"placeholders": {
"color": {
"type": "css_color",
"default": "#cc3300"
}
},
layergroup: {
layers: [
wadusLayer
]
}
};
var tokenAuthTemplateName = 'auth_valid_template';
var tokenAuthTemplate = {
version: '0.0.1',
name: tokenAuthTemplateName,
auth: {
method: 'token',
valid_tokens: ['valid1', 'valid2']
},
layergroup: {
layers: [
wadusLayer
]
}
};
var multipleLayersTemplateName = 'multiple_valid_template';
var multipleLayersTemplate = {
version: '0.0.1',
name: multipleLayersTemplateName,
auth: {
method: 'token',
valid_tokens: ['valid1', 'valid2']
},
"placeholders": {
"polygon_color": {
"type": "css_color",
"default": "green"
},
"color": {
"type": "css_color",
"default": "red"
}
},
layergroup: {
layers: [
wadusMapnikLayer,
wadusLayer
]
}
};
var namedMapLayer = {
type: 'named',
options: {
name: templateName,
config: {},
auth_tokens: []
}
};
var nestedNamedMapTemplateName = 'nested_template';
var nestedNamedMapTemplate = {
version: '0.0.1',
name: nestedNamedMapTemplateName,
auth: {
method: 'open'
},
layergroup: {
layers: [
namedMapLayer
]
}
};
function makeNamedMapLayerConfig(options) {
return {
version: '1.3.0',
layers: [
{
type: 'named',
options: options
}
]
};
}
suiteSetup(function(done) {
templateMaps.addTemplate(username, template, done);
});
test('should fail for named map layer with missing name', function(done) {
var missingNamedMapLayerConfig = makeNamedMapLayerConfig({
config: {}
});
mapConfigNamedLayersAdapter.getLayers(username, missingNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(err);
assert.ok(!layers);
assert.ok(!datasource);
assert.equal(err.message, 'Missing Named Map `name` in layer options');
done();
});
});
test('should fail for non-existing template name', function(done) {
var missingTemplateName = 'wadus';
var nonExistentNamedMapLayerConfig = makeNamedMapLayerConfig({
name: missingTemplateName
});
mapConfigNamedLayersAdapter.getLayers(username, nonExistentNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(err);
assert.ok(!layers);
assert.ok(!datasource);
assert.equal(err.message, "Template '" + missingTemplateName + "' of user '" + username + "' not found");
done();
});
});
test('should fail if not properly authorized', function(done) {
templateMaps.addTemplate(username, tokenAuthTemplate, function(err) {
if (err) {
return done(err);
}
var nonAuthTokensNamedMapLayerConfig = makeNamedMapLayerConfig({
name: tokenAuthTemplateName
});
mapConfigNamedLayersAdapter.getLayers(username, nonAuthTokensNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(err);
assert.ok(!layers);
assert.ok(!datasource);
assert.equal(err.message, "Unauthorized '" + tokenAuthTemplateName + "' template instantiation");
templateMaps.delTemplate(username, tokenAuthTemplateName, done);
});
});
});
test('should fail for nested named map layers', function(done) {
templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) {
if (err) {
return done(err);
}
var nestedNamedMapLayerConfig = makeNamedMapLayerConfig({
name: nestedNamedMapTemplateName
});
mapConfigNamedLayersAdapter.getLayers(username, nestedNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(err);
assert.ok(!layers);
assert.ok(!datasource);
assert.equal(err.message, 'Nested named layers are not allowed');
templateMaps.delTemplate(username, nestedNamedMapTemplateName, done);
});
});
});
test('should return an expanded list of layers for a named map layer', function(done) {
var validNamedMapMapLayerConfig = makeNamedMapLayerConfig({
name: templateName
});
mapConfigNamedLayersAdapter.getLayers(username, validNamedMapMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(!err);
assert.ok(layers.length, 1);
assert.ok(layers[0].type, 'cartodb');
assert.notEqual(datasource.getLayerDatasource(0), undefined);
done();
});
});
test('should return on auth=token with valid tokens provided', function(done) {
templateMaps.addTemplate(username, tokenAuthTemplate, function(err) {
if (err) {
return done(err);
}
var validAuthTokensNamedMapLayerConfig = makeNamedMapLayerConfig({
name: tokenAuthTemplateName,
auth_tokens: ['valid1']
});
mapConfigNamedLayersAdapter.getLayers(username, validAuthTokensNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(!err);
assert.equal(layers.length, 1);
assert.notEqual(datasource.getLayerDatasource(0), undefined);
templateMaps.delTemplate(username, tokenAuthTemplateName, done);
});
});
});
test('should return an expanded list of layers for a named map layer, multiple layers version', function(done) {
templateMaps.addTemplate(username, multipleLayersTemplate, function(err) {
if (err) {
return done(err);
}
var multipleLayersNamedMapLayerConfig = makeNamedMapLayerConfig({
name: multipleLayersTemplateName,
auth_tokens: ['valid2']
});
mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(!err);
assert.equal(layers.length, 2);
assert.equal(layers[0].type, 'mapnik');
assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: green; }');
assert.notEqual(datasource.getLayerDatasource(0), undefined);
assert.equal(layers[1].type, 'cartodb');
assert.equal(layers[1].options.cartocss, '#layer { marker-fill: red; }');
assert.notEqual(datasource.getLayerDatasource(1), undefined);
templateMaps.delTemplate(username, multipleLayersTemplateName, done);
});
});
});
test('should replace template params with the given config', function(done) {
templateMaps.addTemplate(username, multipleLayersTemplate, function(err) {
if (err) {
return done(err);
}
var color = '#cc3300',
polygonColor = '#ff9900';
var multipleLayersNamedMapLayerConfig = makeNamedMapLayerConfig({
name: multipleLayersTemplateName,
config: {
polygon_color: polygonColor,
color: color
},
auth_tokens: ['valid2']
});
mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) {
assert.ok(!err);
assert.equal(layers.length, 2);
assert.equal(layers[0].type, 'mapnik');
assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: ' + polygonColor + '; }');
assert.notEqual(datasource.getLayerDatasource(0), undefined);
assert.equal(layers[1].type, 'cartodb');
assert.equal(layers[1].options.cartocss, '#layer { marker-fill: ' + color + '; }');
assert.notEqual(datasource.getLayerDatasource(1), undefined);
templateMaps.delTemplate(username, multipleLayersTemplateName, done);
});
});
});
suiteTeardown(function(done) {
templateMaps.delTemplate(username, templateName, done);
});
});

View File

@@ -246,9 +246,10 @@ assert.response = function(server, req, res, msg){
assert.equal(
response.statusCode,
status,
msg + 'Invalid response status code.\n'
msg + colorize('Invalid response status code.\n'
+ ' Expected: [green]{' + status + '}\n'
+ ' Got: [red]{' + response.statusCode + '}'
+ ' Got: [red]{' + response.statusCode + '}\n'
+ ' Response body: ' + response.body)
);
}
@@ -280,3 +281,16 @@ assert.response = function(server, req, res, msg){
}
};
/**
* Colorize the given string using ansi-escape sequences.
* Disabled when --boring is set.
*
* @param {String} str
* @return {String}
*/
function colorize(str) {
var colors = { bold: 1, red: 31, green: 32, yellow: 33 };
return str.replace(/\[(\w+)\]\{([^]*?)\}/g, function(_, color, str) {
return '\x1B[' + colors[color] + 'm' + str + '\x1B[0m';
});
}

View File

@@ -78,6 +78,12 @@ if test x"$PREPARE_PGSQL" = xyes; then
sed "s/:TESTPASS/${TESTPASS}/" |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB}
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
fi
if test x"$PREPARE_REDIS" = xyes; then

View File

@@ -0,0 +1,14 @@
-- Return an array of statements found in the given query text
--
-- Regexp curtesy of Hubert Lubaczewski (depesz)
-- Implemented in plpython for performance reasons
--
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
RETURNS SETOF TEXT AS $$
import re
pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL )
for match in pat.findall(query):
cleaned = match[0].strip()
if ( cleaned ):
yield cleaned
$$ language 'plpythonu' IMMUTABLE STRICT;

View File

@@ -0,0 +1,67 @@
-- Return an array of table names scanned by a given query
--
-- Requires PostgreSQL 9.x+
--
CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
RETURNS name[]
AS $$
DECLARE
exp XML;
tables NAME[];
rec RECORD;
rec2 RECORD;
BEGIN
tables := '{}';
FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
IF NOT ( rec.q ilike 'select %' or rec.q ilike 'with %' ) THEN
--RAISE WARNING 'Skipping %', rec.q;
CONTINUE;
END IF;
BEGIN
EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
EXCEPTION WHEN others THEN
-- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
-- the affected table ?
RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
RAISE EXCEPTION '%', SQLERRM;
CONTINUE;
END;
-- Now need to extract all values of <Relation-Name>
-- RAISE DEBUG 'Explain: %', exp;
FOR rec2 IN WITH
inp AS (
SELECT
xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
)
SELECT unnest(x)::name as p, unnest(s)::name as sc from inp
LOOP
-- RAISE DEBUG 'tab: %', rec2.p;
-- RAISE DEBUG 'sc: %', rec2.sc;
tables := array_append(tables, (rec2.sc || '.' || rec2.p)::name);
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
-- Remove duplicates and sort by name
IF array_upper(tables, 1) > 0 THEN
WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
SELECT array_agg(p) from dist into tables;
END IF;
--RAISE DEBUG 'Tables: %', tables;
return tables;
END
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

View File

@@ -177,3 +177,15 @@ CREATE TABLE test_table_private_1 (
INSERT INTO test_table_private_1 SELECT * from test_table;
GRANT ALL ON TABLE test_table_private_1 TO :TESTUSER;
CREATE TABLE IF NOT EXISTS
CDB_TableMetadata (
tabname regclass not null primary key,
updated_at timestamp with time zone not null default now()
);
INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('test_table'::regclass, '2009-02-13T23:31:30.123Z');
INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('test_table_private_1'::regclass, '2009-02-13T23:31:30.123Z');
-- GRANT SELECT ON CDB_TableMetadata TO :PUBLICUSER;
GRANT SELECT ON CDB_TableMetadata TO :TESTUSER;

View File

@@ -92,4 +92,14 @@ suite('template_maps_auth', function() {
})
});
test("auth as 'open' string is authorized", function(done) {
var template = {
name: 'wadus_template',
auth: 'open'
};
assert.ok(templateMaps.isAuthorized(template));
done();
});
});

View File

@@ -1,59 +0,0 @@
#!/usr/bin/env node
var path = require('path');
var request = require('request');
function usage(me, exitcode) {
console.log("Usage: " + me + " [--env <environment>] <username> <tablename>");
process.exit(exitcode);
}
var node_path = process.argv.shift();
var script_path = process.argv.shift();
var basedir = path.dirname(script_path);
var me = path.basename(script_path);
var ENV = 'development.js';
var username, table;
var arg;
while ( arg = process.argv.shift() ) {
if ( arg == '--env' ) {
ENV = process.argv.shift();
}
else if ( ! username ) {
username = arg;
}
else if ( ! table ) {
table = arg;
}
else {
console.warn("Unused parameter " + arg);
}
}
if ( ! table ) {
usage(me, 1);
}
global.environment = require('../config/environments/' + ENV);
// _after_ setting global.environment
var serverOptions = require('../lib/cartodb/server_options');
var host = global.environment.host;
var port = global.environment.port;
var re = ''+serverOptions.re_userFromHost;
var hostname = re.replace(/^\/\^/, '')
.replace(/\/$$/, '')
.replace(/\\/g,'')
.replace(/\([^)]*\)/,username)
;
//console.log("re: " + re);
//console.log("hostname: " + hostname);
var url = 'http://' + host + ':' + port + '/tiles/' + table + '/flush_cache';
request.del({ url: url, headers: { host: hostname } },
function(err, res, body) {
if ( err ) throw err;
console.log(res.body);
});