Compare commits

...

200 Commits

Author SHA1 Message Date
Mario de Frutos
6177f0ac76 Remove info trace not needed in metrics 2016-11-07 11:02:09 +01:00
Mario de Frutos
2c49f09aad Merge pull request #301 from CartoDB/development
Release 0.17.0
2016-11-07 10:48:15 +01:00
Mario de Frutos
22fdbd0f4e Added schema option to define in the integration tests 2016-11-07 10:33:15 +01:00
Mario de Frutos
352d4217bc Fixed typo in the config cache for observatory snapshot functions 2016-11-07 10:31:53 +01:00
Mario de Frutos
2fd560f3cf Release 0.17.0 artifact 2016-11-04 13:41:12 +01:00
Mario de Frutos
a4858bbf3c Merge pull request #299 from CartoDB/add_metrics
Data services metrics enhanced
2016-11-04 13:40:31 +01:00
Mario de Frutos
88ed777785 Added plpy_mock and check multiple instances for metricsdatagatherer 2016-11-04 13:39:11 +01:00
Mario de Frutos
6f0db20163 Added here integration test for geocoder street 2016-11-04 13:39:11 +01:00
Mario de Frutos
73aa26590b Make the data gather singleton support multiple instances
Due to we use pgbouncer, we could share the same instance of the metrics
gatherer singleton in multiples requests. Added support to have
multiple instances identified by transaction id
2016-11-04 13:39:11 +01:00
Mario de Frutos
c571edb21e Split response and function data, added uuid and type of element 2016-11-04 13:39:11 +01:00
Mario de Frutos
e878031744 Added url to the response data 2016-11-04 13:39:11 +01:00
Mario de Frutos
49fd51f6da Added logger to metrics context manager 2016-11-04 13:39:10 +01:00
Mario de Frutos
708bee2912 Added metrics context manager to public functions 2016-11-04 13:39:10 +01:00
Mario de Frutos
cf3c6f2ce5 New metrics logger system
- It's not mandatory to have the log_path defined for all the services
- Now we add metrics to a MetricsGatherer using the add method in order
  to gather data from all the function scope, not just in one defined
  point
- There is a `metrics` context manager to wrap the function block. This
  context manager get the execution time for the block among other
  things
2016-11-02 15:26:39 +01:00
Mario de Frutos
93579532e3 Move old versions 2016-10-28 10:42:25 +02:00
Mario de Frutos
0533018326 Merge pull request #298 from CartoDB/development
Release Python 0.9.4
2016-10-27 16:21:16 +02:00
Mario de Frutos
e380d51bec Merge pull request #296 from CartoDB/add_timeouts_requests
Release python 0.9.4
2016-10-27 16:19:12 +02:00
Mario de Frutos
6058960ec5 Added timeout for all the third-party connections 2016-10-27 16:17:37 +02:00
Mario de Frutos
336d8be977 Hotfix: Typo in status code check 2016-10-26 17:49:07 +02:00
Mario de Frutos
75557837b0 Merge pull request #295 from CartoDB/development
Release python library 0.9.3
2016-10-26 16:58:39 +02:00
Mario de Frutos
e7c35457e1 Merge pull request #294 from CartoDB/293_matrix_without_json
Fix empty response from matrix
2016-10-26 16:41:17 +02:00
Mario de Frutos
80963e2589 504 errors return empty data instead of raise exception
Due to some problems in Mapzen, we're receiving 504 errors from their
servers. To mitigate this problem, instead of raise an exception, we're
going to return empty data for that point
2016-10-26 16:39:35 +02:00
Mario de Frutos
19d6cacdb3 Fix empty response from matrix
https://github.com/CartoDB/dataservices-api/issues/293
2016-10-26 16:00:50 +02:00
Rafa de la Torre
0d22942a72 Update version of python lib server package to 0.9.2 2016-10-21 17:38:32 +02:00
Rafa de la Torre
e8122c6728 Merge pull request #291 from CartoDB/281-quota-mapzen-routing
281 quota mapzen routing
2016-10-21 17:32:39 +02:00
Rafa de la Torre
d4ac2eb5e6 Tests for ServicesRedisConfig (mapzen quota) #281 2016-10-21 15:40:22 +02:00
Rafa de la Torre
db80d389e0 Do not require org geocoder and isolines quota to be present 2016-10-21 15:38:58 +02:00
Rafa de la Torre
8e02c64aeb Tests for UserMetricsService, mapzen routing #281 2016-10-21 15:21:03 +02:00
Rafa de la Torre
cc2ab1bc0c Fix test: make it independent of current date #281 2016-10-21 15:08:17 +02:00
Mario de Frutos
948463f836 Merge pull request #292 from CartoDB/docs-fixed-broken-hyperlink
fixed broken hyperlink
2016-10-20 19:17:16 +02:00
Rafa de la Torre
e1a7d1751c Fix corner case: used_quota == monthly_quota #281 2016-10-20 15:36:48 +02:00
Rafa de la Torre
0c49107f96 Some tests for the QuotaChecker #281 2016-10-20 15:31:21 +02:00
csobier
05dc69af34 fixed broken hyperlink
Fixed one broken hyperlink that broken when the rebranding baseurl was applied.
2016-10-19 15:03:57 -04:00
Rafa de la Torre
247034c21e Some tests for RoutingConfig #281 2016-10-19 18:40:17 +02:00
Rafa de la Torre
2b1b1c981f Add routing to UserMetricsService.used_quota 2016-10-19 17:41:43 +02:00
Rafa de la Torre
aaff5564ec Use soft_limit for __check_routing_quota #281 2016-10-18 17:44:56 +02:00
Rafa de la Torre
72998c324a Simplify the code for regularity's sake #281 2016-10-18 17:40:50 +02:00
Rafa de la Torre
bbd9b6b98e Add soft limit to config object #281 2016-10-18 17:35:36 +02:00
Rafa de la Torre
27be704bd6 Changes needed to read quota from redis #281
Use per-user/org quota from redis, if present. Otherwise default to the
quota stored in server database (as it did to date).
2016-10-18 17:26:45 +02:00
csobier
03f4a1f4f7 Merge pull request #290 from CartoDB/doc_update
Docs issue #1115
2016-10-14 10:48:08 -04:00
csobier
91131488c5 Docs issue #1115
Updated description for IP address
2016-10-14 16:45:23 +02:00
csobier
7d137f3efc Merge pull request #289 from CartoDB/docs-1115-ip-address
Docs issue #1115
2016-10-13 08:54:13 -04:00
csobier
93a5de5f20 Docs issue #1115
Updated description for IP address
2016-10-13 08:38:03 -04:00
Rafa de la Torre
fc35aac639 Merge pull request #288 from CartoDB/development
[server] Refactor of configurations for mapzen geocoder entry point
2016-10-06 15:18:04 +02:00
Rafa de la Torre
98d533b707 Bump version of python pip package to 0.9.1 2016-10-06 15:05:00 +02:00
Rafa de la Torre
6d0ad85d48 Fix for crash when sentinel_master_id is absent in config 2016-10-06 15:04:25 +02:00
Rafa de la Torre
00e6cace76 Update version of the python pip package 2016-10-06 11:06:28 +02:00
Rafa de la Torre
e9ad35ba1d Merge pull request #284 from CartoDB/redis-refactor-take2
Redis refactor: Take 2
2016-10-05 18:32:08 +02:00
Rafa de la Torre
dcb3935021 Upgrade/downgrade files for server extension 2016-10-05 18:28:24 +02:00
Rafa de la Torre
cded6c2f08 Test for NullConfigStorage 2016-10-04 15:20:41 +02:00
Rafa de la Torre
e1b357137a Rename s/storage/backend for consistency 2016-10-04 14:59:57 +02:00
Rafa de la Torre
3844cfc226 Tests for environment module 2016-10-04 14:56:06 +02:00
Rafa de la Torre
2a1276f4f1 Add test for redis_config module 2016-10-04 12:53:52 +02:00
Rafa de la Torre
35da7e48fd Implement a ServerConfigBackendFactory
mostly to keep layers separated.
2016-10-03 17:42:19 +02:00
Rafa de la Torre
12aebb7eee Rename ConfigStorageInterface to ConfigBackendInterface 2016-10-03 17:35:33 +02:00
Rafa de la Torre
0d87a95270 Inject the environment into the LoggerConfigBuilder 2016-10-03 17:06:57 +02:00
Rafa de la Torre
18e1a5c7c9 Remove the class User and its tests
as it is not used.
2016-10-03 16:54:40 +02:00
Rafa de la Torre
fcca5da302 Add a RedisMetricsConnectionFactory class 2016-10-03 16:47:45 +02:00
Rafa de la Torre
1aec541906 Add ServerEnvironment and ServerEnvironmentBuilder 2016-10-03 16:21:28 +02:00
Rafa de la Torre
9e98e0794d Add backends for user and org configs 2016-10-03 16:05:08 +02:00
Rafa de la Torre
8fbb41742c Rename redis conf factories to builders 2016-10-03 15:25:01 +02:00
Rafa de la Torre
275a6dc27f Move redis_config related classes to separate file 2016-10-03 13:33:46 +02:00
Rafa de la Torre
d522083d5c Rename redis_connection_config test file as well 2016-10-03 13:22:47 +02:00
Rafa de la Torre
073163eb1a Rename redis_config.py to redis_connection_config.py 2016-10-03 13:20:30 +02:00
Rafa de la Torre
0c62c4bada Move the NullConfigStorage to a separate file 2016-10-03 13:17:04 +02:00
Rafa de la Torre
3361960cfc Move InMemoryConfigStorage to a separate file 2016-10-03 13:12:49 +02:00
Rafa de la Torre
86ab3abc53 Rename mapzen_geocoder module to mapzen_geocoder_config 2016-10-03 12:58:15 +02:00
Rafa de la Torre
b1f3405cd0 Rename variable s/user_geocoder_config/mapzen_geocoder_config 2016-10-03 12:52:18 +02:00
Rafa de la Torre
fb812ee15e Add tests for redis connection configs 2016-10-03 12:33:38 +02:00
Rafa de la Torre
c1dd410201 Make RedisConnectionConfigBuilder abstract
and also use custom ConfigException instead of generic one.
2016-10-03 12:21:45 +02:00
Rafa de la Torre
34ddd28e6b Move ConfigException to a separate file 2016-10-03 12:20:17 +02:00
Rafa de la Torre
d85bc65bf8 Ignore coverage files 2016-10-03 12:18:47 +02:00
Carla
443fe88d5a Add newline to end fle 2016-09-30 13:08:27 +02:00
Carla Iriberri
6c61626214 Remove RedisMock TODO 2016-09-30 12:29:00 +02:00
Carla Iriberri
74d2fba763 Add Redis Mock -- tested and working 2016-09-30 12:28:27 +02:00
Rafa de la Torre
e24819f193 Take the environment into account 2016-09-30 11:29:44 +02:00
Rafa de la Torre
1e6ee8d5c1 Add an Environment class 2016-09-30 11:21:35 +02:00
Rafa de la Torre
3a6cc4c364 Add mapzen config and integrate into legacy code 2016-09-29 18:41:16 +02:00
Rafa de la Torre
8ad2434b1d Add instantiation of configs to mapzen geocoder 2016-09-29 17:47:36 +02:00
Rafa de la Torre
0b7b44d8a5 Add a couple of factories to abstract user/org configs 2016-09-29 17:46:17 +02:00
Rafa de la Torre
02a2619b45 Fix instantiation of redis metadata connection 2016-09-29 17:19:18 +02:00
Rafa de la Torre
4b4a02905c Implement the RedisConfigStorage 2016-09-29 17:18:45 +02:00
Rafa de la Torre
1f3a655ae5 Add all the code to instantiate a user_config_storage (WIP) 2016-09-29 16:57:31 +02:00
Rafa de la Torre
9d60fde0b8 A stubb for RedisConfigStorage (WIP) 2016-09-29 13:07:40 +02:00
Rafa de la Torre
efdc151282 Add things to get logger config from storage 2016-09-28 18:22:51 +02:00
Rafa de la Torre
fd2cc21942 Add server config storage classes 2016-09-28 17:32:26 +02:00
Rafa de la Torre
18f05fbd4f User class with tests 2016-09-28 16:30:37 +02:00
Carla
d2f4586bae Merge pull request #283 from CartoDB/master
Getting master back to develop: Update NEWS.md for new Python release
2016-09-28 15:53:59 +02:00
Carla
54eb279ae8 Update NEWS.md 2016-09-28 15:40:25 +02:00
Carla
85d6c2a54e Update NEWS.md 2016-09-28 15:38:48 +02:00
Carla
cad2051efe Merge pull request #282 from CartoDB/development
Release Python server package version 0.8.1
2016-09-28 15:37:58 +02:00
Carla Iriberri
96a93e3c56 Python package version 0.8.1 2016-09-28 15:09:29 +02:00
Rafa de la Torre
facda9e8be Merge pull request #275 from CartoDB/fix_qps_retry
Fixed QPS retry decorator
2016-09-28 13:47:45 +02:00
Rafa de la Torre
64fc18b9e0 Fix and improve test speed 2016-09-28 13:43:50 +02:00
Mario de Frutos
9381d5644b Fixed QPS retry decorator 2016-09-16 13:46:02 +02:00
Carla
9f55f2ee3b Update README.md 2016-09-09 11:44:57 +02:00
Carla
1087c1266b Merge pull request #272 from CartoDB/development
Release client 0.11.1
2016-09-08 12:42:26 +02:00
Carla
d5a296a30c Update NEWS.md 2016-09-08 10:39:48 +02:00
Carla Iriberri
f8caf4314d Release client 0.11.1 2016-09-07 17:52:53 +02:00
Carla
d7910fbbf1 Merge pull request #271 from CartoDB/new_framework_obs_compat
Use real function name for compatibility with observatory
2016-09-07 17:45:32 +02:00
Carla
d47049c813 Escape characters in example 2016-09-07 16:01:45 +02:00
Carla
cc8f93c535 Use real function name for compatibility 2016-09-07 15:40:20 +02:00
Carla
3f9441de7e Merge pull request #269 from CartoDB/development
New release client 0.11.0 and server  0.15.2
2016-09-02 12:20:24 +02:00
Carla
fe41359a1f Merge pull request #268 from CartoDB/release_obs_measure
Release DST OBS_Measure
2016-09-02 11:31:06 +02:00
Carla Iriberri
46a934b178 Client 0.11.0 2016-09-01 18:07:44 +02:00
Carla Iriberri
184358bdec Server 0.15.1 2016-09-01 18:06:56 +02:00
Carla Iriberri
a6d546f2ee Bump version client 0.11.0 2016-09-01 17:42:57 +02:00
Carla
fc99f7aba9 Merge pull request #267 from CartoDB/augment_revamp_analysis
Table-level OBS_GetMeasue revamp
2016-09-01 17:21:56 +02:00
Carla Iriberri
e959873b32 Check server return 2016-09-01 16:37:54 +02:00
Mario de Frutos
a98093540d Fix table level tests for server 2016-09-01 15:53:19 +02:00
Carla Iriberri
78add220cd New tests for restructuration - client 2016-09-01 15:46:23 +02:00
Carla Iriberri
cf2f86136b Update signatures in functions 2016-09-01 15:37:08 +02:00
Carla Iriberri
fb183b07ee Update grants for new signatures 2016-09-01 15:34:57 +02:00
Carla Iriberri
5ab727bcb6 Remove warnign and fix server_conn_str path 2016-09-01 14:36:12 +02:00
Carla Iriberri
1e9b551160 Add populate table function and several fixes 2016-08-30 16:38:44 +02:00
Mario de Frutos
699dc9bf0e Merge pull request #266 from CartoDB/development
Release 0.15.0
2016-08-30 12:16:20 +02:00
Carla Iriberri
fc291a7c63 First steps 2016-08-29 18:07:58 +02:00
Mario de Frutos
d73af32c2c Version 0.15.0 relase artifacts 2016-08-29 17:12:00 +02:00
Mario de Frutos
7ea88fa051 Merge pull request #265 from CartoDB/221_mapzen_for_city_names
Mapzen to geocode city names
2016-08-29 14:41:48 +02:00
Mario de Frutos
d2ca40cf38 Removed observatory unnecessary dependency in tests 2016-08-29 14:38:38 +02:00
Mario de Frutos
18ae2525b6 Add search type filter for Mapzen geocoder 2016-08-29 14:32:49 +02:00
Mario de Frutos
06462fdf7a Control file for version 0.15 and empty upgrade/downgrade 2016-08-29 14:04:08 +02:00
Mario de Frutos
71d5ce951a Use mapzen as first option for the namedplace geocoding 2016-08-26 16:39:25 +02:00
Mario de Frutos
d602c42559 Strip spaces in the mapzen geocode parameters 2016-08-19 13:32:11 +02:00
Mario de Frutos
d88763f889 Merge pull request #262 from CartoDB/development
Release python 0.7.4.2
2016-08-19 12:04:34 +02:00
Mario de Frutos
c86496478a Updated NEWS.md 2016-08-19 12:00:35 +02:00
Mario de Frutos
1851b2b716 Merge pull request #256 from CartoDB/add_more_logs_for_mapzen_services
Add more logs for mapzen services
2016-08-19 11:51:28 +02:00
Mario de Frutos
3cc4e06420 Add more logs for mapzen services 2016-08-19 11:44:59 +02:00
Carla
a1f7dccb07 Update NEWS.md 2016-08-17 15:41:08 +02:00
Carla
0c0f231db3 Update NEWS.md 2016-08-17 12:33:38 +02:00
Carla
38439065fd Update NEWS.md 2016-08-17 12:33:17 +02:00
Carla
13b10063e2 Update NEWS.md 2016-08-17 12:24:30 +02:00
Carla
a2aeac9c3b Update NEWS.md 2016-08-17 12:08:46 +02:00
Carla
46c96e02b4 Add semicolons at the end of set conf 2016-08-16 16:07:49 +02:00
Mario de Frutos
2117b931bf Merge pull request #255 from CartoDB/development
Release 0.14.2 for server and 0.10.2 for client
2016-08-11 10:53:13 +02:00
Mario de Frutos
a5696f84b1 Release 0.10.2 client artifacts 2016-08-10 18:22:03 +02:00
Mario de Frutos
8b4b04a669 Release 0.14.2-server artifacts 2016-08-10 17:33:37 +02:00
Mario de Frutos
a1fc7ba3fd Merge pull request #237 from CartoDB/obs-null-default
Replace all 'area' defaults for "normalize" with NULL.
2016-08-10 17:13:45 +02:00
Carla Iriberri
19b3e05661 Remove default intersect for OBS functions 2016-08-10 16:25:26 +02:00
Carla Iriberri
1107371f6f Edit tests accordingly to default values removal 2016-08-10 16:25:26 +02:00
Carla Iriberri
fd80f3b99f Remove all default values, including intersects and geomids 2016-08-10 16:25:26 +02:00
John Krauss
d63f2ba4d5 Replace all 'area' defaults for with NULL. Fixes #235 2016-08-10 16:25:26 +02:00
Mario de Frutos
0fb3cdd8b1 JSON response is not always present, better show the reponse object complete 2016-08-05 11:35:16 +02:00
Mario de Frutos
0ef333b274 Merge pull request #252 from CartoDB/development
Release 0.14.1
2016-08-05 11:18:29 +02:00
Mario de Frutos
6f68014eab Release 0.14.1 artifact 2016-08-05 11:14:10 +02:00
Mario de Frutos
91b35e04a6 Merge pull request #250 from CartoDB/fix_logger_missing_method
Change notice for info in the logger method
2016-08-05 11:12:59 +02:00
Mario de Frutos
7f2f6e7d99 Change notice for info in the logger method 2016-08-04 17:54:57 +02:00
Mario de Frutos
5f8416d166 Add logger config function with proper permission to get logger config 2016-08-03 17:37:40 +02:00
Mario de Frutos
02a8825c49 Merge pull request #249 from CartoDB/development
Release 0.14
2016-08-03 16:57:03 +02:00
Mario de Frutos
7cca8e95c3 Remove testing exception 2016-08-03 14:45:57 +02:00
Mario de Frutos
f97294cfc1 Release 0.14 artifacts 2016-08-03 12:19:49 +02:00
Mario de Frutos
fcee6f440c Merge pull request #248 from CartoDB/rollbar_integration
Improve logger capability
2016-08-03 12:17:26 +02:00
Mario de Frutos
a33999ed04 Dont re-raise the exception to avoid leak sensitive information 2016-08-03 12:08:23 +02:00
Mario de Frutos
9186eb5d5b Log part decoupled from the main config class 2016-08-02 18:40:01 +02:00
Mario de Frutos
ed5cf25e9c Added new logger to all the functions 2016-08-02 16:44:03 +02:00
Mario de Frutos
5d5e8d6f9a Added logger with rollbar integration 2016-08-01 18:03:42 +02:00
Mario de Frutos
993059eafb Added server config with environment data 2016-08-01 14:00:32 +02:00
Mario de Frutos
f9f39dcf9c Refactor metrics logger 2016-08-01 13:17:06 +02:00
Mario de Frutos
e456ff9170 Merge pull request #247 from CartoDB/development
Remove the gateway error from QPS for Mapzen provider
2016-08-01 11:25:06 +02:00
Mario de Frutos
4d47b905e8 Revert include the gateway error in QPS 2016-08-01 11:23:06 +02:00
Carla
0e556ee9ac Update README.md 2016-07-29 17:52:18 +02:00
Carla
70f290cdcf Add user conf in cdb conf 2016-07-29 17:51:38 +02:00
Mario de Frutos
2417441ef4 Merge pull request #246 from CartoDB/development
Release 0.13.3.1
2016-07-29 17:05:57 +02:00
Mario de Frutos
83493e4dc6 Release 0.13.3.1 artifact 2016-07-29 17:04:47 +02:00
Mario de Frutos
9c982c3a97 Merge pull request #244 from CartoDB/fix_isolines_one_row_limit
Remove 1 row limit for isolines functions
2016-07-29 17:04:19 +02:00
Mario de Frutos
6349be476d Remove 1 row limit for isolines functions 2016-07-29 17:00:08 +02:00
Mario de Frutos
073be9dae4 Merge pull request #243 from CartoDB/development
Added QPS for mapzen isolines
2016-07-29 16:28:13 +02:00
Mario de Frutos
4fdca01a73 Merge pull request #242 from CartoDB/add_qps_to_isolines
Add qps to isolines
2016-07-29 16:25:38 +02:00
Mario de Frutos
c23a57c03f Add QPS retry for mapzen isolines and add the 504 error to retry 2016-07-29 16:22:34 +02:00
Carla
ca7d0ba239 Merge pull request #240 from CartoDB/development
Release client 0.10.1
2016-07-27 10:02:05 +02:00
Carla
d1b17a6603 Merge pull request #239 from CartoDB/client_10_1
Release client 0.10.1
2016-07-26 18:31:58 +02:00
Carla Iriberri
a68ab22c89 Release client 0.10.1 2016-07-26 17:53:51 +02:00
Carla
e3b053d4cd Merge pull request #230 from CartoDB/add_index_augment_table_tmp
Add index to temporary table in AugmentTable
2016-07-26 17:04:09 +02:00
Carla
ae568ed741 Create NEWS.md 2016-07-26 11:36:02 +02:00
Mario de Frutos
341ee05f87 Removed value comparation in observatory tests due fast changing pace in the values 2016-07-25 14:06:13 +02:00
Mario de Frutos
efde2ee1b7 Fix typo in the default value for service providers 2016-07-25 14:00:29 +02:00
Mario de Frutos
fc399c12c1 Change notices for debug 2016-07-25 13:58:01 +02:00
Mario de Frutos
f03897257e Better than lower, change to empty string if not provider setting in redis 2016-07-25 13:40:06 +02:00
Mario de Frutos
90998d4819 Fix users without provider setting setted 2016-07-25 13:35:59 +02:00
Mario de Frutos
5c5bed5557 Merge pull request #236 from CartoDB/development
Release 0.13.3
2016-07-25 13:26:27 +02:00
Mario de Frutos
d0f8797ccb Release 0.13.3 complete artifact 2016-07-25 13:25:36 +02:00
Mario de Frutos
eb2c713b0e Merge pull request #231 from CartoDB/user_service_providers
Add service providers that come from the user configuration
2016-07-25 13:19:44 +02:00
Mario de Frutos
79d0b5ba7c Add service providers that come from the user configuration 2016-07-25 12:25:52 +02:00
Mario de Frutos
15eb8633d7 Merge pull request #234 from CartoDB/development
Release 0.13.2
2016-07-22 16:30:35 +02:00
Mario de Frutos
ea4ee402a8 Release 0.13.2 to fix range isolines problem 2016-07-22 16:27:44 +02:00
Mario de Frutos
e8d55ec68a Merge pull request #232 from CartoDB/fix_isolines_range_bug
Fix isolines range bug
2016-07-22 16:24:48 +02:00
Mario de Frutos
69548eaa25 Fixed return error in case NULL value is returned for isolines shape 2016-07-22 14:55:01 +02:00
Mario de Frutos
ff8eed9750 Fix empty component return in isolines 2016-07-22 14:54:53 +02:00
Mario de Frutos
91c715f9f5 Update README.md 2016-07-20 18:14:41 +02:00
Carla
a5bf57a197 fix alias 2016-07-19 17:59:29 +02:00
Carla
5f32f7e2cd add indexes in temp table not in user table 2016-07-19 17:57:34 +02:00
Carla
91b1648f12 test add index 2016-07-19 12:55:40 +02:00
Carla
d5b6d1f5f1 Merge pull request #227 from CartoDB/development
Release server 0.13.1
2016-07-15 18:12:39 +02:00
Carla
28f49bdad2 Merge pull request #226 from CartoDB/release_server_13_1
Release server 0.13.1
2016-07-15 15:33:30 +02:00
Carla
3c63c4ec43 Merge pull request #225 from CartoDB/fix_client_addr_augment
Fix client address recognition in augment functions
2016-07-15 15:31:16 +02:00
Carla Iriberri
8e4c1554bb Release 0.13.1 server 2016-07-15 15:19:10 +02:00
Carla Iriberri
39a7cc955a Move IP address reading to DS server 2016-07-15 11:54:38 +02:00
Carla
1ee7992504 fix typo goecodes -> geocodes 2016-07-12 17:00:55 +02:00
Carla
6cd53b519b Merge pull request #224 from CartoDB/development
Release Client 0.10.0, Server 0.13.0
2016-07-12 14:06:40 +02:00
Carla
64e02e2f77 Merge pull request #218 from CartoDB/development
Release client 0.9.0 and server 0.12.0
2016-07-07 11:38:19 +02:00
172 changed files with 44205 additions and 1484 deletions

1
.gitignore vendored
View File

@@ -1,5 +1,6 @@
.DS_Store
*.pyc
.coverage
cartodb_services.egg-info/
build/
dist/

429
NEWS.md Normal file
View File

@@ -0,0 +1,429 @@
October 21st, 2016
==================
* Version 0.9.2 of the python package
* mapzen routing quota now is configurable per user
September 28, 2016
==========
* Released version 0.8.1 of Python package cartodb\_services
* Improvements in QPS retry decorator for requests to external services
https://github.com/CartoDB/dataservices-api/releases/tag/python-0.8.1
September 8, 2016
===========
* Released version 0.11.1 of the client
* Minor change in the name of the function parameter sent to server and Observatory backend for compatibility with the last observatory-extension framework updates
September 1, 2016
===========
* Released version 0.11.0 of the client
* Include DS table functions to create and populate a table with the GetMeasure function in observatory
* Released version 0.15.1 of the server
* Rename DS table functions
August 29, 2016
===========
* Released version 0.15.0 of the server
* Geocode namedplace point functions uses Mapzen search service and in case of error
it'll use the internal geocoder
August 19, 2016
===========
* Released version 0.7.4.2 of the server python library
* Now connection errors, that intermittently come from Mapzen geocoding service, are caught and treated
* Added more information to the logs from response
* Fixed some errors in the QPS manager when the response object is None
August 11, 2016
===========
* Released server version 0.14.2
* Released client version 0.10.2
* Always default arguments for DO functions to NULL, which prevents duplication & overwrite
August 5, 2016
===========
* Released server version 0.14.1
* Fix problem with calling a logger method that doesn't exists
August 3, 2016
===========
* Released server version 0.14
* New logger with: plpy, rollbar integration and file
* Added min log level to notify as config option
* Server config to define dataservices environment for: log, third party servers, etc
* Added logger to the SQL functions
* Raise exception on events that should not be logged like reach the quota limit
July 28, 2016
===========
* Release server 0.13.3.1
* Fixed limit to 1 row for isolines with multiple range
https://github.com/CartoDB/dataservices-api/releases/tag/0.13.3.1-server
July 25, 2016
===========
* Release client 0.10.1
* Includes an update of the `__AugmentTable` function of the client which creates an index on `cartodb_id` for the temporary table that stores the augmented results that will be afterwards joined with the original table by using this same key, `cartodb_id`.
https://github.com/CartoDB/dataservices-api/releases/tag/0.10.1-client
July 25, 2016
===========
* Release server 0.13.3
* Add provider per service
* Default provider in case the provider is not setted
* Refactor and improvements in the multiprovider services functions
https://github.com/CartoDB/dataservices-api/releases/tag/0.13.3-server
July 22, 2016
===========
* Release server 0.13.2
* Fixes bug with multirange isolines #233
https://github.com/CartoDB/dataservices-api/releases/tag/0.13.2-server
July 15, 2016:
===========
* Release server 0.13.1
* Includes a fix for the table augmentation functions in the server, which will now retrieve the client IP address and send it to the observatory functions as a new parameter. The affected functions are:
* _OBS_ConnectUserTable
* __OBS_ConnectUserTable
This change does not require any client change.
https://github.com/CartoDB/dataservices-api/releases/tag/0.13.1-server
Jul 12, 2016:
===========
* Release server 0.13.0
* [Server] Add beta augment functions, isoline fixes, observatory dump version
https://github.com/CartoDB/dataservices-api/releases/tag/0.13.0-server
Jul 12, 2016:
===========
* Release client 0.10.0
* [Client] Add beta augment functions, isoline fixes, observatory dump version
https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-client
Jul 7, 2016:
===========
* Release client 0.9.0
* This release adds two new functions in the Data Services extension client:
* cdb_mapzen_isodistance
* cdb_mapzen_isochrone
it also includes a bugfix for the previous release in which the explicit GRANTs to the new functions for the here, google and mapzen geocoder providers was missing in the upgrade file.
https://github.com/CartoDB/dataservices-api/releases/tag/0.9.0-client
Jul 7, 2016:
===========
* Release server 0.12.0
* This release adds four new functions in the Data Services extension server:
* cdb_mapzen_isodistance
* cdb_mapzen_isochrone
* _cdb_mapzen_isolines, which contains the real isoline logic.
* _get_mapzen_isolines_config which retrieves the explicit configuration for the Mapzen matrix service.
* In the Python end, this release adds the new Mapzen Matrix logic as well as the additions in the configuration and metrics file for the new service type mapzen_isolines.
https://github.com/CartoDB/dataservices-api/releases/tag/0.12.0-server
Jul 5, 2016:
===========
* Release server 0.11.0
* Added three new public functions for each geocoding provider:
* cdb_here_geocode_street_point
* cdb_google_geocode_street_point
* cdb_mapzen_geocode_street_point
* Added new function to retrieve specifically Mapzen configuration:
* _get_mapzen_geocoder_config
which uses the new class MapzenGeocoderConfig in the Python library.
https://github.com/CartoDB/dataservices-api/releases/tag/0.11.0-server
Jul 5, 2016:
===========
* Release client 0.8.0
* Expose providers in high-precision geocoder functions
https://github.com/CartoDB/dataservices-api/releases/tag/0.8.0-client
Jun 15, 2016:
===========
* [server python] Write quota info from services with zero padding. Closes issue #204.
https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-server3
Jun 13, 2016:
===========
* [server python] Read quota info from services with and without zero padding. Closes issue #201.
https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-server2
May 31, 2016:
===========
* Release client 0.7.0
* Adds new function OBS_GetMeasureById
https://github.com/CartoDB/dataservices-api/releases/tag/0.7.0-client
May 31, 2016:
===========
* Release server 0.10.0
* Adds new function OBS_GetMeasureById
https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-server
May 25, 2016:
===========
* Release server 0.9.0
* Added a new routing function which allows to generate routes from an origin to a destination, which passes through a set of defined locations:
* cdb_dataservices_server.cdb_route_with_waypoints (username text, organization_name text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
* cdb_dataservices_server._cdb_mapzen_route_with_waypoints(waypoints geometry(username text, orgname text, Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
and updates the old cdb_route_point_to_point function to convert the input origin and destination geometries into an array of geometries.
* Support arrays of geometries as input for the Mapzen routing Python client.
* __parse_directions will now generate the locations JSON for the service from an array of geometries.
https://github.com/CartoDB/dataservices-api/releases/tag/0.9.0-server
May 25, 2016:
===========
* Release client 0.6.0
* Includes new client function to obtain a route with waypoints:
* cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
https://github.com/CartoDB/dataservices-api/releases/tag/0.6.0-client
May 18, 2016:
===========
* Release client 0.5.0
* Added new functions for the data observatory:
* obs_getdemographicsnapshot(geometry);
* obs_getsegmentsnapshot(geometry);
* obs_getboundary(geometry, text);
* obs_getboundaryid(geometry, text);
* obs_getboundarybyid(text, text);
* obs_getboundariesbygeometry(geometry, text);
* obs_getboundariesbypointandradius(geometry, numeric, text);
* obs_getpointsbygeometry(geometry, text);
* obs_getpointsbypointandradius(geometry, numeric, text);
* obs_getmeasure(geometry, text);
* obs_getcategory(geometry, text);
* obs_getuscensusmeasure(geometry, text);
* obs_getuscensuscategory(geometry, text);
* obs_getpopulation(geometry);
* obs_search(text);
* obs_getavailableboundaries(geometry);
https://github.com/CartoDB/dataservices-api/releases/tag/0.5.0-client
May 18, 2016:
===========
* Release server 0.8.0: Data Observatory release
* Added new functions for the data observatory:
* obs_getdemographicsnapshot(geometry);
* obs_getsegmentsnapshot(geometry);
* obs_getboundary(geometry, text);
* obs_getboundaryid(geometry, text);
* obs_getboundarybyid(text, text);
* obs_getboundariesbygeometry(geometry, text);
* obs_getboundariesbypointandradius(geometry, numeric, text);
* obs_getpointsbygeometry(geometry, text);
* obs_getpointsbypointandradius(geometry, numeric, text);
* obs_getmeasure(geometry, text);
* obs_getcategory(geometry, text);
* obs_getuscensusmeasure(geometry, text);
* obs_getuscensuscategory(geometry, text);
* obs_getpopulation(geometry);
* obs_search(text);
* obs_getavailableboundaries(geometry);
* Added quota manage for these new functions
https://github.com/CartoDB/dataservices-api/releases/tag/0.8.0-server
May 10, 2016:
===========
* Release server 0.7.4
* In case we receive a 4xx error from one of the services: isolines, here geocoder, etc we have to return an empty value an increment the empty counter. We have to raise exception in 5xx or unhandled exceptions
https://github.com/CartoDB/dataservices-api/releases/tag/0.7.4-server
May 10, 2016:
===========
* Release server 0.7.3
* Change how the blue/green system is working in the server side. Now the loopback is only in the observatory extension functions call instead in all the dataservices-api function for observatory
https://github.com/CartoDB/dataservices-api/releases/tag/0.7.3-server
May 4, 2016:
===========
* Release server 0.7.2
* Added Blue/Green capability to the data observatory functions in order to be able to use staging or production databases
https://github.com/CartoDB/dataservices-api/releases/tag/0.7.2-server
Apr 25, 2016:
===========
* Release server 0.7.1
* Use redis based config if exists, if not use the db config value
* Refactor key to segregate more, now the services is called obs_snapshot
https://github.com/CartoDB/dataservices-api/releases/tag/0.7.1-server
Apr 21, 2016:
===========
* Release client 0.4.0
* Remove old versioning system for client side
* Added obs_get_demography_snapshot function
* Added obs_get_segment snapshot function
* Integrated quota checking
https://github.com/CartoDB/dataservices-api/releases/tag/0.4.0-client
Apr 21, 2016:
===========
* Release server 0.7.0
* Added obs_get_demography_snapshot function
* Added obs_get_segment snapshot function
* Integrated quota checking
https://github.com/CartoDB/dataservices-api/releases/tag/0.7.0-server
Apr 19, 2016:
===========
* Release server 0.6.2
* Add Mapzen routing and geocoder quota check
https://github.com/CartoDB/dataservices-api/releases/tag/0.6.2-server
Apr 14, 2016:
===========
* Release server 0.6.1
* Now the implementation knows how to get the iso3 for the passed country in order to pass it to Mapzen
* The city an the state/province parameters are used for mapzen too
https://github.com/CartoDB/dataservices-api/releases/tag/0.6.1-server
Apr 1, 2016:
===========
* Release server 0.6.0.1
* Use specific isoline routing credentials for a provider for isoline functions, which were previously using the general credentials from the provider.
https://github.com/CartoDB/dataservices-api/releases/tag/0.6.0.1-server
Mar 28, 2016:
===========
* Release server 0.6.0
* Integrated Mapzen geocoder for street level geocoding function
https://github.com/CartoDB/dataservices-api/releases/tag/0.6.0-server
Mar 23, 2016:
===========
* Release server 0.5.2
* Deleted old versioning system
* 4xx responses returns empty routes despite to raise an exception
* In some cases we return and empty response: one of the inputs is null, there is no generated shape for the route, etc
https://github.com/CartoDB/dataservices-api/releases/tag/0.5.2-server
Mar 17, 2016:
===========
* Release server 0.5.1
* Renamed the python library metrics functions
* Create old version's folder to store the last versions
* Refactor: Move redis and DB config logic to the python library
* Generate the metrics log file
https://github.com/CartoDB/dataservices-api/releases/tag/0.5.1-server
Mar 14, 2016:
===========
* Release server 0.5.0
* Mapzen routing functions to calculate a route point to point
* Use of Sentinel transparently
https://github.com/CartoDB/dataservices-api/releases/tag/0.5.0-server
Mar 14, 2016:
===========
* Release client 0.3.0
* Added cdb_routing_point_to_point function using Mapzen as provider
https://github.com/CartoDB/dataservices-api/releases/tag/0.3.0-client
Feb 26, 2016:
===========
* Release client 0.2.0
* Added routing isolines capabilities to the client and public API
https://github.com/CartoDB/dataservices-api/releases/tag/0.2.0-client
Feb 26, 2016:
===========
* Release server 0.4.0
* Added routing isolines capabilities
https://github.com/CartoDB/dataservices-api/releases/tag/0.4.0-server
Feb 11, 2016:
===========
* Release server 0.3.0
* Extension refactor, now is called cdb_dataservices_[client|server] in order to include more services aside the geocoder.
* Add logic to save the metrics for the internal geocoder services as we have for the nokia and google geocoders
* Trimmed all the inputs to avoid empty results
https://github.com/CartoDB/dataservices-api/releases/tag/0.3.0-server
Feb 4, 2016:
===========
* Release server 0.2.0
* Logic for the google geocoder so the users with this geocoder setted up can use street level geocoding too
* Refactor of the python library in order to reflect the change to a services extension more than only geocoder
https://github.com/CartoDB/dataservices-api/releases/tag/0.2.0-server
Jan 25, 2016:
===========
* Release Geocoder API 0.1.0
* Street geocoding available through the cdb_geocoder_street_point_v2 function (only working Heremaps geocoder)
* User config comes from Redis database
* Increment of usage metrics for the cdb_geocoder_street_v2
https://github.com/CartoDB/dataservices-api/releases/tag/0.1.0
Jan 25, 2016:
===========
* Release Geocoder API 0.0.1 production ready
https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1
Dec 23, 2015:
===========
* Release Geocoder API 0.0.2 beta
https://github.com/CartoDB/dataservices-api/releases/tag/0.0.2
Dec 3, 2015:
===========
* Release Geocoder API 0.0.1 Beta2
https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1-beta2
Nov 27, 2015:
===========
* Release Geocoder API BETA 1
* Added the organization public user to the api key check
https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1-beta

View File

@@ -5,7 +5,7 @@ The CARTO Data Services SQL API
Steps to deploy a new Data Services API version :
- Deploy new version of dataservices API to all servers
- Update the server user using: ALTER EXTENSION cdb_dataservices_server UPDATE TO '<CURRENT_VERSION>';
- Update the server user using: ALTER EXTENSION cdb_dataservices_server UPDATE TO '\<CURRENT_VERSION\>';
- Update the python dependencies if needed: **cartodb_geocoder** and **heremaps**
- Add the needed config in the `cdb_conf` table:
- `redis_metadata_config` and `redis_metrics_conf`
@@ -76,9 +76,12 @@ Steps to deploy a new Data Services API version :
SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}');
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}}');
SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}')
SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}')
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}');
SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}');
SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}');
# Environment to decide: rollbar message, which servers for third party use, etc. If not setted uses production by default (current behavior)
SELECT CDB_Conf_SetConf('server_conf', '{"environment": "[development|staging|production]"}')
```
- configure the user DB:

View File

@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS) \
cdb_dataservices_client--0.9.0--0.10.0.sql \
cdb_dataservices_client--0.10.0--0.9.0.sql
cdb_dataservices_client--0.11.0--0.11.1.sql \
cdb_dataservices_client--0.11.1--0.11.0.sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))

View File

@@ -41,3 +41,14 @@ CREATE EXTENSION cdb_dataservices_client;
```
The extension creation in the user's db requires **superuser** privileges.
## User database configuration
After installing the client extension in a database, you will need to set up your configuration to be able to connect with the server.
```
-- Point to the dataservices server DB (you can use a specific database for the server or your same user's):
SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}');
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}');
```

View File

@@ -0,0 +1,140 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.11.1'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'OBS_GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)
)
)
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Create a new table with the required columns
plpy.execute('CREATE TABLE "{schema}".{table_name} ( '
'cartodb_id int, the_geom geometry, {columns_with_types} '
');'
.format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types)
)
plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
.format(schema=user_schema, table_name=output_table_name, user=user_db_role)
)
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
dbname text,
table_name text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'OBS_GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute(
"SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)))
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [
colnames_arr[i] +
' ' +
coltypes_arr[i] for i in range(
0,
len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
aliased_colname_list = ','.join(
['result.' + name for name in colnames_arr])
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute(
"SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, "
"{schema}::text, {dbname}::text, {table_name}::text);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
user_db_role=plpy.quote_literal(user_db_role),
schema=plpy.quote_literal(user_schema),
dbname=plpy.quote_literal(dbname),
table_name=plpy.quote_literal(table_name)))
if ds_fdw_metadata[0]["schemaname"]:
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
else:
raise Exception('Error connecting dataset via FDW')
# Create a new table with the required columns
plpy.execute(
'INSERT INTO "{schema}".{analysis_table_name} '
'SELECT ut.cartodb_id, ut.the_geom, {colname_list} '
'FROM "{schema}".{table_name} ut '
'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, '
'{function_name}::text, {params}::json) '
'AS result ({columns_with_types}, cartodb_id int) '
'ON result.cartodb_id = ut.cartodb_id;' .format(
schema=user_schema,
analysis_table_name=output_table_name,
colname_list=aliased_colname_list,
table_name=table_name,
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params),
columns_with_types=columns_with_types))
# Wipe user FDW data from the server
wiped = plpy.execute(
"SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, "
"{server_table_name}::text, {fdw_server}::text)" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
fdw_server=plpy.quote_literal(server_name)))
return True
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,140 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.11.0'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)
)
)
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Create a new table with the required columns
plpy.execute('CREATE TABLE "{schema}".{table_name} ( '
'cartodb_id int, the_geom geometry, {columns_with_types} '
');'
.format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types)
)
plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
.format(schema=user_schema, table_name=output_table_name, user=user_db_role)
)
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
dbname text,
table_name text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute(
"SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)))
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [
colnames_arr[i] +
' ' +
coltypes_arr[i] for i in range(
0,
len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
aliased_colname_list = ','.join(
['result.' + name for name in colnames_arr])
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute(
"SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, "
"{schema}::text, {dbname}::text, {table_name}::text);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
user_db_role=plpy.quote_literal(user_db_role),
schema=plpy.quote_literal(user_schema),
dbname=plpy.quote_literal(dbname),
table_name=plpy.quote_literal(table_name)))
if ds_fdw_metadata[0]["schemaname"]:
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
else:
raise Exception('Error connecting dataset via FDW')
# Create a new table with the required columns
plpy.execute(
'INSERT INTO "{schema}".{analysis_table_name} '
'SELECT ut.cartodb_id, ut.the_geom, {colname_list} '
'FROM "{schema}".{table_name} ut '
'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, '
'{function_name}::text, {params}::json) '
'AS result ({columns_with_types}, cartodb_id int) '
'ON result.cartodb_id = ut.cartodb_id;' .format(
schema=user_schema,
analysis_table_name=output_table_name,
colname_list=aliased_colname_list,
table_name=table_name,
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params),
columns_with_types=columns_with_types))
# Wipe user FDW data from the server
wiped = plpy.execute(
"SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, "
"{server_table_name}::text, {fdw_server}::text)" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
fdw_server=plpy.quote_literal(server_name)))
return True
$$ LANGUAGE plpythonu;

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices client API extension'
default_version = '0.10.0'
default_version = '0.11.1'
requires = 'plproxy, cartodb'
superuser = true
schema = cdb_dataservices_client

View File

@@ -0,0 +1,96 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.10.1'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json)
RETURNS boolean AS $$
from time import strftime
try:
server_table_name = None
temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
)
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
# Prepare column and type strings required in the SQL queries
colnames = ','.join(colnames_arr)
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
# Create temporary table with the augmented results
plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS '
'(SELECT {columns}, cartodb_id '
'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData('
'{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) '
'AS results({columns_with_types}, cartodb_id int) )'
.format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types,
temp_table_name=temporary_table_name)
)
# Wipe user FDW data from the server
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
# Add index to cartodb_id
plpy.execute('CREATE UNIQUE INDEX {temp_table_name}_pkey ON "{user_schema}".{temp_table_name} (cartodb_id)'
.format(user_schema=user_schema, temp_table_name=temporary_table_name)
)
# Prepare table to receive augmented results in new columns
for idx, column in enumerate(colnames_arr):
if colnames_arr[idx] is not 'the_geom':
plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}'
.format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx])
)
# Populate the user table with the augmented results
plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = '
'(SELECT {columns} FROM "{user_schema}".{temporary_table_name} '
'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)'
.format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types,
temporary_table_name=temporary_table_name)
)
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
return True
except Exception as e:
plpy.warning('Error trying to augment table {0}'.format(e))
# Wipe user FDW data from the server in case of failure if the table was connected
if server_table_name:
# Wipe local temporary table
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return False
$$ LANGUAGE plpythonu;

View File

@@ -89,7 +89,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -117,7 +116,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -145,7 +143,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -173,7 +170,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -201,7 +197,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -229,7 +224,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -257,7 +251,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -285,7 +278,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -313,7 +305,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -341,7 +332,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -369,7 +359,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -397,7 +386,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -425,7 +413,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -453,7 +440,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -481,7 +467,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -509,7 +494,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -537,7 +521,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -565,7 +548,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -593,14 +575,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT '"us.census.tiger".block_group'::text)
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL)
RETURNS json AS $$
DECLARE
ret json;
@@ -621,14 +602,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326), geometry_level text DEFAULT '"us.census.tiger".census_tract'::text)
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL)
RETURNS json AS $$
DECLARE
ret json;
@@ -649,7 +629,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -677,7 +656,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -705,7 +683,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -733,7 +710,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -761,7 +737,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -789,14 +764,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
@@ -817,14 +791,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
@@ -845,14 +818,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
@@ -873,14 +845,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
@@ -901,14 +872,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry, measure_id text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -929,7 +899,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -957,7 +926,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -985,14 +953,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry, name text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -1013,7 +980,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -1041,14 +1007,13 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -1069,7 +1034,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -1097,7 +1061,6 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
@@ -1125,7 +1088,33 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion ()
RETURNS text AS $$
DECLARE
ret text;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT * FROM cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text);
CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]);
@@ -1389,7 +1378,9 @@ RETURNS boolean AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_DisconnectUserTable;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1398,6 +1389,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1406,6 +1398,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1414,6 +1407,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1422,6 +1416,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1430,6 +1425,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1438,6 +1434,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1446,6 +1443,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1454,6 +1452,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1462,6 +1461,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1470,6 +1470,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1478,6 +1479,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1486,6 +1488,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1494,6 +1497,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1502,6 +1506,7 @@ RETURNS SETOF cdb_dataservices_client.isoline AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1510,6 +1515,7 @@ RETURNS SETOF cdb_dataservices_client.isoline AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1518,6 +1524,7 @@ RETURNS SETOF cdb_dataservices_client.isoline AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1526,6 +1533,7 @@ RETURNS SETOF cdb_dataservices_client.isoline AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, organization_name text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_client.simple_route AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1534,6 +1542,7 @@ RETURNS cdb_dataservices_client.simple_route AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_with_waypoints (username text, organization_name text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_client.simple_route AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1541,7 +1550,8 @@ RETURNS cdb_dataservices_client.simple_route AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT '"us.census.tiger".block_group'::text)
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL)
RETURNS json AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1549,7 +1559,8 @@ RETURNS json AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), geometry_level text DEFAULT '"us.census.tiger".census_tract'::text)
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL)
RETURNS json AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1558,6 +1569,7 @@ RETURNS json AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot (username text, organization_name text, geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL)
RETURNS SETOF JSON AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1566,6 +1578,7 @@ RETURNS SETOF JSON AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot (username text, organization_name text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL)
RETURNS SETOF JSON AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1574,6 +1587,7 @@ RETURNS SETOF JSON AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundary (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1582,6 +1596,7 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundaryid (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL)
RETURNS text AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1590,6 +1605,7 @@ RETURNS text AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundarybyid (username text, organization_name text, geometry_id text, boundary_id text, time_span text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1597,7 +1613,8 @@ RETURNS Geometry AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1605,7 +1622,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1613,7 +1631,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1621,7 +1640,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1629,7 +1649,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, organization_name text, geom Geometry, measure_id text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, organization_name text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1638,6 +1659,7 @@ RETURNS numeric AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid (username text, organization_name text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1646,6 +1668,7 @@ RETURNS numeric AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getcategory (username text, organization_name text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS text AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1653,7 +1676,8 @@ RETURNS text AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, organization_name text, geom Geometry, name text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, organization_name text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1662,6 +1686,7 @@ RETURNS numeric AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensuscategory (username text, organization_name text, geom Geometry, name text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS text AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1669,7 +1694,8 @@ RETURNS text AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, organization_name text, geom Geometry, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, organization_name text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1678,6 +1704,7 @@ RETURNS numeric AS $$
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_search (username text, organization_name text, search_term text, relevant_boundary text DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1686,6 +1713,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getavailableboundaries (username text, organization_name text, geom Geometry, timespan text DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
@@ -1693,6 +1721,14 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, organization_name text)
RETURNS text AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_dumpversion (username, organization_name);
$$ LANGUAGE plproxy;
-- Make sure by default there are no permissions for publicuser
-- NOTE: this happens at extension creation time, as part of an implicit transaction.
REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE;
@@ -1739,5 +1775,6 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom G
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom Geometry, normalize text, boundary_id text, time_span text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom Geometry, timespan text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_dumpversion() TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_augmenttable(table_name text, function_name text, params json) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_gettable(table_name text, output_table_name text, function_name text, params json) TO publicuser;

View File

@@ -0,0 +1,91 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.10.0'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json)
RETURNS boolean AS $$
from time import strftime
try:
server_table_name = None
temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
)
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
# Prepare column and type strings required in the SQL queries
colnames = ','.join(colnames_arr)
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
# Create temporary table with the augmented results
plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS '
'(SELECT {columns}, cartodb_id '
'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData('
'{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) '
'AS results({columns_with_types}, cartodb_id int) )'
.format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types,
temp_table_name=temporary_table_name)
)
# Wipe user FDW data from the server
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
# Prepare table to receive augmented results in new columns
for idx, column in enumerate(colnames_arr):
if colnames_arr[idx] is not 'the_geom':
plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}'
.format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx])
)
# Populate the user table with the augmented results
plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = '
'(SELECT {columns} FROM "{user_schema}".{temporary_table_name} '
'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)'
.format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types,
temporary_table_name=temporary_table_name)
)
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
return True
except Exception as e:
plpy.warning('Error trying to augment table {0}'.format(e))
# Wipe user FDW data from the server in case of failure if the table was connected
if server_table_name:
# Wipe local temporary table
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return False
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,286 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.10.2'" to load this file. \quit
-- 20 public functions
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL)
RETURNS json AS $$
DECLARE
ret json;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL)
RETURNS json AS $$
DECLARE
ret json;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
-- 30_plproxy_functions
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT NULL)
RETURNS json AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_get_demographic_snapshot (username, organization_name, geom, time_span, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL)
RETURNS json AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_get_segment_snapshot (username, organization_name, geom, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getboundariesbygeometry (username, organization_name, geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getboundariesbypointandradius (username, organization_name, geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getpointsbygeometry (username, organization_name, geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getpointsbypointandradius (username, organization_name, geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, organization_name text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_getmeasure (username, organization_name, geom, measure_id, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, organization_name text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_getuscensusmeasure (username, organization_name, geom, name, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, organization_name text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_getpopulation (username, organization_name, geom, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,286 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.10.1'" to load this file. \quit
-- 20 public functions
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot (geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT '"us.census.tiger".block_group'::text)
RETURNS json AS $$
DECLARE
ret json;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_get_segment_snapshot (geom geometry(Geometry, 4326), geometry_level text DEFAULT '"us.census.tiger".census_tract'::text)
RETURNS json AS $$
DECLARE
ret json;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getboundariesbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getboundariesbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbygeometry (geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getpointsbygeometry(username, orgname, geom, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius (geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._obs_getpointsbypointandradius(username, orgname, geom, radius, boundary_id, time_span, overlap_type);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasure (geom Geometry, measure_id text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getuscensusmeasure (geom Geometry, name text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getpopulation (geom Geometry, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
-- 30_plproxy_functions
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013'::text, geometry_level text DEFAULT '"us.census.tiger".block_group'::text)
RETURNS json AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_get_demographic_snapshot (username, organization_name, geom, time_span, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_get_segment_snapshot (username text, organization_name text, geom geometry(Geometry, 4326), geometry_level text DEFAULT '"us.census.tiger".census_tract'::text)
RETURNS json AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_get_segment_snapshot (username, organization_name, geom, geometry_level);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getboundariesbygeometry (username, organization_name, geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getboundariesbypointandradius (username, organization_name, geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbygeometry (username text, organization_name text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getpointsbygeometry (username, organization_name, geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius (username text, organization_name text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.obs_getpointsbypointandradius (username, organization_name, geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasure (username text, organization_name text, geom Geometry, measure_id text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_getmeasure (username, organization_name, geom, measure_id, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getuscensusmeasure (username text, organization_name text, geom Geometry, name text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_getuscensusmeasure (username, organization_name, geom, name, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getpopulation (username text, organization_name text, geom Geometry, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.obs_getpopulation (username, organization_name, geom, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;

View File

@@ -0,0 +1,289 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.11.0'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetTable(text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_AugmentTable(text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_AugmentTable(text, text, text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_GetTable(text, text, text, text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_ConnectUserTable(text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetReturnMetadata(text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_FetchJoinFdwTableData(text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_DisconnectUserTable(text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(
output_table_name text,
params json
) RETURNS boolean AS $$
DECLARE
username text;
user_db_role text;
orgname text;
user_schema text;
result boolean;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT session_user INTO user_db_role;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument';
END IF;
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
user_schema := 'public';
ELSE
user_schema := username;
END IF;
SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
username,
orgname,
user_db_role,
user_schema,
output_table_name,
params
) INTO result;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(
table_name text,
output_table_name text,
params json
) RETURNS boolean AS $$
DECLARE
username text;
user_db_role text;
orgname text;
dbname text;
user_schema text;
result boolean;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT session_user INTO user_db_role;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument';
END IF;
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
user_schema := 'public';
ELSE
user_schema := username;
END IF;
SELECT current_database() INTO dbname;
SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
username,
orgname,
user_db_role,
user_schema,
dbname,
table_name,
output_table_name,
params
) INTO result;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)
)
)
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Create a new table with the required columns
plpy.execute('CREATE TABLE "{schema}".{table_name} ( '
'cartodb_id int, the_geom geometry, {columns_with_types} '
');'
.format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types)
)
plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
.format(schema=user_schema, table_name=output_table_name, user=user_db_role)
)
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
dbname text,
table_name text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute(
"SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)))
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [
colnames_arr[i] +
' ' +
coltypes_arr[i] for i in range(
0,
len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
aliased_colname_list = ','.join(
['result.' + name for name in colnames_arr])
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute(
"SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, "
"{schema}::text, {dbname}::text, {table_name}::text);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
user_db_role=plpy.quote_literal(user_db_role),
schema=plpy.quote_literal(user_schema),
dbname=plpy.quote_literal(dbname),
table_name=plpy.quote_literal(table_name)))
if ds_fdw_metadata[0]["schemaname"]:
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
else:
raise Exception('Error connecting dataset via FDW')
# Create a new table with the required columns
plpy.execute(
'INSERT INTO "{schema}".{analysis_table_name} '
'SELECT ut.cartodb_id, ut.the_geom, {colname_list} '
'FROM "{schema}".{table_name} ut '
'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, '
'{function_name}::text, {params}::json) '
'AS result ({columns_with_types}, cartodb_id int) '
'ON result.cartodb_id = ut.cartodb_id;' .format(
schema=user_schema,
analysis_table_name=output_table_name,
colname_list=aliased_colname_list,
table_name=table_name,
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params),
columns_with_types=columns_with_types))
# Wipe user FDW data from the server
wiped = plpy.execute(
"SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, "
"{server_table_name}::text, {fdw_server}::text)" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
fdw_server=plpy.quote_literal(server_name)))
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable(
username text,
orgname text,
user_db_role text,
user_schema text,
dbname text,
table_name text
)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_ConnectUserTable;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata(
username text,
orgname text,
function_name text,
params json
) RETURNS cdb_dataservices_client.ds_return_metadata AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_GetReturnMetadata;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData(
username text,
orgname text,
table_schema text,
table_name text,
function_name text,
params json
) RETURNS SETOF record AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable(
username text,
orgname text,
table_schema text,
table_name text,
server_name text
) RETURNS boolean AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_DisconnectUserTable;
$$ LANGUAGE plproxy;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,281 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.10.2'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(text, text, text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_ConnectUserTable(text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_GetReturnMetadata(text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_FetchJoinFdwTableData(text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_DisconnectUserTable(text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json)
RETURNS boolean AS $$
DECLARE
username text;
user_db_role text;
orgname text;
dbname text;
user_schema text;
result boolean;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT session_user INTO user_db_role;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument';
END IF;
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
user_schema := 'public';
ELSE
user_schema := username;
END IF;
SELECT current_database() INTO dbname;
SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json)
RETURNS boolean AS $$
DECLARE
username text;
user_db_role text;
orgname text;
dbname text;
user_schema text;
result boolean;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT session_user INTO user_db_role;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument';
END IF;
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
user_schema := 'public';
ELSE
user_schema := username;
END IF;
SELECT current_database() INTO dbname;
SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json)
RETURNS boolean AS $$
from time import strftime
try:
server_table_name = None
temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
)
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
# Prepare column and type strings required in the SQL queries
colnames = ','.join(colnames_arr)
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
# Create temporary table with the augmented results
plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS '
'(SELECT {columns}, cartodb_id '
'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData('
'{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) '
'AS results({columns_with_types}, cartodb_id int) )'
.format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types,
temp_table_name=temporary_table_name)
)
# Wipe user FDW data from the server
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
# Add index to cartodb_id
plpy.execute('CREATE UNIQUE INDEX {temp_table_name}_pkey ON "{user_schema}".{temp_table_name} (cartodb_id)'
.format(user_schema=user_schema, temp_table_name=temporary_table_name)
)
# Prepare table to receive augmented results in new columns
for idx, column in enumerate(colnames_arr):
if colnames_arr[idx] is not 'the_geom':
plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}'
.format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx])
)
# Populate the user table with the augmented results
plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = '
'(SELECT {columns} FROM "{user_schema}".{temporary_table_name} '
'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)'
.format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types,
temporary_table_name=temporary_table_name)
)
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
return True
except Exception as e:
plpy.warning('Error trying to augment table {0}'.format(e))
# Wipe user FDW data from the server in case of failure if the table was connected
if server_table_name:
# Wipe local temporary table
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return False
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json)
RETURNS boolean AS $$
try:
server_table_name = None
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
)
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
# Prepare column and type strings required in the SQL queries
colnames = ','.join(colnames_arr)
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
# Get list of user columns to include in the new table
user_table_columns = ','.join(
plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns '
'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass '
'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' '
'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));'
.format(user_schema=user_schema, table_name=table_name, colnames=colnames)
)[0]["columns"]
)
# Populate a new table with the augmented results
plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS '
'(SELECT results.{columns}, {user_table_columns} '
'FROM {table_name} AS user_table '
'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) '
'ON results.cartodb_id = user_table.cartodb_id)'
.format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name),
table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types)
)
plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
.format(schema=user_schema, table_name=output_table_name, user=user_db_role)
)
# Wipe user FDW data from the server
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return True
except Exception as e:
plpy.warning('Error trying to get table {0}'.format(e))
# Wipe user FDW data from the server in case of failure if the table was connected
if server_table_name:
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return False
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_client.ds_return_metadata AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_GetReturnMetadata;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS SETOF record AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text)
RETURNS boolean AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_DisconnectUserTable;
$$ LANGUAGE plproxy;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_augmenttable(table_name text, function_name text, params json) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_gettable(table_name text, output_table_name text, function_name text, params json) TO publicuser;

File diff suppressed because it is too large Load Diff

View File

@@ -33,7 +33,6 @@
- { name: admin1_name, type: text}
- { name: country_name, type: text}
- name: cdb_geocode_postalcode_polygon
return_type: Geometry
params:
@@ -147,13 +146,13 @@
params:
- { name: geom, type: "geometry(Geometry, 4326)" }
- { name: time_span, type: "text", default: "'2009 - 2013'::text" }
- { name: geometry_level, type: text, default: "'\"us.census.tiger\".block_group'::text" }
- { name: geometry_level, type: text, default: 'NULL' }
- name: obs_get_segment_snapshot
return_type: json
params:
- { name: geom, type: "geometry(Geometry, 4326)" }
- { name: geometry_level, type: text, default: "'\"us.census.tiger\".census_tract'::text" }
- { name: geometry_level, type: text, default: 'NULL' }
- name: obs_getdemographicsnapshot
return_type: SETOF JSON
@@ -202,7 +201,7 @@
- { name: geom, type: "geometry(Geometry, 4326)" }
- { name: boundary_id, type: text }
- { name: time_span, type: text, default: 'NULL'}
- { name: overlap_type, type: text, default: "'intersects'"}
- { name: overlap_type, type: text, default: 'NULL'}
- name: obs_getboundariesbypointandradius
return_type: TABLE(the_geom geometry, geom_refs text)
@@ -216,7 +215,7 @@
- { name: radius, type: numeric }
- { name: boundary_id, type: text }
- { name: time_span, type: text, default: 'NULL'}
- { name: overlap_type, type: text, default: "'intersects'"}
- { name: overlap_type, type: text, default: 'NULL'}
- name: obs_getpointsbygeometry
return_type: TABLE(the_geom geometry, geom_refs text)
@@ -229,7 +228,7 @@
- { name: geom, type: "geometry(Geometry, 4326)" }
- { name: boundary_id, type: text }
- { name: time_span, type: text, default: 'NULL'}
- { name: overlap_type, type: text, default: "'intersects'"}
- { name: overlap_type, type: text, default: 'NULL'}
- name: obs_getpointsbypointandradius
return_type: TABLE(the_geom geometry, geom_refs text)
@@ -243,14 +242,14 @@
- { name: radius, type: numeric }
- { name: boundary_id, type: text }
- { name: time_span, type: text, default: 'NULL'}
- { name: overlap_type, type: text, default: "'intersects'"}
- { name: overlap_type, type: text, default: 'NULL'}
- name: obs_getmeasure
return_type: numeric
params:
- { name: geom, type: Geometry }
- { name: measure_id, type: text }
- { name: normalize, type: text, default: "'area'"}
- { name: normalize, type: text, default: 'NULL'}
- { name: boundary_id, type: text, default: 'NULL' }
- { name: time_span, type: text, default: 'NULL'}
@@ -275,7 +274,7 @@
params:
- { name: geom, type: Geometry }
- { name: name, type: text }
- { name: normalize, type: text, default: "'area'"}
- { name: normalize, type: text, default: 'NULL'}
- { name: boundary_id, type: text, default: 'NULL' }
- { name: time_span, type: text, default: 'NULL'}
@@ -291,7 +290,7 @@
return_type: numeric
params:
- { name: geom, type: Geometry }
- { name: normalize, type: text, default: "'area'"}
- { name: normalize, type: text, default: 'NULL'}
- { name: boundary_id, type: text, default: 'NULL' }
- { name: time_span, type: text, default: 'NULL'}

View File

@@ -1,8 +1,53 @@
CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text);
CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json)
RETURNS boolean AS $$
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(
output_table_name text,
params json
) RETURNS boolean AS $$
DECLARE
username text;
user_db_role text;
orgname text;
user_schema text;
result boolean;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT session_user INTO user_db_role;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument';
END IF;
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
user_schema := 'public';
ELSE
user_schema := username;
END IF;
SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
username,
orgname,
user_db_role,
user_schema,
output_table_name,
params
) INTO result;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(
table_name text,
output_table_name text,
params json
) RETURNS boolean AS $$
DECLARE
username text;
user_db_role text;
@@ -31,233 +76,200 @@ BEGIN
SELECT current_database() INTO dbname;
SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result;
SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
username,
orgname,
user_db_role,
user_schema,
dbname,
table_name,
output_table_name,
params
) INTO result;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json)
RETURNS boolean AS $$
DECLARE
username text;
user_db_role text;
orgname text;
dbname text;
user_schema text;
result boolean;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT session_user INTO user_db_role;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument';
END IF;
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
user_schema := 'public';
ELSE
user_schema := username;
END IF;
SELECT current_database() INTO dbname;
SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json)
RETURNS boolean AS $$
from time import strftime
try:
server_table_name = None
temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'OBS_GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)
)
)
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
# Prepare column and type strings required in the SQL queries
colnames = ','.join(colnames_arr)
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
# Create temporary table with the augmented results
plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS '
'(SELECT {columns}, cartodb_id '
'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData('
'{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) '
'AS results({columns_with_types}, cartodb_id int) )'
.format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types,
temp_table_name=temporary_table_name)
)
# Wipe user FDW data from the server
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
# Prepare table to receive augmented results in new columns
for idx, column in enumerate(colnames_arr):
if colnames_arr[idx] is not 'the_geom':
plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}'
.format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx])
)
# Populate the user table with the augmented results
plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = '
'(SELECT {columns} FROM "{user_schema}".{temporary_table_name} '
'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)'
.format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types,
temporary_table_name=temporary_table_name)
)
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
return True
except Exception as e:
plpy.warning('Error trying to augment table {0}'.format(e))
# Wipe user FDW data from the server in case of failure if the table was connected
if server_table_name:
# Wipe local temporary table
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
)
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return False
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json)
RETURNS boolean AS $$
try:
server_table_name = None
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
)
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
# Prepare column and type strings required in the SQL queries
colnames = ','.join(colnames_arr)
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
# Get list of user columns to include in the new table
user_table_columns = ','.join(
plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns '
'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass '
'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' '
'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));'
.format(user_schema=user_schema, table_name=table_name, colnames=colnames)
)[0]["columns"]
# Create a new table with the required columns
plpy.execute('CREATE TABLE "{schema}".{table_name} ( '
'cartodb_id int, the_geom geometry, {columns_with_types} '
');'
.format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types)
)
# Populate a new table with the augmented results
plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS '
'(SELECT results.{columns}, {user_table_columns} '
'FROM {table_name} AS user_table '
'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) '
'ON results.cartodb_id = user_table.cartodb_id)'
.format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name),
table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types)
)
plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
.format(schema=user_schema, table_name=output_table_name, user=user_db_role)
)
plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
.format(schema=user_schema, table_name=output_table_name, user=user_db_role)
)
# Wipe user FDW data from the server
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return True
except Exception as e:
plpy.warning('Error trying to get table {0}'.format(e))
# Wipe user FDW data from the server in case of failure if the table was connected
if server_table_name:
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
)
return False
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
username text,
orgname text,
user_db_role text,
user_schema text,
dbname text,
table_name text,
output_table_name text,
params json
) RETURNS boolean AS $$
function_name = 'OBS_GetMeasure'
# Obtain return types for augmentation procedure
ds_return_metadata = plpy.execute(
"SELECT colnames, coltypes "
"FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params)))
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_ConnectUserTable;
if ds_return_metadata[0]["colnames"]:
colnames_arr = ds_return_metadata[0]["colnames"]
coltypes_arr = ds_return_metadata[0]["coltypes"]
else:
raise Exception('Error retrieving OBS_GetMeasure metadata')
# Prepare column and type strings required in the SQL queries
columns_with_types_arr = [
colnames_arr[i] +
' ' +
coltypes_arr[i] for i in range(
0,
len(colnames_arr))]
columns_with_types = ','.join(columns_with_types_arr)
aliased_colname_list = ','.join(
['result.' + name for name in colnames_arr])
# Instruct the OBS server side to establish a FDW
# The metadata is obtained as well in order to:
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
# - (b) be able to tell OBS to free resources when done.
ds_fdw_metadata = plpy.execute(
"SELECT schemaname, tabname, servername "
"FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, "
"{schema}::text, {dbname}::text, {table_name}::text);" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
user_db_role=plpy.quote_literal(user_db_role),
schema=plpy.quote_literal(user_schema),
dbname=plpy.quote_literal(dbname),
table_name=plpy.quote_literal(table_name)))
if ds_fdw_metadata[0]["schemaname"]:
server_schema = ds_fdw_metadata[0]["schemaname"]
server_table_name = ds_fdw_metadata[0]["tabname"]
server_name = ds_fdw_metadata[0]["servername"]
else:
raise Exception('Error connecting dataset via FDW')
# Create a new table with the required columns
plpy.execute(
'INSERT INTO "{schema}".{analysis_table_name} '
'SELECT ut.cartodb_id, ut.the_geom, {colname_list} '
'FROM "{schema}".{table_name} ut '
'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, '
'{function_name}::text, {params}::json) '
'AS result ({columns_with_types}, cartodb_id int) '
'ON result.cartodb_id = ut.cartodb_id;' .format(
schema=user_schema,
analysis_table_name=output_table_name,
colname_list=aliased_colname_list,
table_name=table_name,
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
function_name=plpy.quote_literal(function_name),
params=plpy.quote_literal(params),
columns_with_types=columns_with_types))
# Wipe user FDW data from the server
wiped = plpy.execute(
"SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, "
"{server_table_name}::text, {fdw_server}::text)" .format(
username=plpy.quote_nullable(username),
orgname=plpy.quote_nullable(orgname),
server_schema=plpy.quote_literal(server_schema),
server_table_name=plpy.quote_literal(server_table_name),
fdw_server=plpy.quote_literal(server_name)))
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable(
username text,
orgname text,
user_db_role text,
user_schema text,
dbname text,
table_name text
)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_ConnectUserTable;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_client.ds_return_metadata AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_GetReturnMetadata;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata(
username text,
orgname text,
function_name text,
params json
) RETURNS cdb_dataservices_client.ds_return_metadata AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_GetReturnMetadata;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS SETOF record AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData(
username text,
orgname text,
table_schema text,
table_name text,
function_name text,
params json
) RETURNS SETOF record AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text)
RETURNS boolean AS $$
CONNECT _server_conn_str();
TARGET cdb_dataservices_server._OBS_DisconnectUserTable;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable(
username text,
orgname text,
table_schema text,
table_name text,
server_name text
) RETURNS boolean AS $$
CONNECT cdb_dataservices_client._server_conn_str();
TARGET cdb_dataservices_server._DST_DisconnectUserTable;
$$ LANGUAGE plproxy;

View File

@@ -1,2 +1,2 @@
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_augmenttable(table_name text, function_name text, params json) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_gettable(table_name text, output_table_name text, function_name text, params json) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser;

View File

@@ -71,28 +71,28 @@ BEGIN
RETURN ret;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getboundariesbypointandradius invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, radius, boundary_id, time_span, overlap_type;
RETURN QUERY SELECT '0106000020E6100000010000000103000000010000001500000083A3E4D5397C52C09946938B315A44404AED45B41D7C52C0785DBF60375A444099F221A81A7C52C0D0EFFB372F5A4440B9C15087157C52C034643C4A255A4440BB438A01127C52C0DE1CAED51E5A44400FECF82F107C52C0876D8B321B5A4440510FD1E80E7C52C0C4D32B65195A44407FDAA84E077C52C0E0675C38105A444092AD2EA7047C52C08E75711B0D5A4440CF2D7425027C52C0A86DC328085A4440EFFCA204FD7B52C07E384888F259444034BC5983F77B52C0205ED72FD8594440E76F4221027C52C0F3A96395D259444081AD122C0E7C52C00ED6FF39CC594440CB63CDC8207C52C031276893C359444047205ED72F7C52C0287D21E4BC594440BD1C76DF317C52C0F8325184D4594440CD1FD3DA347C52C0B0A9F3A8F8594440C72E51BD357C52C0C3651536035A444043024697377C52C028999CDA195A444083A3E4D5397C52C09946938B315A4440'::geometry as the_geom, '36047048900'::text as geom_refs;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getpointsbygeometry invoked with params (%, %, %, %, %, %)', username, orgname, geom, boundary_id, time_span, overlap_type;
RETURN QUERY SELECT '0101000020E610000037CB75821A7C52C0BA8784EFFD594440'::geometry as the_geom, '36047048900'::text as geom_refs;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getpointsbypointandradius invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, radius, boundary_id, time_span, overlap_type;
RETURN QUERY SELECT '0101000020E610000037CB75821A7C52C0BA8784EFFD594440'::geometry as the_geom, '36047048900'::text as geom_refs;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -122,7 +122,7 @@ BEGIN
RETURN ret;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -142,7 +142,7 @@ BEGIN
RETURN ret;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -166,7 +166,7 @@ BEGIN
RETURN QUERY SELECT 'us.census.tiger.place'::text as boundary_id, 'Incorporated places are those reported to the Census Bureau as legally in existence as of January 1, 2010, as reported in the latest Boundary and Annexation Survey (BAS), under the laws of their respective states.'::text as description, '2014'::text as timespan, 'obs_7c9493c41fa8f4bd178ab993ea3d5891c1977667'::text as tablename;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getboundariesbygeometry invoked with params (%, %, %, %, %, %)', username, orgname, geom, boundary_id, time_span, overlap_type;
@@ -236,7 +236,7 @@ PL/pgSQL function obs_getboundarybyid(text,text,text) line 16 at SQL statement
(1 row)
SELECT obs_getboundariesbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text);
NOTICE: cdb_dataservices_client._obs_getboundariesbygeometry(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getboundariesbygeometry invoked with params (test_user, <NULL>, 0103000020E61000000100000005000000C7F8FFD37E7C52C0F860AE1175594440C7F8FFD37E7C52C0BF0E0D64E55A4440A70300E0647B52C0BF0E0D64E55A4440A70300E0647B52C0F860AE1175594440C7F8FFD37E7C52C0F860AE1175594440, us.census.tiger.census_tract, <NULL>, intersects)
NOTICE: cdb_dataservices_client._obs_getboundariesbygeometry(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getboundariesbygeometry invoked with params (test_user, <NULL>, 0103000020E61000000100000005000000C7F8FFD37E7C52C0F860AE1175594440C7F8FFD37E7C52C0BF0E0D64E55A4440A70300E0647B52C0BF0E0D64E55A4440A70300E0647B52C0F860AE1175594440C7F8FFD37E7C52C0F860AE1175594440, us.census.tiger.census_tract, <NULL>, <NULL>)
CONTEXT: PL/pgSQL function obs_getboundariesbygeometry(geometry,text,text,text) line 16 at RETURN QUERY
obs_getboundariesbygeometry
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -244,7 +244,7 @@ CONTEXT: PL/pgSQL function obs_getboundariesbygeometry(geometry,text,text,text)
(1 row)
SELECT obs_getboundariesbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500, 'us.census.tiger.census_tract'::text);
NOTICE: cdb_dataservices_client._obs_getboundariesbypointandradius(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getboundariesbypointandradius invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 500, us.census.tiger.census_tract, <NULL>, intersects)
NOTICE: cdb_dataservices_client._obs_getboundariesbypointandradius(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getboundariesbypointandradius invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 500, us.census.tiger.census_tract, <NULL>, <NULL>)
CONTEXT: PL/pgSQL function obs_getboundariesbypointandradius(geometry,numeric,text,text,text) line 16 at RETURN QUERY
obs_getboundariesbypointandradius
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -252,7 +252,7 @@ CONTEXT: PL/pgSQL function obs_getboundariesbypointandradius(geometry,numeric,t
(1 row)
SELECT obs_getpointsbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text);
NOTICE: cdb_dataservices_client._obs_getpointsbygeometry(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getpointsbygeometry invoked with params (test_user, <NULL>, 0103000020E61000000100000005000000C7F8FFD37E7C52C0F860AE1175594440C7F8FFD37E7C52C0BF0E0D64E55A4440A70300E0647B52C0BF0E0D64E55A4440A70300E0647B52C0F860AE1175594440C7F8FFD37E7C52C0F860AE1175594440, us.census.tiger.census_tract, <NULL>, intersects)
NOTICE: cdb_dataservices_client._obs_getpointsbygeometry(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getpointsbygeometry invoked with params (test_user, <NULL>, 0103000020E61000000100000005000000C7F8FFD37E7C52C0F860AE1175594440C7F8FFD37E7C52C0BF0E0D64E55A4440A70300E0647B52C0BF0E0D64E55A4440A70300E0647B52C0F860AE1175594440C7F8FFD37E7C52C0F860AE1175594440, us.census.tiger.census_tract, <NULL>, <NULL>)
CONTEXT: PL/pgSQL function obs_getpointsbygeometry(geometry,text,text,text) line 16 at RETURN QUERY
obs_getpointsbygeometry
------------------------------------------------------------------
@@ -260,7 +260,7 @@ CONTEXT: PL/pgSQL function obs_getpointsbygeometry(geometry,text,text,text) lin
(1 row)
SELECT obs_getpointsbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500::numeric, 'us.census.tiger.census_tract'::text);
NOTICE: cdb_dataservices_client._obs_getpointsbypointandradius(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getpointsbypointandradius invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 500, us.census.tiger.census_tract, <NULL>, intersects)
NOTICE: cdb_dataservices_client._obs_getpointsbypointandradius(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getpointsbypointandradius invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 500, us.census.tiger.census_tract, <NULL>, <NULL>)
CONTEXT: PL/pgSQL function obs_getpointsbypointandradius(geometry,numeric,text,text,text) line 16 at RETURN QUERY
obs_getpointsbypointandradius
------------------------------------------------------------------
@@ -268,7 +268,7 @@ CONTEXT: PL/pgSQL function obs_getpointsbypointandradius(geometry,numeric,text,
(1 row)
SELECT obs_getmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.acs.B01001001'::text);
NOTICE: cdb_dataservices_client._obs_getmeasure(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeasure invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, us.census.acs.B01001001, area, <NULL>, <NULL>)
NOTICE: cdb_dataservices_client._obs_getmeasure(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeasure invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, us.census.acs.B01001001, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeasure(username, orgname, geom, measure_id, normalize, boundary_id, time_span)"
PL/pgSQL function obs_getmeasure(geometry,text,text,text,text) line 16 at SQL statement
obs_getmeasure
@@ -295,7 +295,7 @@ PL/pgSQL function obs_getcategory(geometry,text,text,text) line 16 at SQL statem
(1 row)
SELECT obs_getuscensusmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'male population'::text);
NOTICE: cdb_dataservices_client._obs_getuscensusmeasure(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getuscensusmeasure invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, male population, area, <NULL>, <NULL>)
NOTICE: cdb_dataservices_client._obs_getuscensusmeasure(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getuscensusmeasure invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, male population, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getuscensusmeasure(username, orgname, geom, name, normalize, boundary_id, time_span)"
PL/pgSQL function obs_getuscensusmeasure(geometry,text,text,text,text) line 16 at SQL statement
obs_getuscensusmeasure
@@ -313,7 +313,7 @@ PL/pgSQL function obs_getuscensuscategory(geometry,text,text,text) line 16 at SQ
(1 row)
SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326));
NOTICE: cdb_dataservices_client._obs_getpopulation(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getpopulation invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, area, <NULL>, <NULL>)
NOTICE: cdb_dataservices_client._obs_getpopulation(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getpopulation invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getpopulation(username, orgname, geom, normalize, boundary_id, time_span)"
PL/pgSQL function obs_getpopulation(geometry,text,text,text) line 16 at SQL statement
obs_getpopulation

View File

@@ -3,65 +3,63 @@ SET search_path TO public,cartodb,cdb_dataservices_client;
CREATE TABLE my_table(cartodb_id int);
INSERT INTO my_table (cartodb_id) VALUES (1);
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
BEGIN
RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_client.ds_return_metadata AS $$
BEGIN
RETURN (Array['total_pop'], Array['double precision']);
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS RECORD AS $$
BEGIN
RETURN (23.4::double precision, 1::int);
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
RETURNS boolean AS $$
BEGIN
RETURN true;
END;
$$ LANGUAGE 'plpgsql';
-- Augment a table with the total_pop column
SELECT cdb_dataservices_client._OBS_AugmentTable('my_table', 'dummy', '{"dummy":"dummy"}'::json);
_obs_augmenttable
-------------------
-- Create a sample user table
CREATE TABLE user_table (cartodb_id int, the_geom geometry);
INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
INSERT INTO user_table(cartodb_id, the_geom) VALUES (2, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
INSERT INTO user_table(cartodb_id, the_geom) VALUES (3, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-- Prepare a table with the total_pop column
SELECT cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure('my_table_dst', '{"dummy":"dummy"}'::json);
_dst_preparetableobs_getmeasure
---------------------------------
t
(1 row)
-- The results of the table should return the mocked value of 23.4 in the total_pop column
SELECT * FROM my_table;
cartodb_id | total_pop
------------+-----------
1 | 23.4
(1 row)
-- The table should now exist and be empty
SELECT * FROM my_table_dst;
cartodb_id | the_geom | total_pop
------------+----------+-----------
(0 rows)
-- Mock again the function for it to return a different value now
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS RECORD AS $$
BEGIN
RETURN (577777.4::double precision, 1::int);
END;
$$ LANGUAGE 'plpgsql';
-- Augment a new table with total_pop
SELECT cdb_dataservices_client._OBS_GetTable('my_table', 'my_table_new', 'dummy', '{"dummy":"dummy"}'::json);
_obs_gettable
---------------
-- Populate the table with measurement data
SELECT cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure('user_table', 'my_table_dst', '{"dummy":"dummy"}'::json);
_dst_populatetableobs_getmeasure
----------------------------------
t
(1 row)
-- Check that the table contains the new value for total_pop and not the value already existent in the table
SELECT * FROM my_table_new;
total_pop | cartodb_id
-----------+------------
577777.4 | 1
(1 row)
-- The table should now show the results
SELECT * FROM my_table_dst;
cartodb_id | the_geom | total_pop
------------+----------------------------------------------------+-----------
1 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 | 23.4
2 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |
3 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |
(3 rows)
-- Clean tables
DROP TABLE my_table;
DROP TABLE my_table_new;
DROP TABLE my_table_dst;

View File

@@ -80,7 +80,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getboundariesbypointandradius invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, radius, boundary_id, time_span, overlap_type;
@@ -88,7 +88,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getpointsbygeometry invoked with params (%, %, %, %, %, %)', username, orgname, geom, boundary_id, time_span, overlap_type;
@@ -96,7 +96,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpointsbypointandradius (username text, orgname text, geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getpointsbypointandradius invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, radius, boundary_id, time_span, overlap_type;
@@ -104,7 +104,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeasure (username text, orgname text, geom Geometry, measure_id text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -137,7 +137,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getuscensusmeasure (username text, orgname text, geom Geometry, name text, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -159,7 +159,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT 'area', boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getpopulation (username text, orgname text, geom Geometry, normalize text DEFAULT NULL, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
RETURNS numeric AS $$
DECLARE
ret numeric;
@@ -186,7 +186,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT 'intersects')
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getboundariesbygeometry (username text, orgname text, geom geometry(Geometry, 4326), boundary_id text, time_span text DEFAULT NULL, overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getboundariesbygeometry invoked with params (%, %, %, %, %, %)', username, orgname, geom, boundary_id, time_span, overlap_type;

View File

@@ -6,54 +6,51 @@ CREATE TABLE my_table(cartodb_id int);
INSERT INTO my_table (cartodb_id) VALUES (1);
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
BEGIN
RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_client.ds_return_metadata AS $$
BEGIN
RETURN (Array['total_pop'], Array['double precision']);
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS RECORD AS $$
BEGIN
RETURN (23.4::double precision, 1::int);
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
RETURNS boolean AS $$
BEGIN
RETURN true;
END;
$$ LANGUAGE 'plpgsql';
-- Augment a table with the total_pop column
SELECT cdb_dataservices_client._OBS_AugmentTable('my_table', 'dummy', '{"dummy":"dummy"}'::json);
-- Create a sample user table
CREATE TABLE user_table (cartodb_id int, the_geom geometry);
INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
INSERT INTO user_table(cartodb_id, the_geom) VALUES (2, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
INSERT INTO user_table(cartodb_id, the_geom) VALUES (3, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-- The results of the table should return the mocked value of 23.4 in the total_pop column
SELECT * FROM my_table;
-- Prepare a table with the total_pop column
SELECT cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure('my_table_dst', '{"dummy":"dummy"}'::json);
-- Mock again the function for it to return a different value now
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS RECORD AS $$
BEGIN
RETURN (577777.4::double precision, 1::int);
END;
$$ LANGUAGE 'plpgsql';
-- The table should now exist and be empty
SELECT * FROM my_table_dst;
-- Augment a new table with total_pop
SELECT cdb_dataservices_client._OBS_GetTable('my_table', 'my_table_new', 'dummy', '{"dummy":"dummy"}'::json);
-- Populate the table with measurement data
SELECT cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure('user_table', 'my_table_dst', '{"dummy":"dummy"}'::json);
-- Check that the table contains the new value for total_pop and not the value already existent in the table
SELECT * FROM my_table_new;
-- The table should now show the results
SELECT * FROM my_table_dst;
-- Clean tables
DROP TABLE my_table;
DROP TABLE my_table_new;
DROP TABLE my_table_dst;

View File

@@ -1,6 +1,6 @@
# Demographic Functions
The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](/carto-engine/dataservices-api/geocoding-functions/) documentation.
The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/) documentation.
_**Note:** The Demographic Snapshot functions are only available for the United States._

View File

@@ -221,7 +221,7 @@ This function geocodes your data into point, or polygon, geometries for postal c
### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
Goecodes the postal code for a specified country into a **polygon** geometry.
Geocodes the postal code for a specified country into a **polygon** geometry.
#### Arguments
@@ -250,7 +250,7 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_postalcode_polygon('11211'
### cdb_geocode_postalcode_point(_code text, country_name text_)
Goecodes the postal code for a specified country into a **point** geometry.
Geocodes the postal code for a specified country into a **point** geometry.
#### Arguments
@@ -290,8 +290,7 @@ Geocodes a postal code from a specified country into an IP address, displayed as
Name | Type | Description
--- | --- | ---
`ip_address` | `text` | Postal code
`country_name` | `text` | IPv4 or IPv6 address
`ip_address` | `text` | IPv4 or IPv6 address
#### Returns

View File

@@ -8,16 +8,18 @@ ERB = erb
REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
TEST_DIR = test/
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
# DATA is a special variable used by postgres build infrastructure
# These are the files to be installed in the server shared dir,
# for installation from scratch, upgrades and downgrades.
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(EXTENSION)--*.sql
OLD_VERSIONS = $(wildcard old_versions/*.sql)
DATA = $(NEW_EXTENSION_ARTIFACT) \
$(EXTENSION)--*--*.sql \
$(OLD_VERSIONS)
SOURCES_DATA_DIR = sql/
SOURCES_DATA = $(wildcard sql/*.sql)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.13.0'
default_version = '0.17.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

View File

@@ -0,0 +1,21 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;

View File

@@ -0,0 +1,21 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.12.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;

View File

@@ -0,0 +1,37 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,48 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,240 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.3'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_geocoder_config(text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_isolines_config(text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- mapzen isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- mapzen isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = isoline
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,223 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
-- isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = isoline
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -0,0 +1,49 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.3.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,49 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.3'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,65 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.14.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
import json
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
from cartodb_services.mapzen.types import polyline_to_linestring
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_routing_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
client = MapzenRouting(user_routing_config.mapzen_api_key, logger)
if not waypoints or len(waypoints) < 2:
logger.info("Empty origin or destination")
quota_service.increment_empty_service_use()
return [None, None, None]
waypoint_coords = []
for waypoint in waypoints:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
quota_service.increment_success_service_use()
return [shape_linestring, resp.length, resp.duration]
else:
quota_service.increment_empty_service_use()
return [None, None, None]
else:
quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate mapzen routing')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -0,0 +1,403 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.14.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['population']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,65 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.14'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
import json
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
from cartodb_services.mapzen.types import polyline_to_linestring
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_routing_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
client = MapzenRouting(user_routing_config.mapzen_api_key, logger)
if not waypoints or len(waypoints) < 2:
logger.notice("Empty origin or destination")
quota_service.increment_empty_service_use()
return [None, None, None]
waypoint_coords = []
for waypoint in waypoints:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
quota_service.increment_success_service_use()
return [shape_linestring, resp.length, resp.duration]
else:
quota_service.increment_empty_service_use()
return [None, None, None]
else:
quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate mapzen routing')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,403 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.14.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT 'area',
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT 'area',
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT 'area',
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT 'area',
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT 'area',
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT 'area',
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['population']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type text DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius(
username TEXT,
orgname TEXT,
geom geometry(Point, 4326),
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,181 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.15.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_geocoder_config(text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import GeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider)
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
try:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
except BaseException as e:
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
try:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
except BaseException as e:
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
try:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
except BaseException as e:
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen')))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country_name:
country_iso3 = country_to_iso3(country_name)
coordinates = geocoder.geocode(searchtext=city_name, city=None,
state_province=admin1_name,
country=country_iso3, search_type='locality')
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode city point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
if admin1_name and country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
elif country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
else:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3, search_type='address')
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,169 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.14.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_geocoder_config(text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_geocode_namedplace(text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_internal_geocode_namedplace(text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import GeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,44 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.15.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_ConnectUserTable(text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetReturnMetadata(text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_FetchJoinFdwTableData(text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_DisconnectUserTable(text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_server.ds_return_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_GetReturnMetadata;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS SETOF record AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_FetchJoinFdwTableData;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
RETURNS boolean AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_DisconnectUserTable;
$$ LANGUAGE plproxy;

View File

@@ -0,0 +1,44 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.15.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._DST_ConnectUserTable(text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server.__DST_ConnectUserTable(text, text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._DST_GetReturnMetadata(text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_server._DST_FetchJoinFdwTableData(text, text, text, text, text, json);
DROP FUNCTION IF EXISTS cdb_dataservices_server._DST_DisconnectUserTable(text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_server.ds_return_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_GetReturnMetadata;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS SETOF record AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_FetchJoinFdwTableData;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
RETURNS boolean AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_DisconnectUserTable;
$$ LANGUAGE plproxy;

View File

@@ -0,0 +1,69 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.16.0'" to load this file. \quit
-- Here goes your code to upgrade/downgrade
-- This is done in order to avoid an undesired depedency on cartodb extension
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text)
RETURNS JSON AS $$
SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key;
$$ LANGUAGE SQL STABLE SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
import cartodb_services
cartodb_services.init(plpy, GD)
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
server_config_backend = ServerConfigBackendFactory().get()
environment = ServerEnvironmentBuilder(server_config_backend).get()
user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
logger = Logger(logger_config)
mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3, search_type='address')
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,54 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.15.1'" to load this file. \quit
-- Here goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_conf_getconf(text);
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
import cartodb_services
cartodb_services.init(plpy, GD)
from cartodb_services.config.user import User
from cartodb_services.config.configs import ConfigsFactory
from cartodb_services.config.hires_geocoder_config import HiResGeocoderConfigFactory
from cartodb_services.request.request import RequestFactory
user = User(username, orgname)
configs = ConfigsFactory.get(user)
request = RequestFactory().create(user, configs, 'cdb_geocode_street_point')
# TODO change to hires_geocoder_config = HiResGeocoderConfigFactory.get(request)
hires_geocoder_config = HiResGeocoderConfigFactory(configs).get(user)
if hires_geocoder_config.provider == 'here':
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif hires_geocoder_config.provider == 'google':
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif hires_geocoder_config.provider == 'mapzen':
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Here geocoder is not available for your account.')
$$ LANGUAGE plpythonu;

File diff suppressed because it is too large Load Diff

View File

@@ -17,19 +17,24 @@ RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.mapzen.types import polyline_to_linestring
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = MapzenRouting(user_routing_config.mapzen_api_key)
client = MapzenRouting(user_routing_config.mapzen_api_key, logger)
if not waypoints or len(waypoints) < 2:
plpy.notice("Empty origin or destination")
logger.info("Empty origin or destination")
quota_service.increment_empty_service_use()
return [None, None, None]
@@ -52,12 +57,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$
quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate mapzen routing')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -7,15 +7,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
waypoints = [origin, destination]
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
with metrics('cdb_route_with_point', user_routing_config, logger):
waypoints = [origin, destination]
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
$$ LANGUAGE plpythonu;
@@ -27,12 +33,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
with metrics('cdb_route_with_waypoints', user_routing_config, logger):
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
$$ LANGUAGE plpythonu;

View File

@@ -11,9 +11,9 @@ RETURNS text AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
return user_obs_snapshot_config.connection_str
return user_obs_config.connection_str
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
@@ -34,36 +34,40 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
@@ -84,39 +88,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
@@ -135,36 +143,40 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_segment_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
@@ -183,39 +195,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetSegmentSnapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
@@ -223,7 +239,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT 'area',
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
@@ -236,39 +252,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
orgname TEXT,
geom geometry(Geometry, 4326),
measure_id TEXT,
normalize TEXT DEFAULT 'area',
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetMeasure: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
@@ -291,35 +311,39 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetCategory: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getcategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
@@ -327,7 +351,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT 'area',
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
@@ -340,39 +364,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
orgname TEXT,
geom geometry(Geometry, 4326),
name TEXT,
normalize TEXT DEFAULT 'area',
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetUSCensusMeasure: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getuscensusmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
@@ -395,42 +423,46 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetUSCensusCategory: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getuscensuscategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT 'area',
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
@@ -442,39 +474,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
normalize TEXT DEFAULT 'area',
normalize TEXT DEFAULT NULL,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['population']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPopulation: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getpopulation', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['population']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
@@ -497,33 +533,37 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetMeasureById: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getmeasurebyid', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -14,43 +14,47 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
search_term TEXT,
relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
if result:
resp = []
for element in result:
id = element['id']
description = element['description']
name = element['name']
aggregate = element['aggregate']
source = element['source']
resp.append([id, description, name, aggregate, source])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return [None, None, None, None, None]
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_Search: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_search', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
if result:
resp = []
for element in result:
id = element['id']
description = element['description']
name = element['name']
aggregate = element['aggregate']
source = element['source']
resp.append([id, description, name, aggregate, source])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return [None, None, None, None, None]
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_Search')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
@@ -69,40 +73,44 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
if result:
resp = []
for element in result:
id = element['boundary_id']
description = element['description']
tspan = element['time_span']
tablename = element['tablename']
resp.append([id, description, tspan, tablename])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetAvailableBoundaries: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
if result:
resp = []
for element in result:
id = element['boundary_id']
description = element['description']
tspan = element['time_span']
tablename = element['tablename']
resp.append([id, description, tspan, tablename])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -16,35 +16,39 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundary: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundary', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundary')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
@@ -65,35 +69,39 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use obs_search: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundaryid', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryId')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
@@ -114,35 +122,39 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundaryById: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
@@ -151,7 +163,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type text DEFAULT 'intersects')
overlap_type text DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type);
@@ -163,42 +175,46 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundariesByGeometry: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
@@ -208,7 +224,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndR
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
@@ -221,42 +237,46 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundariesByPointAndRadius: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
@@ -265,7 +285,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type);
@@ -277,42 +297,46 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
geom geometry(Point, 4326),
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPointsByGeometry: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
@@ -322,7 +346,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadiu
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type);
@@ -335,40 +359,44 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius
radius NUMERIC,
boundary_id TEXT,
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT 'intersects')
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPointsByPointAndRadius: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -2,33 +2,34 @@ CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname
CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
RETURNS cdb_dataservices_server.ds_return_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_GetReturnMetadata;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
RETURNS SETOF record AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_FetchJoinFdwTableData;
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
RETURNS boolean AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_DisconnectUserTable;

View File

@@ -1,4 +1,22 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config()
RETURNS boolean AS $$
cache_key = "logger_config"
if cache_key in GD:
return False
else:
from cartodb_services.tools import LoggerConfig
logger_config = LoggerConfig(plpy)
GD[cache_key] = logger_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- This is done in order to avoid an undesired depedency on cartodb extension
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text)
RETURNS JSON AS $$
SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key;
$$ LANGUAGE SQL STABLE SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
@@ -7,39 +25,11 @@ RETURNS boolean AS $$
from cartodb_services.metrics import GeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname)
geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider)
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_internal_geocoder_config_{0}".format(username)

View File

@@ -1,22 +1,28 @@
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
@@ -32,7 +38,7 @@ RETURNS Geometry AS $$
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Here geocoder is not available for your account.')
raise Exception('Here geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
@@ -47,7 +53,7 @@ RETURNS Geometry AS $$
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Google geocoder is not available for your account.')
raise Exception('Google geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
@@ -56,8 +62,8 @@ RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
@@ -68,17 +74,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin
RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_service_use()
@@ -89,12 +99,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using here maps')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -103,13 +111,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_po
RETURNS Geometry AS $$
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_service_use()
@@ -120,36 +133,54 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using google maps')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
import cartodb_services
cartodb_services.init(plpy, GD)
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
server_config_backend = ServerConfigBackendFactory().get()
environment = ServerEnvironmentBuilder(server_config_backend).get()
user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
logger = Logger(logger_config)
mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
country=country_iso3, search_type='address')
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@@ -159,12 +190,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -1,14 +1,20 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
@@ -20,12 +26,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin0 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -1,15 +1,22 @@
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
@@ -21,12 +28,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -34,34 +39,39 @@ $$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------

View File

@@ -1,84 +1,108 @@
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
try:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
except BaseException as e:
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
try:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
except BaseException as e:
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
try:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
except BaseException as e:
internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
$$ LANGUAGE plpythonu;
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService, metrics
from cartodb_services.tools import Logger,LoggerConfig
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen')))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country_name:
country_iso3 = country_to_iso3(country_name)
coordinates = geocoder.geocode(searchtext=city_name, city=None,
state_province=admin1_name,
country=country_iso3, search_type='locality')
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode city point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig, metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
if admin1_name and country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
elif country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
else:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
@@ -87,12 +111,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -1,14 +1,20 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [code], 1)
@@ -20,27 +26,31 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
@@ -52,27 +62,31 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [code], 1)
@@ -84,27 +98,31 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
@@ -116,12 +134,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -1,14 +1,20 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
@@ -20,12 +26,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -6,17 +6,22 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.here import HereMapsRoutingIsoline
from cartodb_services.metrics import QuotaService
from cartodb_services.here.types import geo_polyline_to_multipolygon
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL)
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id,
user_isolines_routing_config.heremaps_app_code, logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@@ -44,12 +49,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;
@@ -65,21 +68,24 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.mapzen import MatrixClient, MapzenIsolines
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger)
mapzen_isolines = MapzenIsolines(client, logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@@ -117,13 +123,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -1,4 +1,25 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
raise Exception('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -6,18 +27,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
return result
$$ LANGUAGE plpythonu;
-- mapzen isodistance
@@ -25,11 +38,11 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(userna
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result

View File

@@ -1,4 +1,31 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.metrics import metrics
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
if user_isolines_config.google_services_user:
raise Exception('This service is not available for google service users.')
with metrics('cb_isochrone', user_isolines_config, logger):
if user_isolines_config.heremaps_provider:
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -6,32 +33,22 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
return result
$$ LANGUAGE plpythonu;
-- mapzen isochrones
-- mapzen isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;

View File

@@ -4,7 +4,6 @@ CREATE EXTENSION plpythonu;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
CREATE EXTENSION observatory VERSION 'dev';
-- Install the extension
CREATE EXTENSION cdb_dataservices_server;
-- Mock the redis server connection to point to this very test db

View File

@@ -2,7 +2,7 @@ SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_connectusertable'
AND proname = '_dst_connectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
exists
--------
@@ -13,7 +13,7 @@ SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_getreturnmetadata'
AND proname = '_dst_getreturnmetadata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
exists
--------
@@ -24,7 +24,7 @@ SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_fetchjoinfdwtabledata'
AND proname = '_dst_fetchjoinfdwtabledata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, json');
exists
--------
@@ -35,7 +35,7 @@ SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_disconnectusertable'
AND proname = '_dst_disconnectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text');
exists
--------

View File

@@ -27,7 +27,7 @@ INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lower
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'ES');
-- Insert dummy data into admin1 decoder table
INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
-- This should return the point inserted above

View File

@@ -4,7 +4,6 @@ CREATE EXTENSION plpythonu;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
CREATE EXTENSION observatory VERSION 'dev';
-- Install the extension
CREATE EXTENSION cdb_dataservices_server;

View File

@@ -2,27 +2,27 @@ SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_connectusertable'
AND proname = '_dst_connectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_getreturnmetadata'
AND proname = '_dst_getreturnmetadata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, json');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_fetchjoinfdwtabledata'
AND proname = '_dst_fetchjoinfdwtabledata'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, json');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = '_obs_disconnectusertable'
AND proname = '_dst_disconnectusertable'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text');

View File

@@ -15,7 +15,7 @@ INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lower
);
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'ES');
-- Insert dummy data into admin1 decoder table
INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');

View File

@@ -1,4 +1,4 @@
# CartoDB dataservices API python module
# CARTO dataservices API python module
This directory contains the python library used by the server side of CARTO LDS (Location Data Services).

View File

@@ -0,0 +1,35 @@
# NOTE: This init function must be called from plpythonu entry points to
# initialize cartodb_services module properly. E.g:
#
# CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(...)
# RETURNS SETOF cdb_dataservices_server.isoline AS $$
#
# import cartodb_services
# cartodb_services.init(plpy, GD)
#
# # rest of the code here
# cartodb_services.GD[key] = val
# cartodb_services.plpy.execute('SELECT * FROM ...')
#
# $$ LANGUAGE plpythonu;
plpy = None
GD = None
def init(_plpy, _GD):
global plpy
global GD
if plpy is None:
plpy = _plpy
if GD is None:
GD = _GD
def _reset():
# NOTE: just for testing
global plpy
global GD
plpy = None
GD = None

View File

@@ -9,11 +9,12 @@ from exceptions import MalformedResult
class GoogleMapsGeocoder:
"""A Google Maps Geocoder wrapper for python"""
def __init__(self, client_id, client_secret):
def __init__(self, client_id, client_secret, logger):
self.client_id = self._clean_client_id(client_id)
self.client_secret = client_secret
self.geocoder = googlemaps.Client(
client_id=self.client_id, client_secret=self.client_secret)
self._logger = logger
def geocode(self, searchtext, city=None, state=None,
country=None):

View File

@@ -5,15 +5,18 @@ import json
import requests
from exceptions import *
from cartodb_services.metrics import Traceable
class HereMapsGeocoder:
class HereMapsGeocoder(Traceable):
'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
DEFAULT_MAXRESULTS = 1
DEFAULT_GEN = 9
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
ADDRESS_PARAMS = [
'city',
@@ -47,10 +50,11 @@ class HereMapsGeocoder:
'strictlanguagemode'
] + ADDRESS_PARAMS
def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS,
def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS,
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
self.app_id = app_id
self.app_code = app_code
self._logger = logger
self.maxresults = maxresults
self.gen = gen
self.host = host
@@ -84,13 +88,21 @@ class HereMapsGeocoder:
'gen': self.gen
}
request_params.update(params)
response = requests.get(self.host, params=request_params)
response = requests.get(self.host, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return json.loads(response.text)
elif response.status_code == requests.codes.bad_request:
self._logger.warning('Error 4xx trying to geocode street using HERE',
data={"response": response.json(), "params":
params})
return []
else:
response.raise_for_status()
self._logger.error('Error trying to geocode street using HERE',
data={"response": response.json(), "params":
params})
raise Exception('Error trying to geocode street using Here')
def _extract_lng_lat_from_result(self, result):
location = result['Location']

View File

@@ -2,14 +2,17 @@ import requests
import json
from exceptions import WrongParams
from cartodb_services.metrics import Traceable
class HereMapsRoutingIsoline:
class HereMapsRoutingIsoline(Traceable):
'A Here Maps Routing wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
STAGING_ROUTING_BASE_URL = 'https://isoline.route.cit.api.here.com'
ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
ACCEPTED_MODES = {
"walk": "pedestrian",
@@ -25,9 +28,11 @@ class HereMapsRoutingIsoline:
'quality'
]
def __init__(self, app_id, app_code, base_url=PRODUCTION_ROUTING_BASE_URL):
def __init__(self, app_id, app_code, logger,
base_url=PRODUCTION_ROUTING_BASE_URL):
self._app_id = app_id
self._app_code = app_code
self._logger = logger
self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH)
def calculate_isodistance(self, source, mode, data_range, options=[]):
@@ -48,13 +53,25 @@ class HereMapsRoutingIsoline:
data_range,
range_type,
parsed_options)
response = requests.get(self._url, params=request_params)
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_isolines_response(response.text)
elif response.status_code == requests.codes.bad_request:
return []
else:
response.raise_for_status()
self._logger.error('Error trying to calculate HERE isolines',
data={"response_status": response.status_code,
"response_reason": response.reason,
"response_content": response.text,
"reponse_url": response.url,
"response_headers": response.headers,
"source": source, "mode": mode,
"data_range": data_range,
"range_type": range_type,
"options": options})
raise Exception('Error trying to calculate HERE isolines')
def __parse_options(self, options):
return dict(option.split('=') for option in options)
@@ -77,8 +94,12 @@ class HereMapsRoutingIsoline:
isolines_response = parsed_response['response']['isoline']
isolines = []
for isoline in isolines_response:
if not isoline['component']:
geom_value = []
else:
geom_value = isoline['component'][0]['shape']
isolines.append({'range': isoline['range'],
'geom': isoline['component'][0]['shape']})
'geom': geom_value})
return isolines

View File

@@ -4,13 +4,18 @@ import plpy
def geo_polyline_to_multipolygon(polyline):
"""Convert a HERE polyline shape to a PostGIS multipolygon"""
coordinates = []
for point in polyline:
lat, lon = point.split(',')
coordinates.append("%s %s" % (lon, lat))
wkt_coordinates = ','.join(coordinates)
# In case we receive an empty polyline from here and we don't want to
# change this kind of thing in the extension sql
if not polyline:
sql = "SELECT ST_MPolyFromText(NULL, 4326) as geom"
else:
coordinates = []
for point in polyline:
lat, lon = point.split(',')
coordinates.append("%s %s" % (lon, lat))
wkt_coordinates = ','.join(coordinates)
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
geometry = plpy.execute(sql, 1)[0]['geom']
return geometry

View File

@@ -19,3 +19,15 @@ class MalformedResult(Exception):
class TimeoutException(Exception):
def __str__(self):
return repr('Timeout requesting to mapzen server')
class ServiceException(Exception):
def __init__(self, message, response):
self.message = message
self.response = response
def response(self):
return self.response
def __str__(self):
return self.message

View File

@@ -2,38 +2,73 @@ import requests
import json
import re
from exceptions import WrongParams, MalformedResult
from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine
from cartodb_services.metrics import Traceable
class MapzenGeocoder:
class MapzenGeocoder(Traceable):
'A Mapzen Geocoder wrapper for python'
BASE_URL = 'https://search.mapzen.com/v1/search'
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
def __init__(self, app_key, base_url=BASE_URL):
def __init__(self, app_key, logger, base_url=BASE_URL):
self._app_key = app_key
self._url = base_url
self._logger = logger
@qps_retry
def geocode(self, searchtext, city=None, state_province=None, country=None):
request_params = self._build_requests_parameters(searchtext, city, state_province, country)
response = requests.get(self._url, params=request_params)
if response.status_code == requests.codes.ok:
return self.__parse_response(response.text)
elif response.status_code == requests.codes.bad_request:
@qps_retry(qps=20)
def geocode(self, searchtext, city=None, state_province=None,
country=None, search_type=None):
request_params = self._build_requests_parameters(searchtext, city,
state_province,
country, search_type)
try:
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_response(response.text)
elif response.status_code == requests.codes.bad_request:
return []
else:
self._logger.error('Error trying to geocode using mapzen',
data={"response_status": response.status_code,
"response_reason": response.reason,
"response_content": response.text,
"reponse_url": response.url,
"response_headers": response.headers,
"searchtext": searchtext,
"city": city, "country": country,
"state_province": state_province})
raise ServiceException('Error trying to geocode {0} using mapzen'.format(searchtext),
response)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
self._logger.error('Timeout connecting to Mapzen geocoding server')
raise ServiceException('Error trying to geocode {0} using mapzen'.format(searchtext),
None)
except requests.ConnectionError as e:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapzen geocoding server',
exception=e)
return []
else:
response.raise_for_status()
def _build_requests_parameters(self, searchtext, city=None,
state_province=None, country=None):
state_province=None, country=None,
search_type=None):
request_params = {}
search_string = self._build_search_text(searchtext, city, state_province)
search_string = self._build_search_text(searchtext.strip(),
city,
state_province)
request_params['text'] = search_string
request_params['layers'] = 'address'
request_params['api_key'] = self._app_key
if search_type:
request_params['layers'] = search_type
if country:
request_params['boundary.country'] = country
return request_params

View File

@@ -1,5 +1,5 @@
from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2
import logging
class MapzenIsolines:
@@ -9,8 +9,9 @@ class MapzenIsolines:
EARTH_RADIUS_METERS = 6367444
def __init__(self, matrix_client):
def __init__(self, matrix_client, logger):
self._matrix_client = matrix_client
self._logger = logger
"""Get an isochrone using mapzen API.
@@ -85,11 +86,7 @@ class MapzenIsolines:
def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0):
# NOTE: not for production
#logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log')
#logging.basicConfig(level=logging.DEBUG)
logging.debug('origin = %s' % origin)
logging.debug('costing_model = %s' % costing_model)
logging.debug('isorange = %d' % isorange)
# self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange})
# Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
# we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE
@@ -108,14 +105,16 @@ class MapzenIsolines:
response = self._matrix_client.one_to_many([origin] + location_estimates, costing_model)
costs = [None] * self.NUMBER_OF_ANGLES
if not response:
# In case the matrix client doesn't return any data
break
for idx, c in enumerate(response['one_to_many'][0][1:]):
if c[cost_variable]:
costs[idx] = c[cost_variable]*unit_factor
else:
costs[idx] = isorange
logging.debug('i = %d, costs = %s' % (i, costs))
errors = [(cost - isorange) / float(isorange) for cost in costs]
max_abs_error = max([abs(e) for e in errors])
if max_abs_error <= self.TOLERANCE:

View File

@@ -1,7 +1,11 @@
import requests
import json
from qps import qps_retry
from exceptions import ServiceException
from cartodb_services.metrics import Traceable
class MatrixClient:
class MatrixClient(Traceable):
"""
A minimal client for Mapzen Time-Distance Matrix Service
@@ -16,9 +20,12 @@ class MatrixClient:
"""
ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many'
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
def __init__(self, matrix_key):
def __init__(self, matrix_key, logger):
self._matrix_key = matrix_key
self._logger = logger
"""Get distances and times to a set of locations.
See https://mapzen.com/documentation/matrix/api-reference/
@@ -30,14 +37,42 @@ class MatrixClient:
Returns:
A dict with one_to_many, units and locations
"""
@qps_retry
def one_to_many(self, locations, costing):
request_params = {
'json': json.dumps({'locations': locations}),
'costing': costing,
'api_key': self._matrix_key
}
response = requests.get(self.ONE_TO_MANY_URL, params=request_params)
response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
response.raise_for_status() # raise exception if not 200 OK
if response.status_code != requests.codes.ok:
self._logger.error('Error trying to get matrix distance from mapzen',
data={"response_status": response.status_code,
"response_reason": response.reason,
"response_content": response.text,
"reponse_url": response.url,
"response_headers": response.headers,
"locations": locations,
"costing": costing})
# In case 4xx error we return empty because the error comes from
# the provided info by the user and we don't want to top the
# isolines generation
if response.status_code == requests.codes.bad_request:
return {}
elif response.status_code == 504:
# Due to some unsolved problems in the Mapzen Matrix API we're
# getting randomly 504, probably timeouts. To avoid raise an
# exception in all the jobs, for now we're going to return
# empty in that case
return {}
else:
raise ServiceException("Error trying to get matrix distance from mapzen", response)
return response.json()
# response could return with empty json
try:
return response.json()
except:
return {}

View File

@@ -4,18 +4,38 @@ from datetime import datetime
from exceptions import TimeoutException
DEFAULT_RETRY_TIMEOUT = 60
DEFAULT_QUERIES_PER_SECOND = 10
def qps_retry(f):
def wrapped_f(*args, **kw):
return QPSService().call(f, *args, **kw)
return wrapped_f
def qps_retry(original_function=None,**options):
""" Query Per Second retry decorator
The intention of this decorator is to retry requests against third
party services that has QPS restriction.
Parameters:
- timeout: Maximum number of seconds to retry
- qps: Allowed queries per second. This parameter is used to
calculate the next time to retry the request
"""
if original_function is not None:
def wrapped_function(*args, **kwargs):
if 'timeout' in options:
timeout = options['timeout']
else:
timeout = DEFAULT_RETRY_TIMEOUT
if 'qps' in options:
qps = options['qps']
else:
qps = DEFAULT_QUERIES_PER_SECOND
return QPSService(retry_timeout=timeout, queries_per_second=qps).call(original_function, *args, **kwargs)
return wrapped_function
else:
def partial_wrapper(func):
return qps_retry(func, **options)
return partial_wrapper
class QPSService:
def __init__(self, queries_per_second=10,
retry_timeout=DEFAULT_RETRY_TIMEOUT):
def __init__(self, queries_per_second, retry_timeout):
self._queries_per_second = queries_per_second
self._retry_timeout = retry_timeout
@@ -26,7 +46,8 @@ class QPSService:
try:
return fn(*args, **kwargs)
except Exception as e:
if hasattr(e, 'response') and e.response.status_code == 429:
response = getattr(e, 'response', None)
if response is not None and (response.status_code == 429):
self.retry(start_time, attempt_number)
else:
raise e
@@ -34,7 +55,7 @@ class QPSService:
def retry(self, first_request_time, retry_count):
elapsed = datetime.now() - first_request_time
if elapsed.seconds > self._retry_timeout:
if elapsed.microseconds > (self._retry_timeout * 1000.0):
raise TimeoutException()
# inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per

Some files were not shown because too many files have changed in this diff Show More