cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Server data observatory functions

Browse Source
This commit is contained in:
Mario de Frutos
2016-04-20 15:58:38 +02:00
parent dda13033b0
commit 3c5325c1d7
12 changed files with 212 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/extension/sql/110_data_observatory.sql
+76
View File
@@ -0,0 +1,76 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
quota_service = QuotaService(user_data_observatory_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
quota_service = QuotaService(user_data_observatory_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
server/extension/sql/15_config_helper.sql
+14 -4
View File
@@ -1,4 +1,3 @@
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
@@ -13,7 +12,6 @@ RETURNS boolean AS $$
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_internal_geocoder_config_{0}".format(username)
@@ -28,7 +26,6 @@ RETURNS boolean AS $$
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_isolines_routing_config_{0}".format(username)
@@ -43,7 +40,6 @@ RETURNS boolean AS $$
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_routing_config_{0}".format(username)
@@ -57,3 +53,17 @@ RETURNS boolean AS $$
GD[cache_key] = routing_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_data_observatory_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import DataObservatoryConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = data_observatory_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
server/extension/test/expected/00_install_test.out
+7
View File
@@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
CREATE EXTENSION observatory VERSION 'dev';
-- Install the extension
CREATE EXTENSION cdb_dataservices_server;
-- Mock the redis server connection to point to this very test db
@@ -37,6 +38,12 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null
(1 row)
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
cdb_conf_setconf
------------------
(1 row)
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)
CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$
server/extension/test/expected/100_data_observatory_test.out
+22
View File
@@ -0,0 +1,22 @@
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_get_demographic_snapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_get_segment_snapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text');
exists
--------
t
(1 row)
server/extension/test/sql/00_install_test.sql
+2
View File
@@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
CREATE EXTENSION observatory VERSION 'dev';
-- Install the extension
CREATE EXTENSION cdb_dataservices_server;
@@ -14,6 +15,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)
server/extension/test/sql/100_data_observatory_test.sql
+13
View File
@@ -0,0 +1,13 @@
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_get_demographic_snapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_get_segment_snapshot'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text');