Server data observatory functions

server/extension/sql/110_data_observatory.sql

@@ -0,0 +1,76 @@
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT '2009 - 2013',
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_data_observatory_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reach the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
+      result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_data_observatory_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reach the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
+      result = plpy.execute(obs_plan, [geom, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/sql/15_config_helper.sql

@@ -1,4 +1,3 @@
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_geocoder_config_{0}".format(username)
@@ -13,7 +12,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_internal_geocoder_config_{0}".format(username)
@@ -28,7 +26,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_isolines_routing_config_{0}".format(username)
@@ -43,7 +40,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_routing_config_{0}".format(username)
@@ -57,3 +53,17 @@ RETURNS boolean AS $$
     GD[cache_key] = routing_config
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_data_observatory_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import DataObservatoryConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = data_observatory_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/test/expected/00_install_test.out

@@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
+CREATE EXTENSION observatory VERSION 'dev';
 -- Install the extension
 CREATE EXTENSION cdb_dataservices_server;
 -- Mock the redis server connection to point to this very test db
@@ -37,6 +38,12 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null
  
 (1 row)
 
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 -- Mock the varnish invalidation function
 -- (used by cdb_geocoder tests)
 CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$

server/extension/test/expected/100_data_observatory_test.out

@@ -0,0 +1,22 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_demographic_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text, text');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_segment_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text');
+ exists 
+--------
+ t
+(1 row)
+

server/extension/test/sql/00_install_test.sql

@@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
+CREATE EXTENSION observatory VERSION 'dev';
 
 -- Install the extension
 CREATE EXTENSION cdb_dataservices_server;
@@ -14,6 +15,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
 SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
 SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
 
 -- Mock the varnish invalidation function
 -- (used by cdb_geocoder tests)

server/extension/test/sql/100_data_observatory_test.sql

@@ -0,0 +1,13 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_demographic_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text, text');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_segment_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text');

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,3 +1,3 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException
+from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, DataObservatoryConfig
 from quota import QuotaService
 from user import UserMetricsService

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -32,6 +32,28 @@ class ServiceConfig(object):
     def organization(self):
         return self._orgname
 
+class DataObservatoryConfig(ServiceConfig):
+
+    PERIOD_END_DATE = 'period_end_date'
+
+    def __init__(self, redis_connection, db_conn, username, orgname=None):
+        super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
+                                            username, orgname)
+        self._monthly_quota = self._db_config.data_observatory_monthly_quota
+        self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
+
+    @property
+    def service_type(self):
+        return 'data_observatory'
+
+    @property
+    def monthly_quota(self):
+        return self._monthly_quota
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
 
 class RoutingConfig(ServiceConfig):
 
@@ -315,6 +337,7 @@ class ServicesDBConfig:
         self._get_here_config()
         self._get_mapzen_config()
         self._get_logger_config()
+        self._get_data_observatory_config()
 
     def _get_here_config(self):
         heremaps_conf_json = self._get_conf('heremaps_conf')
@@ -340,6 +363,14 @@ class ServicesDBConfig:
             self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
             self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
 
+    def _get_data_observatory_config(self):
+        do_conf_json = self._get_conf('data_observatory_conf')
+        if not do_conf_json:
+            raise ConfigException('Data Observatory configuration missing')
+        else:
+            do_conf = json.loads(do_conf_json)
+            self._data_observatory_monthly_quota = do_conf['monthly_quota']
+
     def _get_logger_config(self):
         logger_conf_json = self._get_conf('logger_conf')
         if not logger_conf_json:
@@ -396,6 +427,10 @@ class ServicesDBConfig:
     def geocoder_log_path(self):
         return self._geocoder_log_path
 
+    @property
+    def data_observatory_monthly_quota(self):
+        return self._data_observatory_monthly_quota
+
 
 class ServicesRedisConfig:
 

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -73,6 +73,9 @@ class QuotaChecker:
         elif re.match('routing_mapzen',
                       self._user_service_config.service_type) is not None:
             return self.__check_routing_quota()
+        elif re.match('data_observatory',
+                      self._user_service_config.service_type) is not None:
+            return self.__check_data_observatory_quota()
         else:
             return False
 
@@ -114,3 +117,14 @@ class QuotaChecker:
             return True
         else:
             return False
+
+    def __check_data_observatory_quota(self):
+        user_quota = self._user_service_config.monthly_quota
+        today = date.today()
+        service_type = self._user_service_config.service_type
+        current_used = self._user_service.used_quota(service_type, today)
+
+        if (user_quota > 0 and current_used <= user_quota):
+            return True
+        else:
+            return False

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.4.5',
+    version='0.5.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_helper.py

@@ -51,3 +51,5 @@ def _plpy_execute_side_effect(*args, **kwargs):
         return [{'conf': '{"routing": {"api_key": "valhalla-Z61FWEs", "monthly_quota": 1500000}, "geocoder": {"api_key": "search-d744tp0", "monthly_quota": 1500000}}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
         return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
+    elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
+        return [{'conf': '{"monthly_quota": 100000}'}]

server/lib/python/cartodb_services/test/test_quota_service.py

@@ -1,7 +1,7 @@
 import test_helper
 from mockredis import MockRedis
 from cartodb_services.metrics import QuotaService
-from cartodb_services.metrics import GeocoderConfig, RoutingConfig
+from cartodb_services.metrics import GeocoderConfig, RoutingConfig, DataObservatoryConfig
 from unittest import TestCase
 from nose.tools import assert_raises
 from datetime import datetime, date
@@ -109,6 +109,20 @@ class TestQuotaService(TestCase):
         qs.increment_success_service_use(amount=1500000)
         assert qs.check_user_quota() is False
 
+    def test_should_check_user_data_observatory_quota_correctly(self):
+        qs = self.__build_data_observatory_quota_service('test_user')
+        qs.increment_success_service_use()
+        assert qs.check_user_quota() is True
+        qs.increment_success_service_use(amount=100000)
+        assert qs.check_user_quota() is False
+
+    def test_should_check_org_data_observatory_quota_correctly(self):
+        qs = self.__build_data_observatory_quota_service('test_user', orgname='testorg')
+        qs.increment_success_service_use()
+        assert qs.check_user_quota() is True
+        qs.increment_success_service_use(amount=100000)
+        assert qs.check_user_quota() is False
+
     def __prepare_quota_service(self, username, quota, service, orgname,
                                 soft_limit, end_date):
         test_helper.build_redis_user_config(self.redis_conn, username,
@@ -139,3 +153,13 @@ class TestQuotaService(TestCase):
         routing_config = RoutingConfig(self.redis_conn, self._plpy_mock,
                                        username, orgname)
         return QuotaService(routing_config, redis_connection=self.redis_conn)
+
+    def __build_data_observatory_quota_service(self, username, quota=100,
+                                               service='data_observatory', orgname=None,
+                                               soft_limit=False,
+                                               end_date=datetime.today()):
+        self.__prepare_quota_service(username, quota, service, orgname,
+                                     soft_limit, end_date)
+        do_config = DataObservatoryConfig(self.redis_conn, self._plpy_mock,
+                                          username, orgname)
+        return QuotaService(do_config, redis_connection=self.redis_conn)