Merge pull request #155 from CartoDB/do_integration

Data observatory integration

client/renderer/interface.yaml

@@ -89,3 +89,15 @@
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
     - { name: units, type: "text", default: "'kilometers'"}
 
+- name: obs_get_demographic_snapshot
+  return_type: json
+  params:
+    - { name: geom,  type: "geometry(Geometry, 4326)" }
+    - { name: time_span,  type: "text", default: "'2009 - 2013'::text" }
+    - { name: geometry_level, type: text, default: "'\"us.census.tiger\".block_group'::text" }
+
+- name: obs_get_segment_snapshot
+  return_type: json
+  params:
+    - { name: geom,  type: "geometry(Geometry, 4326)" }
+    - { name: geometry_level, type: text, default: "'\"us.census.tiger\".census_tract'::text" }

client/renderer/templates/20_public_functions.erb

@@ -1,5 +1,5 @@
 --
--- Public geocoder API function
+-- Public dataservices API function
 --
 -- These are the only ones with permissions to publicuser role
 -- and should also be the only ones with SECURITY DEFINER

client/test/expected/90_data_observatory_test.out

@@ -0,0 +1,42 @@
+-- Add to the search path the schema
+SET search_path TO public,cartodb,cdb_dataservices_client;
+-- Mock the server functions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013', geometry_level text DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+DECLARE
+  ret json;
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (%, %, %, %, %)', username, orgname, geom, time_span, geometry_level;
+  SELECT '{"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null}'::json INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT '"us.census.tiger".census_tract')
+RETURNS json AS $$
+DECLARE
+  ret json;
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_get_segment_snapshot invoked with params (%, %, %, %)', username, orgname, geom, geometry_level;
+  SELECT '{"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null}'::json INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+-- Exercise the public and the proxied function
+SELECT obs_get_demographic_snapshot('POINT(-87.81406 41.89308)'::geometry);
+NOTICE:  cdb_dataservices_client._obs_get_demographic_snapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (test_user, <NULL>, 0101000000D53E1D8F19F455C0185B087250F24440, 2009 - 2013, "us.census.tiger".block_group)
+CONTEXT:  SQL statement "SELECT cdb_dataservices_client._obs_get_demographic_snapshot(username, orgname, geom, time_span, geometry_level)"
+PL/pgSQL function obs_get_demographic_snapshot(geometry,text,text) line 16 at SQL statement
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        obs_get_demographic_snapshot                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null}
+(1 row)
+
+SELECT obs_get_segment_snapshot('POINT(-87.81406 41.89308)'::geometry);
+NOTICE:  cdb_dataservices_client._obs_get_segment_snapshot(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_segment_snapshot invoked with params (test_user, <NULL>, 0101000000D53E1D8F19F455C0185B087250F24440, "us.census.tiger".census_tract)
+CONTEXT:  SQL statement "SELECT cdb_dataservices_client._obs_get_segment_snapshot(username, orgname, geom, geometry_level)"
+PL/pgSQL function obs_get_segment_snapshot(geometry,text) line 16 at SQL statement
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          obs_get_segment_snapshot                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null}
+(1 row)
+

client/test/sql/90_data_observatory_test.sql

@@ -0,0 +1,31 @@
+-- Add to the search path the schema
+SET search_path TO public,cartodb,cdb_dataservices_client;
+
+-- Mock the server functions
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot (username text, orgname text, geom geometry(Geometry, 4326), time_span text DEFAULT '2009 - 2013', geometry_level text DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+DECLARE
+  ret json;
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (%, %, %, %, %)', username, orgname, geom, time_span, geometry_level;
+  SELECT '{"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null}'::json INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot (username text, orgname text, geom geometry(Geometry, 4326), geometry_level text DEFAULT '"us.census.tiger".census_tract')
+RETURNS json AS $$
+DECLARE
+  ret json;
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_get_segment_snapshot invoked with params (%, %, %, %)', username, orgname, geom, geometry_level;
+  SELECT '{"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null}'::json INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+-- Exercise the public and the proxied function
+SELECT obs_get_demographic_snapshot('POINT(-87.81406 41.89308)'::geometry);
+SELECT obs_get_segment_snapshot('POINT(-87.81406 41.89308)'::geometry);

server/extension/sql/100_routing_helper.sql

@@ -25,7 +25,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
 
   quota_service = QuotaService(user_routing_config, redis_conn)
   if not quota_service.check_user_quota():
-    plpy.error('You have reach the limit of your quota')
+    plpy.error('You have reached the limit of your quota')
 
   try:
     client = MapzenRouting(user_routing_config.mapzen_api_key)

server/extension/sql/110_data_observatory.sql

@@ -0,0 +1,76 @@
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT '2009 - 2013',
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_data_observatory_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
+      result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
+RETURNS json AS $$
+  from cartodb_services.metrics import QuotaService
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_data_observatory_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
+      result = plpy.execute(obs_plan, [geom, geometry_level])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['snapshot']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/sql/15_config_helper.sql

@@ -1,4 +1,3 @@
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_geocoder_config_{0}".format(username)
@@ -13,7 +12,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_internal_geocoder_config_{0}".format(username)
@@ -28,7 +26,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_isolines_routing_config_{0}".format(username)
@@ -43,7 +40,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
--- Get the Redis configuration from the _conf table --
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_routing_config_{0}".format(username)
@@ -57,3 +53,17 @@ RETURNS boolean AS $$
     GD[cache_key] = routing_config
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_data_observatory_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import DataObservatoryConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = data_observatory_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/sql/20_geocode_street.sql

@@ -31,7 +31,7 @@ RETURNS Geometry AS $$
   # -- Check the quota
   quota_service = QuotaService(user_geocoder_config, redis_conn)
   if not quota_service.check_user_quota():
-    plpy.error('You have reach the limit of your quota')
+    plpy.error('You have reached the limit of your quota')
 
   try:
     geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
@@ -96,7 +96,7 @@ RETURNS Geometry AS $$
   user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
   quota_service = QuotaService(user_geocoder_config, redis_conn)
   if not quota_service.check_user_quota():
-    plpy.error('You have reach the limit of your quota')
+    plpy.error('You have reached the limit of your quota')
 
   try:
     geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)

server/extension/sql/80_isolines_helper.sql

@@ -13,7 +13,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   # -- Check the quota
   quota_service = QuotaService(user_isolines_routing_config, redis_conn)
   if not quota_service.check_user_quota():
-    plpy.error('You have reach the limit of your quota')
+    plpy.error('You have reached the limit of your quota')
 
   try:
     client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL)

server/extension/test/expected/00_install_test.out

@@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
+CREATE EXTENSION observatory VERSION 'dev';
 -- Install the extension
 CREATE EXTENSION cdb_dataservices_server;
 -- Mock the redis server connection to point to this very test db
@@ -37,6 +38,12 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null
  
 (1 row)
 
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 -- Mock the varnish invalidation function
 -- (used by cdb_geocoder tests)
 CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$

server/extension/test/expected/100_data_observatory_test.out

@@ -0,0 +1,22 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_demographic_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text, text');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_segment_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text');
+ exists 
+--------
+ t
+(1 row)
+

server/extension/test/sql/00_install_test.sql

@@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
+CREATE EXTENSION observatory VERSION 'dev';
 
 -- Install the extension
 CREATE EXTENSION cdb_dataservices_server;
@@ -14,6 +15,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
 SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
 SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}');
 
 -- Mock the varnish invalidation function
 -- (used by cdb_geocoder tests)

server/extension/test/sql/100_data_observatory_test.sql

@@ -0,0 +1,13 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_demographic_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text, text');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_get_segment_snapshot'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text');

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,3 +1,3 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException
+from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, DataObservatoryConfig
 from quota import QuotaService
 from user import UserMetricsService

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -32,6 +32,28 @@ class ServiceConfig(object):
     def organization(self):
         return self._orgname
 
+class DataObservatoryConfig(ServiceConfig):
+
+    PERIOD_END_DATE = 'period_end_date'
+
+    def __init__(self, redis_connection, db_conn, username, orgname=None):
+        super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
+                                            username, orgname)
+        self._monthly_quota = self._db_config.data_observatory_monthly_quota
+        self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
+
+    @property
+    def service_type(self):
+        return 'data_observatory'
+
+    @property
+    def monthly_quota(self):
+        return self._monthly_quota
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
 
 class RoutingConfig(ServiceConfig):
 
@@ -315,6 +337,7 @@ class ServicesDBConfig:
         self._get_here_config()
         self._get_mapzen_config()
         self._get_logger_config()
+        self._get_data_observatory_config()
 
     def _get_here_config(self):
         heremaps_conf_json = self._get_conf('heremaps_conf')
@@ -340,6 +363,14 @@ class ServicesDBConfig:
             self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
             self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
 
+    def _get_data_observatory_config(self):
+        do_conf_json = self._get_conf('data_observatory_conf')
+        if not do_conf_json:
+            raise ConfigException('Data Observatory configuration missing')
+        else:
+            do_conf = json.loads(do_conf_json)
+            self._data_observatory_monthly_quota = do_conf['monthly_quota']
+
     def _get_logger_config(self):
         logger_conf_json = self._get_conf('logger_conf')
         if not logger_conf_json:
@@ -396,6 +427,10 @@ class ServicesDBConfig:
     def geocoder_log_path(self):
         return self._geocoder_log_path
 
+    @property
+    def data_observatory_monthly_quota(self):
+        return self._data_observatory_monthly_quota
+
 
 class ServicesRedisConfig:
 

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -73,6 +73,9 @@ class QuotaChecker:
         elif re.match('routing_mapzen',
                       self._user_service_config.service_type) is not None:
             return self.__check_routing_quota()
+        elif re.match('data_observatory',
+                      self._user_service_config.service_type) is not None:
+            return self.__check_data_observatory_quota()
         else:
             return False
 
@@ -114,3 +117,14 @@ class QuotaChecker:
             return True
         else:
             return False
+
+    def __check_data_observatory_quota(self):
+        user_quota = self._user_service_config.monthly_quota
+        today = date.today()
+        service_type = self._user_service_config.service_type
+        current_used = self._user_service.used_quota(service_type, today)
+
+        if (user_quota > 0 and current_used <= user_quota):
+            return True
+        else:
+            return False

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.4.5',
+    version='0.5.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_helper.py

@@ -51,3 +51,5 @@ def _plpy_execute_side_effect(*args, **kwargs):
         return [{'conf': '{"routing": {"api_key": "valhalla-Z61FWEs", "monthly_quota": 1500000}, "geocoder": {"api_key": "search-d744tp0", "monthly_quota": 1500000}}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
         return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
+    elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
+        return [{'conf': '{"monthly_quota": 100000}'}]

server/lib/python/cartodb_services/test/test_quota_service.py

@@ -1,7 +1,7 @@
 import test_helper
 from mockredis import MockRedis
 from cartodb_services.metrics import QuotaService
-from cartodb_services.metrics import GeocoderConfig, RoutingConfig
+from cartodb_services.metrics import GeocoderConfig, RoutingConfig, DataObservatoryConfig
 from unittest import TestCase
 from nose.tools import assert_raises
 from datetime import datetime, date
@@ -109,6 +109,20 @@ class TestQuotaService(TestCase):
         qs.increment_success_service_use(amount=1500000)
         assert qs.check_user_quota() is False
 
+    def test_should_check_user_data_observatory_quota_correctly(self):
+        qs = self.__build_data_observatory_quota_service('test_user')
+        qs.increment_success_service_use()
+        assert qs.check_user_quota() is True
+        qs.increment_success_service_use(amount=100000)
+        assert qs.check_user_quota() is False
+
+    def test_should_check_org_data_observatory_quota_correctly(self):
+        qs = self.__build_data_observatory_quota_service('test_user', orgname='testorg')
+        qs.increment_success_service_use()
+        assert qs.check_user_quota() is True
+        qs.increment_success_service_use(amount=100000)
+        assert qs.check_user_quota() is False
+
     def __prepare_quota_service(self, username, quota, service, orgname,
                                 soft_limit, end_date):
         test_helper.build_redis_user_config(self.redis_conn, username,
@@ -139,3 +153,13 @@ class TestQuotaService(TestCase):
         routing_config = RoutingConfig(self.redis_conn, self._plpy_mock,
                                        username, orgname)
         return QuotaService(routing_config, redis_connection=self.redis_conn)
+
+    def __build_data_observatory_quota_service(self, username, quota=100,
+                                               service='data_observatory', orgname=None,
+                                               soft_limit=False,
+                                               end_date=datetime.today()):
+        self.__prepare_quota_service(username, quota, service, orgname,
+                                     soft_limit, end_date)
+        do_config = DataObservatoryConfig(self.redis_conn, self._plpy_mock,
+                                          username, orgname)
+        return QuotaService(do_config, redis_connection=self.redis_conn)

test/integration/test_admin0_functions.py

@@ -8,7 +8,7 @@ class TestAdmin0Functions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_admin1_functions.py

@@ -8,7 +8,7 @@ class TestAdmin1Functions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_data_observatory_functions.py

@@ -0,0 +1,45 @@
+from unittest import TestCase
+from nose.tools import assert_raises
+from nose.tools import assert_not_equal, assert_equal
+from ..helpers.integration_test_helper import IntegrationTestHelper
+
+
+class TestDataObservatoryFunctions(TestCase):
+
+    def setUp(self):
+        self.env_variables = IntegrationTestHelper.get_environment_variables()
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+            self.env_variables['username'],
+            self.env_variables['host'],
+            self.env_variables['api_key']
+        )
+
+    def test_if_get_demographic_snapshot_is_ok(self):
+        query = "SELECT duration, length, shape as the_geom " \
+                "FROM cdb_do_get_demographic_snapshot(CDB_LatLng(40.704512, -73.936669))".format(
+                    self.env_variables['api_key'])
+        routing = IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        assert_not_equal(routing['the_geom'], None)
+
+    def test_if_get_demographic_snapshot_without_api_key_raise_error(self):
+        query = "SELECT duration, length, shape as the_geom " \
+                "FROM cdb_do_get_demographic_snapshot(CDB_LatLng(40.704512, -73.936669))"
+        try:
+            IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        except Exception as e:
+            assert_equal(e.message[0], "The api_key must be provided")
+
+    def test_if_get_segment_snapshot_is_ok(self):
+        query = "SELECT duration, length, shape as the_geom " \
+                "FROM cdb_do_get_segment_snapshot(CDB_LatLng(40.704512, -73.936669))".format(
+                    self.env_variables['api_key'])
+        routing = IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        assert_not_equal(routing['the_geom'], None)
+
+    def test_if_get_segment_snapshot_without_api_key_raise_error(self):
+        query = "SELECT duration, length, shape as the_geom " \
+                "FROM cdb_do_get_segment_snapshot(CDB_LatLng(40.704512, -73.936669))"
+        try:
+            IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        except Exception as e:
+            assert_equal(e.message[0], "The api_key must be provided")

test/integration/test_ipaddress_functions.py

@@ -8,7 +8,7 @@ class TestPostalcodeFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_isolines_functions.py

@@ -8,7 +8,7 @@ class TestIsolinesFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_namedplace_functions.py

@@ -8,7 +8,7 @@ class TestNameplaceFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_postalcode_functions.py

@@ -8,7 +8,7 @@ class TestPostalcodeFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_routing_functions.py

@@ -8,7 +8,7 @@ class TestRoutingFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_street_functions.py

@@ -8,7 +8,7 @@ class TestStreetFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v2/sql".format(
+        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']