We let the old qlik snapshots functions but we make new ones with the new snapshot part from observatory'

2016-05-12 18:17:26 +02:00
parent f59217779e
commit 8680c9cbd0
10 changed files with 92 additions and 185 deletions
--- a/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql
+++ b/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql
@@ -1,6 +1,8 @@
 --DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
 -- Complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.8.0'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetDemographicSnapshot(text, text, geometry(Geometry, 4326), text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetSegmentSnapshot(text, text, geometry(Geometry, 4326), text);

 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -16,17 +18,16 @@ RETURNS boolean AS $$
    return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;

-CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
  username TEXT,
  orgname TEXT,
  geom geometry(Geometry, 4326),
  time_span TEXT DEFAULT NULL,
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
-  obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
-  result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
-  return result
-$$ LANGUAGE plpythonu;
+RETURNS SETOF json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level);
+$$ LANGUAGE plproxy;

 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
  username TEXT,
@@ -34,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
  geom geometry(Geometry, 4326),
  time_span TEXT DEFAULT NULL,
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
+RETURNS SETOF JSON AS $$
  from cartodb_services.metrics import QuotaService

  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -50,11 +51,15 @@ RETURNS json AS $$
      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
      result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
      if result:
+        resp = []
+        for element in result:
+          value = element['snapshot']
+          resp.append(value)
        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
+        return resp
      else:
        quota_service.increment_empty_service_use()
-        return None
+        return []
  except BaseException as e:
      import sys, traceback
      type_, value_, traceback_ = sys.exc_info()
@@ -66,23 +71,22 @@ RETURNS json AS $$
      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;

-CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
  username TEXT,
  orgname TEXT,
  geom geometry(Geometry, 4326),
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
-  obs_plan = plpy.prepare("SELECT cdb_dataservices_server.OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
-  result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
-  return result
-$$ LANGUAGE plpythonu;
+RETURNS SETOF json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
+$$ LANGUAGE plproxy;

 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
  username TEXT,
  orgname TEXT,
  geom geometry(Geometry, 4326),
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
+RETURNS SETOF JSON AS $$
  from cartodb_services.metrics import QuotaService

  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -95,14 +99,18 @@ RETURNS json AS $$
    plpy.error('You have reached the limit of your quota')

  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
      result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
      if result:
+        resp = []
+        for element in result:
+          value = element['snapshot']
+          resp.append(value)
        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
+        return resp
      else:
        quota_service.increment_empty_service_use()
-        return None
+        return []
  except BaseException as e:
      import sys, traceback
      type_, value_, traceback_ = sys.exc_info()
--- a/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql
+++ b/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql
@@ -3,7 +3,9 @@
 \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.4'" to load this file. \quit

 DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_config(text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetDemographicSnapshot(text, text, geometry(Geometry, 4326), text, text);
 DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetDemographicSnapshot(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetSegmentSnapshot(text, text, geometry(Geometry, 4326), text);
 DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetSegmentSnapshot(TEXT, TEXT, geometry(Geometry, 4326), TEXT);
 DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT);
 DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT);
@@ -34,79 +36,24 @@ DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetPointsByGeometry(TEXT, T
 DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetPointsByPointAndRadius(TEXT, TEXT, geometry(Point, 4326), NUMERIC, TEXT, TEXT, TEXT);
 DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetPointsByPointAndRadius(TEXT, TEXT, geometry(Point, 4326), NUMERIC, TEXT, TEXT, TEXT);

-CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
  username TEXT,
  orgname TEXT,
  geom geometry(Geometry, 4326),
  time_span TEXT DEFAULT '2009 - 2013',
  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
 RETURNS json AS $$
-  from cartodb_services.metrics import QuotaService
-  import json
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level);
+$$ LANGUAGE plproxy;

-  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
-  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
-
-  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
-  if not quota_service.check_user_quota():
-    plpy.error('You have reached the limit of your quota')
-
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys, traceback
-      type_, value_, traceback_ = sys.exc_info()
-      quota_service.increment_failed_service_use()
-      error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
-      plpy.notice(traceback.format_tb(traceback_))
-      plpy.error(error_msg)
-  finally:
-      quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
  username TEXT,
  orgname TEXT,
  geom geometry(Geometry, 4326),
  geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
 RETURNS json AS $$
-  from cartodb_services.metrics import QuotaService
-  import json
-
-  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
-  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
-
-  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
-  if not quota_service.check_user_quota():
-    plpy.error('You have reached the limit of your quota')
-
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys, traceback
-      type_, value_, traceback_ = sys.exc_info()
-      quota_service.increment_failed_service_use()
-      error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
-      plpy.notice(traceback.format_tb(traceback_))
-      plpy.error(error_msg)
-  finally:
-      quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu;
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
+$$ LANGUAGE plproxy;
--- a/server/extension/sql/110_data_observatory_augmentation.sql
+++ b/server/extension/sql/110_data_observatory_augmentation.sql
@@ -22,30 +22,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
  geom geometry(Geometry, 4326),
  time_span TEXT DEFAULT NULL,
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
+RETURNS SETOF json AS $$
  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
-  SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level);
+  SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level);
 $$ LANGUAGE plproxy;

-CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
-  username TEXT,
-  orgname TEXT,
-  geom geometry(Geometry, 4326),
-  time_span TEXT DEFAULT NULL,
-  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
-  obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
-  result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
-  return result
-$$ LANGUAGE plpythonu;
-
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
  username TEXT,
  orgname TEXT,
  geom geometry(Geometry, 4326),
  time_span TEXT DEFAULT NULL,
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
+RETURNS SETOF JSON AS $$
  from cartodb_services.metrics import QuotaService

  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -61,11 +49,15 @@ RETURNS json AS $$
      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
      result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
      if result:
+        resp = []
+        for element in result:
+          value = element['snapshot']
+          resp.append(value)
        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
+        return resp
      else:
        quota_service.increment_empty_service_use()
-        return None
+        return []
  except BaseException as e:
      import sys, traceback
      type_, value_, traceback_ = sys.exc_info()
@@ -82,28 +74,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
  orgname TEXT,
  geom geometry(Geometry, 4326),
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
+RETURNS SETOF json AS $$
  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
-  SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
+  SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
 $$ LANGUAGE plproxy;

-CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
-  username TEXT,
-  orgname TEXT,
-  geom geometry(Geometry, 4326),
-  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
-  obs_plan = plpy.prepare("SELECT cdb_dataservices_server.OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
-  result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
-  return result
-$$ LANGUAGE plpythonu;
-
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
  username TEXT,
  orgname TEXT,
  geom geometry(Geometry, 4326),
  geometry_level TEXT DEFAULT NULL)
-RETURNS json AS $$
+RETURNS SETOF JSON AS $$
  from cartodb_services.metrics import QuotaService

  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -116,14 +97,18 @@ RETURNS json AS $$
    plpy.error('You have reached the limit of your quota')

  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
      result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
      if result:
+        resp = []
+        for element in result:
+          value = element['snapshot']
+          resp.append(value)
        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
+        return resp
      else:
        quota_service.increment_empty_service_use()
-        return None
+        return []
  except BaseException as e:
      import sys, traceback
      type_, value_, traceback_ = sys.exc_info()
--- a/server/extension/test/expected/100_data_observatory_test.out
+++ b/server/extension/test/expected/100_data_observatory_test.out
@@ -1,25 +1,3 @@
-SELECT exists(SELECT *
-              FROM pg_proc p
-              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
-              WHERE ns.nspname = 'cdb_dataservices_server'
-              AND proname = 'obs_get_demographic_snapshot'
-              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text, text');
- exists 
--------
- t
-(1 row)
-
-SELECT exists(SELECT *
-              FROM pg_proc p
-              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
-              WHERE ns.nspname = 'cdb_dataservices_server'
-              AND proname = 'obs_get_segment_snapshot'
-              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text');
- exists 
--------
- t
-(1 row)
-
 SELECT exists(SELECT *
              FROM pg_proc p
              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
--- a/server/extension/test/sql/100_data_observatory_test.sql
+++ b/server/extension/test/sql/100_data_observatory_test.sql
@@ -1,17 +1,3 @@
-SELECT exists(SELECT *
-              FROM pg_proc p
-              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
-              WHERE ns.nspname = 'cdb_dataservices_server'
-              AND proname = 'obs_get_demographic_snapshot'
-              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text, text');
-
-SELECT exists(SELECT *
-              FROM pg_proc p
-              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
-              WHERE ns.nspname = 'cdb_dataservices_server'
-              AND proname = 'obs_get_segment_snapshot'
-              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text');
-
 SELECT exists(SELECT *
              FROM pg_proc p
              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)