Merge pull request #517 from CartoDB/development

Release 0.33 server and 0.20 for python library

NEWS.md

@@ -1,3 +1,9 @@
+
+Aug 27th, 2018
+==============
+* Version `0.33.0` of the server, and `0.20.0` of the Python library.
+    * Remove the obs_snapshot quota and now the snapshot functions uses obs_general quota
+
 Jul 19th, 2018
 ==============
 * Version `0.25.0` of the client, `0.32.0` of the server, and `0.19.1` of the Python library.

client/carto-package.json

@@ -0,0 +1,13 @@
+{
+  "name": "dataservices-api-client-extension",
+  "current_version": {
+    "requires": {
+      "postgresql": "^10.0.0",
+      "postgis": "^2.4.0.0",
+      "carto_postgresql_ext": "^0.23.0"
+    },
+    "works_with": {
+      "dataservices-api-server-extension": "^0.32.0"
+    }
+  }
+}

server/extension/carto-package.json

@@ -0,0 +1,14 @@
+{
+  "name": "dataservices-api-server-extension",
+  "current_version": {
+    "requires": {
+      "postgresql": "^10.0.0",
+      "postgis": "^2.4.0.0",
+      "carto_postgresql_ext": "^0.23.0"
+    },
+    "works_with": {
+      "dataservices-api-server-python-lib": "^0.19.1",
+      "observatory-server-extension": "^1.9.0"
+    }
+  }
+}

server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql

@@ -0,0 +1,204 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.33.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_snapshot_config;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
+  username TEXT,
+  orgname TEXT)
+RETURNS text AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  return user_obs_config.connection_str
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_segment_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetSegmentSnapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql

@@ -0,0 +1,216 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.32.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_obs_snapshot_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import ObservatorySnapshotConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = obs_snapshot_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
+  username TEXT,
+  orgname TEXT)
+RETURNS text AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  return user_obs_config.connection_str
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_segment_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetSegmentSnapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.32.0'
+default_version = '0.33.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/110_data_observatory_augmentation.sql

@@ -10,8 +10,8 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
 RETURNS text AS $$
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   return user_obs_config.connection_str
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
@@ -41,8 +41,8 @@ RETURNS json AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
@@ -94,8 +94,8 @@ RETURNS SETOF JSON AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
@@ -150,8 +150,8 @@ RETURNS json AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
@@ -201,8 +201,8 @@ RETURNS SETOF JSON AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]

server/extension/sql/15_config_helper.sql

@@ -90,20 +90,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text)
-RETURNS boolean AS $$
-  cache_key = "user_obs_snapshot_config_{0}".format(username)
-  if cache_key in GD:
-    return False
-  else:
-    from cartodb_services.metrics import ObservatorySnapshotConfig
-    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
-    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
-    obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname)
-    GD[cache_key] = obs_snapshot_config
-    return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
-
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_obs_config_{0}".format(username)

server/lib/python/cartodb_services/carto-package.json

@@ -0,0 +1,8 @@
+{
+  "name": "dataservices-api-server-python-lib",
+  "current_version": {
+    "requires": {
+      "python": "~2.7.0"
+    }
+  }
+}

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,4 +1,4 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
+from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
 from quota import QuotaService
 from user import UserMetricsService
 from log import metrics, MetricsDataGatherer, Traceable

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -86,28 +86,6 @@ class DataObservatoryConfig(ServiceConfig):
         return 'data observatory'
 
 
-class ObservatorySnapshotConfig(DataObservatoryConfig):
-
-    SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
-    QUOTA_KEY = 'obs_snapshot_quota'
-    PERIOD_END_DATE = 'period_end_date'
-
-    def __init__(self, redis_connection, db_conn, username, orgname=None):
-        super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn,
-                                            username, orgname)
-        self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
-        if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
-            self._soft_limit = True
-        else:
-            self._soft_limit = False
-        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
-        self._connection_str = self._db_config.data_observatory_connection_str
-
-    @property
-    def service_type(self):
-        return 'obs_snapshot'
-
-
 class ObservatoryConfig(DataObservatoryConfig):
 
     SOFT_LIMIT_KEY = 'soft_obs_general_limit'
@@ -890,7 +868,6 @@ class ServicesRedisConfig:
     QUOTA_KEY = 'geocoding_quota'
     ISOLINES_QUOTA_KEY = 'here_isolines_quota'
     ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
-    OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
     OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
     PERIOD_END_DATE = 'period_end_date'
     GEOCODER_PROVIDER_KEY = 'geocoder_provider'
@@ -934,8 +911,6 @@ class ServicesRedisConfig:
                 user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
             if self.ROUTING_QUOTA_KEY in org_config:
                 user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
-            if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
-                user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
             if self.OBS_GENERAL_QUOTA_KEY in org_config:
                 user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY]
             if self.PERIOD_END_DATE in org_config:

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.19.1',
+    version='0.20.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -173,7 +173,8 @@ class TestGeocoderOrgConfig(TestCase):
 
 
 class TestIsolinesUserConfig(TestCase):
-    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom']
+    # Don't test mapbox. See CartoDB/cartodb-management/issues/5199"
+    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -233,8 +234,8 @@ class TestIsolinesUserConfig(TestCase):
 
 
 class TestIsolinesOrgConfig(TestCase):
-
-    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom']
+    # Don't test mapbox. See CartoDB/cartodb-management/issues/5199"
+    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -343,41 +344,7 @@ class TestDataObservatoryUserConfig(TestCase):
         self.redis_conn = MockRedis()
         plpy_mock_config()
 
-    def test_should_return_config_for_obs_snapshot(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=100, end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.monthly_quota == 100
-        assert do_config.soft_limit is False
-        assert do_config.period_end_date.date() == yesterday.date()
-
-    def test_should_return_true_if_soft_limit_is_true_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=0, soft_limit=True, end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.soft_limit is True
-
-    def test_should_return_0_if_quota_is_0_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=0, end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                              'test_user')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_0_if_quota_is_empty_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota='', end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                              'test_user')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_config_for_obs_snapshot(self):
+    def test_should_return_config_for_obs_config(self):
         yesterday = datetime.today() - timedelta(days=1)
         build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
                                 quota=100, end_date=yesterday)

server/lib/python/cartodb_services/test/test_helper.py

@@ -28,9 +28,7 @@ def build_redis_user_config(redis_conn, username, service, quota=100,
         redis_conn.hset(user_redis_name, 'mapzen_routing_quota', str(quota))
         redis_conn.hset(user_redis_name, 'soft_mapzen_routing_limit', str(soft_limit).lower())
     elif service is 'data_observatory':
-        redis_conn.hset(user_redis_name, 'obs_snapshot_quota', str(quota))
         redis_conn.hset(user_redis_name, 'obs_general_quota', str(quota))
-        redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit', str(soft_limit).lower())
         redis_conn.hset(user_redis_name, 'soft_obs_general_limit', str(soft_limit).lower())
 
     redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
@@ -57,7 +55,6 @@ def build_redis_org_config(redis_conn, orgname, service, quota=100,
             redis_conn.hset(org_redis_name, 'mapzen_routing_quota', str(quota))
     elif service is 'data_observatory':
         if quota is not None:
-            redis_conn.hset(org_redis_name, 'obs_snapshot_quota', str(quota))
             redis_conn.hset(org_redis_name, 'obs_general_quota', str(quota))
 
     redis_conn.hset(org_redis_name, 'google_maps_client_id', '')

server/lib/python/cartodb_services/test/test_mapboxisoline.py

@@ -3,15 +3,13 @@ from mock import Mock
 from cartodb_services.mapbox.isolines import MapboxIsolines
 from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
 from cartodb_services.mapbox.matrix_client import MapboxMatrixClient
-from cartodb_services.mapbox.routing import MapboxRouting
 from cartodb_services.tools import Coordinate
-from cartodb_services.tools.coordinates import (validate_coordinates,
-                                                marshall_coordinates)
 from credentials import mapbox_api_key
 
 VALID_ORIGIN = Coordinate(-73.989, 40.733)
 
 
+@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
 class MapboxIsolinesTestCase(unittest.TestCase):
 
     def setUp(self):

server/lib/python/cartodb_services/test/test_mapboxmatrix.py

@@ -20,6 +20,7 @@ VALID_PROFILE = DEFAULT_PROFILE
 INVALID_PROFILE = 'invalid_profile'
 
 
+@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
 class MapboxMatrixTestCase(unittest.TestCase):
     def setUp(self):
         self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(),

server/lib/python/cartodb_services/test/test_mapboxrouting.py

@@ -17,16 +17,17 @@ VALID_PROFILE = DEFAULT_PROFILE
 INVALID_PROFILE = 'invalid_profile'
 
 WELL_KNOWN_SHAPE = [(40.73312, -73.98891), (40.73353, -73.98987),
-                    (40.73398, -73.99095), (40.73321, -73.99111),
-                    (40.73245, -73.99129), (40.7333, -73.99332),
-                    (40.7338, -73.99449),  (40.73403, -73.99505),
-                    (40.73344, -73.99549), (40.73286, -73.9959),
-                    (40.73226, -73.99635), (40.73186, -73.99664),
-                    (40.73147, -73.99693), (40.73141, -73.99698),
-                    (40.73147, -73.99707), (40.73219, -73.99856),
-                    (40.73222, -73.99861), (40.73225, -73.99868),
-                    (40.73293, -74.00007), (40.733, -74.00001)]
-WELL_KNOWN_LENGTH = 1384.8
+                    (40.73398, -73.99095), (40.73453, -73.99227),
+                    (40.73531, -73.99412), (40.73467, -73.99459),
+                    (40.73442, -73.99477), (40.73435, -73.99482),
+                    (40.73403, -73.99505), (40.73344, -73.99549),
+                    (40.73286, -73.9959), (40.73226, -73.99635),
+                    (40.73186, -73.99664), (40.73147, -73.99693),
+                    (40.73141, -73.99698), (40.73147, -73.99707),
+                    (40.73219, -73.99856), (40.73222, -73.99861),
+                    (40.73225, -73.99868), (40.73293, -74.00007),
+                    (40.733, -74.00001)]
+WELL_KNOWN_LENGTH = 1317.9
 
 
 class MapboxRoutingTestCase(unittest.TestCase):

server/lib/python/cartodb_services/test/test_quota_service.py

@@ -181,28 +181,6 @@ class TestQuotaService(TestCase):
     # Quick workaround so we don't take into account numer of credits
     # spent for users that have defined the quota.
     # See https://github.com/CartoDB/bigmetadata/issues/215
-    def test_should_check_user_obs_snapshot_quota_correctly(self):
-        qs = self.__build_obs_snapshot_quota_service('test_user')
-        qs.increment_success_service_use()
-        assert qs.check_user_quota() is True
-        qs.increment_success_service_use(amount=100000)
-        assert qs.check_user_quota() is True
-
-    def test_should_check_org_obs_snapshot_quota_correctly(self):
-        qs = self.__build_obs_snapshot_quota_service('test_user',
-                                                     orgname='testorg')
-        qs.increment_success_service_use()
-        assert qs.check_user_quota() is True
-        qs.increment_success_service_use(amount=100000)
-        assert qs.check_user_quota() is True
-
-    def test_should_check_user_obs_quota_correctly(self):
-        qs = self.__build_obs_snapshot_quota_service('test_user')
-        qs.increment_success_service_use()
-        assert qs.check_user_quota() is True
-        qs.increment_success_service_use(amount=100000)
-        assert qs.check_user_quota() is True
-
     def test_should_check_org_obs_quota_correctly(self):
         qs = self.__build_obs_quota_service('test_user',
                                             orgname='testorg')
@@ -249,17 +227,6 @@ class TestQuotaService(TestCase):
                                                username, orgname)
         return QuotaService(isolines_config, redis_connection=self.redis_conn)
 
-    def __build_obs_snapshot_quota_service(self, username, quota=100,
-                                           provider='obs_snapshot',
-                                           orgname=None,
-                                           soft_limit=False,
-                                           end_date=datetime.today()):
-        self.__prepare_quota_service(username, 'data_observatory', quota,
-                                     None, orgname, soft_limit, end_date)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                          username, orgname)
-        return QuotaService(do_config, redis_connection=self.redis_conn)
-
     def __build_obs_quota_service(self, username, quota=100,
                                   provider='obs_general',
                                   orgname=None,