TL;DR: safer deployment and minor fixes

- Instead of modifying cdb_service_quota_info to return max_batch_size,
a new type (service_quota_info_batch) and a new function
(cdb_service_quota_info_batch) are created. That makes deployment safe.
- Fixes geocoding with forced batch size 1.
- Improves namespacing for count_estimate (->
cdb_dataservices_client.cdb_count_estimate).
- Improves namespacing for jsonb_array_casttext (->
cdb_dataservices_client.cdb_jsonb_array_casttext).
This commit is contained in:
Juan Ignacio Sánchez Lara
2018-07-18 14:29:50 +02:00
parent 0c5e9da028
commit b0c1948c14
16 changed files with 375 additions and 197 deletions

View File

@@ -4,75 +4,48 @@
-- HERE goes your code to upgrade/downgrade
ALTER TYPE cdb_dataservices_server.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
WHERE pg_type.typname = 'service_quota_info_batch'
AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS (
service cdb_dataservices_server.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
soft_limit BOOLEAN,
provider TEXT,
max_batch_size NUMERIC
);
END IF;
END $$;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info(
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
username TEXT,
orgname TEXT)
RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
from cartodb_services.metrics.user import UserMetricsService
from datetime import date
RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
today = date.today()
ret = []
for info in sqi:
if info['service'] == 'hires_geocoder':
provider = info['provider']
batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
else:
max_batch_size = 1
#-- Isolines
service = 'isolines'
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
user_service = UserMetricsService(user_isolines_config, redis_conn)
info['max_batch_size'] = max_batch_size
else:
info['max_batch_size'] = 1
monthly_quota = user_isolines_config.isolines_quota
used_quota = user_service.used_quota(user_isolines_config.service_type, today)
soft_limit = user_isolines_config.soft_isolines_limit
provider = user_isolines_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
#-- Hires Geocoder
service = 'hires_geocoder'
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
user_service = UserMetricsService(user_geocoder_config, redis_conn)
monthly_quota = user_geocoder_config.geocoding_quota
used_quota = user_service.used_quota(user_geocoder_config.service_type, today)
soft_limit = user_geocoder_config.soft_geocoding_limit
provider = user_geocoder_config.provider
batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
else:
max_batch_size = 1
ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]]
#-- Routing
service = 'routing'
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
user_service = UserMetricsService(user_routing_config, redis_conn)
monthly_quota = user_routing_config.monthly_quota
used_quota = user_service.used_quota(user_routing_config.service_type, today)
soft_limit = user_routing_config.soft_limit
provider = user_routing_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
#-- Observatory
service = 'observatory'
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
user_service = UserMetricsService(user_obs_config, redis_conn)
monthly_quota = user_obs_config.monthly_quota
used_quota = user_service.used_quota(user_obs_config.service_type, today)
soft_limit = user_obs_config.soft_limit
provider = user_obs_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
@@ -114,8 +87,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
raise Exception('Requested geocoder is not available')
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
result = plpy.execute(plan, [username, orgname, searches])
return result
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

View File

@@ -4,75 +4,12 @@
-- HERE goes your code to upgrade/downgrade
ALTER TYPE cdb_dataservices_server.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info(
username TEXT,
orgname TEXT)
RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
from cartodb_services.metrics.user import UserMetricsService
from datetime import date
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
today = date.today()
ret = []
#-- Isolines
service = 'isolines'
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
user_service = UserMetricsService(user_isolines_config, redis_conn)
monthly_quota = user_isolines_config.isolines_quota
used_quota = user_service.used_quota(user_isolines_config.service_type, today)
soft_limit = user_isolines_config.soft_isolines_limit
provider = user_isolines_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Hires Geocoder
service = 'hires_geocoder'
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
user_service = UserMetricsService(user_geocoder_config, redis_conn)
monthly_quota = user_geocoder_config.geocoding_quota
used_quota = user_service.used_quota(user_geocoder_config.service_type, today)
soft_limit = user_geocoder_config.soft_geocoding_limit
provider = user_geocoder_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Routing
service = 'routing'
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
user_service = UserMetricsService(user_routing_config, redis_conn)
monthly_quota = user_routing_config.monthly_quota
used_quota = user_service.used_quota(user_routing_config.service_type, today)
soft_limit = user_routing_config.soft_limit
provider = user_routing_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Observatory
service = 'observatory'
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
user_service = UserMetricsService(user_obs_config, redis_conn)
monthly_quota = user_obs_config.monthly_quota
used_quota = user_service.used_quota(user_obs_config.service_type, today)
soft_limit = user_obs_config.soft_limit
provider = user_obs_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_quota_info_batch(TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocode_street_point(TEXT, TEXT, jsonb);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_google_geocode_street_point(TEXT, TEXT, jsonb);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(TEXT, TEXT, jsonb);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(TEXT, TEXT, jsonb);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(TEXT, TEXT, jsonb);
DROP TYPE IF EXISTS cdb_dataservices_server.geocoding;
DROP TYPE IF EXISTS cdb_dataservices_server.service_quota_info_batch;

View File

@@ -1857,6 +1857,21 @@ BEGIN
WHERE pg_type.typname = 'service_quota_info'
AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
CREATE TYPE cdb_dataservices_server.service_quota_info AS (
service cdb_dataservices_server.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
soft_limit BOOLEAN,
provider TEXT
);
END IF;
END $$;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
WHERE pg_type.typname = 'service_quota_info_batch'
AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS (
service cdb_dataservices_server.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
@@ -1867,13 +1882,13 @@ BEGIN
END IF;
END $$;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info(
username TEXT,
orgname TEXT)
RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
from cartodb_services.metrics.user import UserMetricsService
from datetime import date
from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -1891,7 +1906,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_isolines_config.service_type, today)
soft_limit = user_isolines_config.soft_isolines_limit
provider = user_isolines_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Hires Geocoder
service = 'hires_geocoder'
@@ -1903,12 +1918,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_geocoder_config.service_type, today)
soft_limit = user_geocoder_config.soft_geocoding_limit
provider = user_geocoder_config.provider
batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
else:
max_batch_size = 1
ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Routing
service = 'routing'
@@ -1920,7 +1930,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_routing_config.service_type, today)
soft_limit = user_routing_config.soft_limit
provider = user_routing_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Observatory
service = 'observatory'
@@ -1932,12 +1942,41 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_obs_config.service_type, today)
soft_limit = user_obs_config.soft_limit
provider = user_obs_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
username TEXT,
orgname TEXT)
RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
ret = []
for info in sqi:
if info['service'] == 'hires_geocoder':
provider = info['provider']
batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
else:
max_batch_size = 1
info['max_batch_size'] = max_batch_size
else:
info['max_batch_size'] = 1
ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota(
username TEXT,
orgname TEXT,
@@ -2385,8 +2424,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
raise Exception('Requested geocoder is not available')
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
result = plpy.execute(plan, [username, orgname, searches])
return result
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

View File

@@ -18,6 +18,21 @@ BEGIN
WHERE pg_type.typname = 'service_quota_info'
AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
CREATE TYPE cdb_dataservices_server.service_quota_info AS (
service cdb_dataservices_server.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
soft_limit BOOLEAN,
provider TEXT
);
END IF;
END $$;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
WHERE pg_type.typname = 'service_quota_info_batch'
AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS (
service cdb_dataservices_server.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
@@ -28,13 +43,13 @@ BEGIN
END IF;
END $$;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info(
username TEXT,
orgname TEXT)
RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
from cartodb_services.metrics.user import UserMetricsService
from datetime import date
from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -52,7 +67,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_isolines_config.service_type, today)
soft_limit = user_isolines_config.soft_isolines_limit
provider = user_isolines_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Hires Geocoder
service = 'hires_geocoder'
@@ -64,12 +79,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_geocoder_config.service_type, today)
soft_limit = user_geocoder_config.soft_geocoding_limit
provider = user_geocoder_config.provider
batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
else:
max_batch_size = 1
ret += [[service, monthly_quota, used_quota, soft_limit, provider, max_batch_size]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Routing
service = 'routing'
@@ -81,7 +91,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_routing_config.service_type, today)
soft_limit = user_routing_config.soft_limit
provider = user_routing_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
#-- Observatory
service = 'observatory'
@@ -93,12 +103,41 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
used_quota = user_service.used_quota(user_obs_config.service_type, today)
soft_limit = user_obs_config.soft_limit
provider = user_obs_config.provider
ret += [[service, monthly_quota, used_quota, soft_limit, provider, 1]]
ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
username TEXT,
orgname TEXT)
RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
ret = []
for info in sqi:
if info['service'] == 'hires_geocoder':
provider = info['provider']
batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
else:
max_batch_size = 1
info['max_batch_size'] = max_batch_size
else:
info['max_batch_size'] = 1
ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
return ret
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota(
username TEXT,
orgname TEXT,

View File

@@ -35,8 +35,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
raise Exception('Requested geocoder is not available')
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
result = plpy.execute(plan, [username, orgname, searches])
return result
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

View File

@@ -94,7 +94,11 @@ class Logger:
else:
exception_message = ''
# Adding trace breaks tests
# trace = traceback.format_exc(15)
# message = '{}{}. Trace: {}'.format(text, exception_message, trace)
message = '{}{}'.format(text, exception_message)
if self._check_plpy():
if level == 'debug':
plpy.debug(message)