cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TL;DR: safer deployment and minor fixes

Browse Source 
- Instead of modifying cdb_service_quota_info to return max_batch_size,
a new type (service_quota_info_batch) and a new function
(cdb_service_quota_info_batch) are created. That makes deployment safe.
- Fixes geocoding with forced batch size 1.
- Improves namespacing for count_estimate (->
cdb_dataservices_client.cdb_count_estimate).
- Improves namespacing for jsonb_array_casttext (->
cdb_dataservices_client.cdb_jsonb_array_casttext).
This commit is contained in:
Juan Ignacio Sánchez Lara
2018-07-18 14:29:50 +02:00
parent 0c5e9da028
commit b0c1948c14
16 changed files with 375 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
client/cdb_dataservices_client--0.24.0--0.25.0.sql
View File

@@ -7,7 +7,7 @@ SET search_path = "$user",cartodb,public,cdb_dataservices_client;
-- HERE goes your code to upgrade/downgrade
-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS
CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
$func$
DECLARE
rec record;
@@ -23,18 +23,30 @@ END
$func$ LANGUAGE plpgsql;
-- Taken from https://stackoverflow.com/a/48013356/351721
CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
$f$ LANGUAGE sql IMMUTABLE;--
CREATE TYPE cdb_dataservices_client.geocoding AS (
cartodb_id integer,
the_geom geometry(Multipolygon,4326),
metadata jsonb
);
ALTER TYPE cdb_dataservices_client.service_quota_info ADD ATTRIBUTE max_batch_size NUMERIC;
CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
service cdb_dataservices_client.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
soft_limit BOOLEAN,
provider TEXT,
max_batch_size NUMERIC
);
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
@@ -55,12 +67,33 @@ BEGIN
RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
RETURNS SETOF service_quota_info_batch AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
@@ -74,13 +107,12 @@ DECLARE
batches_n integer;
DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
current_row_count integer ;
temp_table_name text;
BEGIN
SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
INTO remaining_quota, max_batch_size
FROM cdb_dataservices_client.cdb_service_quota_info() csqi
FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
WHERE service = 'hires_geocoder';
RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
@@ -120,12 +152,11 @@ BEGIN
IF batches_n > 0 THEN
FOR cartodb_id_batch in 0..(batches_n - 1)
LOOP
EXECUTE format(
'WITH geocoding_data as (' ||
' SELECT ' ||
' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
' floor((row_number() over ())::float/$1) as batch' ||
' floor((row_number() over () - 1)::float/$1) as batch' ||
' FROM (%s) _x' ||
') ' ||
'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
@@ -133,9 +164,6 @@ BEGIN
'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
USING batch_size, cartodb_id_batch;
GET DIAGNOSTICS current_row_count = ROW_COUNT;
RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count;
END LOOP;
END IF;
@@ -143,6 +171,10 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;
--
-- Exception-safe private DataServices API function
--
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
DECLARE
@@ -175,19 +207,69 @@ BEGIN
END;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
--
-- Exception-safe private DataServices API function
--
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe ()
RETURNS SETOF service_quota_info_batch AS $$
DECLARE
username text;
orgname text;
_returned_sqlstate TEXT;
_message_text TEXT;
_pg_exception_context TEXT;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
BEGIN
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
EXCEPTION
WHEN OTHERS THEN
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
_message_text = MESSAGE_TEXT,
_pg_exception_context = PG_EXCEPTION_CONTEXT;
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
END;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb);
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text)
RETURNS SETOF service_quota_info_batch AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb ) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser;

18
client/cdb_dataservices_client--0.25.0--0.24.0.sql
View File

@@ -6,20 +6,24 @@
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS count_estimate(query text);
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_count_estimate(query text);
DROP FUNCTION IF EXISTS jsonb_array_casttext(jsonb);
ALTER TYPE cdb_dataservices_client.service_quota_info DROP ATTRIBUTE IF EXISTS max_batch_size;
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb);
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (jsonb);
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text,
text, text, text, text, integer);
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_quota_info_batch();
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, text, text, text, text, integer);
DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (jsonb);
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe ();
DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (text, text, jsonb);
DROP TYPE IF EXISTS cdb_dataservices_client.geocoding;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (text, text);
DROP TYPE IF EXISTS cdb_dataservices_client.service_quota_info_batch;
DROP TYPE IF EXISTS cdb_dataservices_client.geocoding;

93
client/cdb_dataservices_client--0.25.0.sql
View File

@@ -5,7 +5,7 @@
-- Make sure we have a sane search path to create/update the extension
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS
CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
$func$
DECLARE
rec record;
@@ -21,7 +21,7 @@ END
$func$ LANGUAGE plpgsql;
-- Taken from https://stackoverflow.com/a/48013356/351721
CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
$f$ LANGUAGE sql IMMUTABLE;--
-- Geocoder server connection config
@@ -113,6 +113,14 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM (
);
CREATE TYPE cdb_dataservices_client.service_quota_info AS (
service cdb_dataservices_client.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
soft_limit BOOLEAN,
provider TEXT
);
CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
service cdb_dataservices_client.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
@@ -1601,6 +1609,31 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
RETURNS SETOF service_quota_info_batch AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
RETURNS BOOLEAN AS $$
DECLARE
@@ -2005,13 +2038,12 @@ DECLARE
batches_n integer;
DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
current_row_count integer ;
temp_table_name text;
BEGIN
SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
INTO remaining_quota, max_batch_size
FROM cdb_dataservices_client.cdb_service_quota_info() csqi
FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
WHERE service = 'hires_geocoder';
RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
@@ -2051,12 +2083,11 @@ BEGIN
IF batches_n > 0 THEN
FOR cartodb_id_batch in 0..(batches_n - 1)
LOOP
EXECUTE format(
'WITH geocoding_data as (' ||
' SELECT ' ||
' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
' floor((row_number() over ())::float/$1) as batch' ||
' floor((row_number() over () - 1)::float/$1) as batch' ||
' FROM (%s) _x' ||
') ' ||
'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
@@ -2064,9 +2095,6 @@ BEGIN
'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
USING batch_size, cartodb_id_batch;
GET DIAGNOSTICS current_row_count = ROW_COUNT;
RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count;
END LOOP;
END IF;
@@ -4201,6 +4229,42 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
-- Exception-safe private DataServices API function
--
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe ()
RETURNS SETOF service_quota_info_batch AS $$
DECLARE
username text;
orgname text;
_returned_sqlstate TEXT;
_message_text TEXT;
_pg_exception_context TEXT;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
BEGIN
RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
EXCEPTION
WHEN OTHERS THEN
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
_message_text = MESSAGE_TEXT,
_pg_exception_context = PG_EXCEPTION_CONTEXT;
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
END;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
--
-- Exception-safe private DataServices API function
--
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe (service TEXT ,input_size NUMERIC)
RETURNS BOOLEAN AS $$
DECLARE
@@ -4845,6 +4909,14 @@ RETURNS SETOF service_quota_info AS $$
SELECT * FROM cdb_dataservices_server.cdb_service_quota_info (username, orgname);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text)
RETURNS SETOF service_quota_info_batch AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_enough_quota (username text, orgname text, service TEXT, input_size NUMERIC)
@@ -5126,6 +5198,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exc
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info() TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_exception_safe( ) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( ) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_enough_quota(service TEXT, input_size NUMERIC) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC ) TO publicuser;

7
client/renderer/interface.yaml
View File

@@ -517,6 +517,13 @@
params:
- {}
- name: cdb_service_quota_info_batch
return_type: SETOF service_quota_info_batch
multi_row: true
multi_field: true
params:
- {}
- name: cdb_enough_quota
return_type: BOOLEAN
params:

4
client/sql/05_utils.sql
View File

@@ -1,5 +1,5 @@
-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS
CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
$func$
DECLARE
rec record;
@@ -15,6 +15,6 @@ END
$func$ LANGUAGE plpgsql;
-- Taken from https://stackoverflow.com/a/48013356/351721
CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
$f$ LANGUAGE sql IMMUTABLE;

8
client/sql/16_custom_types.sql
View File

@@ -35,6 +35,14 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM (
);
CREATE TYPE cdb_dataservices_client.service_quota_info AS (
service cdb_dataservices_client.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
soft_limit BOOLEAN,
provider TEXT
);
CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
service cdb_dataservices_client.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,

9
client/sql/21_bulk_geocoding_functions.sql
View File

@@ -11,13 +11,12 @@ DECLARE
batches_n integer;
DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
current_row_count integer ;
temp_table_name text;
BEGIN
SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
INTO remaining_quota, max_batch_size
FROM cdb_dataservices_client.cdb_service_quota_info() csqi
FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
WHERE service = 'hires_geocoder';
RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
@@ -57,12 +56,11 @@ BEGIN
IF batches_n > 0 THEN
FOR cartodb_id_batch in 0..(batches_n - 1)
LOOP
EXECUTE format(
'WITH geocoding_data as (' ||
' SELECT ' ||
' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
' floor((row_number() over ())::float/$1) as batch' ||
' floor((row_number() over () - 1)::float/$1) as batch' ||
' FROM (%s) _x' ||
') ' ||
'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
@@ -70,9 +68,6 @@ BEGIN
'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
USING batch_size, cartodb_id_batch;
GET DIAGNOSTICS current_row_count = ROW_COUNT;
RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count;
END LOOP;
END IF;

10
client/test/expected/21_bulk_geocoding_functions_test.out
View File

@@ -1,7 +1,7 @@
\set VERBOSITY terse
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info ()
RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$
SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size;
$$ LANGUAGE SQL;
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked;
@@ -15,7 +15,7 @@ ERROR: Remaining quota: 0. Estimated cost: 1
-- Test quota check by mocking quota 0
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
ERROR: Remaining quota: 0. Estimated cost: 1
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info;
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;

10
client/test/sql/21_bulk_geocoding_functions_test.sql
View File

@@ -1,8 +1,8 @@
\set VERBOSITY terse
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info() RENAME TO cdb_service_quota_info_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info ()
RETURNS SETOF cdb_dataservices_client.service_quota_info AS $$
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked;
CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$
SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size;
$$ LANGUAGE SQL;
@@ -18,9 +18,9 @@ SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartod
-- Test quota check by mocking quota 0
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info;
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_mocked() RENAME TO cdb_service_quota_info;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;