cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #543 from CartoDB/fix-bulk-geocoding-soft-limit

Browse Source 
Fix bulk geocoding soft limit
This commit is contained in:
Javier Goizueta
2019-02-25 16:39:29 +01:00
committed by GitHub
parent 246ec135db dfaf0d5245
commit 00f77cca8a
3 changed files with 40 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client/sql/21_bulk_geocoding_functions.sql
View File

@@ -49,7 +49,7 @@ BEGIN
RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
query_row_count, query, country_column, state_column, city_column, street_column;
SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
IF remaining_quota < query_row_count THEN
IF enough_quota IS NULL OR NOT enough_quota THEN
RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
END IF;

21
client/test/expected/21_bulk_geocoding_functions_test.out
View File

@@ -9,6 +9,13 @@ CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_si
RETURNS BOOLEAN as $$
SELECT FALSE;
$$ LANGUAGE SQL;
ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point_mocked;
CREATE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
BEGIN
RAISE NOTICE 'called with this searches: %', searches;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
-- No permissions granted
-- Test bulk size not mandatory (it will get the optimal)
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
@@ -42,6 +49,18 @@ ERROR: Remaining quota: 0. Estimated cost: 1
-- Test quota check by mocking quota 0
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
ERROR: Remaining quota: 0. Estimated cost: 1
-- Check that when cdb_enough_quota returns true (ie. when soft_limit is set to true, even if not enough quota)
-- it is able to proceed with the bulk geocode
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
RETURNS BOOLEAN as $$
SELECT TRUE;
$$ LANGUAGE SQL;
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
NOTICE: called with this searches: [{"id": 1, "city": "", "state": "", "address": "Valladolid, Spain", "country": ""}]
cdb_bulk_geocode_street_point
-------------------------------
(0 rows)
-- Remove permissions
SELECT CDB_Conf_RemoveConf('api_keys_postgres');
cdb_conf_removeconf
@@ -51,5 +70,7 @@ SELECT CDB_Conf_RemoveConf('api_keys_postgres');
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
DROP FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point;
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;
ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_mocked(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point;

19
client/test/sql/21_bulk_geocoding_functions_test.sql
View File

@@ -12,6 +12,14 @@ RETURNS BOOLEAN as $$
SELECT FALSE;
$$ LANGUAGE SQL;
ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point_mocked;
CREATE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb)
RETURNS SETOF cdb_dataservices_client.geocoding AS $$
BEGIN
RAISE NOTICE 'called with this searches: %', searches;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
-- No permissions granted
-- Test bulk size not mandatory (it will get the optimal)
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
@@ -32,12 +40,21 @@ SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartod
-- Test quota check by mocking quota 0
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
-- Check that when cdb_enough_quota returns true (ie. when soft_limit is set to true, even if not enough quota)
-- it is able to proceed with the bulk geocode
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
RETURNS BOOLEAN as $$
SELECT TRUE;
$$ LANGUAGE SQL;
SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
-- Remove permissions
SELECT CDB_Conf_RemoveConf('api_keys_postgres');
DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
DROP FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point;
ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;
ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_mocked(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point;