cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TL;DR: safer deployment and minor fixes

Browse Source 
- Instead of modifying cdb_service_quota_info to return max_batch_size,
a new type (service_quota_info_batch) and a new function
(cdb_service_quota_info_batch) are created. That makes deployment safe.
- Fixes geocoding with forced batch size 1.
- Improves namespacing for count_estimate (->
cdb_dataservices_client.cdb_count_estimate).
- Improves namespacing for jsonb_array_casttext (->
cdb_dataservices_client.cdb_jsonb_array_casttext).
This commit is contained in:
Juan Ignacio Sánchez Lara
2018-07-18 14:29:50 +02:00
parent 0c5e9da028
commit b0c1948c14
16 changed files with 375 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client/sql/05_utils.sql
View File

@@ -1,5 +1,5 @@
-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
CREATE FUNCTION count_estimate(query text) RETURNS INTEGER AS
CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
$func$
DECLARE
rec record;
@@ -15,6 +15,6 @@ END
$func$ LANGUAGE plpgsql;
-- Taken from https://stackoverflow.com/a/48013356/351721
CREATE OR REPLACE FUNCTION jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
$f$ LANGUAGE sql IMMUTABLE;

8
client/sql/16_custom_types.sql
View File

@@ -35,6 +35,14 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM (
);
CREATE TYPE cdb_dataservices_client.service_quota_info AS (
service cdb_dataservices_client.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,
soft_limit BOOLEAN,
provider TEXT
);
CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
service cdb_dataservices_client.service_type,
monthly_quota NUMERIC,
used_quota NUMERIC,

9
client/sql/21_bulk_geocoding_functions.sql
View File

@@ -11,13 +11,12 @@ DECLARE
batches_n integer;
DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
current_row_count integer ;
temp_table_name text;
BEGIN
SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
INTO remaining_quota, max_batch_size
FROM cdb_dataservices_client.cdb_service_quota_info() csqi
FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
WHERE service = 'hires_geocoder';
RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
@@ -57,12 +56,11 @@ BEGIN
IF batches_n > 0 THEN
FOR cartodb_id_batch in 0..(batches_n - 1)
LOOP
EXECUTE format(
'WITH geocoding_data as (' ||
' SELECT ' ||
' json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
' floor((row_number() over ())::float/$1) as batch' ||
' floor((row_number() over () - 1)::float/$1) as batch' ||
' FROM (%s) _x' ||
') ' ||
'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
@@ -70,9 +68,6 @@ BEGIN
'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
USING batch_size, cartodb_id_batch;
GET DIAGNOSTICS current_row_count = ROW_COUNT;
RAISE DEBUG 'Batch % --> %', cartodb_id_batch, current_row_count;
END LOOP;
END IF;