diff --git a/client/cdb_dataservices_client--0.7.0--0.8.0.sql b/client/cdb_dataservices_client--0.7.0--0.8.0.sql index b6cedea..2fabf50 100644 --- a/client/cdb_dataservices_client--0.7.0--0.8.0.sql +++ b/client/cdb_dataservices_client--0.7.0--0.8.0.sql @@ -2,266 +2,113 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.8.0'" to load this file. \quit -CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); -CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json) -RETURNS boolean AS $$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ DECLARE + ret Geometry; username text; - user_db_role text; orgname text; - dbname text; - user_schema text; - result boolean; BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT session_user INTO user_db_role; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument'; + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - - IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN - user_schema := 'public'; - ELSE - user_schema := username; - END IF; - - SELECT current_database() INTO dbname; - - SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result; - - RETURN result; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json) -RETURNS boolean AS $$ +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ DECLARE + ret Geometry; username text; - user_db_role text; orgname text; - dbname text; - user_schema text; - result boolean; BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT session_user INTO user_db_role; - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument'; + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - - IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN - user_schema := 'public'; - ELSE - user_schema := username; - END IF; - - SELECT current_database() INTO dbname; - - SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result; - - RETURN result; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; -CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json) -RETURNS boolean AS $$ - from time import strftime - try: - server_table_name = None - temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER - # Obtain return types for augmentation procedure - ds_return_metadata = plpy.execute("SELECT colnames, coltypes " - "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params)) - ) - - colnames_arr = ds_return_metadata[0]["colnames"] - coltypes_arr = ds_return_metadata[0]["coltypes"] - - # Prepare column and type strings required in the SQL queries - colnames = ','.join(colnames_arr) - columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] - columns_with_types = ','.join(columns_with_types_arr) +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - # Instruct the OBS server side to establish a FDW - # The metadata is obtained as well in order to: - # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, - # - (b) be able to tell OBS to free resources when done. - ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername " - "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) - ) - - server_schema = ds_fdw_metadata[0]["schemaname"] - server_table_name = ds_fdw_metadata[0]["tabname"] - server_name = ds_fdw_metadata[0]["servername"] - - # Create temporary table with the augmented results - plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS ' - '(SELECT {columns}, cartodb_id ' - 'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData(' - '{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) ' - 'AS results({columns_with_types}, cartodb_id int) )' - .format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), - user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name), - function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types, - temp_table_name=temporary_table_name) - ) - - # Wipe user FDW data from the server - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - - # Prepare table to receive augmented results in new columns - for idx, column in enumerate(colnames_arr): - if colnames_arr[idx] is not 'the_geom': - plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}' - .format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx]) - ) - - # Populate the user table with the augmented results - plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = ' - '(SELECT {columns} FROM "{user_schema}".{temporary_table_name} ' - 'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)' - .format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), - user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types, - temporary_table_name=temporary_table_name) - ) - - plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}' - .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name) - ) - - return True - except Exception as e: - plpy.warning('Error trying to augment table {0}'.format(e)) - # Wipe user FDW data from the server in case of failure if the table was connected - if server_table_name: - # Wipe local temporary table - plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}' - .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name) - ) - - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - return False -$$ LANGUAGE plpythonu; - - - -CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json) -RETURNS boolean AS $$ - try: - server_table_name = None - # Obtain return types for augmentation procedure - ds_return_metadata = plpy.execute("SELECT colnames, coltypes " - "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params)) - ) - - colnames_arr = ds_return_metadata[0]["colnames"] - coltypes_arr = ds_return_metadata[0]["coltypes"] - - # Prepare column and type strings required in the SQL queries - colnames = ','.join(colnames_arr) - columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] - columns_with_types = ','.join(columns_with_types_arr) - - - # Instruct the OBS server side to establish a FDW - # The metadata is obtained as well in order to: - # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, - # - (b) be able to tell OBS to free resources when done. - ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername " - "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) - ) - - server_schema = ds_fdw_metadata[0]["schemaname"] - server_table_name = ds_fdw_metadata[0]["tabname"] - server_name = ds_fdw_metadata[0]["servername"] - - # Get list of user columns to include in the new table - user_table_columns = ','.join( - plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns ' - 'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass ' - 'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' ' - 'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));' - .format(user_schema=user_schema, table_name=table_name, colnames=colnames) - )[0]["columns"] - ) - - # Populate a new table with the augmented results - plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS ' - '(SELECT results.{columns}, {user_table_columns} ' - 'FROM {table_name} AS user_table ' - 'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) ' - 'ON results.cartodb_id = user_table.cartodb_id)' - .format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username), - orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), - table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types) - ) - - plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' - .format(schema=user_schema, table_name=output_table_name, user=user_db_role) - ) - - # Wipe user FDW data from the server - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - - return True - except Exception as e: - plpy.warning('Error trying to get table {0}'.format(e)) - # Wipe user FDW data from the server in case of failure if the table was connected - if server_table_name: - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - return False -$$ LANGUAGE plpythonu; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text) -RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_ConnectUserTable; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + $$ LANGUAGE plproxy; -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json) -RETURNS cdb_dataservices_client.ds_return_metadata AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_GetReturnMetadata; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + $$ LANGUAGE plproxy; -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) -RETURNS SETOF record AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData; -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text) -RETURNS boolean AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_DisconnectUserTable; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + $$ LANGUAGE plproxy; diff --git a/client/cdb_dataservices_client--0.8.0--0.7.0.sql b/client/cdb_dataservices_client--0.8.0--0.7.0.sql index 2227420..43071aa 100644 --- a/client/cdb_dataservices_client--0.8.0--0.7.0.sql +++ b/client/cdb_dataservices_client--0.8.0--0.7.0.sql @@ -2,14 +2,9 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.7.0'" to load this file. \quit -DROP TYPE IF EXISTS cdb_dataservices_client.ds_fdw_metadata; -DROP TYPE IF EXISTS cdb_dataservices_client.ds_return_metadata; - -DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetTable(text, text, text, json); -DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_AugmentTable(text, text, json) -DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_AugmentTable(text, text, text, text, text, text, text, json); -DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_GetTable(text, text, text, text, text, text, text, text, json); -DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_ConnectUserTable(text, text, text, text, text, text); -DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetReturnMetadata(text, text, text, json); -DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_FetchJoinFdwTableData(text, text, text, text, text, json); -DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_DisconnectUserTable(text, text, text, text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_here_geocode_street_point (text, text, text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_google_geocode_street_point (text, text, text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapzen_geocode_street_point (text, text, text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (text, text, text, text, text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (text, text, text, text, text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (text, text, text, text, text, text); diff --git a/client/cdb_dataservices_client--0.8.0.sql b/client/cdb_dataservices_client--0.8.0.sql index 7020727..af9e686 100644 --- a/client/cdb_dataservices_client--0.8.0.sql +++ b/client/cdb_dataservices_client--0.8.0.sql @@ -348,6 +348,90 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER; -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public dataservices API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) RETURNS SETOF cdb_dataservices_client.isoline AS $$ DECLARE @@ -986,269 +1070,6 @@ BEGIN END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; -CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); -CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json) -RETURNS boolean AS $$ -DECLARE - username text; - user_db_role text; - orgname text; - dbname text; - user_schema text; - result boolean; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - - SELECT session_user INTO user_db_role; - - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument'; - END IF; - - IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN - user_schema := 'public'; - ELSE - user_schema := username; - END IF; - - SELECT current_database() INTO dbname; - - SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result; - - RETURN result; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json) -RETURNS boolean AS $$ -DECLARE - username text; - user_db_role text; - orgname text; - dbname text; - user_schema text; - result boolean; -BEGIN - IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN - RAISE EXCEPTION 'The api_key must be provided'; - END IF; - - SELECT session_user INTO user_db_role; - - SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); - -- JSON value stored "" is taken as literal - IF username IS NULL OR username = '' OR username = '""' THEN - RAISE EXCEPTION 'Username is a mandatory argument'; - END IF; - - IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN - user_schema := 'public'; - ELSE - user_schema := username; - END IF; - - SELECT current_database() INTO dbname; - - SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result; - - RETURN result; -END; -$$ LANGUAGE 'plpgsql' SECURITY DEFINER; - -CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json) -RETURNS boolean AS $$ - from time import strftime - try: - server_table_name = None - temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name - - # Obtain return types for augmentation procedure - ds_return_metadata = plpy.execute("SELECT colnames, coltypes " - "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params)) - ) - - colnames_arr = ds_return_metadata[0]["colnames"] - coltypes_arr = ds_return_metadata[0]["coltypes"] - - # Prepare column and type strings required in the SQL queries - colnames = ','.join(colnames_arr) - columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] - columns_with_types = ','.join(columns_with_types_arr) - - - # Instruct the OBS server side to establish a FDW - # The metadata is obtained as well in order to: - # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, - # - (b) be able to tell OBS to free resources when done. - ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername " - "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) - ) - - server_schema = ds_fdw_metadata[0]["schemaname"] - server_table_name = ds_fdw_metadata[0]["tabname"] - server_name = ds_fdw_metadata[0]["servername"] - - # Create temporary table with the augmented results - plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS ' - '(SELECT {columns}, cartodb_id ' - 'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData(' - '{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) ' - 'AS results({columns_with_types}, cartodb_id int) )' - .format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), - user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name), - function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types, - temp_table_name=temporary_table_name) - ) - - # Wipe user FDW data from the server - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - - # Prepare table to receive augmented results in new columns - for idx, column in enumerate(colnames_arr): - if colnames_arr[idx] is not 'the_geom': - plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}' - .format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx]) - ) - - # Populate the user table with the augmented results - plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = ' - '(SELECT {columns} FROM "{user_schema}".{temporary_table_name} ' - 'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)' - .format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), - user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types, - temporary_table_name=temporary_table_name) - ) - - plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}' - .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name) - ) - - return True - except Exception as e: - plpy.warning('Error trying to augment table {0}'.format(e)) - # Wipe user FDW data from the server in case of failure if the table was connected - if server_table_name: - # Wipe local temporary table - plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}' - .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name) - ) - - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - return False -$$ LANGUAGE plpythonu; - - - -CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json) -RETURNS boolean AS $$ - try: - server_table_name = None - # Obtain return types for augmentation procedure - ds_return_metadata = plpy.execute("SELECT colnames, coltypes " - "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params)) - ) - - colnames_arr = ds_return_metadata[0]["colnames"] - coltypes_arr = ds_return_metadata[0]["coltypes"] - - # Prepare column and type strings required in the SQL queries - colnames = ','.join(colnames_arr) - columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] - columns_with_types = ','.join(columns_with_types_arr) - - - # Instruct the OBS server side to establish a FDW - # The metadata is obtained as well in order to: - # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, - # - (b) be able to tell OBS to free resources when done. - ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername " - "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) - ) - - server_schema = ds_fdw_metadata[0]["schemaname"] - server_table_name = ds_fdw_metadata[0]["tabname"] - server_name = ds_fdw_metadata[0]["servername"] - - # Get list of user columns to include in the new table - user_table_columns = ','.join( - plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns ' - 'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass ' - 'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' ' - 'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));' - .format(user_schema=user_schema, table_name=table_name, colnames=colnames) - )[0]["columns"] - ) - - # Populate a new table with the augmented results - plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS ' - '(SELECT results.{columns}, {user_table_columns} ' - 'FROM {table_name} AS user_table ' - 'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) ' - 'ON results.cartodb_id = user_table.cartodb_id)' - .format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username), - orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), - table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types) - ) - - plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' - .format(schema=user_schema, table_name=output_table_name, user=user_db_role) - ) - - # Wipe user FDW data from the server - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - - return True - except Exception as e: - plpy.warning('Error trying to get table {0}'.format(e)) - # Wipe user FDW data from the server in case of failure if the table was connected - if server_table_name: - wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) - ) - return False -$$ LANGUAGE plpythonu; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text) -RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_ConnectUserTable; -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json) -RETURNS cdb_dataservices_client.ds_return_metadata AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_GetReturnMetadata; -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) -RETURNS SETOF record AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData; -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text) -RETURNS boolean AS $$ - CONNECT _server_conn_str(); - TARGET cdb_dataservices_server._OBS_DisconnectUserTable; -$$ LANGUAGE plproxy; CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text) RETURNS Geometry AS $$ CONNECT cdb_dataservices_client._server_conn_str(); @@ -1329,6 +1150,30 @@ RETURNS Geometry AS $$ $$ LANGUAGE plproxy; +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy; + CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) RETURNS SETOF cdb_dataservices_client.isoline AS $$ CONNECT cdb_dataservices_client._server_conn_str(); @@ -1531,6 +1376,9 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; @@ -1554,5 +1402,3 @@ GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom G GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom Geometry, normalize text, boundary_id text, time_span text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) TO publicuser; GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom Geometry, timespan text) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_augmenttable(table_name text, function_name text, params json) TO publicuser; -GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_gettable(table_name text, output_table_name text, function_name text, params json) TO publicuser; diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index a2990bc..b2917ca 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -59,6 +59,30 @@ - { name: state_province, type: text, default: 'NULL'} - { name: country, type: text, default: 'NULL'} +- name: cdb_here_geocode_street_point + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} + +- name: cdb_google_geocode_street_point + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} + +- name: cdb_mapzen_geocode_street_point + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} + - name: cdb_isodistance return_type: SETOF cdb_dataservices_client.isoline multi_row: true diff --git a/server/extension/cdb_dataservices_server--0.10.0--0.11.0.sql b/server/extension/cdb_dataservices_server--0.10.0--0.11.0.sql index a52b3d4..0635e8d 100644 --- a/server/extension/cdb_dataservices_server--0.10.0--0.11.0.sql +++ b/server/extension/cdb_dataservices_server--0.10.0--0.11.0.sql @@ -2,39 +2,100 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.11.0'" to load this file. \quit --- HERE goes your code to upgrade/downgrade -CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] -CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Here geocoder is not available for your account.') -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) -RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ - return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)" - .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) - )[0] $$ LANGUAGE plpythonu; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) -RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_ConnectUserTable; -$$ LANGUAGE plproxy; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json) -RETURNS cdb_dataservices_server.ds_return_metadata AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_GetReturnMetadata; -$$ LANGUAGE plproxy; + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Google geocoder is not available for your account.') -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) -RETURNS SETOF record AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_FetchJoinFdwTableData; -$$ LANGUAGE plproxy; +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text) RETURNS boolean AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - TARGET cdb_observatory._OBS_DisconnectUserTable; -$$ LANGUAGE plproxy; + cache_key = "user_mapzen_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import MapzenGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = mapzen_geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.11.0--0.10.0.sql b/server/extension/cdb_dataservices_server--0.11.0--0.10.0.sql index 6ff0b8e..139215f 100644 --- a/server/extension/cdb_dataservices_server--0.11.0--0.10.0.sql +++ b/server/extension/cdb_dataservices_server--0.11.0--0.10.0.sql @@ -2,12 +2,46 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.10.0'" to load this file. \quit --- HERE goes your code to upgrade/downgrade -DROP TYPE IF EXISTS cdb_dataservices_server.ds_fdw_metadata; -DROP TYPE IF EXISTS cdb_dataservices_server.ds_return_metadata; +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapzen_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_geocoder_config(text, text); -DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_ConnectUserTable(text, text, text, text, text, text); -DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text); -DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetReturnMetadata(text, text, text, json); -DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_FetchJoinFdwTableData(text, text, text, text, text, json); -DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_DisconnectUserTable(text, text, text, text, text); +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_dataservices_server--0.11.0.sql b/server/extension/cdb_dataservices_server--0.11.0.sql index eb2f86f..d80d838 100644 --- a/server/extension/cdb_dataservices_server--0.11.0.sql +++ b/server/extension/cdb_dataservices_server--0.11.0.sql @@ -1131,6 +1131,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; +<<<<<<< HEAD CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); @@ -1166,6 +1167,8 @@ RETURNS boolean AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); TARGET cdb_observatory._OBS_DisconnectUserTable; $$ LANGUAGE plproxy; +======= +>>>>>>> master CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_geocoder_config_{0}".format(username) @@ -1180,6 +1183,23 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; +<<<<<<< HEAD +======= +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_mapzen_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import MapzenGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = mapzen_geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +>>>>>>> master CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_internal_geocoder_config_{0}".format(username) @@ -1271,6 +1291,53 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu; +<<<<<<< HEAD +======= + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu; + +>>>>>>> master CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder @@ -1344,13 +1411,22 @@ RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] +<<<<<<< HEAD user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] quota_service = QuotaService(user_geocoder_config, redis_conn) +======= + user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn) +>>>>>>> master if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') try: +<<<<<<< HEAD geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) +======= + geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key) +>>>>>>> master country_iso3 = None if country: country_iso3 = country_to_iso3(country) diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 68726ff..3d6fd72 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -12,6 +12,20 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_mapzen_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import MapzenGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = mapzen_geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_internal_geocoder_config_{0}".format(username) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index a955f78..606233e 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -20,6 +20,50 @@ RETURNS Geometry AS $$ $$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder @@ -93,13 +137,13 @@ RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - quota_service = QuotaService(user_geocoder_config, redis_conn) + user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) + geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key) country_iso3 = None if country: country_iso3 = country_to_iso3(country) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 078bbbd..9bba57c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig +from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 5ecb832..340e5f6 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -128,6 +128,60 @@ class RoutingConfig(ServiceConfig): def period_end_date(self): return self._period_end_date +# Explicit class for the geocoder configuration for Mapzen +# which does not require users to be configured to use the service +class MapzenGeocoderConfig(ServiceConfig): + + PERIOD_END_DATE = 'period_end_date' + + def __init__(self, redis_connection, db_conn, username, orgname=None): + super(MapzenGeocoderConfig, self).__init__(redis_connection, db_conn, + username, orgname) + self._log_path = self._db_config.geocoder_log_path + try: + self._mapzen_api_key = self._db_config.mapzen_geocoder_api_key + self._monthly_quota = self._db_config.mapzen_geocoder_monthly_quota + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) + self._cost_per_hit = 0 + except Exception as e: + raise ConfigException("Malformed config for Mapzen geocoder: {1}".format(key, e)) + + @property + def service_type(self): + return 'geocoder_mapzen' + + @property + def mapzen_api_key(self): + return self._mapzen_api_key + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def google_geocoder(self): + return None + + @property + def geocoding_quota(self): + return self._monthly_quota + + @property + def soft_geocoding_limit(self): + return False + + @property + def is_high_resolution(self): + return True + + @property + def log_path(self): + return self._log_path + class IsolinesRoutingConfig(ServiceConfig): diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index a31ecb6..663c5ec 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.6.4', + version='0.6.5', description='CartoDB Services API Python Library', diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index c9647ff..263aaa6 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -22,6 +22,14 @@ class TestStreetFunctions(TestCase): geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(geometry['geometry'], None) + def test_if_select_with_mapzen_provider_street_point_is_ok(self): + query = "SELECT cdb_mapzen_geocode_street_point(street) " \ + "as geometry FROM {0} LIMIT 1&api_key={1}".format( + self.env_variables['table_name'], + self.env_variables['api_key']) + geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) + assert_not_equal(geometry['geometry'], None) + def test_if_select_with_street_without_api_key_raise_error(self): query = "SELECT cdb_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1".format(