From 18f05fbd4fc5c9131646191fb2fe80f16ee840d3 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 28 Sep 2016 16:30:37 +0200 Subject: [PATCH 01/39] User class with tests --- .../cartodb_services/refactor/__init__.py | 0 .../refactor/core/__init__.py | 0 .../cartodb_services/refactor/core/user.py | 24 +++++++++++ .../test/refactor/__init__.py | 0 .../test/refactor/core/__init__.py | 0 .../test/refactor/core/test_user.py | 43 +++++++++++++++++++ 6 files changed, 67 insertions(+) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/core/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py create mode 100644 server/lib/python/cartodb_services/test/refactor/__init__.py create mode 100644 server/lib/python/cartodb_services/test/refactor/core/__init__.py create mode 100644 server/lib/python/cartodb_services/test/refactor/core/test_user.py diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/__init__.py b/server/lib/python/cartodb_services/cartodb_services/refactor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/__init__.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py new file mode 100644 index 0000000..a36a350 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py @@ -0,0 +1,24 @@ +class User(object): + + def __init__(self, username, orgname = None): + self.__username = username + self.__orgname = orgname + + @property + def username(self): + return self.__username + + @property + def orgname(self): + return self.__orgname + + @property + def is_org_user(self): + return self.__orgname is not None + + def __eq__(self, other): + # NOTE: usernames are unique in the system + eq = (self.__username == other.__username) + if eq: + assert self.__orgname == other.__orgname, 'Found two users with same name and different orgs!' + return eq diff --git a/server/lib/python/cartodb_services/test/refactor/__init__.py b/server/lib/python/cartodb_services/test/refactor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/test/refactor/core/__init__.py b/server/lib/python/cartodb_services/test/refactor/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/test/refactor/core/test_user.py b/server/lib/python/cartodb_services/test/refactor/core/test_user.py new file mode 100644 index 0000000..70d4359 --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/core/test_user.py @@ -0,0 +1,43 @@ +from unittest import TestCase +from cartodb_services.refactor.core.user import User +from nose.tools import assert_raises, raises + +class TestUser(TestCase): + + def test_can_retrieve_properties(self): + u = User('pepito', 'acme') + assert u.username == 'pepito' + assert u.orgname == 'acme' + + @raises(AttributeError) + def test_cannot_modify_properties(self): + u = User('pepito') + u.username = 'juanito' + + def test_is_org_user_returns_true_for_org_users(self): + u = User('pepito', 'acme') + assert u.is_org_user == True + + def test_is_org_user_returns_false_for_non_org_users(self): + u = User('pepito') + assert u.is_org_user == False + + def test_different_users_are_not_equal(self): + u1 = User('pepito') + u2 = User('juanito') + assert u1 != u2 + + def test_same_user_are_equal(self): + u1 = User('pepito') + u2 = User('pepito') + assert u1 == u2 + + o1 = User('juanito', 'acme') + o2 = User('juanito', 'acme') + assert o1 == o2 + + @raises(AssertionError) + def test_raises_exception_if_same_user_is_in_two_orgs(self): + u1 = User('juanito', 'acme') + u2 = User('juanito', 'some_other_org') + u1 == u2 From fd2cc219424e460517d15ec938301cb939a96e5b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 28 Sep 2016 17:32:26 +0200 Subject: [PATCH 02/39] Add server config storage classes --- .../cartodb_services/__init__.py | 35 ++++++++++++++++ .../refactor/storage/__init__.py | 0 .../refactor/storage/interfaces.py | 11 +++++ .../refactor/storage/server_config.py | 26 ++++++++++++ .../test/refactor/storage/__init__.py | 0 .../refactor/storage/test_server_config.py | 42 +++++++++++++++++++ 6 files changed, 114 insertions(+) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/interfaces.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py create mode 100644 server/lib/python/cartodb_services/test/refactor/storage/__init__.py create mode 100644 server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py diff --git a/server/lib/python/cartodb_services/cartodb_services/__init__.py b/server/lib/python/cartodb_services/cartodb_services/__init__.py index e69de29..61670c2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/__init__.py @@ -0,0 +1,35 @@ +# NOTE: This init function must be called from plpythonu entry points to +# initialize cartodb_services module properly. E.g: +# +# CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(...) +# RETURNS SETOF cdb_dataservices_server.isoline AS $$ +# +# import cartodb_services +# cartodb_services.init(plpy, GD) +# +# # rest of the code here +# cartodb_services.GD[key] = val +# cartodb_services.plpy.execute('SELECT * FROM ...') +# +# $$ LANGUAGE plpythonu; + +plpy = None +GD = None + +def init(_plpy, _GD): + global plpy + global GD + + if plpy is None: + plpy = _plpy + + if GD is None: + GD = _GD + +def _reset(): + # NOTE: just for testing + global plpy + global GD + + plpy = None + GD = None diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/__init__.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/interfaces.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/interfaces.py new file mode 100644 index 0000000..5f00f53 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/interfaces.py @@ -0,0 +1,11 @@ +import abc + +class ConfigStorageInterface(object): + """This is an interface that all config storages must abide to""" + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def get(self, key): + """Return a value based on the key supplied from some storage""" + pass diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py new file mode 100644 index 0000000..435588d --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -0,0 +1,26 @@ +import json +import cartodb_services +from interfaces import ConfigStorageInterface + +class InDbServerConfigStorage(ConfigStorageInterface): + + def get(self, key): + sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key) + rows = cartodb_services.plpy.execute(sql, 1) + json_output = rows[0]['conf'] + if json_output: + return json.loads(json_output) + else: + return None + + +class InMemoryConfigStorage(ConfigStorageInterface): + + def __init__(self, config_hash={}): + self._config_hash = config_hash + + def get(self, key): + try: + return self._config_hash[key] + except KeyError: + return None diff --git a/server/lib/python/cartodb_services/test/refactor/storage/__init__.py b/server/lib/python/cartodb_services/test/refactor/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py new file mode 100644 index 0000000..4e3d27c --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py @@ -0,0 +1,42 @@ +from unittest import TestCase +from mock import Mock, MagicMock +from nose.tools import raises +from cartodb_services.refactor.storage.server_config import * +import cartodb_services + +class TestInDbServerConfigStorage(TestCase): + + def setUp(self): + self.plpy_mock = Mock() + cartodb_services.init(self.plpy_mock, _GD={}) + + def tearDown(self): + cartodb_services._reset() + + def test_gets_configs_from_db(self): + self.plpy_mock.execute = MagicMock(return_value=[{'conf': '"any_value"'}]) + server_config = InDbServerConfigStorage() + assert server_config.get('any_config') == 'any_value' + self.plpy_mock.execute.assert_called_once_with("SELECT cdb_dataservices_server.cdb_conf_getconf('any_config') as conf", 1) + + def test_gets_none_if_cannot_retrieve_key(self): + self.plpy_mock.execute = MagicMock(return_value=[{'conf': None}]) + server_config = InDbServerConfigStorage() + assert server_config.get('any_non_existing_key') is None + + def test_deserializes_from_db_to_plain_dict(self): + self.plpy_mock.execute = MagicMock(return_value=[{'conf': '{"environment": "testing"}'}]) + server_config = InDbServerConfigStorage() + assert server_config.get('server_conf') == {'environment': 'testing'} + self.plpy_mock.execute.assert_called_once_with("SELECT cdb_dataservices_server.cdb_conf_getconf('server_conf') as conf", 1) + + +class TestInMemoryConfigStorage(TestCase): + + def test_can_provide_values_from_hash(self): + server_config = InMemoryConfigStorage({'any_key': 'any_value'}) + assert server_config.get('any_key') == 'any_value' + + def test_gets_none_if_cannot_retrieve_key(self): + server_config = InMemoryConfigStorage() + assert server_config.get('any_non_existing_key') == None From efdc1512824c8015eed9aa01feb23b5660639c9e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 28 Sep 2016 18:22:51 +0200 Subject: [PATCH 03/39] Add things to get logger config from storage --- server/extension/sql/15_config_helper.sql | 6 ++ server/extension/sql/20_geocode_street.sql | 14 +++- .../refactor/tools/__init__.py | 0 .../cartodb_services/refactor/tools/logger.py | 67 +++++++++++++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/tools/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 5f6fc77..9d83ebd 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -10,6 +10,12 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL STABLE SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) RETURNS boolean AS $$ cache_key = "user_geocoder_config_{0}".format(username) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 02e2a2f..2c4df2f 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -137,17 +137,25 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + import cartodb_services + cartodb_services.init(plpy, GD) from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.tools import Logger + from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage + from cartodb_services.refactor.tools.logger import LoggerConfigBuilder redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] + server_config_storage = InDbServerConfigStorage() + + logger_config = LoggerConfigBuilder(server_config_storage).get() logger = Logger(logger_config) + + + quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py new file mode 100644 index 0000000..febde62 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py @@ -0,0 +1,67 @@ +class LoggerConfig(object): + + """This class is a value object needed to setup a Logger""" + + def __init__(self, server_environment, rollbar_api_key, log_file_path, min_log_level): + self._server_environment = server_environment + self._rollbar_api_key = rollbar_api_key + self._log_file_path = log_file_path + self._min_log_level = min_log_level + + @property + def environment(self): + return self._server_environment + + @property + def rollbar_api_key(self): + return self._rollbar_api_key + + @property + def log_file_path(self): + return self._log_file_path + + @property + def min_log_level(self): + return self._min_log_level + +class ConfigException(Exception): + pass + +# TODO this needs tests +# TODO FTM this is just config, maybe move around +class LoggerConfigBuilder(object): + + def __init__(self, server_config_storage): + self._server_config_storage = server_config_storage + + def get(self): + server_environment = self._get_server_environment() + + logger_conf = self._server_config_storage.get('logger_conf') + if not logger_conf: + raise ConfigException('Logger configuration missing') + + rollbar_api_key = self._get_value_or_none(logger_conf, 'rollbar_api_key') + log_file_path = self._get_value_or_none(logger_conf, 'log_file_path') + min_log_level = self._get_value_or_none(logger_conf, 'min_log_level') or 'warning' + + logger_config = LoggerConfig(server_environment, rollbar_api_key, log_file_path, min_log_level) + return logger_config + + def _get_server_environment(self): + server_config = self._server_config_storage.get('server_conf') + if not server_config: + environment = 'development' + else: + if 'environment' in server_config: + environment = server_config['environment'] + else: + environment = 'development' + + return environment + + def _get_value_or_none(self, logger_conf, key): + value = None + if key in logger_conf: + value = logger_conf[key] + return value From 9d60fde0b8149e34b91c698be5832ecdc1aafdb1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 13:07:40 +0200 Subject: [PATCH 04/39] A stubb for RedisConfigStorage (WIP) --- .../refactor/storage/server_config.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 435588d..0cea760 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -24,3 +24,14 @@ class InMemoryConfigStorage(ConfigStorageInterface): return self._config_hash[key] except KeyError: return None + + +class RedisConfigStorage(ConfigStorageInterface): + + def __init__(self, connection, prefix): + self._connection = connection + self._prefix = prefix + + def get(self, key): + # TODO: Implement. this is just a stub + return None From 1f3a655ae590d90c9f7e1bafbd33b4c35051f8bf Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 16:57:31 +0200 Subject: [PATCH 05/39] Add all the code to instantiate a user_config_storage (WIP) --- server/extension/sql/20_geocode_street.sql | 13 ++- .../refactor/storage/redis_config.py | 85 +++++++++++++++++++ .../refactor/storage/redis_connection.py | 22 +++++ 3 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 2c4df2f..acaff8e 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -143,17 +143,22 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger - from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage + from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, RedisConfigStorage from cartodb_services.refactor.tools.logger import LoggerConfigBuilder - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder + from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder server_config_storage = InDbServerConfigStorage() logger_config = LoggerConfigBuilder(server_config_storage).get() logger = Logger(logger_config) + redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() + redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config) + user_config_storage = RedisConfigStorage(redis_metadata_connection, 'rails:users:{0}'.format(username)) + + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] quota_service = QuotaService(user_geocoder_config, redis_conn) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py new file mode 100644 index 0000000..8c156eb --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -0,0 +1,85 @@ +""" +How to use this (just a draft, WIP): + + redis_connection_config = RedisConnectionConfigBuilder(server_config_storage).get() + connection = RedisConnectionBuilder(redis_connection_config) + + user_config_storage = RedisConfigStorage(connection, user) + user_config_storage = UserConfigStorageFactory(environment).get() +""" + +class RedisConnectionConfig(object): + """ + This represents a value object to contain configuration needed to set up + a connection to a redis server. + """ + + + def __init__(self, host, port, timeout, db, sentinel_id): + self._host = host + self._port = port + self._timeout = timeout + self._db = db + self._sentinel_id = sentinel_id + + @property + def host(self): + return self._host + + @property + def port(self): + return self._port + + @property + def timeout(self): + return self._timeout + + @property + def db(self): + return self._db + + @property + def sentinel_id(self): + return self._sentinel_id + + +# TODO make it abstract +class RedisConnectionConfigBuilder(object): + + DEFAULT_USER_DB = 5 + DEFAULT_TIMEOUT = 1.5 # seconds + + def __init__(self, server_config_storage, config_key): + self._server_config_storage = server_config_storage + self._config_key = config_key + + def get(self): + conf = self._server_config_storage.get(self._config_key) + if conf is None: + raise Exception("There is no redis configuration defined") + + host = conf['redis_host'] + port = conf['redis_port'] + timeout = conf['timeout'] or self.DEFAULT_TIMEOUT + db = conf['redis_db'] or self.DEFAULT_USER_DB + sentinel_id = conf['sentinel_master_id'] + + return RedisConnectionConfig(host, port, timeout, db, sentinel_id) + + +class RedisMetadataConnectionConfigBuilder(RedisConnectionConfigBuilder): + + def __init__(self, server_config_storage): + super(RedisMetadataConnectionConfigBuilder, self).__init__( + server_config_storage, + 'redis_metadata_config' + ) + + +class RedisMetricsConnectionConfigBuilder(RedisConnectionConfigBuilder): + + def __init__(self, server_config_storage): + super(RedisMetricsConnectionConfigBuilder, self).__init__( + server_config_storage, + 'redis_metrics_config' + ) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py new file mode 100644 index 0000000..a125af3 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py @@ -0,0 +1,22 @@ +from redis.sentinel import Sentinel +from redis import StrictRedis + +class RedisConnectionBuilder(): + + def __init__(self, connection_config): + self._config = connection_config + + def get(self): + if self._config.sentinel_id: + sentinel = Sentinel([(self._config.host, + self._config.port)], + socket_timeout=self._config.timeout) + return sentinel.master_for(self._config.sentinel_id, + socket_timeout=self._config.timeout, + db=self._config.db, + retry_on_timeout=True) + else: + conn = StrictRedis(host=self._config.host, port=self._config.port, + db=self._config.db, retry_on_timeout=True, + socket_timeout=self._config.timeout) + return conn From 4b4a02905cc6718e57c69cf065b812d2f0d5da0b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 17:18:45 +0200 Subject: [PATCH 06/39] Implement the RedisConfigStorage --- .../refactor/storage/server_config.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 0cea760..3637e3d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -25,13 +25,15 @@ class InMemoryConfigStorage(ConfigStorageInterface): except KeyError: return None - +# TODO move out of this file. In general this is config but either user or org config class RedisConfigStorage(ConfigStorageInterface): - def __init__(self, connection, prefix): + def __init__(self, connection, config_key): self._connection = connection - self._prefix = prefix + self._config_key = config_key + self._data = None def get(self, key): - # TODO: Implement. this is just a stub - return None + if not self._data: + self._data = self._connection.hgetall(self._config_key) + return self._data[key] From 02a2619b450a28b8d4baea03225aad013ff76e8a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 17:19:18 +0200 Subject: [PATCH 07/39] Fix instantiation of redis metadata connection --- server/extension/sql/20_geocode_street.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index acaff8e..45265db 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -154,7 +154,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() - redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config) + redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() user_config_storage = RedisConfigStorage(redis_metadata_connection, 'rails:users:{0}'.format(username)) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] From 0b7b44d8a5ec126f688db38e4669b3a763f0c14d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 17:46:17 +0200 Subject: [PATCH 08/39] Add a couple of factories to abstract user/org configs --- .../refactor/storage/server_config.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 3637e3d..7a65946 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -25,6 +25,11 @@ class InMemoryConfigStorage(ConfigStorageInterface): except KeyError: return None +class NullConfigStorage(ConfigStorageInterface): + + def get(self, key): + return None + # TODO move out of this file. In general this is config but either user or org config class RedisConfigStorage(ConfigStorageInterface): @@ -37,3 +42,24 @@ class RedisConfigStorage(ConfigStorageInterface): if not self._data: self._data = self._connection.hgetall(self._config_key) return self._data[key] + +class UserConfigStorageFactory(object): + # TODO rework to support onpremise and InDbStorage + def __init__(self, redis_connection, username): + self._redis_connection = redis_connection + self._username = username + + def get(self): + return RedisConfigStorage(self._redis_connection, 'rails:users:{0}'.format(self._username)) + +class OrgConfigStorageFactory(object): + # TODO rework to support onpremise and InDbStorage + def __init__(self, redis_connection, orgname): + self._redis_connection = redis_connection + self._orgname = orgname + + def get(self): + if self._orgname: + return RedisConfigStorage(self._redis_connection, 'rails:orgs:{0}'.format(self._orgname)) + else: + return NullConfigStorage() From 8ad2434b1d96171e605b233fb96a38636d8f32d2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 17:47:36 +0200 Subject: [PATCH 09/39] Add instantiation of configs to mapzen geocoder --- server/extension/sql/20_geocode_street.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 45265db..f0dfeec 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -143,7 +143,7 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger - from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, RedisConfigStorage + from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, UserConfigStorageFactory, OrgConfigStorageFactory from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder @@ -155,7 +155,8 @@ RETURNS Geometry AS $$ redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() - user_config_storage = RedisConfigStorage(redis_metadata_connection, 'rails:users:{0}'.format(username)) + user_config_storage = UserConfigStorageFactory(redis_metadata_connection, username).get() + org_config_storage = OrgConfigStorageFactory(redis_metadata_connection, orgname).get() user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] From 3a6cc4c364a310c63931bf990c395e3b0f7077b9 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 18:41:16 +0200 Subject: [PATCH 10/39] Add mapzen config and integrate into legacy code --- server/extension/sql/20_geocode_street.sql | 5 +- .../refactor/service/__init__.py | 0 .../refactor/service/mapzen_geocoder.py | 113 ++++++++++++++++++ 3 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index f0dfeec..4546a78 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -147,6 +147,7 @@ RETURNS Geometry AS $$ from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder + from cartodb_services.refactor.service.mapzen_geocoder import MapzenGeocoderConfigBuilder server_config_storage = InDbServerConfigStorage() @@ -158,7 +159,9 @@ RETURNS Geometry AS $$ user_config_storage = UserConfigStorageFactory(redis_metadata_connection, username).get() org_config_storage = OrgConfigStorageFactory(redis_metadata_connection, orgname).get() - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + # TODO rename this variable + user_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_storage, org_config_storage, username, orgname).get() + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/__init__.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder.py new file mode 100644 index 0000000..2db5784 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder.py @@ -0,0 +1,113 @@ +from dateutil.parser import parse as date_parse + +# TODO: rename this file +class MapzenGeocoderConfig(object): + """ + Value object that represents the configuration needed to operate the mapzen service. + """ + + def __init__(self, + geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + mapzen_api_key, + username, + organization): + self._geocoding_quota = geocoding_quota + self._soft_geocoding_limit = soft_geocoding_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._mapzen_api_key = mapzen_api_key + self._username = username + self._organization = organization + + # Kind of generic properties. Note which ones are for actually running the + # service and which ones are needed for quota stuff. + @property + def service_type(self): + return 'geocoder_mapzen' + + @property + def provider(self): + return 'mapzen' + + @property + def is_high_resolution(self): + return True + + @property + def geocoding_quota(self): + return self._geocoding_quota + + @property + def soft_geocoding_limit(self): + return self._soft_geocoding_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + # Server config, TODO: locate where this is actually used + @property + def log_path(self): + return self._log_path + + # This is actually the specific one to run requests against the remote endpoitn + @property + def mapzen_api_key(self): + return self._mapzen_api_key + + # These two identify the user + @property + def username(self): + return self._username + @property + def organization(self): + return self._organization + + # TODO: for BW compat, remove + @property + def google_geocoder(self): + return False + + +class MapzenGeocoderConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + + + def get(self): + mapzen_server_conf = self._server_conf.get('mapzen_conf') + geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota'] + mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] + + soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit') + + cost_per_hit=0 + + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf['geocoder_log_path'] + + return MapzenGeocoderConfig(geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + mapzen_api_key, + self._username, + self._orgname) From 1e6ee8d5c1db66f6020df04d16bb2dc3820d5b02 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 30 Sep 2016 11:21:35 +0200 Subject: [PATCH 11/39] Add an Environment class --- .../cartodb_services/refactor/core/environment.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py new file mode 100644 index 0000000..5d33d95 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py @@ -0,0 +1,13 @@ +class Environment: + def __init__(self, server_config_storage): + self._server_config_storage = server_config_storage + + def get(self): + server_config = self._server_config_storage.get('server_conf') + + if not server_config or 'environment' not in server_config: + environment = 'development' + else: + environment = server_config['environment'] + + return environment From e24819f19373a8dd0a6b2b652e485cba65a10f5a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 30 Sep 2016 11:29:44 +0200 Subject: [PATCH 12/39] Take the environment into account --- server/extension/sql/20_geocode_street.sql | 27 +++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 4546a78..eb8e965 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -145,27 +145,38 @@ RETURNS Geometry AS $$ from cartodb_services.tools import Logger from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, UserConfigStorageFactory, OrgConfigStorageFactory from cartodb_services.refactor.tools.logger import LoggerConfigBuilder - from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder + from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder from cartodb_services.refactor.service.mapzen_geocoder import MapzenGeocoderConfigBuilder + from cartodb_services.refactor.core.environment import Environment server_config_storage = InDbServerConfigStorage() logger_config = LoggerConfigBuilder(server_config_storage).get() logger = Logger(logger_config) - redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() - redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() - user_config_storage = UserConfigStorageFactory(redis_metadata_connection, username).get() - org_config_storage = OrgConfigStorageFactory(redis_metadata_connection, orgname).get() + # TODO encapsulate construction of user_config_storage and org_config_storage + environment = Environment(server_config_storage).get() + if environment == 'onpremise': + user_config_storage = org_config_storage = server_config_storage + else: + redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() + redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() + user_config_storage = UserConfigStorageFactory(redis_metadata_connection, username).get() + org_config_storage = OrgConfigStorageFactory(redis_metadata_connection, orgname).get() # TODO rename this variable user_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_storage, org_config_storage, username, orgname).get() - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + # TODO encapsulate the connection creation + # TODO implement the RedisConnectionMock + if environment == 'onpremise': + redis_metrics_connection = RedisConnectionMock() + else: + redis_metrics_connection_config = RedisMetricsConnectionConfigBuilder(server_config_storage).get() + redis_metrics_connection = RedisConnectionBuilder(redis_metrics_connection_config).get() - - quota_service = QuotaService(user_geocoder_config, redis_conn) + quota_service = QuotaService(user_geocoder_config, redis_metrics_connection) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') From 74d2fba763d23b06a4bd4d19a4013a08cdddafb5 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Fri, 30 Sep 2016 12:28:27 +0200 Subject: [PATCH 13/39] Add Redis Mock -- tested and working --- server/extension/sql/20_geocode_street.sql | 1 + .../cartodb_services/refactor/tools/redis_mock.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index eb8e965..e3bc55c 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -145,6 +145,7 @@ RETURNS Geometry AS $$ from cartodb_services.tools import Logger from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, UserConfigStorageFactory, OrgConfigStorageFactory from cartodb_services.refactor.tools.logger import LoggerConfigBuilder + from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder from cartodb_services.refactor.service.mapzen_geocoder import MapzenGeocoderConfigBuilder diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py new file mode 100644 index 0000000..016e0fb --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py @@ -0,0 +1,8 @@ +class RedisConnectionMock(object): + """ Simple class to mock a dummy behaviour for Redis related functions """ + + def zscore(self, redis_prefix, day): + pass + + def zincrby(self, redis_prefix, day, amount): + pass \ No newline at end of file From 6c61626214d6dc23f25b18a5fc35218d01aab1c3 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Fri, 30 Sep 2016 12:29:00 +0200 Subject: [PATCH 14/39] Remove RedisMock TODO --- server/extension/sql/20_geocode_street.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index e3bc55c..afaabee 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -170,7 +170,6 @@ RETURNS Geometry AS $$ user_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_storage, org_config_storage, username, orgname).get() # TODO encapsulate the connection creation - # TODO implement the RedisConnectionMock if environment == 'onpremise': redis_metrics_connection = RedisConnectionMock() else: From 443fe88d5a548033321232b866388ca92f8ef3d7 Mon Sep 17 00:00:00 2001 From: Carla Date: Fri, 30 Sep 2016 13:08:27 +0200 Subject: [PATCH 15/39] Add newline to end fle --- .../cartodb_services/refactor/tools/redis_mock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py index 016e0fb..4f07c92 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/redis_mock.py @@ -5,4 +5,4 @@ class RedisConnectionMock(object): pass def zincrby(self, redis_prefix, day, amount): - pass \ No newline at end of file + pass From d85bc65bf8eb1a63d02d272fb44e33e83ca04dbe Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 12:18:47 +0200 Subject: [PATCH 16/39] Ignore coverage files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f33df99..16d9ab5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store *.pyc +.coverage cartodb_services.egg-info/ build/ dist/ From 34ddd28e6b64ff169840842b1344daca37cdff33 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 12:20:17 +0200 Subject: [PATCH 17/39] Move ConfigException to a separate file --- .../cartodb_services/refactor/config/__init__.py | 0 .../cartodb_services/refactor/config/exceptions.py | 2 ++ .../cartodb_services/refactor/tools/logger.py | 5 ++--- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/config/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/config/exceptions.py diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/config/__init__.py b/server/lib/python/cartodb_services/cartodb_services/refactor/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/config/exceptions.py b/server/lib/python/cartodb_services/cartodb_services/refactor/config/exceptions.py new file mode 100644 index 0000000..722f035 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/config/exceptions.py @@ -0,0 +1,2 @@ +class ConfigException(Exception): + pass diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py index febde62..3f802aa 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py @@ -1,3 +1,5 @@ +from cartodb_services.refactor.config.exceptions import ConfigException + class LoggerConfig(object): """This class is a value object needed to setup a Logger""" @@ -24,9 +26,6 @@ class LoggerConfig(object): def min_log_level(self): return self._min_log_level -class ConfigException(Exception): - pass - # TODO this needs tests # TODO FTM this is just config, maybe move around class LoggerConfigBuilder(object): From c1dd41020122cf2d0bf15791bf138bb55671240c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 12:21:44 +0200 Subject: [PATCH 18/39] Make RedisConnectionConfigBuilder abstract and also use custom ConfigException instead of generic one. --- .../cartodb_services/refactor/storage/redis_config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index 8c156eb..96346eb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -1,3 +1,6 @@ +from cartodb_services.refactor.config.exceptions import ConfigException +from abc import ABCMeta, abstractmethod + """ How to use this (just a draft, WIP): @@ -43,12 +46,14 @@ class RedisConnectionConfig(object): return self._sentinel_id -# TODO make it abstract class RedisConnectionConfigBuilder(object): + __metaclass__ = ABCMeta + DEFAULT_USER_DB = 5 DEFAULT_TIMEOUT = 1.5 # seconds + @abstractmethod def __init__(self, server_config_storage, config_key): self._server_config_storage = server_config_storage self._config_key = config_key @@ -56,7 +61,7 @@ class RedisConnectionConfigBuilder(object): def get(self): conf = self._server_config_storage.get(self._config_key) if conf is None: - raise Exception("There is no redis configuration defined") + raise ConfigException("There is no redis configuration defined") host = conf['redis_host'] port = conf['redis_port'] From fb812ee15ec0a758a3959006e52b4c51406179a8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 12:33:38 +0200 Subject: [PATCH 19/39] Add tests for redis connection configs --- .../refactor/storage/test_redis_config.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py new file mode 100644 index 0000000..292874e --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py @@ -0,0 +1,101 @@ +from unittest import TestCase +from cartodb_services.refactor.storage.redis_config import * +from cartodb_services.refactor.storage.server_config import InMemoryConfigStorage +from cartodb_services.refactor.config.exceptions import ConfigException + +class TestRedisConnectionConfig(TestCase): + + def test_config_holds_values(self): + # this is mostly for completeness, dummy class, dummy test + config = RedisConnectionConfig('myhost.com', 6379, 0.1, 5, None) + assert config.host == 'myhost.com' + assert config.port == 6379 + assert config.timeout == 0.1 + assert config.db == 5 + assert config.sentinel_id is None + + +class TestRedisConnectionConfigBuilder(TestCase): + + def test_it_raises_exception_as_it_is_abstract(self): + server_config_storage = InMemoryConfigStorage() + self.assertRaises(TypeError, RedisConnectionConfigBuilder, server_config_storage, 'whatever_key') + + +class TestRedisMetadataConnectionConfigBuilder(TestCase): + + def test_it_raises_exception_if_config_is_missing(self): + server_config_storage = InMemoryConfigStorage() + config_builder = RedisMetadataConnectionConfigBuilder(server_config_storage) + self.assertRaises(ConfigException, config_builder.get) + + def test_it_gets_a_valid_config_from_the_server_storage(self): + server_config_storage = InMemoryConfigStorage({ + 'redis_metadata_config': { + 'redis_host': 'myhost.com', + 'redis_port': 6379, + 'timeout': 0.2, + 'redis_db': 3, + 'sentinel_master_id': None + } + }) + config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() + assert config.host == 'myhost.com' + assert config.port == 6379 + assert config.timeout == 0.2 + assert config.db == 3 + assert config.sentinel_id is None + + def test_it_gets_a_default_timeout_if_none(self): + server_config_storage = InMemoryConfigStorage({ + 'redis_metadata_config': { + 'redis_host': 'myhost.com', + 'redis_port': 6379, + 'timeout': None, + 'redis_db': 3, + 'sentinel_master_id': None + } + }) + config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() + assert config.host == 'myhost.com' + assert config.port == 6379 + assert config.timeout == RedisConnectionConfigBuilder.DEFAULT_TIMEOUT + assert config.db == 3 + assert config.sentinel_id is None + + def test_it_gets_a_default_db_if_none(self): + server_config_storage = InMemoryConfigStorage({ + 'redis_metadata_config': { + 'redis_host': 'myhost.com', + 'redis_port': 6379, + 'timeout': 0.2, + 'redis_db': None, + 'sentinel_master_id': None + } + }) + config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() + assert config.host == 'myhost.com' + assert config.port == 6379 + assert config.timeout == 0.2 + assert config.db == RedisConnectionConfigBuilder.DEFAULT_USER_DB + assert config.sentinel_id is None + + +class TestRedisMetricsConnectionConfigBuilder(TestCase): + + def test_it_gets_a_valid_config_from_the_server_storage(self): + server_config_storage = InMemoryConfigStorage({ + 'redis_metrics_config': { + 'redis_host': 'myhost.com', + 'redis_port': 6379, + 'timeout': 0.2, + 'redis_db': 3, + 'sentinel_master_id': 'some_master_id' + } + }) + config = RedisMetricsConnectionConfigBuilder(server_config_storage).get() + assert config.host == 'myhost.com' + assert config.port == 6379 + assert config.timeout == 0.2 + assert config.db == 3 + assert config.sentinel_id == 'some_master_id' From b1f3405cd0b61051806a9cacd17f6d6bb7f2af3d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 12:52:18 +0200 Subject: [PATCH 20/39] Rename variable s/user_geocoder_config/mapzen_geocoder_config --- server/extension/sql/20_geocode_street.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index afaabee..b89acdd 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -166,8 +166,7 @@ RETURNS Geometry AS $$ user_config_storage = UserConfigStorageFactory(redis_metadata_connection, username).get() org_config_storage = OrgConfigStorageFactory(redis_metadata_connection, orgname).get() - # TODO rename this variable - user_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_storage, org_config_storage, username, orgname).get() + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_storage, org_config_storage, username, orgname).get() # TODO encapsulate the connection creation if environment == 'onpremise': @@ -176,12 +175,12 @@ RETURNS Geometry AS $$ redis_metrics_connection_config = RedisMetricsConnectionConfigBuilder(server_config_storage).get() redis_metrics_connection = RedisConnectionBuilder(redis_metrics_connection_config).get() - quota_service = QuotaService(user_geocoder_config, redis_metrics_connection) + quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger) country_iso3 = None if country: country_iso3 = country_to_iso3(country) From 86ab3abc53cf1fd74cff5dd5cce3a5dd6fb74a8f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 12:58:15 +0200 Subject: [PATCH 21/39] Rename mapzen_geocoder module to mapzen_geocoder_config --- server/extension/sql/20_geocode_street.sql | 2 +- .../{mapzen_geocoder.py => mapzen_geocoder_config.py} | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) rename server/lib/python/cartodb_services/cartodb_services/refactor/service/{mapzen_geocoder.py => mapzen_geocoder_config.py} (98%) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index b89acdd..314d899 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -148,7 +148,7 @@ RETURNS Geometry AS $$ from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder - from cartodb_services.refactor.service.mapzen_geocoder import MapzenGeocoderConfigBuilder + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder from cartodb_services.refactor.core.environment import Environment server_config_storage = InDbServerConfigStorage() diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py similarity index 98% rename from server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder.py rename to server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index 2db5784..71e2d35 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -1,6 +1,5 @@ from dateutil.parser import parse as date_parse -# TODO: rename this file class MapzenGeocoderConfig(object): """ Value object that represents the configuration needed to operate the mapzen service. @@ -86,7 +85,7 @@ class MapzenGeocoderConfigBuilder(object): self._org_conf = org_conf self._username = username self._orgname = orgname - + def get(self): mapzen_server_conf = self._server_conf.get('mapzen_conf') @@ -96,7 +95,7 @@ class MapzenGeocoderConfigBuilder(object): soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit') cost_per_hit=0 - + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') period_end_date = date_parse(period_end_date_str) From 3361960cfc241b4f749ce7ad1aa43c9f043fec58 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 13:12:49 +0200 Subject: [PATCH 22/39] Move InMemoryConfigStorage to a separate file --- .../cartodb_services/refactor/storage/mem_config.py | 12 ++++++++++++ .../refactor/storage/server_config.py | 12 ------------ .../test/refactor/storage/test_mem_config.py | 12 ++++++++++++ .../test/refactor/storage/test_redis_config.py | 2 +- .../test/refactor/storage/test_server_config.py | 11 ----------- 5 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py create mode 100644 server/lib/python/cartodb_services/test/refactor/storage/test_mem_config.py diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py new file mode 100644 index 0000000..01d3981 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py @@ -0,0 +1,12 @@ +from interfaces import ConfigStorageInterface + +class InMemoryConfigStorage(ConfigStorageInterface): + + def __init__(self, config_hash={}): + self._config_hash = config_hash + + def get(self, key): + try: + return self._config_hash[key] + except KeyError: + return None diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 7a65946..e02259b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -13,18 +13,6 @@ class InDbServerConfigStorage(ConfigStorageInterface): else: return None - -class InMemoryConfigStorage(ConfigStorageInterface): - - def __init__(self, config_hash={}): - self._config_hash = config_hash - - def get(self, key): - try: - return self._config_hash[key] - except KeyError: - return None - class NullConfigStorage(ConfigStorageInterface): def get(self, key): diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_mem_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_mem_config.py new file mode 100644 index 0000000..3caf3e3 --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_mem_config.py @@ -0,0 +1,12 @@ +from unittest import TestCase +from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage + +class TestInMemoryConfigStorage(TestCase): + + def test_can_provide_values_from_hash(self): + server_config = InMemoryConfigStorage({'any_key': 'any_value'}) + assert server_config.get('any_key') == 'any_value' + + def test_gets_none_if_cannot_retrieve_key(self): + server_config = InMemoryConfigStorage() + assert server_config.get('any_non_existing_key') == None diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py index 292874e..a99956e 100644 --- a/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py @@ -1,6 +1,6 @@ from unittest import TestCase from cartodb_services.refactor.storage.redis_config import * -from cartodb_services.refactor.storage.server_config import InMemoryConfigStorage +from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage from cartodb_services.refactor.config.exceptions import ConfigException class TestRedisConnectionConfig(TestCase): diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py index 4e3d27c..ece3c6b 100644 --- a/server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_server_config.py @@ -29,14 +29,3 @@ class TestInDbServerConfigStorage(TestCase): server_config = InDbServerConfigStorage() assert server_config.get('server_conf') == {'environment': 'testing'} self.plpy_mock.execute.assert_called_once_with("SELECT cdb_dataservices_server.cdb_conf_getconf('server_conf') as conf", 1) - - -class TestInMemoryConfigStorage(TestCase): - - def test_can_provide_values_from_hash(self): - server_config = InMemoryConfigStorage({'any_key': 'any_value'}) - assert server_config.get('any_key') == 'any_value' - - def test_gets_none_if_cannot_retrieve_key(self): - server_config = InMemoryConfigStorage() - assert server_config.get('any_non_existing_key') == None From 0c62c4bada4c2419c0ccd98b2030563de293f5b8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 13:17:04 +0200 Subject: [PATCH 23/39] Move the NullConfigStorage to a separate file --- .../cartodb_services/refactor/storage/null_config.py | 6 ++++++ .../cartodb_services/refactor/storage/server_config.py | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py new file mode 100644 index 0000000..88e0cad --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py @@ -0,0 +1,6 @@ +from interfaces import ConfigStorageInterface + +class NullConfigStorage(ConfigStorageInterface): + + def get(self, key): + return None diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index e02259b..eb54bac 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -1,6 +1,7 @@ import json import cartodb_services from interfaces import ConfigStorageInterface +from null_config import NullConfigStorage class InDbServerConfigStorage(ConfigStorageInterface): @@ -13,10 +14,6 @@ class InDbServerConfigStorage(ConfigStorageInterface): else: return None -class NullConfigStorage(ConfigStorageInterface): - - def get(self, key): - return None # TODO move out of this file. In general this is config but either user or org config class RedisConfigStorage(ConfigStorageInterface): From 073163eb1abff9c9036017580706299ffba28095 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 13:20:30 +0200 Subject: [PATCH 24/39] Rename redis_config.py to redis_connection_config.py --- server/extension/sql/20_geocode_street.sql | 2 +- .../storage/{redis_config.py => redis_connection_config.py} | 0 .../cartodb_services/test/refactor/storage/test_redis_config.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename server/lib/python/cartodb_services/cartodb_services/refactor/storage/{redis_config.py => redis_connection_config.py} (100%) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 314d899..9ab6900 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -146,7 +146,7 @@ RETURNS Geometry AS $$ from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, UserConfigStorageFactory, OrgConfigStorageFactory from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock - from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder + from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder from cartodb_services.refactor.core.environment import Environment diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection_config.py similarity index 100% rename from server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py rename to server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection_config.py diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py index a99956e..9106cdc 100644 --- a/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py @@ -1,5 +1,5 @@ from unittest import TestCase -from cartodb_services.refactor.storage.redis_config import * +from cartodb_services.refactor.storage.redis_connection_config import * from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage from cartodb_services.refactor.config.exceptions import ConfigException From d522083d5c732d9e4c414c6c762e462ca7a42bac Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 13:22:47 +0200 Subject: [PATCH 25/39] Rename redis_connection_config test file as well --- .../{test_redis_config.py => test_redis_connection_config.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/lib/python/cartodb_services/test/refactor/storage/{test_redis_config.py => test_redis_connection_config.py} (100%) diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_connection_config.py similarity index 100% rename from server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py rename to server/lib/python/cartodb_services/test/refactor/storage/test_redis_connection_config.py From 275a6dc27f82be36218128ab4a5811e13fce06ba Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 13:33:46 +0200 Subject: [PATCH 26/39] Move redis_config related classes to separate file --- server/extension/sql/20_geocode_street.sql | 3 +- .../refactor/storage/redis_config.py | 38 +++++++++++++++++++ .../storage/redis_connection_config.py | 10 ----- .../refactor/storage/server_config.py | 36 ------------------ 4 files changed, 40 insertions(+), 47 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 9ab6900..e457206 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -143,7 +143,8 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger - from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, UserConfigStorageFactory, OrgConfigStorageFactory + from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage + from cartodb_services.refactor.storage.redis_config import UserConfigStorageFactory, OrgConfigStorageFactory from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py new file mode 100644 index 0000000..43e61bb --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -0,0 +1,38 @@ +from interfaces import ConfigStorageInterface +from null_config import NullConfigStorage + + +class RedisConfigStorage(ConfigStorageInterface): + + def __init__(self, connection, config_key): + self._connection = connection + self._config_key = config_key + self._data = None + + def get(self, key): + if not self._data: + self._data = self._connection.hgetall(self._config_key) + return self._data[key] + + +class UserConfigStorageFactory(object): + # TODO rework to support onpremise and InDbStorage + def __init__(self, redis_connection, username): + self._redis_connection = redis_connection + self._username = username + + def get(self): + return RedisConfigStorage(self._redis_connection, 'rails:users:{0}'.format(self._username)) + + +class OrgConfigStorageFactory(object): + # TODO rework to support onpremise and InDbStorage + def __init__(self, redis_connection, orgname): + self._redis_connection = redis_connection + self._orgname = orgname + + def get(self): + if self._orgname: + return RedisConfigStorage(self._redis_connection, 'rails:orgs:{0}'.format(self._orgname)) + else: + return NullConfigStorage() diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection_config.py index 96346eb..97eb25c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection_config.py @@ -1,15 +1,6 @@ from cartodb_services.refactor.config.exceptions import ConfigException from abc import ABCMeta, abstractmethod -""" -How to use this (just a draft, WIP): - - redis_connection_config = RedisConnectionConfigBuilder(server_config_storage).get() - connection = RedisConnectionBuilder(redis_connection_config) - - user_config_storage = RedisConfigStorage(connection, user) - user_config_storage = UserConfigStorageFactory(environment).get() -""" class RedisConnectionConfig(object): """ @@ -17,7 +8,6 @@ class RedisConnectionConfig(object): a connection to a redis server. """ - def __init__(self, host, port, timeout, db, sentinel_id): self._host = host self._port = port diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index eb54bac..0597d2b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -1,7 +1,6 @@ import json import cartodb_services from interfaces import ConfigStorageInterface -from null_config import NullConfigStorage class InDbServerConfigStorage(ConfigStorageInterface): @@ -13,38 +12,3 @@ class InDbServerConfigStorage(ConfigStorageInterface): return json.loads(json_output) else: return None - - -# TODO move out of this file. In general this is config but either user or org config -class RedisConfigStorage(ConfigStorageInterface): - - def __init__(self, connection, config_key): - self._connection = connection - self._config_key = config_key - self._data = None - - def get(self, key): - if not self._data: - self._data = self._connection.hgetall(self._config_key) - return self._data[key] - -class UserConfigStorageFactory(object): - # TODO rework to support onpremise and InDbStorage - def __init__(self, redis_connection, username): - self._redis_connection = redis_connection - self._username = username - - def get(self): - return RedisConfigStorage(self._redis_connection, 'rails:users:{0}'.format(self._username)) - -class OrgConfigStorageFactory(object): - # TODO rework to support onpremise and InDbStorage - def __init__(self, redis_connection, orgname): - self._redis_connection = redis_connection - self._orgname = orgname - - def get(self): - if self._orgname: - return RedisConfigStorage(self._redis_connection, 'rails:orgs:{0}'.format(self._orgname)) - else: - return NullConfigStorage() From 8fbb41742c8224cdc6890084dd956c4cfc5614ab Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 15:25:01 +0200 Subject: [PATCH 27/39] Rename redis conf factories to builders --- server/extension/sql/20_geocode_street.sql | 6 +++--- .../cartodb_services/refactor/storage/redis_config.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index e457206..e65e8b5 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -144,7 +144,7 @@ RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage - from cartodb_services.refactor.storage.redis_config import UserConfigStorageFactory, OrgConfigStorageFactory + from cartodb_services.refactor.storage.redis_config import RedisUserConfigStorageBuilder, RedisOrgConfigStorageBuilder from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder @@ -164,8 +164,8 @@ RETURNS Geometry AS $$ else: redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() - user_config_storage = UserConfigStorageFactory(redis_metadata_connection, username).get() - org_config_storage = OrgConfigStorageFactory(redis_metadata_connection, orgname).get() + user_config_storage = RedisUserConfigStorageBuilder(redis_metadata_connection, username).get() + org_config_storage = RedisOrgConfigStorageBuilder(redis_metadata_connection, orgname).get() mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_storage, org_config_storage, username, orgname).get() diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index 43e61bb..35d0645 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -15,7 +15,7 @@ class RedisConfigStorage(ConfigStorageInterface): return self._data[key] -class UserConfigStorageFactory(object): +class RedisUserConfigStorageBuilder(object): # TODO rework to support onpremise and InDbStorage def __init__(self, redis_connection, username): self._redis_connection = redis_connection @@ -25,7 +25,7 @@ class UserConfigStorageFactory(object): return RedisConfigStorage(self._redis_connection, 'rails:users:{0}'.format(self._username)) -class OrgConfigStorageFactory(object): +class RedisOrgConfigStorageBuilder(object): # TODO rework to support onpremise and InDbStorage def __init__(self, redis_connection, orgname): self._redis_connection = redis_connection From 9e98e0794db34efa450b6414319498ee44dd2f2a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 16:05:08 +0200 Subject: [PATCH 28/39] Add backends for user and org configs --- server/extension/sql/20_geocode_street.sql | 17 +++++-------- .../refactor/backend/__init__.py | 0 .../refactor/backend/org_config.py | 25 +++++++++++++++++++ .../refactor/backend/user_config.py | 25 +++++++++++++++++++ .../refactor/storage/redis_config.py | 2 -- 5 files changed, 56 insertions(+), 13 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/backend/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index e65e8b5..44654a5 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -144,30 +144,25 @@ RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage - from cartodb_services.refactor.storage.redis_config import RedisUserConfigStorageBuilder, RedisOrgConfigStorageBuilder from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock - from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder, RedisMetricsConnectionConfigBuilder + from cartodb_services.refactor.storage.redis_connection_config import RedisMetricsConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder from cartodb_services.refactor.core.environment import Environment + from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory + from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory server_config_storage = InDbServerConfigStorage() logger_config = LoggerConfigBuilder(server_config_storage).get() logger = Logger(logger_config) - # TODO encapsulate construction of user_config_storage and org_config_storage environment = Environment(server_config_storage).get() - if environment == 'onpremise': - user_config_storage = org_config_storage = server_config_storage - else: - redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() - redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() - user_config_storage = RedisUserConfigStorageBuilder(redis_metadata_connection, username).get() - org_config_storage = RedisOrgConfigStorageBuilder(redis_metadata_connection, orgname).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_storage).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_storage).get() - mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_storage, org_config_storage, username, orgname).get() + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_backend, org_config_backend, username, orgname).get() # TODO encapsulate the connection creation if environment == 'onpremise': diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/__init__.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py new file mode 100644 index 0000000..2aa8f3d --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py @@ -0,0 +1,25 @@ +from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder +from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder +from cartodb_services.refactor.storage.redis_config import RedisOrgConfigStorageBuilder + +class OrgConfigBackendFactory(object): + """ + This class abstracts the creation of an org configuration storage. It will return + an implementation of the ConfigStorageInterface appropriate to the org, depending + on the environment. + """ + + def __init__(self, orgname, environment, server_config_storage): + self._orgname = orgname + self._environment = environment + self._server_config_storage = server_config_storage + + def get(self): + # TODO rename Environment class to ServerEnvironment and add accessors instead of checking against plain str + if self._environment == 'onpremise': + org_config_backend = self._server_config_storage + else: + redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_storage).get() + redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() + org_config_backend = RedisOrgConfigStorageBuilder(redis_metadata_connection, self._orgname).get() + return org_config_backend diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py new file mode 100644 index 0000000..19ffbaf --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py @@ -0,0 +1,25 @@ +from cartodb_services.refactor.storage.redis_connection_config import RedisMetadataConnectionConfigBuilder +from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder +from cartodb_services.refactor.storage.redis_config import RedisUserConfigStorageBuilder + +class UserConfigBackendFactory(object): + """ + This class abstracts the creation of a user configuration storage. It will return + an implementation of the ConfigStorageInterface appropriate to the user, depending + on the environment. + """ + + def __init__(self, username, environment, server_config_storage): + self._username = username + self._environment = environment + self._server_config_storage = server_config_storage + + def get(self): + # TODO rename Environment class to ServerEnvironment and add accessors instead of checking against plain str + if self._environment == 'onpremise': + user_config_backend = self._server_config_storage + else: + redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_storage).get() + redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() + user_config_backend = RedisUserConfigStorageBuilder(redis_metadata_connection, self._username).get() + return user_config_backend diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index 35d0645..f01d492 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -16,7 +16,6 @@ class RedisConfigStorage(ConfigStorageInterface): class RedisUserConfigStorageBuilder(object): - # TODO rework to support onpremise and InDbStorage def __init__(self, redis_connection, username): self._redis_connection = redis_connection self._username = username @@ -26,7 +25,6 @@ class RedisUserConfigStorageBuilder(object): class RedisOrgConfigStorageBuilder(object): - # TODO rework to support onpremise and InDbStorage def __init__(self, redis_connection, orgname): self._redis_connection = redis_connection self._orgname = orgname From 1aec541906d2e08491c4dde1700e8a0337f9c65b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 16:21:28 +0200 Subject: [PATCH 29/39] Add ServerEnvironment and ServerEnvironmentBuilder --- server/extension/sql/20_geocode_street.sql | 6 +-- .../refactor/backend/org_config.py | 3 +- .../refactor/backend/user_config.py | 3 +- .../refactor/core/environment.py | 37 +++++++++++++++++-- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 44654a5..d912320 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -149,7 +149,7 @@ RETURNS Geometry AS $$ from cartodb_services.refactor.storage.redis_connection_config import RedisMetricsConnectionConfigBuilder from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder - from cartodb_services.refactor.core.environment import Environment + from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory @@ -158,14 +158,14 @@ RETURNS Geometry AS $$ logger_config = LoggerConfigBuilder(server_config_storage).get() logger = Logger(logger_config) - environment = Environment(server_config_storage).get() + environment = ServerEnvironmentBuilder(server_config_storage).get() user_config_backend = UserConfigBackendFactory(username, environment, server_config_storage).get() org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_storage).get() mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_backend, org_config_backend, username, orgname).get() # TODO encapsulate the connection creation - if environment == 'onpremise': + if environment.is_onpremise: redis_metrics_connection = RedisConnectionMock() else: redis_metrics_connection_config = RedisMetricsConnectionConfigBuilder(server_config_storage).get() diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py index 2aa8f3d..d712c7b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py @@ -15,8 +15,7 @@ class OrgConfigBackendFactory(object): self._server_config_storage = server_config_storage def get(self): - # TODO rename Environment class to ServerEnvironment and add accessors instead of checking against plain str - if self._environment == 'onpremise': + if self._environment.is_onpremise: org_config_backend = self._server_config_storage else: redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_storage).get() diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py index 19ffbaf..438cb6c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py @@ -15,8 +15,7 @@ class UserConfigBackendFactory(object): self._server_config_storage = server_config_storage def get(self): - # TODO rename Environment class to ServerEnvironment and add accessors instead of checking against plain str - if self._environment == 'onpremise': + if self._environment.is_onpremise: user_config_backend = self._server_config_storage else: redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_storage).get() diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py index 5d33d95..e8e3552 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py @@ -1,4 +1,33 @@ -class Environment: +class ServerEnvironment(object): + + DEVELOPMENT = 'development' + STAGING = 'staging' + PRODUCTION = 'production' + ONPREMISE = 'onpremise' + + VALID_ENVIRONMENTS = [ + DEVELOPMENT, + STAGING, + PRODUCTION, + ONPREMISE + ] + + def __init__(self, environment_str): + assert environment_str in self.VALID_ENVIRONMENTS + self._environment_str = environment_str + + def __str__(self): + return self._environment_str + + @property + def is_onpremise(self): + return self._environment_str == self.ONPREMISE + + +class ServerEnvironmentBuilder(object): + + DEFAULT_ENVIRONMENT = ServerEnvironment.DEVELOPMENT + def __init__(self, server_config_storage): self._server_config_storage = server_config_storage @@ -6,8 +35,8 @@ class Environment: server_config = self._server_config_storage.get('server_conf') if not server_config or 'environment' not in server_config: - environment = 'development' + environment_str = self.DEFAULT_ENVIRONMENT else: - environment = server_config['environment'] + environment_str = server_config['environment'] - return environment + return ServerEnvironment(environment_str) From fcca5da302646b52b1e6b8f150dcfc8d4a7d38f8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 16:47:45 +0200 Subject: [PATCH 30/39] Add a RedisMetricsConnectionFactory class --- server/extension/sql/20_geocode_street.sql | 11 ++--------- .../backend/redis_metrics_connection.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/backend/redis_metrics_connection.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index d912320..20f7e16 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -145,13 +145,11 @@ RETURNS Geometry AS $$ from cartodb_services.tools import Logger from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage from cartodb_services.refactor.tools.logger import LoggerConfigBuilder - from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock - from cartodb_services.refactor.storage.redis_connection_config import RedisMetricsConnectionConfigBuilder - from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory server_config_storage = InDbServerConfigStorage() @@ -164,12 +162,7 @@ RETURNS Geometry AS $$ mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_backend, org_config_backend, username, orgname).get() - # TODO encapsulate the connection creation - if environment.is_onpremise: - redis_metrics_connection = RedisConnectionMock() - else: - redis_metrics_connection_config = RedisMetricsConnectionConfigBuilder(server_config_storage).get() - redis_metrics_connection = RedisConnectionBuilder(redis_metrics_connection_config).get() + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_storage).get() quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) if not quota_service.check_user_quota(): diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/redis_metrics_connection.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/redis_metrics_connection.py new file mode 100644 index 0000000..c32283b --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/redis_metrics_connection.py @@ -0,0 +1,17 @@ +from cartodb_services.refactor.tools.redis_mock import RedisConnectionMock +from cartodb_services.refactor.storage.redis_connection_config import RedisMetricsConnectionConfigBuilder +from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder + +class RedisMetricsConnectionFactory(object): + def __init__(self, environment, server_config_storage): + self._environment = environment + self._server_config_storage = server_config_storage + + def get(self): + if self._environment.is_onpremise: + redis_metrics_connection = RedisConnectionMock() + else: + redis_metrics_connection_config = RedisMetricsConnectionConfigBuilder(self._server_config_storage).get() + redis_metrics_connection = RedisConnectionBuilder(redis_metrics_connection_config).get() + return redis_metrics_connection + From 18e1a5c7c92a05718e4f5a0af5a600cdfcdf0bd8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 16:54:38 +0200 Subject: [PATCH 31/39] Remove the class User and its tests as it is not used. --- .../cartodb_services/refactor/core/user.py | 24 ----------- .../test/refactor/core/test_user.py | 43 ------------------- 2 files changed, 67 deletions(-) delete mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py delete mode 100644 server/lib/python/cartodb_services/test/refactor/core/test_user.py diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py deleted file mode 100644 index a36a350..0000000 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/core/user.py +++ /dev/null @@ -1,24 +0,0 @@ -class User(object): - - def __init__(self, username, orgname = None): - self.__username = username - self.__orgname = orgname - - @property - def username(self): - return self.__username - - @property - def orgname(self): - return self.__orgname - - @property - def is_org_user(self): - return self.__orgname is not None - - def __eq__(self, other): - # NOTE: usernames are unique in the system - eq = (self.__username == other.__username) - if eq: - assert self.__orgname == other.__orgname, 'Found two users with same name and different orgs!' - return eq diff --git a/server/lib/python/cartodb_services/test/refactor/core/test_user.py b/server/lib/python/cartodb_services/test/refactor/core/test_user.py deleted file mode 100644 index 70d4359..0000000 --- a/server/lib/python/cartodb_services/test/refactor/core/test_user.py +++ /dev/null @@ -1,43 +0,0 @@ -from unittest import TestCase -from cartodb_services.refactor.core.user import User -from nose.tools import assert_raises, raises - -class TestUser(TestCase): - - def test_can_retrieve_properties(self): - u = User('pepito', 'acme') - assert u.username == 'pepito' - assert u.orgname == 'acme' - - @raises(AttributeError) - def test_cannot_modify_properties(self): - u = User('pepito') - u.username = 'juanito' - - def test_is_org_user_returns_true_for_org_users(self): - u = User('pepito', 'acme') - assert u.is_org_user == True - - def test_is_org_user_returns_false_for_non_org_users(self): - u = User('pepito') - assert u.is_org_user == False - - def test_different_users_are_not_equal(self): - u1 = User('pepito') - u2 = User('juanito') - assert u1 != u2 - - def test_same_user_are_equal(self): - u1 = User('pepito') - u2 = User('pepito') - assert u1 == u2 - - o1 = User('juanito', 'acme') - o2 = User('juanito', 'acme') - assert o1 == o2 - - @raises(AssertionError) - def test_raises_exception_if_same_user_is_in_two_orgs(self): - u1 = User('juanito', 'acme') - u2 = User('juanito', 'some_other_org') - u1 == u2 From 0d87a95270ea8a2f824750fe92ae9770a72a2ad4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 17:06:57 +0200 Subject: [PATCH 32/39] Inject the environment into the LoggerConfigBuilder --- server/extension/sql/20_geocode_street.sql | 7 +++---- .../cartodb_services/refactor/tools/logger.py | 20 +++---------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 20f7e16..b752a5f 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -152,14 +152,13 @@ RETURNS Geometry AS $$ from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory server_config_storage = InDbServerConfigStorage() - - logger_config = LoggerConfigBuilder(server_config_storage).get() - logger = Logger(logger_config) - environment = ServerEnvironmentBuilder(server_config_storage).get() user_config_backend = UserConfigBackendFactory(username, environment, server_config_storage).get() org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_storage).get() + logger_config = LoggerConfigBuilder(environment, server_config_storage).get() + logger = Logger(logger_config) + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_backend, org_config_backend, username, orgname).get() redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_storage).get() diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py index 3f802aa..0b6523a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/tools/logger.py @@ -27,15 +27,13 @@ class LoggerConfig(object): return self._min_log_level # TODO this needs tests -# TODO FTM this is just config, maybe move around class LoggerConfigBuilder(object): - def __init__(self, server_config_storage): + def __init__(self, environment, server_config_storage): + self._server_environment = environment self._server_config_storage = server_config_storage def get(self): - server_environment = self._get_server_environment() - logger_conf = self._server_config_storage.get('logger_conf') if not logger_conf: raise ConfigException('Logger configuration missing') @@ -44,21 +42,9 @@ class LoggerConfigBuilder(object): log_file_path = self._get_value_or_none(logger_conf, 'log_file_path') min_log_level = self._get_value_or_none(logger_conf, 'min_log_level') or 'warning' - logger_config = LoggerConfig(server_environment, rollbar_api_key, log_file_path, min_log_level) + logger_config = LoggerConfig(str(self._server_environment), rollbar_api_key, log_file_path, min_log_level) return logger_config - def _get_server_environment(self): - server_config = self._server_config_storage.get('server_conf') - if not server_config: - environment = 'development' - else: - if 'environment' in server_config: - environment = server_config['environment'] - else: - environment = 'development' - - return environment - def _get_value_or_none(self, logger_conf, key): value = None if key in logger_conf: From 12aebb7eee4f31d9ae3719217c2eb07368bfdd66 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 17:35:33 +0200 Subject: [PATCH 33/39] Rename ConfigStorageInterface to ConfigBackendInterface --- .../cartodb_services/refactor/backend/org_config.py | 2 +- .../cartodb_services/refactor/backend/user_config.py | 2 +- .../cartodb_services/refactor/{storage => core}/interfaces.py | 4 ++-- .../cartodb_services/refactor/storage/mem_config.py | 4 ++-- .../cartodb_services/refactor/storage/null_config.py | 4 ++-- .../cartodb_services/refactor/storage/redis_config.py | 4 ++-- .../cartodb_services/refactor/storage/server_config.py | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) rename server/lib/python/cartodb_services/cartodb_services/refactor/{storage => core}/interfaces.py (64%) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py index d712c7b..0635585 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py @@ -5,7 +5,7 @@ from cartodb_services.refactor.storage.redis_config import RedisOrgConfigStorage class OrgConfigBackendFactory(object): """ This class abstracts the creation of an org configuration storage. It will return - an implementation of the ConfigStorageInterface appropriate to the org, depending + an implementation of the ConfigBackendInterface appropriate to the org, depending on the environment. """ diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py index 438cb6c..5bb2f6b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py @@ -5,7 +5,7 @@ from cartodb_services.refactor.storage.redis_config import RedisUserConfigStorag class UserConfigBackendFactory(object): """ This class abstracts the creation of a user configuration storage. It will return - an implementation of the ConfigStorageInterface appropriate to the user, depending + an implementation of the ConfigBackendInterface appropriate to the user, depending on the environment. """ diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/interfaces.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py similarity index 64% rename from server/lib/python/cartodb_services/cartodb_services/refactor/storage/interfaces.py rename to server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py index 5f00f53..46dcd42 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/interfaces.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py @@ -1,7 +1,7 @@ import abc -class ConfigStorageInterface(object): - """This is an interface that all config storages must abide to""" +class ConfigBackendInterface(object): + """This is an interface that all config backends must abide to""" __metaclass__ = abc.ABCMeta diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py index 01d3981..0661626 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py @@ -1,6 +1,6 @@ -from interfaces import ConfigStorageInterface +from ..core.interfaces import ConfigBackendInterface -class InMemoryConfigStorage(ConfigStorageInterface): +class InMemoryConfigStorage(ConfigBackendInterface): def __init__(self, config_hash={}): self._config_hash = config_hash diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py index 88e0cad..b9c11e6 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py @@ -1,6 +1,6 @@ -from interfaces import ConfigStorageInterface +from ..core.interfaces import ConfigBackendInterface -class NullConfigStorage(ConfigStorageInterface): +class NullConfigStorage(ConfigBackendInterface): def get(self, key): return None diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index f01d492..cdb2e57 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -1,8 +1,8 @@ -from interfaces import ConfigStorageInterface +from ..core.interfaces import ConfigBackendInterface from null_config import NullConfigStorage -class RedisConfigStorage(ConfigStorageInterface): +class RedisConfigStorage(ConfigBackendInterface): def __init__(self, connection, config_key): self._connection = connection diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 0597d2b..50137ca 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -1,8 +1,8 @@ import json import cartodb_services -from interfaces import ConfigStorageInterface +from ..core.interfaces import ConfigBackendInterface -class InDbServerConfigStorage(ConfigStorageInterface): +class InDbServerConfigStorage(ConfigBackendInterface): def get(self, key): sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key) From 35da7e48fdc5c2973f54dc3de98313b140d8f956 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 3 Oct 2016 17:42:17 +0200 Subject: [PATCH 34/39] Implement a ServerConfigBackendFactory mostly to keep layers separated. --- server/extension/sql/20_geocode_street.sql | 16 ++++++++-------- .../refactor/backend/server_config.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/backend/server_config.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index b752a5f..ef1fc9b 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -143,25 +143,25 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger - from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder + from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory - server_config_storage = InDbServerConfigStorage() - environment = ServerEnvironmentBuilder(server_config_storage).get() - user_config_backend = UserConfigBackendFactory(username, environment, server_config_storage).get() - org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_storage).get() + server_config_backend = ServerConfigBackendFactory().get() + environment = ServerEnvironmentBuilder(server_config_backend).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() - logger_config = LoggerConfigBuilder(environment, server_config_storage).get() + logger_config = LoggerConfigBuilder(environment, server_config_backend).get() logger = Logger(logger_config) - mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_storage, user_config_backend, org_config_backend, username, orgname).get() + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() - redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_storage).get() + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) if not quota_service.check_user_quota(): diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/server_config.py new file mode 100644 index 0000000..4a1e33c --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/server_config.py @@ -0,0 +1,13 @@ +from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage + + +class ServerConfigBackendFactory(object): + """ + This class creates a backend to retrieve server configurations (implementing the ConfigBackendInterface). + + At this moment it will always return an InDbServerConfigStorage, but nothing prevents from changing the + implementation. To something that reads from a file, memory or whatever. It is mostly there to keep + the layers separated. + """ + def get(self): + return InDbServerConfigStorage() From 2a1276f4f1467db763d92f77ece1ef6974703474 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 4 Oct 2016 12:53:39 +0200 Subject: [PATCH 35/39] Add test for redis_config module --- .../refactor/storage/test_redis_config.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py new file mode 100644 index 0000000..ec5fcd4 --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_redis_config.py @@ -0,0 +1,77 @@ +from unittest import TestCase +from cartodb_services.refactor.storage.redis_config import * +from mockredis import MockRedis +from mock import Mock, MagicMock +from nose.tools import raises + + +class TestRedisConfigStorage(TestCase): + + CONFIG_HASH_KEY = 'mykey' + + def test_can_get_a_config_field(self): + connection = MockRedis() + connection.hset(self.CONFIG_HASH_KEY, 'field1', 42) + redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY) + + value = redis_config.get('field1') + assert type(value) == str # this is something to take into account, redis always returns strings + assert value == '42' + + @raises(KeyError) + def test_raises_an_exception_if_config_key_not_present(self): + connection = MockRedis() + redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY) + redis_config.get('whatever_field') + + @raises(KeyError) + def test_returns_nothing_if_field_not_present(self): + connection = MockRedis() + connection.hmset(self.CONFIG_HASH_KEY, {'field1': 42, 'field2': 43}) + redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY) + redis_config.get('whatever_field') + + def test_it_reads_the_config_hash_just_once(self): + connection = Mock() + connection.hgetall = MagicMock(return_value={'field1': '42'}) + redis_config = RedisConfigStorage(connection, self.CONFIG_HASH_KEY) + + assert redis_config.get('field1') == '42' + assert redis_config.get('field1') == '42' + + connection.hgetall.assert_called_once_with(self.CONFIG_HASH_KEY) + + +class TestRedisUserConfigStorageBuilder(TestCase): + + USERNAME = 'john' + EXPECTED_REDIS_CONFIG_HASH_KEY = 'rails:users:john' + + def test_it_reads_the_correct_hash_key(self): + connection = Mock() + connection.hgetall = MagicMock(return_value={'an_user_config_field': 'nice'}) + redis_config = RedisConfigStorage(connection, self.EXPECTED_REDIS_CONFIG_HASH_KEY) + + redis_config = RedisUserConfigStorageBuilder(connection, self.USERNAME).get() + assert redis_config.get('an_user_config_field') == 'nice' + connection.hgetall.assert_called_once_with(self.EXPECTED_REDIS_CONFIG_HASH_KEY) + + +class TestRedisOrgConfigStorageBuilder(TestCase): + + ORGNAME = 'smith' + EXPECTED_REDIS_CONFIG_HASH_KEY = 'rails:orgs:smith' + + def test_it_reads_the_correct_hash_key(self): + connection = Mock() + connection.hgetall = MagicMock(return_value={'an_org_config_field': 'awesome'}) + redis_config = RedisConfigStorage(connection, self.EXPECTED_REDIS_CONFIG_HASH_KEY) + + redis_config = RedisOrgConfigStorageBuilder(connection, self.ORGNAME).get() + assert redis_config.get('an_org_config_field') == 'awesome' + connection.hgetall.assert_called_once_with(self.EXPECTED_REDIS_CONFIG_HASH_KEY) + + def test_it_returns_a_null_config_storage_if_theres_no_orgname(self): + redis_config = RedisOrgConfigStorageBuilder(None, None).get() + assert type(redis_config) == NullConfigStorage + assert redis_config.get('whatever') == None From 3844cfc226479d32401c644f848a15290e998e71 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 4 Oct 2016 14:56:06 +0200 Subject: [PATCH 36/39] Tests for environment module --- .../refactor/core/environment.py | 15 ++++++ .../test/refactor/core/test_environment.py | 47 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/refactor/core/test_environment.py diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py index e8e3552..b45b610 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/core/environment.py @@ -19,10 +19,25 @@ class ServerEnvironment(object): def __str__(self): return self._environment_str + @property + def is_development(self): + return self._environment_str == self.DEVELOPMENT + + @property + def is_staging(self): + return self._environment_str == self.STAGING + + @property + def is_production(self): + return self._environment_str == self.PRODUCTION + @property def is_onpremise(self): return self._environment_str == self.ONPREMISE + def __eq__(self, other): + return self._environment_str == other._environment_str + class ServerEnvironmentBuilder(object): diff --git a/server/lib/python/cartodb_services/test/refactor/core/test_environment.py b/server/lib/python/cartodb_services/test/refactor/core/test_environment.py new file mode 100644 index 0000000..befd62b --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/core/test_environment.py @@ -0,0 +1,47 @@ +from unittest import TestCase +from cartodb_services.refactor.core.environment import * +from nose.tools import raises +from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage + +class TestServerEnvironment(TestCase): + + def test_can_be_a_valid_one(self): + env_dev = ServerEnvironment('development') + env_staging = ServerEnvironment('staging') + env_prod = ServerEnvironment('production') + env_onpremise = ServerEnvironment('onpremise') + + @raises(AssertionError) + def test_cannot_be_a_non_valid_one(self): + env_whatever = ServerEnvironment('whatever') + + def test_is_on_premise_returns_true_when_onpremise(self): + assert ServerEnvironment('onpremise').is_onpremise == True + + def test_is_on_premise_returns_true_when_any_other(self): + assert ServerEnvironment('development').is_onpremise == False + assert ServerEnvironment('staging').is_onpremise == False + assert ServerEnvironment('production').is_onpremise == False + + def test_equality(self): + assert ServerEnvironment('development') == ServerEnvironment('development') + assert ServerEnvironment('development') <> ServerEnvironment('onpremise') + + +class TestServerEnvironmentBuilder(TestCase): + + def test_returns_env_according_to_configuration(self): + server_config_storage = InMemoryConfigStorage({ + 'server_conf': { + 'environment': 'staging' + } + }) + server_env = ServerEnvironmentBuilder(server_config_storage).get() + assert server_env.is_staging == True + + def test_returns_default_when_no_server_conf(self): + server_config_storage = InMemoryConfigStorage({}) + server_env = ServerEnvironmentBuilder(server_config_storage).get() + + assert server_env.is_development == True + assert str(server_env) == ServerEnvironmentBuilder.DEFAULT_ENVIRONMENT From e1b357137adf5d5af9212421f8ee935bbe81e7f5 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 4 Oct 2016 14:59:57 +0200 Subject: [PATCH 37/39] Rename s/storage/backend for consistency --- .../cartodb_services/refactor/backend/org_config.py | 10 +++++----- .../cartodb_services/refactor/backend/user_config.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py index 0635585..d380c38 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/org_config.py @@ -4,21 +4,21 @@ from cartodb_services.refactor.storage.redis_config import RedisOrgConfigStorage class OrgConfigBackendFactory(object): """ - This class abstracts the creation of an org configuration storage. It will return + This class abstracts the creation of an org configuration backend. It will return an implementation of the ConfigBackendInterface appropriate to the org, depending on the environment. """ - def __init__(self, orgname, environment, server_config_storage): + def __init__(self, orgname, environment, server_config_backend): self._orgname = orgname self._environment = environment - self._server_config_storage = server_config_storage + self._server_config_backend = server_config_backend def get(self): if self._environment.is_onpremise: - org_config_backend = self._server_config_storage + org_config_backend = self._server_config_backend else: - redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_storage).get() + redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_backend).get() redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() org_config_backend = RedisOrgConfigStorageBuilder(redis_metadata_connection, self._orgname).get() return org_config_backend diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py index 5bb2f6b..949d26f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/backend/user_config.py @@ -4,21 +4,21 @@ from cartodb_services.refactor.storage.redis_config import RedisUserConfigStorag class UserConfigBackendFactory(object): """ - This class abstracts the creation of a user configuration storage. It will return + This class abstracts the creation of a user configuration backend. It will return an implementation of the ConfigBackendInterface appropriate to the user, depending on the environment. """ - def __init__(self, username, environment, server_config_storage): + def __init__(self, username, environment, server_config_backend): self._username = username self._environment = environment - self._server_config_storage = server_config_storage + self._server_config_backend = server_config_backend def get(self): if self._environment.is_onpremise: - user_config_backend = self._server_config_storage + user_config_backend = self._server_config_backend else: - redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_storage).get() + redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(self._server_config_backend).get() redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config).get() user_config_backend = RedisUserConfigStorageBuilder(redis_metadata_connection, self._username).get() return user_config_backend From cded6c2f088736ace88c0771a08cd9c8ef6dccef Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 4 Oct 2016 15:20:41 +0200 Subject: [PATCH 38/39] Test for NullConfigStorage --- .../test/refactor/storage/test_null_config.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/refactor/storage/test_null_config.py diff --git a/server/lib/python/cartodb_services/test/refactor/storage/test_null_config.py b/server/lib/python/cartodb_services/test/refactor/storage/test_null_config.py new file mode 100644 index 0000000..dda058d --- /dev/null +++ b/server/lib/python/cartodb_services/test/refactor/storage/test_null_config.py @@ -0,0 +1,14 @@ +from unittest import TestCase +from cartodb_services.refactor.storage.null_config import NullConfigStorage +from cartodb_services.refactor.core.interfaces import ConfigBackendInterface + + +class TestNullConfigStorage(TestCase): + + def test_is_a_config_backend(self): + null_config = NullConfigStorage() + assert isinstance(null_config, ConfigBackendInterface) + + def test_returns_none_regardless_of_input(self): + null_config = NullConfigStorage() + assert null_config.get('whatever') is None From dcb39350217ae52bf149e196c4ca390a655ada5f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 5 Oct 2016 18:28:24 +0200 Subject: [PATCH 39/39] Upgrade/downgrade files for server extension --- ...db_dataservices_server--0.15.1--0.16.0.sql | 69 + ...db_dataservices_server--0.16.0--0.15.1.sql | 54 + .../cdb_dataservices_server--0.16.0.sql | 2411 +++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.15.0--0.15.1.sql | 0 ...db_dataservices_server--0.15.1--0.15.0.sql | 0 .../cdb_dataservices_server--0.15.1.sql | 0 7 files changed, 2535 insertions(+), 1 deletion(-) create mode 100644 server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql create mode 100644 server/extension/cdb_dataservices_server--0.16.0.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.15.0--0.15.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.15.1--0.15.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.15.1.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql b/server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql new file mode 100644 index 0000000..c498615 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql @@ -0,0 +1,69 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.16.0'" to load this file. \quit + +-- Here goes your code to upgrade/downgrade + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL STABLE SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + import cartodb_services + cartodb_services.init(plpy, GD) + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger + from cartodb_services.refactor.tools.logger import LoggerConfigBuilder + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder + from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory + from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory + from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory + + server_config_backend = ServerConfigBackendFactory().get() + environment = ServerEnvironmentBuilder(server_config_backend).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + + logger_config = LoggerConfigBuilder(environment, server_config_backend).get() + logger = Logger(logger_config) + + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() + + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() + + quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql b/server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql new file mode 100644 index 0000000..a9c041a --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql @@ -0,0 +1,54 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.15.1'" to load this file. \quit + +-- Here goes your code to upgrade/downgrade + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_conf_getconf(text); + +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + import cartodb_services + cartodb_services.init(plpy, GD) + + from cartodb_services.config.user import User + from cartodb_services.config.configs import ConfigsFactory + from cartodb_services.config.hires_geocoder_config import HiResGeocoderConfigFactory + from cartodb_services.request.request import RequestFactory + + user = User(username, orgname) + configs = ConfigsFactory.get(user) + request = RequestFactory().create(user, configs, 'cdb_geocode_street_point') + + # TODO change to hires_geocoder_config = HiResGeocoderConfigFactory.get(request) + hires_geocoder_config = HiResGeocoderConfigFactory(configs).get(user) + + if hires_geocoder_config.provider == 'here': + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif hires_geocoder_config.provider == 'google': + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif hires_geocoder_config.provider == 'mapzen': + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_dataservices_server--0.16.0.sql b/server/extension/cdb_dataservices_server--0.16.0.sql new file mode 100644 index 0000000..3bc37ad --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.16.0.sql @@ -0,0 +1,2411 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + quota_service = QuotaService(user_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MapzenRouting(user_routing_config.mapzen_api_key, logger) + + if not waypoints or len(waypoints) < 2: + logger.info("Empty origin or destination") + quota_service.increment_empty_service_use() + return [None, None, None] + + waypoint_coords = [] + for waypoint in waypoints: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon'] + waypoint_coords.append(Coordinate(lon,lat)) + + resp = client.calculate_route_point_to_point(waypoint_coords, mode, options, units) + if resp and resp.shape: + shape_linestring = polyline_to_linestring(resp.shape) + if shape_linestring: + quota_service.increment_success_service_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + else: + quota_service.increment_empty_service_use() + return [None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + + waypoints = [origin, destination] + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( + username TEXT, + orgname TEXT, + waypoints geometry(Point, 4326)[], + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_snapshot_config.connection_str +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( + username TEXT, + orgname TEXT, + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host'] + return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr)) + )[0] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text) +RETURNS text AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.obs_dumpversion(); +$$ LANGUAGE plproxy; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_logger_config() +RETURNS boolean AS $$ + cache_key = "logger_config" + if cache_key in GD: + return False + else: + from cartodb_services.tools import LoggerConfig + logger_config = LoggerConfig(plpy) + GD[cache_key] = logger_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- This is done in order to avoid an undesired depedency on cartodb extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text) +RETURNS JSON AS $$ + SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; +$$ LANGUAGE SQL STABLE SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_internal_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import InternalGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Here geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Google geocoder is not available for your account.') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + import cartodb_services + cartodb_services.init(plpy, GD) + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger + from cartodb_services.refactor.tools.logger import LoggerConfigBuilder + from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder + from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder + from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory + from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory + from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory + + server_config_backend = ServerConfigBackendFactory().get() + environment = ServerEnvironmentBuilder(server_config_backend).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + + logger_config = LoggerConfigBuilder(environment, server_config_backend).get() + logger = Logger(logger_config) + + mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() + + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() + + quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3, search_type='address') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise e + raise Exception('Error trying to geocode street point using mapzen') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + try: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point'] + except BaseException as e: + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point'] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + try: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point'] + except BaseException as e: + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + try: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] + except BaseException as e: + internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"]) + return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen'))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) + country_iso3 = None + if country_name: + country_iso3 = country_to_iso3(country_name) + coordinates = geocoder.geocode(searchtext=city_name, city=None, + state_province=admin1_name, + country=country_iso3, search_type='locality') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode city point using mapzen') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines( + username TEXT, + orgname TEXT, + isotype TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient, MapzenIsolines + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger) + mapzen_isolines = MapzenIsolines(client, logger) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + if isotype == 'isodistance': + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + elif isotype == 'isochrone': + for r in data_range: + isoline = mapzen_isolines.calculate_isochrone(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + if user_isolines_config.google_services_user: + raise Exception('This service is not available for google service users.') + + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + return result +$$ LANGUAGE plpythonu; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index d8cfda4..08a06f3 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.15.1' +default_version = '0.16.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.15.0--0.15.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.15.0--0.15.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.15.0--0.15.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.15.0--0.15.1.sql diff --git a/server/extension/cdb_dataservices_server--0.15.1--0.15.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.15.1--0.15.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.15.1--0.15.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.15.1--0.15.0.sql diff --git a/server/extension/cdb_dataservices_server--0.15.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.15.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.15.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.15.1.sql