Refactor to move logic from SQL functions to Python

- Moved the logic the retrieve the redis connection params to
  RedisDBConfig class
- Moved the logic that retrieve the services configuration to
  ServicesDBConfig

server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql

@@ -1,3 +1,66 @@
+DROP FUNCTION IF EXISTS cdb_dataservices_server._get_redis_conf_v2(text);
+DROP TYPE IF EXISTS cdb_dataservices_server._redis_conf_params;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text)
+RETURNS boolean AS $$
+  cache_key = "redis_connection_{0}".format(user_id)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.tools import RedisConnection, RedisDBConfig
+    metadata_config = RedisDBConfig('redis_metadata_config', plpy)
+    metrics_config = RedisDBConfig('redis_metrics_config', plpy)
+    redis_metadata_connection = RedisConnection(metadata_config).redis_connection()
+    redis_metrics_connection = RedisConnection(metrics_config).redis_connection()
+    GD[cache_key] = {
+      'redis_metadata_connection': redis_metadata_connection,
+      'redis_metrics_connection': redis_metrics_connection,
+    }
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_geocoder_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import GeocoderConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = geocoder_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_isolines_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import IsolinesRoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = isolines_routing_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import RoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    routing_config = RoutingConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = routing_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
   from cartodb_services.here import HereMapsGeocoder
@@ -505,4 +568,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     plpy.error(error_msg)
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER;
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql

@@ -1,3 +1,141 @@
+CREATE TYPE cdb_dataservices_server._redis_conf_params AS (
+    sentinel_master_id text,
+    redis_host text,
+    redis_port int,
+    redis_db text,
+    timeout float
+);
+
+-- Get the Redis configuration from the _conf table --
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text)
+RETURNS cdb_dataservices_server._redis_conf_params AS $$
+    conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
+    conf = plpy.execute(conf_query)[0]['conf']
+    if conf is None:
+      plpy.error("There is no redis configuration defined")
+    else:
+      import json
+      params = json.loads(conf)
+      redis_conf_params = {
+        "redis_host": params['redis_host'],
+        "redis_port": params['redis_port'],
+        "timeout": params['timeout'],
+        "redis_db": params['redis_db']
+      }
+      if "sentinel_master_id" in params:
+        redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"]
+      else:
+        redis_conf_params["sentinel_master_id"] = None
+
+      return redis_conf_params
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text)
+RETURNS boolean AS $$
+  cache_key = "redis_connection_{0}".format(user_id)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.tools import RedisConnection
+    metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, 
+        c.redis_port, c.timeout, c.redis_db
+        from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
+    metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, 
+        c.redis_port, c.timeout, c.redis_db
+        from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
+    redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'],
+        metadata_config_params['redis_host'],
+        metadata_config_params['redis_port'],
+        timeout=metadata_config_params['timeout'],
+        redis_db=metadata_config_params['redis_db']).redis_connection()
+    redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'],
+        metrics_config_params['redis_host'],
+        metrics_config_params['redis_port'],
+        timeout=metrics_config_params['timeout'],
+        redis_db=metrics_config_params['redis_db']).redis_connection()
+    GD[cache_key] = {
+      'redis_metadata_connection': redis_metadata_connection,
+      'redis_metrics_connection': redis_metrics_connection,
+    }
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+-- Get the Redis configuration from the _conf table --
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_geocoder_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    import json
+    from cartodb_services.metrics import GeocoderConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
+    if not heremaps_conf_json:
+      heremaps_app_id = None
+      heremaps_app_code = None
+    else:
+      heremaps_conf = json.loads(heremaps_conf_json)
+      heremaps_app_id = heremaps_conf['app_id']
+      heremaps_app_code = heremaps_conf['app_code']
+    geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
+    # --Think about the security concerns with this kind of global cache, it should be only available
+    # --for this user session but...
+    GD[cache_key] = geocoder_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+-- Get the Redis configuration from the _conf table --
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_isolines_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    import json
+    from cartodb_services.metrics import IsolinesRoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
+    if not heremaps_conf_json:
+      heremaps_app_id = None
+      heremaps_app_code = None
+    else:
+      heremaps_conf = json.loads(heremaps_conf_json)
+      heremaps_app_id = heremaps_conf['app_id']
+      heremaps_app_code = heremaps_conf['app_code']
+    isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
+    # --Think about the security concerns with this kind of global cache, it should be only available
+    # --for this user session but...
+    GD[cache_key] = isolines_routing_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+-- Get the Redis configuration from the _conf table --
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    import json
+    from cartodb_services.metrics import RoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf']
+    if not mapzen_conf_json:
+      mapzen_app_key = None
+    else:
+      mapzen_conf = json.loads(mapzen_conf_json)
+      mapzen_app_key = mapzen_conf['routing_app_key']
+    routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key)
+    # --Think about the security concerns with this kind of global cache, it should be only available
+    # --for this user session but...
+    GD[cache_key] = routing_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
   from cartodb_services.here import HereMapsGeocoder
@@ -505,4 +643,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     plpy.error(error_msg)
   finally:
     quota_service.increment_total_geocoder_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER;
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/cdb_dataservices_server--0.5.1.sql

@@ -74,38 +74,6 @@ RETURNS cdb_dataservices_server.simple_route AS $$
   result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units])
   return [result[0]['shape'],result[0]['length'], result[0]['duration']]
 $$ LANGUAGE plpythonu;
-CREATE TYPE cdb_dataservices_server._redis_conf_params AS (
-    sentinel_master_id text,
-    redis_host text,
-    redis_port int,
-    redis_db text,
-    timeout float
-);
-
--- Get the Redis configuration from the _conf table --
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text)
-RETURNS cdb_dataservices_server._redis_conf_params AS $$
-    conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
-    conf = plpy.execute(conf_query)[0]['conf']
-    if conf is None:
-      plpy.error("There is no redis configuration defined")
-    else:
-      import json
-      params = json.loads(conf)
-      redis_conf_params = {
-        "redis_host": params['redis_host'],
-        "redis_port": params['redis_port'],
-        "timeout": params['timeout'],
-        "redis_db": params['redis_db']
-      }
-      if "sentinel_master_id" in params:
-        redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"]
-      else:
-        redis_conf_params["sentinel_master_id"] = None
-
-      return redis_conf_params
-$$ LANGUAGE plpythonu;
-
 -- Get the connection to redis from cache or create a new one
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text)
 RETURNS boolean AS $$
@@ -113,23 +81,11 @@ RETURNS boolean AS $$
   if cache_key in GD:
     return False
   else:
-    from cartodb_services.tools import RedisConnection
-    metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, 
-        c.redis_port, c.timeout, c.redis_db
-        from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
-    metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, 
-        c.redis_port, c.timeout, c.redis_db
-        from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
-    redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'],
-        metadata_config_params['redis_host'],
-        metadata_config_params['redis_port'],
-        timeout=metadata_config_params['timeout'],
-        redis_db=metadata_config_params['redis_db']).redis_connection()
-    redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'],
-        metrics_config_params['redis_host'],
-        metrics_config_params['redis_port'],
-        timeout=metrics_config_params['timeout'],
-        redis_db=metrics_config_params['redis_db']).redis_connection()
+    from cartodb_services.tools import RedisConnection, RedisDBConfig
+    metadata_config = RedisDBConfig('redis_metadata_config', plpy)
+    metrics_config = RedisDBConfig('redis_metrics_config', plpy)
+    redis_metadata_connection = RedisConnection(metadata_config).redis_connection()
+    redis_metrics_connection = RedisConnection(metrics_config).redis_connection()
     GD[cache_key] = {
       'redis_metadata_connection': redis_metadata_connection,
       'redis_metrics_connection': redis_metrics_connection,
@@ -143,21 +99,10 @@ RETURNS boolean AS $$
   if cache_key in GD:
     return False
   else:
-    import json
     from cartodb_services.metrics import GeocoderConfig
     plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
     redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
-    heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
-    if not heremaps_conf_json:
-      heremaps_app_id = None
-      heremaps_app_code = None
-    else:
-      heremaps_conf = json.loads(heremaps_conf_json)
-      heremaps_app_id = heremaps_conf['app_id']
-      heremaps_app_code = heremaps_conf['app_code']
-    geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
-    # --Think about the security concerns with this kind of global cache, it should be only available
-    # --for this user session but...
+    geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = geocoder_config
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
@@ -169,21 +114,10 @@ RETURNS boolean AS $$
   if cache_key in GD:
     return False
   else:
-    import json
     from cartodb_services.metrics import IsolinesRoutingConfig
     plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
     redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
-    heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
-    if not heremaps_conf_json:
-      heremaps_app_id = None
-      heremaps_app_code = None
-    else:
-      heremaps_conf = json.loads(heremaps_conf_json)
-      heremaps_app_id = heremaps_conf['app_id']
-      heremaps_app_code = heremaps_conf['app_code']
-    isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
-    # --Think about the security concerns with this kind of global cache, it should be only available
-    # --for this user session but...
+    isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = isolines_routing_config
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
@@ -195,19 +129,10 @@ RETURNS boolean AS $$
   if cache_key in GD:
     return False
   else:
-    import json
     from cartodb_services.metrics import RoutingConfig
     plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
     redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
-    mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf']
-    if not mapzen_conf_json:
-      mapzen_app_key = None
-    else:
-      mapzen_conf = json.loads(mapzen_conf_json)
-      mapzen_app_key = mapzen_conf['routing_app_key']
-    routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key)
-    # --Think about the security concerns with this kind of global cache, it should be only available
-    # --for this user session but...
+    routing_config = RoutingConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = routing_config
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/sql/0.5.1/10_redis_helper.sql

@@ -1 +0,0 @@
-../0.5.0/10_redis_helper.sql

server/extension/sql/0.5.1/10_redis_helper.sql

@@ -0,0 +1,18 @@
+-- Get the connection to redis from cache or create a new one
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text)
+RETURNS boolean AS $$
+  cache_key = "redis_connection_{0}".format(user_id)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.tools import RedisConnection, RedisDBConfig
+    metadata_config = RedisDBConfig('redis_metadata_config', plpy)
+    metrics_config = RedisDBConfig('redis_metrics_config', plpy)
+    redis_metadata_connection = RedisConnection(metadata_config).redis_connection()
+    redis_metrics_connection = RedisConnection(metrics_config).redis_connection()
+    GD[cache_key] = {
+      'redis_metadata_connection': redis_metadata_connection,
+      'redis_metrics_connection': redis_metrics_connection,
+    }
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/sql/0.5.1/15_config_helper.sql

@@ -1 +0,0 @@
-../0.5.0/15_config_helper.sql

server/extension/sql/0.5.1/15_config_helper.sql

@@ -0,0 +1,44 @@
+-- Get the Redis configuration from the _conf table --
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_geocoder_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import GeocoderConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = geocoder_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+-- Get the Redis configuration from the _conf table --
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_isolines_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import IsolinesRoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = isolines_routing_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+-- Get the Redis configuration from the _conf table --
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import RoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    routing_config = RoutingConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = routing_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;