Version 0.15.5 of the python library

GoogleMapsClientFactory to persist clients #401

NEWS.md

@@ -1,3 +1,23 @@
+October 6th, 2017
+=================
+* Version `0.15.5` of the python library
+    * googlemaps dependency updated to v2.5.1
+    * Google geocoder performance boost: client connections are now reused between queries. See #401
+    * Fixed issue with Google keys validity check. See #382
+    * Fixed inconsistency in service usage failed requests tracking. See f0a3249
+* Client extension tests are now compatible with PostgreSQL 9.5 and 9.6
+
+August 30th, 2017
+=============
+* Version `0.15.4` of the python library
+    * Fixed invalid geometries for isochrones due to `generalize` option. See #397
+
+August 24th, 2017
+=============
+* Improved the documentation
+* Version `0.15.3` of the python library
+    * Disabled DO quota check for users that have it configured . See #395
+
 August 23th, 2017
 =============
 * Version `0.27.0` of the server

client/Makefile

@@ -21,7 +21,7 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
 SOURCES_DATA_DIR = sql/
 
 
-REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
+REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
 TEST_DIR = test/
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 

client/test/expected/00_installation_test.out

@@ -1,6 +1,5 @@
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION plproxy;

client/test/expected/10_admin0_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server function

client/test/expected/20_admin1_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions

client/test/expected/25_exception_safe_private_functions_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 SET client_min_messages TO warning;
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions to raise exceptions
@@ -31,8 +32,6 @@ SET ROLE test_regular_user;
 -- Exercise the exception safe and the proxied functions
 SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
 WARNING:  cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
-DETAIL:  SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
-PL/pgSQL function _cdb_geocode_street_point_exception_safe(text,text,text,text) line 21 at SQL statement
  _cdb_geocode_street_point_exception_safe 
 ------------------------------------------
  
@@ -40,15 +39,12 @@ PL/pgSQL function _cdb_geocode_street_point_exception_safe(text,text,text,text)
 
 SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
 WARNING:  cdb_dataservices_client._cdb_isodistance(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
-DETAIL:  PL/pgSQL function _cdb_isodistance_exception_safe(geometry,text,integer[],text[]) line 21 at RETURN QUERY
  center | data_range | the_geom 
 --------+------------+----------
 (0 rows)
 
 SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
 WARNING:  cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
-DETAIL:  SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)"
-PL/pgSQL function _cdb_route_point_to_point_exception_safe(geometry,geometry,text,text[],text) line 21 at SQL statement
  shape | length | duration 
 -------+--------+----------
        |        |         

client/test/expected/30_namedplaces_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions

client/test/expected/40_postalcodes_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions

client/test/expected/50_ipaddresses_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions

client/test/expected/60_street_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions

client/test/expected/80_route_point_to_point_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions

client/test/expected/90_data_observatory_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 -- Mock the server functions

client/test/expected/999_permissions_test.out

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Use regular user role
 SET ROLE test_regular_user;
 -- Add to the search path the schema

client/test/sql/00_installation_test.sql

@@ -1,6 +1,5 @@
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION plproxy;

client/test/sql/10_admin0_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/20_admin1_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/25_exception_safe_private_functions_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 SET client_min_messages TO warning;
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/30_namedplaces_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/40_postalcodes_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/50_ipaddresses_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/60_street_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/80_route_point_to_point_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/90_data_observatory_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 

client/test/sql/999_permissions_test.sql

@@ -1,3 +1,4 @@
+\set VERBOSITY terse
 -- Use regular user role
 SET ROLE test_regular_user;
 

server/lib/python/cartodb_services/cartodb_services/google/client_factory.py

@@ -0,0 +1,38 @@
+#!/usr/local/bin/python
+# -*- coding: utf-8 -*-
+
+import googlemaps
+import base64
+from exceptions import InvalidGoogleCredentials
+
+class GoogleMapsClientFactory():
+    clients = {}
+
+    @classmethod
+    def get(cls, client_id, client_secret):
+        cache_key = "{}:{}".format(client_id, client_secret)
+        client = cls.clients.get(cache_key)
+        if not client:
+            cls.assert_valid_crendentials(client_secret)
+            client = googlemaps.Client(
+                client_id=client_id,
+                client_secret=client_secret)
+            cls.clients[cache_key] = client
+        return client
+
+    @classmethod
+    def assert_valid_crendentials(cls, client_secret):
+        if not cls.valid_credentials(client_secret):
+            raise InvalidGoogleCredentials
+
+    @staticmethod
+    def valid_credentials(client_secret):
+        try:
+            # Only fails if the string dont have a correct padding for b64
+            # but this way we could provide a more clear error than
+            # TypeError: Incorrect padding
+            b64_secret = client_secret.replace('-', '+').replace('_', '/')
+            base64.b64decode(b64_secret)
+            return True
+        except TypeError:
+            return False

server/lib/python/cartodb_services/cartodb_services/google/geocoder.py

@@ -1,22 +1,19 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
 
-import base64
 import googlemaps
 
-from exceptions import MalformedResult, InvalidGoogleCredentials
+from exceptions import MalformedResult
+from client_factory import GoogleMapsClientFactory
 
 
 class GoogleMapsGeocoder:
     """A Google Maps Geocoder wrapper for python"""
 
     def __init__(self, client_id, client_secret, logger):
-        if not self._valid_credentials(client_secret):
-            raise InvalidGoogleCredentials('Invalid google secret key')
         self.client_id = self._clean_client_id(client_id)
         self.client_secret = client_secret
-        self.geocoder = googlemaps.Client(
-            client_id=self.client_id, client_secret=self.client_secret)
+        self.geocoder = GoogleMapsClientFactory.get(self.client_id, self.client_secret)
         self._logger = logger
 
     def geocode(self, searchtext, city=None, state=None,
@@ -52,13 +49,3 @@ class GoogleMapsGeocoder:
     def _clean_client_id(self, client_id):
         # Consistency with how the client_id is saved in metadata
         return client_id.replace('client=', '')
-
-    def _valid_credentials(self, private_key):
-        try:
-            # Only fails if the string dont have a correct padding for b64
-            # but this way we could provide a more clear error than
-            # TypeError: Incorrect padding
-            base64.b64decode(private_key)
-            return True
-        except TypeError:
-            return False

server/lib/python/cartodb_services/cartodb_services/mapzen/types.py

@@ -28,7 +28,7 @@ def coordinates_to_polygon(coordinates):
     wkt_coordinates = ','.join(result_coordinates)
 
     try:
-        sql = "SELECT ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326)) as geom".format(wkt_coordinates)
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3) as geom".format(wkt_coordinates)
         geometry = plpy.execute(sql, 1)[0]['geom']
     except BaseException as e:
         plpy.warning("Can't generate POLYGON from coordinates: {0}".format(e))

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -32,7 +32,7 @@ class QuotaService:
 
     def increment_failed_service_use(self, amount=1):
         self._user_service.increment_service_use(
-            self._user_service_config.service_type, "fail_responses",
+            self._user_service_config.service_type, "failed_responses",
             amount=amount)
         self._log_service_process("fail")
 

server/lib/python/cartodb_services/requirements.txt

@@ -1,7 +1,7 @@
 redis==2.10.5
 hiredis==0.1.5
 python-dateutil==2.2
-googlemaps==2.4.2
+googlemaps==2.5.1
 rollbar==0.13.2
 # Dependency for googlemaps package
 requests==2.9.1

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.15.3',
+    version='0.15.5',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_google_client_factory.py

@@ -0,0 +1,61 @@
+#!/usr/local/bin/python
+# -*- coding: utf-8 -*-
+
+import googlemaps
+import unittest
+import base64
+
+from cartodb_services.google.client_factory import GoogleMapsClientFactory
+from cartodb_services.google.exceptions import InvalidGoogleCredentials
+
+class GoogleMapsClientFactoryTestCase(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        # reset client cache
+        GoogleMapsClientFactory.clients = {}
+
+    def test_consecutive_calls_with_same_params_return_same_client(self):
+        id = 'any_id'
+        key = base64.b64encode('any_key')
+        client1 = GoogleMapsClientFactory.get(id, key)
+        client2 = GoogleMapsClientFactory.get(id, key)
+        self.assertEqual(client1, client2)
+
+    def test_consecutive_calls_with_different_key_return_different_clients(self):
+        """
+        This requirement is important for security reasons as well as not to
+        cache a wrong key accidentally.
+        """
+        id = 'any_id'
+        key1 = base64.b64encode('any_key')
+        key2 = base64.b64encode('another_key')
+        client1 = GoogleMapsClientFactory.get(id, key1)
+        client2 = GoogleMapsClientFactory.get(id, key2)
+        self.assertNotEqual(client1, client2)
+
+    def test_consecutive_calls_with_different_ids_return_different_clients(self):
+        """
+        This requirement is important for security reasons as well as not to
+        cache a wrong key accidentally.
+        """
+        id1 = 'any_id'
+        id2 = 'another_id'
+        key = base64.b64encode('any_key')
+        client1 = GoogleMapsClientFactory.get(id1, key)
+        client2 = GoogleMapsClientFactory.get(id2, key)
+        self.assertNotEqual(client1, client2)
+
+    def test_invalid_credentials(self):
+        with self.assertRaises(InvalidGoogleCredentials):
+            GoogleMapsClientFactory.get('dummy_client_id', 'lalala')
+
+    def test_credentials_with_dashes_can_be_valid(self):
+        client = GoogleMapsClientFactory.get('yet_another_dummy_client_id', 'Ola-k-ase---')
+        self.assertIsInstance(client, googlemaps.Client)
+
+    def test_credentials_with_underscores_can_be_valid(self):
+        client = GoogleMapsClientFactory.get('yet_another_dummy_client_id', 'Ola_k_ase___')
+        self.assertIsInstance(client, googlemaps.Client)

server/lib/python/cartodb_services/test/test_googlegeocoder.py

@@ -6,7 +6,7 @@ import requests_mock
 from mock import Mock
 
 from cartodb_services.google import GoogleMapsGeocoder
-from cartodb_services.google.exceptions import MalformedResult, InvalidGoogleCredentials
+from cartodb_services.google.exceptions import MalformedResult
 
 requests_mock.Mocker.TEST_PREFIX = 'test_'
 
@@ -118,9 +118,3 @@ class GoogleGeocoderTestCase(unittest.TestCase):
                 searchtext='Calle Eloy Gonzalo 27',
                 city='Madrid',
                 country='España')
-
-    def test_invalid_credentials(self, req_mock):
-        with self.assertRaises(InvalidGoogleCredentials):
-            GoogleMapsGeocoder('dummy_client_id',
-                               'lalala',
-                               None)

server/lib/python/cartodb_services/test/test_user_service.py

@@ -52,7 +52,7 @@ class TestUserService(TestCase):
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
         us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
-        us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
+        us.increment_service_use(self.NOKIA_GEOCODER, 'failed_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
 
     def test_should_increment_org_used_quota(self):
@@ -61,7 +61,7 @@ class TestUserService(TestCase):
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
         us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
-        us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
+        us.increment_service_use(self.NOKIA_GEOCODER, 'failed_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
 
     def test_should_increment_user_used_quota_in_for_multiples_dates(self):
@@ -81,7 +81,7 @@ class TestUserService(TestCase):
         us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
                                  date=one_day_after)
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
-        us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
+        us.increment_service_use(self.NOKIA_GEOCODER, 'failed_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
 
     @freeze_time("2015-06-01")