Merge pull request #397 from Algunenano/plpy_warnings

Remove leftover plpython warnings

.travis.yml

@@ -6,7 +6,6 @@ env:
     - PGUSER=postgres
     - PGDATABASE=postgres
     - PGOPTIONS='-c client_min_messages=NOTICE'
-    - PGPORT=5432
 
 jobs:
   include:
@@ -22,17 +21,15 @@ jobs:
       dist: bionic
 
 script:
-  - sudo service postgresql stop;
-  - sudo apt-get remove postgresql* -y
   - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
   - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi;
   - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis
   # For pre12, install plpython2. For PG12 install plpython3
   - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi;
   - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
-  - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
-  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password
-  - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
+  - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION /var/ramfs/postgresql/$POSTGRESQL_VERSION
+  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main --start -- --auth-local trust --auth-host password
+  - export PGPORT=$(pg_lsclusters | grep $POSTGRESQL_VERSION | awk '{print $3}')
   - make
   - sudo make install
   - make installcheck
@@ -41,3 +38,5 @@ after_failure:
   - pg_lsclusters
   - cat regression.out
   - cat regression.diffs
+  - echo $PGPORT
+  - cat /var/log/postgresql/postgresql-$POSTGRESQL_VERSION-main.log

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.35.0
+EXTVERSION = 0.36.0
 
 SED = sed
 AWK = awk
@@ -108,6 +108,7 @@ UPGRADABLE = \
   0.33.0 \
   0.34.0 \
   0.35.0 \
+  0.36.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,7 @@
+0.36.0 (2020-02-13)
+* Make `_CDB_Group_API_Auth` python3 compatible by passing bytes representation instead of a string.
+* Make `_CDB_Group_API_Request` python3 compatible by adapting the function signature of `HTTPConnection`.
+
 0.35.0 (2019-12-30)
 * Reapply the changes in 0.33.0 (the issue we were looking for was unrelated)
 * Reapply `Make PG12 depend on plpython3u instead of plpythonu`

scripts-available/CDB_Groups_API.sql

@@ -197,8 +197,17 @@ CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
     RETURNS TEXT AS
 $$
+    import sys
     import base64
-    return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
+
+    data_to_encode = '%s:%s' % (username, password)
+    if sys.version_info[0] < 3:
+      data_encoded = base64.encodestring(data_to_encode)
+    else:
+      data_encoded = base64.b64encode(data_to_encode.encode()).decode()
+
+    data_encoded = data_encoded.replace('\n', '')
+    return data_encoded
 $$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
 
 -- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
@@ -206,10 +215,12 @@ CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
     RETURNS int AS
 $$
+    python_v2 = True
     try:
-        import httplib as client
+      import httplib as client
     except:
-        from http import client
+      from http import client
+      python_v2 = False
 
     params = plpy.execute("select c.host, c.port, c.timeout, c.auth from @extschema@._CDB_Group_API_Conf() c;")[0]
     if params['host'] is None:
@@ -222,7 +233,10 @@ $$
     last_err = None
     while retry > 0:
       try:
-        conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        if python_v2:
+          conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        else:
+          conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], port=params['port'], timeout=params['timeout'])
         database_name = plpy.execute("select current_database();")[0]['current_database']
         conn.request(method, url.format(database_name), body, headers)
         response = conn.getresponse()

test/CDB_FederatedServerDiagnostics.sql

@@ -99,11 +99,12 @@ FROM latency;
 \echo '%% It raises an error if the wrong port is provided'
 SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port');
 
-\echo '%% Latency stats: can get them on default PG port 5432 when not provided'
-WITH latency AS (
-   SELECT CDB_Federated_Server_Diagnostics('loopback-no-port')->'server_latency_ms' ms
-) SELECT '2.4', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0
-FROM latency;
+-- Disabled: It's not compatible with Travis since the target database (self) might be in a different port
+-- \echo '%% Latency stats: can get them on default PG port 5432 when not provided'
+-- WITH latency AS (
+--    SELECT CDB_Federated_Server_Diagnostics('loopback-no-port')->'server_latency_ms' ms
+-- ) SELECT '2.4', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0
+-- FROM latency;
 
 
 -- ===================================================================

test/CDB_FederatedServerDiagnostics_expect

@@ -21,8 +21,6 @@ ERROR:  Server "doesNotExist" does not exist
 2.3|t
 %% It raises an error if the wrong port is provided
 ERROR:  could not connect to server "cdb_fs_wrong-port"
-%% Latency stats: can get them on default PG port 5432 when not provided
-2.4|t|t
 D1|
 D2|
 D3|

test/CDB_FederatedServer_expect

@@ -2,20 +2,20 @@
 ## List non-existent server shows nothing
 ## Create and list a server works
 1.3|
-1.4|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
+1.4|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
 ## Create and list a second server works
 2.1|
-2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
-2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user)
+2.2|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
+2.2|(myRemote2,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,fdw_user)
 ## List server by name works
-2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
+2.3|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
 ## Re-register a second server works
 3.1|
-3.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
-3.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user)
+3.2|(myRemote,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
+3.2|(myRemote2,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,other_remote_user)
 ## Unregister server 1 works
 4.1|
-4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user)
+4.2|(myRemote2,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,other_remote_user)
 ## Unregistering a server that does not exist fails
 ERROR:  Server "doesNotExist" does not exist
 ## Unregister the second server works
@@ -30,25 +30,25 @@ ERROR:  Server information is mandatory
 ERROR:  Credentials are mandatory
 ## Create a server with empty credentials works
 7.3|
-7.4|(empty,postgres_fdw,localhost,5432,fdw_target,read-only,)
+7.4|(empty,postgres_fdw,localhost,@@PGPORT@@,fdw_target,read-only,)
 7.5|
 ## Create a server without options fails
 ERROR:  Server information is mandatory
 ## Create a server with special characters works
 8.1|
-8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user")
+8.2|("myRemote"" or'not",postgres_fdw,localhost,@@PGPORT@@,"fdw target",read-only,"fdw user")
 8.3|
 9.1|
 You are now connected to database "contrib_regression" as user "cdb_fs_tester".
 ## All users are able to list servers
-9.2|(myRemote3,postgres_fdw,localhost,5432,,read-only,)
+9.2|(myRemote3,postgres_fdw,localhost,@@PGPORT@@,,read-only,)
 ## Only superadmins can create servers
 ERROR:  Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw
 You are now connected to database "contrib_regression" as user "postgres".
 ## Granting access to a user works
 9.5|
 You are now connected to database "contrib_regression" as user "cdb_fs_tester".
-9.55|(myRemote3,postgres_fdw,localhost,5432,,read-only,fdw_user)
+9.55|(myRemote3,postgres_fdw,localhost,@@PGPORT@@,,read-only,fdw_user)
 You are now connected to database "contrib_regression" as user "postgres".
 ERROR:  Server "does not exist" does not exist
 ERROR:  Could not grant access on "myRemote3" to "does not exist": role "does not exist" does not exist