From c3a3f5545cf93643f4565aeffe5bd3f371c6b8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Wed, 9 May 2018 19:19:46 +0200 Subject: [PATCH 1/2] IP geocoder table update --- geocoder/ip-addresses/README.md | 2 +- .../ip-addresses/sql/build_data_table.sql | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/geocoder/ip-addresses/README.md b/geocoder/ip-addresses/README.md index af7d897..9c585d7 100644 --- a/geocoder/ip-addresses/README.md +++ b/geocoder/ip-addresses/README.md @@ -11,7 +11,7 @@ SELECT geocode_ip(Array['1.0.16.0', '::ffff:1.0.16.0']) ``` # Creation steps -1. Create the `ip_address_locations` table +1. Create the `ip_address_locations` table (see `40_ipaddr.sql` file) 2. Obtain the file from http://geolite.maxmind.com/download/geoip/database/GeoLite2-City-CSV.zip 3. Uncompress it and upload the `GeoLite2-City-Blocks-IPv4.csv` file 4. Rename the uploaded table as `latest_ip_address_locations` diff --git a/geocoder/ip-addresses/sql/build_data_table.sql b/geocoder/ip-addresses/sql/build_data_table.sql index ec42d0e..92b7c80 100644 --- a/geocoder/ip-addresses/sql/build_data_table.sql +++ b/geocoder/ip-addresses/sql/build_data_table.sql @@ -4,5 +4,23 @@ -- Clear table DELETE FROM ip_address_locations; -- Updates table with new source data -INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, network_start_ip::inet FROM latest_ip_address_locations; +INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, ('::ffff:' || split_part(network, '/', 1))::inet FROM latest_ip_address_locations; DROP TABLE latest_ip_address_locations; + +-- Update table + +-- Assumptions (read https://github.com/CartoDB/data-services/tree/master/geocoder/ip-addresses#creation-steps) +-- - latest_ip_address_locations is the GeoLite2-City-Blocks-IPv4.csv table +-- - latest_ip6_address_locations is the GeoLite2-City-Blocks-IPv6.csv table +-- - For option B (exported file), export from valid `ip_address_locations` table with +-- \copy (select * from ip_address_locations) TO /tmp/ip_address_locations.dump; + +CREATE TABLE ip_address_locations_backup as select * from ip_address_locations; +TRUNCATE ip_address_locations; + +-- OPTION A) From scratch +INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, ('::ffff:' || split_part(network, '/', 1))::inet FROM latest_ip_address_locations; +INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, split_part(network, '/', 1)::inet FROM latest_ip6_address_locations; + +-- OPTION B) From exported file (took 9 minutes in staging) +\copy ip_address_locations from /tmp/ip_address_locations.dump \ No newline at end of file From 1614495919199ca098b12a7d3c9cbe7dbdf83613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Ignacio=20S=C3=A1nchez=20Lara?= Date: Thu, 10 May 2018 15:50:24 +0200 Subject: [PATCH 2/2] Final instructions --- geocoder/ip-addresses/README.md | 49 +++++++++++++++++++ .../ip-addresses/sql/build_data_table.sql | 18 ------- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/geocoder/ip-addresses/README.md b/geocoder/ip-addresses/README.md index 9c585d7..7cd6bfb 100644 --- a/geocoder/ip-addresses/README.md +++ b/geocoder/ip-addresses/README.md @@ -17,6 +17,55 @@ SELECT geocode_ip(Array['1.0.16.0', '::ffff:1.0.16.0']) 4. Rename the uploaded table as `latest_ip_address_locations` 5. Run the `sql/build_data_table` script to update the table +# Update steps + +## Option A: generate a new `ip_address_locations` table at geocoder user + +If the geocoder database is a CARTO user, do these steps: + +1. Import `GeoLite2-City-Blocks-IPv4.csv` and rename it to `latest_ip_address_locations`. +2. Import `GeoLite2-City-Blocks-IPv6.csv` and rename it to `latest_ip6_address_locations`. +3. If you want to create a backup of the previous table, do this: + + ```sql + CREATE TABLE ip_address_locations_backup as + select * from ip_address_locations; + ``` + +4. Clear previous table: + + ```sql + TRUNCATE ip_address_locations; + ``` + +5. Load the new values: + +```sql +set statement_timeout = '20min'; +INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, ('::ffff:' || split_part(network, '/', 1))::inet FROM latest_ip_address_locations; +INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, split_part(network, '/', 1)::inet FROM latest_ip6_address_locations; +``` + +## Option B: load a dump of the table + +If the geocoder database is not a CARTO user, do these steps: + +1. Perform option A in any user (it can even be a staging user). If you need to create the table, check `40_ipaddr.sql`. +2. Generate a dump of the file at that user database: + + ```sql + \copy (select * from ip_address_locations) TO /tmp/ip_address_locations.dump; + ``` + +3. Copy the file to the remote server. +4. Perform steps A.3 and A.4. +5. Load the new data (takes ~10 minutes): + +```sql +set statement_timeout = '20min'; +\copy ip_address_locations from /tmp/ip_address_locations.dump +``` + # Tables ### ip_address_locations diff --git a/geocoder/ip-addresses/sql/build_data_table.sql b/geocoder/ip-addresses/sql/build_data_table.sql index 92b7c80..10c100d 100644 --- a/geocoder/ip-addresses/sql/build_data_table.sql +++ b/geocoder/ip-addresses/sql/build_data_table.sql @@ -6,21 +6,3 @@ DELETE FROM ip_address_locations; -- Updates table with new source data INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, ('::ffff:' || split_part(network, '/', 1))::inet FROM latest_ip_address_locations; DROP TABLE latest_ip_address_locations; - --- Update table - --- Assumptions (read https://github.com/CartoDB/data-services/tree/master/geocoder/ip-addresses#creation-steps) --- - latest_ip_address_locations is the GeoLite2-City-Blocks-IPv4.csv table --- - latest_ip6_address_locations is the GeoLite2-City-Blocks-IPv6.csv table --- - For option B (exported file), export from valid `ip_address_locations` table with --- \copy (select * from ip_address_locations) TO /tmp/ip_address_locations.dump; - -CREATE TABLE ip_address_locations_backup as select * from ip_address_locations; -TRUNCATE ip_address_locations; - --- OPTION A) From scratch -INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, ('::ffff:' || split_part(network, '/', 1))::inet FROM latest_ip_address_locations; -INSERT INTO ip_address_locations (the_geom, network_start_ip) SELECT the_geom, split_part(network, '/', 1)::inet FROM latest_ip6_address_locations; - --- OPTION B) From exported file (took 9 minutes in staging) -\copy ip_address_locations from /tmp/ip_address_locations.dump \ No newline at end of file