cartodb4/cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
zhongjin
2020-06-15 10:58:47 +08:00
commit 4f1dfe7564
8590 changed files with 1516878 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/table-geocoder/spec/factories/database.json
+7
View File
@@ -0,0 +1,7 @@
{
"host" : "localhost",
"port" : 5432,
"user" : "postgres",
"password" : "",
"database" : "testgeocoder"
}
services/table-geocoder/spec/factories/database.json.sample
+7
View File
@@ -0,0 +1,7 @@
{
"host" : "localhost",
"port" : 5432,
"user" : "test",
"password" : "",
"database" : "test"
}
services/table-geocoder/spec/factories/pg_connection.rb
+53
View File
@@ -0,0 +1,53 @@
require 'pg'
require 'sequel'
require 'json'
module CartoDB
module Importer2
module Factories
class PGConnection
def initialize
raise(
"Please configure your database settings " +
"in spec/factories/database.json"
) unless File.exists?(configuration_file)
@pg_options = ::JSON.parse(File.read(configuration_file))
create_db
end #initialize
def create_db
conn = Sequel.postgres(pg_options.reject{|k,v| k == :database})
begin
conn.run("CREATE DATABASE \"#{ pg_options[:database] }\"
WITH TEMPLATE = template_postgis
OWNER = #{ pg_options[:user] }
ENCODING = 'UTF8'
CONNECTION LIMIT=-1")
rescue Sequel::DatabaseError => e
raise unless e.message =~ /database .* already exists/
end
begin
conn.run("CREATE EXTENSION postgis")
rescue Sequel::DatabaseError => e
raise unless e.message =~ /extension \"postgis\" already exists/
end
end
def connection
Sequel.postgres(pg_options)
end #connection
def pg_options
Hash[@pg_options.map { |k, v| [k.to_sym, v] }]
end #pg_options
private
def configuration_file
File.join(File.dirname("#{__FILE__}"), 'database.json')
end #configuration_file
end # PGConnection
end # Factories
end # Importer2
end # CartoDB
services/table-geocoder/spec/fixtures/adm0.csv
Vendored
+14
View File
@@ -0,0 +1,14 @@
1,"Netherlands"
2,"Spain"
3,"The Netherlands"
4,"Russia"
5,"Japan"
6,"France"
7,"Italy"
8,"Sweden"
9," Norway"
10,"Canada"
11,"Brazil"
12,"Spain"
13,"Spain"
14,"Spain"
1 1 Netherlands
2 2 Spain
3 3 The Netherlands
4 4 Russia
5 5 Japan
6 6 France
7 7 Italy
8 8 Sweden
9 9 Norway
10 10 Canada
11 11 Brazil
12 12 Spain
13 13 Spain
14 14 Spain
services/table-geocoder/spec/fixtures/gme_output_empty.json
Vendored
+64
View File
@@ -0,0 +1,64 @@
{
"results": [
{
"address_components": [
{
"long_name": "1600",
"short_name": "1600",
"types": [
"street_number"
]
},
{
"long_name": "Amphitheatre Pkwy",
"short_name": "Amphitheatre Pkwy",
"types": [
"route"
]
},
{
"long_name": "Mountain View",
"short_name": "Mountain View",
"types": [
"locality",
"political"
]
},
{
"long_name": "Santa Clara County",
"short_name": "Santa Clara County",
"types": [
"administrative_area_level_2",
"political"
]
},
{
"long_name": "California",
"short_name": "CA",
"types": [
"administrative_area_level_1",
"political"
]
},
{
"long_name": "United States",
"short_name": "US",
"types": [
"country",
"political"
]
},
{
"long_name": "94043",
"short_name": "94043",
"types": [
"postal_code"
]
}
],
"formatted_address": "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
"geometry": {}
}
],
"status": "ZERO_RESULTS"
}
services/table-geocoder/spec/fixtures/gme_output_error.json
Vendored
+3
View File
@@ -0,0 +1,3 @@
{
"status": "UNKNOWN_ERROR"
}
services/table-geocoder/spec/fixtures/gme_output_error_with_message.json
Vendored
+4
View File
@@ -0,0 +1,4 @@
{
"error_message": "dummy error message",
"status": "UNKNOWN_ERROR"
}
services/table-geocoder/spec/fixtures/gme_output_ok.json
Vendored
+84
View File
@@ -0,0 +1,84 @@
{
"results": [
{
"address_components": [
{
"long_name": "1600",
"short_name": "1600",
"types": [
"street_number"
]
},
{
"long_name": "Amphitheatre Pkwy",
"short_name": "Amphitheatre Pkwy",
"types": [
"route"
]
},
{
"long_name": "Mountain View",
"short_name": "Mountain View",
"types": [
"locality",
"political"
]
},
{
"long_name": "Santa Clara County",
"short_name": "Santa Clara County",
"types": [
"administrative_area_level_2",
"political"
]
},
{
"long_name": "California",
"short_name": "CA",
"types": [
"administrative_area_level_1",
"political"
]
},
{
"long_name": "United States",
"short_name": "US",
"types": [
"country",
"political"
]
},
{
"long_name": "94043",
"short_name": "94043",
"types": [
"postal_code"
]
}
],
"formatted_address": "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
"geometry": {
"location": {
"lat": 37.4224764,
"lng": -122.0842499
},
"location_type": "ROOFTOP",
"viewport": {
"northeast": {
"lat": 37.4238253802915,
"lng": -122.0829009197085
},
"southwest": {
"lat": 37.4211274197085,
"lng": -122.0855988802915
}
}
},
"place_id": "ChIJ2eUgeAK6j4ARbn5u_wAGqWA",
"types": [
"street_address"
]
}
],
"status": "OK"
}
services/table-geocoder/spec/fixtures/kXYkQhuDfxnUSmWFP3dmq6TzTZAzwy4x.zip
Vendored
BIN
View File
Binary file not shown.
services/table-geocoder/spec/fixtures/nokia_input.csv
Vendored
+10
View File
@@ -0,0 +1,10 @@
recid,searchtext
"Østerbrogade 43, 2100 KØBENHAVN, DNK","Østerbrogade 43, 2100 KØBENHAVN, DNK"
"Amster dam, NLD","Amster dam, NLD"
"León, ESP","León, ESP"
"Villablino, León","Villablino, León"
"Leganés , Madrid","Leganés , Madrid"
", ESP",", ESP"
"Groningen, Netherlands,","Groningen, Netherlands,"
"Leganés, Madrid","Leganés, Madrid"
"Leganes, Madrid","Leganes, Madrid"
1 recid searchtext
2 Østerbrogade 43, 2100 KØBENHAVN, DNK Østerbrogade 43, 2100 KØBENHAVN, DNK
3 Amster dam, NLD Amster dam, NLD
4 León, ESP León, ESP
5 Villablino, León Villablino, León
6 Leganés , Madrid Leganés , Madrid
7 , ESP , ESP
8 Groningen, Netherlands, Groningen, Netherlands,
9 Leganés, Madrid Leganés, Madrid
10 Leganes, Madrid Leganes, Madrid
services/table-geocoder/spec/fixtures/nokia_input_.csv
Vendored
+38
View File
@@ -0,0 +1,38 @@
recid,searchtext
5498,"Agadez, Niger"
6604,"Tabriz, Iran"
206,"Ames, United States"
1760,"Apsheronsk, Russia"
2012,"Agapa, Russia"
2163,"Guasave, Mexico"
2225,"Pochutla, Mexico"
2633,"Tongling, China"
2985,"Anlu, China"
3327,"Berekum, Ghana"
3452,"Melun, French Republic"
3666,"Shahrud, Iran"
4081,"Sumy, Ukraine"
4167,"Pinheiro, Brazil"
4400,"Hinton, Canada"
4509,"Salluit, Canada"
5005,"Baranavichy, Belarus"
5098,"Katoomba, Australia"
5217,"Smithton, Australia"
5487,"Bilbao, Kingdom of Spain"
5826,"La Paz, Mexico"
5886,"Wonsan, Korea, North"
5997,"Amman, Jordan"
6397,"Sekondi, Ghana"
6532,"Kirkuk, Iraq"
6841,"Dawson, Canada"
6870,"Fredericton, Canada"
6927,"Santa Fe, Argentina"
7129,"Whyalla, Australia"
7176,"Sittwe, Myanmar"
7266,"Uummannaq, Denmark"
887,"My Tho, Vietnam"
3232,"Otsu, Japan"
3602,"Pori, Finland"
5319,"Paramaribo, Suriname"
5693,"Tver, Russia"
7313,"Oymyakon, Russia"
1 recid searchtext
2 5498 Agadez, Niger
3 6604 Tabriz, Iran
4 206 Ames, United States
5 1760 Apsheronsk, Russia
6 2012 Agapa, Russia
7 2163 Guasave, Mexico
8 2225 Pochutla, Mexico
9 2633 Tongling, China
10 2985 Anlu, China
11 3327 Berekum, Ghana
12 3452 Melun, French Republic
13 3666 Shahrud, Iran
14 4081 Sumy, Ukraine
15 4167 Pinheiro, Brazil
16 4400 Hinton, Canada
17 4509 Salluit, Canada
18 5005 Baranavichy, Belarus
19 5098 Katoomba, Australia
20 5217 Smithton, Australia
21 5487 Bilbao, Kingdom of Spain
22 5826 La Paz, Mexico
23 5886 Wonsan, Korea, North
24 5997 Amman, Jordan
25 6397 Sekondi, Ghana
26 6532 Kirkuk, Iraq
27 6841 Dawson, Canada
28 6870 Fredericton, Canada
29 6927 Santa Fe, Argentina
30 7129 Whyalla, Australia
31 7176 Sittwe, Myanmar
32 7266 Uummannaq, Denmark
33 887 My Tho, Vietnam
34 3232 Otsu, Japan
35 3602 Pori, Finland
36 5319 Paramaribo, Suriname
37 5693 Tver, Russia
38 7313 Oymyakon, Russia
services/table-geocoder/spec/fixtures/nokia_output.txt
Vendored
+44
View File
@@ -0,0 +1,44 @@
"Agadez, Niger",1,1,17.0926609,7.5755601
"Agapa, Russia",1,1,53.3041992,59.1366501
"Ames, United States",1,5,42.02535,-93.6202
"Ames, United States",2,5,30.05339,-94.74227
"Ames, United States",3,5,36.24605,-98.18511
"Ames, United States",4,5,42.83659,-74.59919
"Ames, United States",5,5,41.45263,-96.62508
"Amman, Jordan",1,1,31.95181,35.94042
"Anlu, China",1,1,31.25582,113.68918
"Apsheronsk, Russia",1,1,44.46301,39.72893
"Baranavichy, Belarus",1,1,53.13963,26.02236
"Berekum, Ghana",1,1,7.45498,-2.58425
"Bilbao, Kingdom of Spain",1,1,37.2187881,-5.3498902
"Dawson, Canada",1,2,64.06332,-139.43439
"Dawson, Canada",2,2,48.72222,-94.48447
"Fredericton, Canada",1,1,45.96063,-66.63911
"Guasave, Mexico",1,1,25.5706,-108.47102
"Hinton, Canada",1,1,53.40095,-117.57697
"Katoomba, Australia",1,1,-33.71181,150.31056
"Kirkuk, Iraq",1,1,35.4687309,44.3895187
"La Paz, Mexico",1,2,19.35847,-98.95884
"La Paz, Mexico",2,2,24.15464,-110.31023
"My Tho, Vietnam",1,1,10.35595,106.36643
"Otsu, Japan",1,1,42.73321,143.59012
"Oymyakon, Russia",1,1,63.4606133,142.786026
"Paramaribo, Suriname",1,1,5.82031,-55.16542
"Pinheiro, Brazil",1,1,-2.52138,-45.0832
"Pochutla, Mexico",1,1,15.7434,-96.46694
"Pori, Finland",1,1,61.48596,21.79537
"Salluit, Canada",1,1,62.20523,-75.64252
"Santa Fe, Argentina",1,1,-31.63666,-60.712
"Sekondi, Ghana",1,1,4.9340301,-1.70972
"Shahrud, Iran",1,2,30.9944096,50.0435715
"Shahrud, Iran",2,2,30.1367893,52.5212402
"Sittwe, Myanmar",1,1,20.13819,92.88672
"Smithton, Australia",1,1,-40.85546,145.12035
"Sumy, Ukraine",1,1,50.90787,34.79759
"Tabriz, Iran",1,1,38.0797081,46.3001518
"Tongling, China",1,1,30.94496,117.81258
"Tver, Russia",1,1,56.85238,35.93375
"Whyalla, Australia",1,1,-33.03351,137.58416
"Wonsan, Korea, North",1,3,34.5083313,126.3166122
"Wonsan, Korea, North",2,3,36.1985207,127.0890808
"Wonsan, Korea, North",3,3,35.188839,126.947731
services/table-geocoder/spec/fixtures/populated_places_short.csv
Vendored
+19
View File
@@ -0,0 +1,19 @@
1,"Amster
dam",NLD
2,"Groningen, Netherlands",
3,León,ESP
4,"Østerbrogade 43,
2100 KØBENHAVN",DNK
5,León,"ESP
"
6,Villablino,León
7,Villablino,León
8,Leganés,Madrid
9,Leganés ,Madrid
10,,
11, ,
12,,ESP
13, ,ESP
14,Leganes, Madrid
1 1 Amster dam NLD
2 2 Groningen, Netherlands
3 3 León ESP
4 4 Østerbrogade 43, 2100 KØBENHAVN DNK
5 5 León ESP
6 6 Villablino León
7 7 Villablino León
8 8 Leganés Madrid
9 9 Leganés Madrid
10 10
11 11
12 12 ESP
13 13 ESP
14 14 Leganes Madrid
services/table-geocoder/spec/fixtures/result_20130919-04-55_6.2.46.1_out.txt
Vendored Executable
+45
View File
@@ -0,0 +1,45 @@
206,1,5,42.02535,-93.6202
206,2,5,30.05339,-94.74227
206,3,5,36.24605,-98.18511
206,4,5,42.83659,-74.59919
206,5,5,41.45263,-96.62508
887,1,1,10.35855,106.35897
1760,1,1,44.46301,39.72893
2012,1,1,44.6720581,-73.792572
2163,1,1,25.5706,-108.47102
2225,1,1,15.7434,-96.46694
2633,1,1,30.94496,117.81258
2985,1,1,31.25582,113.68918
3232,1,1,42.73321,143.59012
3327,1,1,7.45498,-2.58425
3602,1,1,61.48596,21.79537
3666,1,4,30.9944096,50.0435715
3666,2,4,30.1367893,52.5212402
3666,3,4,30.4089508,57.7633095
3666,4,4,38.1983109,44.7774811
4081,1,1,50.90787,34.79759
4167,1,1,-2.52138,-45.0832
4400,1,1,53.40095,-117.57697
4509,1,1,62.20523,-75.64252
5005,1,1,53.13963,26.02236
5098,1,1,-33.71181,150.31056
5217,1,1,-40.85546,145.12035
5319,1,1,5.82031,-55.16542
5487,1,1,37.2187881,-5.3498902
5498,1,1,17.0926609,7.5755601
5693,1,1,56.85238,35.93375
5826,1,2,19.35842,-98.95862
5826,2,2,24.15464,-110.31023
5886,1,2,34.5083313,126.3166122
5886,2,2,36.2474098,128.2595367
5997,1,1,31.95181,35.94042
6397,1,1,4.9360099,-1.72823
6532,1,1,35.4687309,44.3895187
6604,1,1,38.0797081,46.3001518
6841,1,2,64.06332,-139.43439
6841,2,2,48.72225,-94.48446
6870,1,1,45.96063,-66.63911
6927,1,1,-31.63666,-60.712
7129,1,1,-33.03351,137.58416
7176,1,1,20.13819,92.88672
7313,1,1,63.4606133,142.786026
services/table-geocoder/spec/geocoder_cache_spec.rb
+73
View File
@@ -0,0 +1,73 @@
require 'open3'
require_relative '../lib/table_geocoder.rb'
require_relative 'factories/pg_connection'
require_relative '../../../spec/rspec_configuration.rb'
RSpec.configure do |config|
config.mock_with :mocha
end
describe CartoDB::GeocoderCache do
before do
conn = CartoDB::Importer2::Factories::PGConnection.new
@db = conn.connection
@pg_options = conn.pg_options
@table_name = "ne_10m_populated_places_simple"
@usage_metrics_stub = stub
@log = mock
@log.stubs(:append)
@log.stubs(:append_and_store)
# Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
filename = 'populated_places_short.csv'
stdout, stderr, status = Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
raise if stderr != ''
load_csv "/tmp/#{filename}"
end
after do
@db.drop_table @table_name
end
let(:default_params) { {
table_name: @table_name,
formatter: "concat(name, iso3)",
connection: @db, sql_api: { table_name: '' },
qualified_table_name: @table_name,
usage_metrics: @usage_metrics_stub,
log: @log
} }
describe '#get_cache_results' do
it "runs the query in batches" do
cache = CartoDB::GeocoderCache.new(default_params.merge(batch_size: 5))
@db.run("alter table #{@table_name} add column cartodb_georef_status BOOLEAN DEFAULT NULL")
cache.expects(:run_query).times(3).returns('')
cache.get_cache_results
end
it "honors max_rows" do
@db.run("alter table #{@table_name} add column cartodb_georef_status BOOLEAN DEFAULT NULL")
cache = CartoDB::GeocoderCache.new(default_params.merge(
batch_size: 5,
max_rows: 10
))
cache.expects(:run_query).times(2).returns('')
cache.get_cache_results
end
end #run
def path_to(filepath = '')
File.expand_path(
File.join(File.dirname(__FILE__), "../spec/fixtures/#{filepath}")
)
end #path_to
def load_csv(path)
@db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
@db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
end # create_table
end # CartoDB::GeocoderCache
services/table-geocoder/spec/internal-geocoder/input_type_resolver_spec.rb
+77
View File
@@ -0,0 +1,77 @@
require_relative '../../lib/internal-geocoder/input_type_resolver.rb'
require_relative '../../../../spec/rspec_configuration.rb'
class String
# We just need this instead of adding the whole rails thing
def present?
self && !empty?
end
end
describe CartoDB::InternalGeocoder::InputTypeResolver do
before(:each) do
@internal_geocoder = mock
@input_type_resolver = CartoDB::InternalGeocoder::InputTypeResolver.new(@internal_geocoder)
end
describe '#type' do
it 'should return an array characterizing the inputs for <namedplace, country_name, region_name, point>' do
@internal_geocoder.stubs('kind').once.returns(:namedplace)
@internal_geocoder.stubs('geometry_type').once.returns(:point)
@internal_geocoder.stubs('country_column').once.returns(nil)
@internal_geocoder.stubs('region_column').once.returns(nil)
@internal_geocoder.stubs('regions').once.returns('region')
@input_type_resolver.type.should == [:namedplace, :text, :text, :point]
end
end
describe '#kind' do
it 'should return the type of the internal geocoding: namedplace' do
@internal_geocoder.stubs('kind').once.returns(:namedplace)
@input_type_resolver.kind.should == :namedplace
end
end
describe '#geometry_type' do
it 'should return the type of the geometry to be geocoded: polygon' do
@internal_geocoder.stubs('geometry_type').once.returns(:polygon)
@input_type_resolver.geometry_type.should == :polygon
end
end
describe '#country_input_type' do
it 'should return column if a column was passed' do
@internal_geocoder.stubs('country_column').once.returns('any_column_name')
@input_type_resolver.country_input_type.should == :column
end
it 'should return column if no column was passed' do
@internal_geocoder.stubs('country_column').once.returns(nil)
@input_type_resolver.country_input_type.should == :text
end
end
describe '#region_input_type' do
it 'should return :column if a column is present' do
@internal_geocoder.stubs('region_column').once.returns('wadus')
@input_type_resolver.region_input_type.should == :column
end
it 'should return :text if regions are present' do
@internal_geocoder.stubs('region_column').once.returns(nil)
@internal_geocoder.stubs('regions').once.returns('minnesota')
@input_type_resolver.region_input_type.should == :text
end
it 'should return nil if no column or text are present' do
@internal_geocoder.stubs('region_column').once.returns(nil)
@internal_geocoder.stubs('regions').once.returns('')
@input_type_resolver.region_input_type.should == nil
end
end
end
services/table-geocoder/spec/internal-geocoder/query_generator_factory_spec.rb
+118
View File
@@ -0,0 +1,118 @@
require_relative '../../lib/internal-geocoder/query_generator_factory.rb'
require_relative '../../lib/internal-geocoder/abstract_query_generator.rb'
require_relative '../../../../spec/rspec_configuration.rb'
require 'active_support/core_ext' # Needed for string.blank?
RSpec.configure do |config|
config.mock_with :mocha
end
=begin
The class should generate queries to be used by the InternalGeocoder depending on the inputs.
* Different types of inputs:
- kind: namedplace, ipaddress, postalcode, admin0, admin1
- country: column, freetext
- geometry_type: point, polygon
* Where queries are needed:
- to query the data-services
- to import results into table
=end
describe CartoDB::InternalGeocoder::QueryGeneratorFactory do
before(:each) do
@internal_geocoder = mock
end
describe '#dataservices' do
it 'should return a query template suitable for <namedplace, country_name, region_name, point>' do
query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:namedplace, :text, :text, :point])
query_generator.should be_a_kind_of CartoDB::InternalGeocoder::AbstractQueryGenerator
@internal_geocoder.expects('countries').once.returns(%Q{'Spain'})
@internal_geocoder.expects('regions').once.returns(%Q{'Madrid'})
search_terms = [{city: %Q{'Madrid'}}, {city: %Q{'Granada'}}]
query = query_generator.dataservices_query(search_terms)
query.should == "WITH geo_function AS (SELECT (geocode_namedplace(Array['Madrid','Granada'], 'Madrid', 'Spain')).*) SELECT q, c, a1, geom, success FROM geo_function"
end
end
describe '#search_terms_query' do
it 'should get the search terms for <namedplace, country_name, region_name, point>' do
@internal_geocoder.stubs('column_name').once.returns('city')
@internal_geocoder.stubs('qualified_table_name').once.returns(%Q{"public"."untitled_table"})
@internal_geocoder.stubs('batch_size').returns(5000)
query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:namedplace, :text, :text, :point])
query = query_generator.search_terms_query(0)
query.squish.should == 'SELECT DISTINCT(trim(quote_nullable("city"))) AS city FROM "public"."untitled_table" WHERE cartodb_georef_status IS NULL LIMIT 5000 OFFSET 0'
end
end
describe '#copy_results_to_table_query' do
it 'should generate a suitable query to update geocoded table with temp table' do
@internal_geocoder.stubs('qualified_table_name').returns(%Q{"public"."untitled_table"})
@internal_geocoder.stubs('temp_table_name').once.returns('any_temp_table')
@internal_geocoder.stubs('column_name').once.returns('any_column_name')
query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:namedplace, :text, :text, :point])
query = query_generator.copy_results_to_table_query
query.squish.should == %Q{
UPDATE "public"."untitled_table" AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM any_temp_table AS orig
WHERE trim(dest."any_column_name"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}.squish
end
end
describe 'CDB-4269' do
it 'should generate a suitable query generator for [:admin1, :column, nil, :polygon]' do
@internal_geocoder.stubs('column_name').twice.returns('region_column_name')
@internal_geocoder.stubs('qualified_table_name').returns('any_table_name')
@internal_geocoder.stubs('country_column').twice.returns('country_column_name')
@internal_geocoder.stubs('batch_size').twice.returns(10)
@internal_geocoder.stubs('temp_table_name').once.returns('any_temp_tablename')
query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:admin1, :column, nil, :polygon])
query_generator.search_terms_query(0).squish.should == %Q{
SELECT DISTINCT
trim(quote_nullable(region_column_name)) as region,
trim(quote_nullable(country_column_name)) as country
FROM any_table_name
WHERE cartodb_georef_status IS NULL
LIMIT 10 OFFSET 0
}.squish
search_terms = [
{ region: %Q{'New York'}, country: %Q{'USA'}},
{ region: %Q{'Sucumbios'}, country: %Q{'Ecuador'}}
]
query_generator.dataservices_query(search_terms).squish.should == %Q{
WITH geo_function AS (SELECT (geocode_admin1_polygons(Array['New York','Sucumbios'], Array['USA','Ecuador'])).*)
SELECT q, c, null AS a1, geom, success FROM geo_function
}.squish
query_generator.copy_results_to_table_query.squish.should == %Q{
UPDATE any_table_name AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM any_temp_tablename AS orig
WHERE trim(dest.region_column_name::text) = trim(orig.geocode_string)
AND trim(dest.country_column_name::text) = trim(orig.country)
AND dest.cartodb_georef_status IS NULL
}.squish
end
end
end
services/table-geocoder/spec/internal_Geocoder_spec.rb
+49
View File
@@ -0,0 +1,49 @@
require 'open3'
require_relative 'factories/pg_connection'
require_relative '../lib/internal_geocoder.rb'
require_relative '../../../spec/rspec_configuration.rb'
describe CartoDB::InternalGeocoder::Geocoder do
before do
conn = CartoDB::Importer2::Factories::PGConnection.new
@db = conn.connection
@pg_options = conn.pg_options
end
let(:default_params) { { connection: @db, internal: { username: 'geocoding' }, kind: 'admin0', geometry_type: 'polygon' } }
describe '#download_results' do
before do
# Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
filename = 'adm0.csv'
stdout, stderr, status = Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
raise if stderr != ''
load_csv "/tmp/#{filename}", 'adm0'
end
after do
@db.drop_table 'adm0'
end
it "generates a csv with geocoded data" do
ig = CartoDB::InternalGeocoder.new(default_params.merge(table_name: 'adm0', formatter: 'geo_string'))
ig.ensure_georef_status_colummn_valid
results = ig.download_results
`wc -l #{results} 2>&1`.to_i.should eq 11
ig.processed_rows.should eq 11
end
end #run
def path_to(filepath = '')
File.expand_path(
File.join(File.dirname(__FILE__), "../spec/fixtures/#{filepath}")
)
end #path_to
def load_csv(path , table_name)
@db.run("DROP TABLE IF EXISTS #{table_name}")
@db.run("CREATE TABLE #{table_name} (the_geom geometry, cartodb_id integer, geo_string text)")
@db.run("COPY #{Sequel.lit(table_name)}(cartodb_id, geo_string) FROM '#{path}' DELIMITER ',' CSV")
end # create_table
end # CartoDB::GeocoderCache
services/table-geocoder/spec/lib/abstract_table_geocoder_spec.rb
+82
View File
@@ -0,0 +1,82 @@
require_relative '../factories/pg_connection'
require_relative '../../lib/abstract_table_geocoder'
require_relative '../../../../spec/rspec_configuration.rb'
describe CartoDB::AbstractTableGeocoder do
before(:all) do
class ConcreteTableGeocoder < CartoDB::AbstractTableGeocoder; end
conn = CartoDB::Importer2::Factories::PGConnection.new
@db = conn.connection
@pg_options = conn.pg_options
@table_name = "ne_10m_populated_places_simple"
load_csv path_to("populated_places_short.csv")
end
after(:all) do
@db.drop_table @table_name
end
describe '#initialize' do
it 'sets the connection timeout to 5 hours' do
tg = ConcreteTableGeocoder.new({
connection: @db,
table_name: @table_name,
qualified_table_name: @table_name
})
timeout = @db.fetch("SHOW statement_timeout").first.fetch(:statement_timeout)
timeout.should == '5h'
end
end
describe '#ensure_georef_status_colummn_valid' do
before(:each) do
@tg = ConcreteTableGeocoder.new({
connection: @db,
table_name: @table_name,
qualified_table_name: @table_name
})
end
it 'adds a georef_status_column if it does not exists' do
@tg.send(:ensure_georef_status_colummn_valid)
assert_correctness_of_georef_status_column
end
it 'does nothing if the column already exists' do
@tg.send(:ensure_georef_status_colummn_valid)
@tg.send(:ensure_georef_status_colummn_valid)
assert_correctness_of_georef_status_column
end
it 'casts its type if the column exists and is not bool' do
@db.add_column @table_name, :georef_status_column, :text
@tg.send(:ensure_georef_status_colummn_valid)
assert_correctness_of_georef_status_column
end
end
def assert_correctness_of_georef_status_column
georef_status_column = @db.schema(@table_name, reload: true).select {|c| c[0] == :cartodb_georef_status}.first
georef_status_column.nil?.should == false
georef_status_column[1][:db_type].should == 'boolean'
end
def load_csv(path)
@db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
@db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
end
def path_to(filepath = '')
File.expand_path(
File.join(File.dirname(__FILE__), "../fixtures/#{filepath}")
)
end
end
services/table-geocoder/spec/lib/gme/table_geocoder_spec.rb
+246
View File
@@ -0,0 +1,246 @@
require 'open3'
require_relative '../../../lib/gme/table_geocoder'
require_relative '../../../../../lib/url_signer'
require_relative '../../../lib/gme/exceptions'
require_relative '../../factories/pg_connection'
require_relative '../../../../../spec/spec_helper.rb'
require_relative '../../../../../spec/rspec_configuration.rb'
describe Carto::Gme::TableGeocoder do
before(:all) do
connection_stub = mock
connection_stub.stubs(:run)
@usage_metrics_stub = stub
@log = mock
@log.stubs(:append)
@log.stubs(:append_and_store)
@geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution', formatter: '{street}')
@mandatory_args = {
connection: connection_stub,
original_formatter: '{mock}',
client_id: 'my_client_id',
private_key: 'my_private_key',
usage_metrics: @usage_metrics_stub,
log: @log,
geocoding_model: @geocoding_model
}
end
describe '#initialize' do
it 'returns an object that responds to AbstractTableGeocoder interface' do
table_geocoder = Carto::Gme::TableGeocoder.new(@mandatory_args)
interface_methods = [:ensure_georef_status_colummn_valid,
:cancel,
:run,
:remote_id,
:update_geocoding_status,
:process_results]
interface_methods.each do |method|
table_geocoder.respond_to?(method, true).should == true
end
end
it 'raises an exception if not fed with mandatory arguments' do
expect { Carto::Gme::TableGeocoder.new }.to raise_error(ArgumentError)
@mandatory_args.each do |arg|
args_missing_one = @mandatory_args.dup
args_missing_one.delete(arg[0])
lambda { Carto::Gme::TableGeocoder.new(args_missing_one) }.should raise_error(KeyError)
end
end
it 'creates a client with the provided credentials' do
gme_client_mock = mock
Carto::Gme::Client.expects(:new).with('my_client_id', 'my_private_key').once.returns(gme_client_mock)
Carto::Gme::GeocoderClient.expects(:new).with(gme_client_mock).once
Carto::Gme::TableGeocoder.new(@mandatory_args)
end
end
describe '#run' do
before(:each) do
Carto::UrlSigner.any_instance.stubs(:sign_url).returns('https://maps.googleapis.com/maps/api/geocode/json')
@table_geocoder = Carto::Gme::TableGeocoder.new(@mandatory_args)
end
it "set's the state to 'processing' when it starts" do
pending 'actually as a requirement this does not make much sense'
end
it "set's the state to 'completed' when it ends" do
# TODO: there's something weird that needs review here
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
@table_geocoder.stubs(:data_input_blocks).returns([])
@table_geocoder.run
@geocoding_model.state.should == 'completed'
end
it "if there's an uncontrolled exception, sets the state to 'failed' and raises it" do
# TODO: there's something weird that needs review here
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
@table_geocoder.stubs(:ensure_georef_status_aolummn_valid)
@table_geocoder.stubs(:data_input_blocks).returns([{ cartodb_id: 1, searchtext: 'dummy text' }])
@table_geocoder.stubs(:geocode).raises(StandardError, 'unexpected exception')
expect { @table_geocoder.run }.to raise_error('unexpected exception')
@geocoding_model.state.should == 'failed'
end
it "processes 1 block at a time, keeping track of processed rows in each block" do
# TODO: there's something weird that needs review here
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 4)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 4)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
mocked_input = Enumerator.new do |enum|
# 2 blocks of 2 rows each as input
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }, { cartodb_id: 2, searchtext: 'dummy text' }]
enum.yield [{ cartodb_id: 3, searchtext: 'dummy text' }, { cartodb_id: 4, searchtext: 'dummy text' }]
end
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_ok.json'))
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
@table_geocoder.expects(:update_table).twice
@table_geocoder.run
@table_geocoder.processed_rows.should == 4
end
it "processes empty response" do
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 1)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
mocked_input = Enumerator.new do |enum|
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
end
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_empty.json'))
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
@table_geocoder.expects(:update_table).once
@table_geocoder.run
@table_geocoder.processed_rows.should == 1
end
it "processes error rows response" do
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 1)
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
mocked_input = Enumerator.new do |enum|
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
end
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_error.json'))
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
@table_geocoder.expects(:update_table).once
@table_geocoder.run
@table_geocoder.processed_rows.should == 1
end
it "processes error with message response" do
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
@usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 1)
@table_geocoder.stubs(:ensure_georef_status_colummn_valid)
mocked_input = Enumerator.new do |enum|
enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
end
@table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_error_with_message.json'))
Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
@table_geocoder.expects(:update_table).once
# CartoDB.expects(:notify_error).once
@table_geocoder.run
@table_geocoder.processed_rows.should == 1
end
end
describe '#data_input_blocks' do
before do
conn = CartoDB::Importer2::Factories::PGConnection.new
@db = conn.connection
@pg_options = conn.pg_options
@table_name = "ne_10m_populated_places_simple_#{rand.to_s[2..11]}"
# Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
filename = 'populated_places_short.csv'
_stdout, stderr, _status = Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
raise if stderr != ''
load_csv "/tmp/#{filename}"
params = {
connection: @db,
table_name: @table_name,
qualified_table_name: @table_name,
sequel_qualified_table_name: @table_name,
original_formatter: "{name}, {iso3}",
client_id: 'my_client_id',
private_key: 'my_private_key',
max_block_size: 4,
usage_metrics: @usage_metrics_stub,
log: @log,
geocoding_model: @geocoding_model
}
@table_geocoder = Carto::Gme::TableGeocoder.new(params)
end
after do
@db.drop_table @table_name
end
it 'performs (floor(rows / max_block_size) + 1) queries' do
rows = @db[@table_name.to_sym].count
@table_geocoder.send(:ensure_georef_status_colummn_valid)
count = 0
@table_geocoder.send(:data_input_blocks).each do |data_block|
data_block.each do |row|
row.merge!(cartodb_georef_status: false)
end
@table_geocoder.send(:update_table, data_block)
count += 1
end
count.should == (rows / @table_geocoder.max_block_size).floor + 1
end
end
def path_to(filepath = '')
File.expand_path(
File.join(File.dirname(__FILE__), "../../fixtures/#{filepath}")
)
end
def read_fixture_file(filename)
File.read(path_to(filename))
end
def load_csv(path)
@db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
@db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
end
end
services/table-geocoder/spec/table_geocoder_spec.rb
+286
View File
@@ -0,0 +1,286 @@
require 'open3'
require_relative '../lib/table_geocoder'
require_relative 'factories/pg_connection'
require 'set'
require_relative '../../../spec/rspec_configuration'
require_relative '../../../spec/spec_helper'
describe CartoDB::TableGeocoder do
let(:default_params) {{
app_id: '',
token: '',
mailto: '',
usage_metrics: mock('usage_metrics')
}}
before do
conn = CartoDB::Importer2::Factories::PGConnection.new
@db = conn.connection
@pg_options = conn.pg_options
@table_name = "ne_10m_populated_places_simple_#{rand.to_s[2..11]}"
@log = mock
@log.stubs(:append)
@log.stubs(:append_and_store)
@geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution',
formatter: '{street}', remote_id: 'dummy_request_id')
# Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
filename = 'populated_places_short.csv'
_stdout, stderr, _status = Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
raise if stderr != ''
load_csv "/tmp/#{filename}"
end
after do
@db.drop_table @table_name
end
describe '#run' do
before do
# TODO: Note the coupling of the geocoder object and the metrics
success_rows = 10
empty_rows = 3
failed_rows = 4
total = success_rows + empty_rows + failed_rows
default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :success_responses, success_rows)
default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :empty_responses, empty_rows)
default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :failed_responses, failed_rows)
default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :total_requests, total)
@tg = CartoDB::TableGeocoder.new(default_params.merge(table_name: @table_name,
qualified_table_name: @table_name,
sequel_qualified_table_name: @table_name,
formatter: "name, ', ', iso3",
connection: @db,
log: @log,
geocoding_model: @geocoding_model,
max_rows: 1000))
geocoder = mock
geocoder.stubs(:upload).returns(true)
geocoder.stubs(:request_id).returns('dummy_request_id')
geocoder.stubs(:run).returns(true)
geocoder.stubs(:status).returns('foo')
# TODO: Note the coupling of the geocoder object and the metrics
geocoder.stubs(:successful_processed_rows).returns(success_rows)
geocoder.stubs(:empty_processed_rows).returns(empty_rows)
geocoder.stubs(:failed_processed_rows).returns(failed_rows)
@tg.stubs(:geocoder).returns(geocoder)
@tg.stubs(:cache_disabled?).returns(true)
@tg.run
end
it "generates a csv file for uploading" do
expected = Set.new(File.readlines(path_to('nokia_input.csv')))
actual = Set.new(File.readlines("#{@tg.working_dir}/wadus.csv"))
actual.should == expected
end
it "assigns a remote_id" do
@tg.remote_id.should == 'dummy_request_id'
end
it "holds a db connection with the specified statement timeout" do
timeout = @tg.connection.fetch("SHOW statement_timeout").all[0][:statement_timeout]
timeout.should == '5h'
end
end
describe '#generate_csv' do
before do
@tg = CartoDB::TableGeocoder.new(default_params.merge(table_name: @table_name,
qualified_table_name: @table_name,
sequel_qualified_table_name: @table_name,
formatter: "name, ', ', iso3",
connection: @db,
log: @log,
geocoding_model: @geocoding_model,
max_rows: 1000))
@tg.send(:ensure_georef_status_colummn_valid)
end
it "generates a csv file with the correct format" do
@tg.send(:mark_rows_to_geocode)
@tg.send(:generate_csv)
File.readlines("#{@tg.working_dir}/wadus.csv").to_set.should == File.readlines(path_to('nokia_input.csv')).to_set
end
it "honors max_rows" do
max_rows = 10
@tg.stubs(:max_rows).returns max_rows
@tg.send(:mark_rows_to_geocode)
@tg.send(:generate_csv)
# Note there might be duplicate input strings but we send unique inputs to the geocoder api.
# Also note the csv file has a header.
File.readlines("#{@tg.working_dir}/wadus.csv").count.should <= (max_rows + 1)
end
end
describe '#download_results' do
it 'gets the geocoder results' do
tg = CartoDB::TableGeocoder.new(table_name: 'a', connection: @db, max_rows: 1000,
usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
geocoder = mock
geocoder.expects(:result).times(1).returns('a')
tg.stubs(:geocoder).returns(geocoder)
tg.send(:download_results)
tg.result.should == 'a'
end
end
describe '#deflate_results' do
it 'does not raise an error if no results file' do
dir = Dir.mktmpdir
tg = CartoDB::TableGeocoder.new(table_name: 'a',
connection: @db, working_dir: dir, max_rows: 1000,
usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
expect { tg.send(:deflate_results) }.to_not raise_error
end
it 'extracts nokia result files' do
dir = Dir.mktmpdir
`cp #{path_to('kXYkQhuDfxnUSmWFP3dmq6TzTZAzwy4x.zip')} #{dir}`
tg = CartoDB::TableGeocoder.new(table_name: 'a',
connection: @db, working_dir: dir, max_rows: 1000,
usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
tg.send(:deflate_results)
filename = 'result_20130919-04-55_6.2.46.1_out.txt'
destfile = File.open(File.join(dir, filename))
destfile.read.should eq File.open(path_to(filename)).read
end
end
describe '#create_temp_table' do
it 'raises error if no remote_id' do
tg = CartoDB::TableGeocoder.new(table_name: 'a', connection: @db, max_rows: 1000,
usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
expect { tg.send(:create_temp_table) }.to raise_error(Sequel::DatabaseError)
end
it 'creates a temporary table' do
tg = CartoDB::TableGeocoder.new(table_name: 'a',
connection: @db,
remote_id: 'geo_HvyxzttLyFhaQ7JKmnrZxdCVySd8N0Ua',
log: @log,
geocoding_model: @geocoding_model,
schema: 'public', max_rows: 1000, usage_metrics: nil)
tg.send(:drop_temp_table)
tg.send(:create_temp_table)
@db.fetch("select * from #{tg.send(:temp_table_name)}").all.should eq []
end
end
describe '#import_results_to_temp_table' do
before do
@tg = CartoDB::TableGeocoder.new(table_name: 'a',
connection: @db,
log: @log,
geocoding_model: @geocoding_model,
remote_id: 'temp_table', schema: 'public', max_rows: 1000, usage_metrics: nil)
@tg.send(:create_temp_table)
end
after do
@tg.send(:drop_temp_table)
end
it 'loads the Nokia output format to an existing temp table' do
@tg.stubs(:deflated_results_path).returns(path_to('nokia_output.txt'))
@tg.send(:import_results_to_temp_table)
@db.fetch(%{
SELECT count(*) FROM #{@tg.send(:temp_table_name)}
WHERE displayLatitude IS NOT NULL AND displayLongitude IS NOT NULL
}).first[:count].should eq 44
end
end
describe '#ensure_georef_status_colummn_valid' do
before do
table_name = 'wwwwww'
@db.run("create table #{table_name} (id integer)")
@tg = CartoDB::TableGeocoder.new(table_name: 'wwwwww',
qualified_table_name: table_name,
sequel_qualified_table_name: table_name,
connection: @db,
remote_id: 'wadus',
max_rows: 1000,
log: @log,
geocoding_model: @geocoding_model,
usage_metrics: nil)
end
after do
@db.run("drop table wwwwww")
end
it 'adds a boolean cartodb_georef_status column' do
@tg.send(:ensure_georef_status_colummn_valid)
@db.run("select cartodb_georef_status from wwwwww").should eq nil
end
it 'does nothing when the column already exists' do
@tg.expects(:cast_georef_status_column).once
@tg.send(:ensure_georef_status_colummn_valid)
@tg.send(:ensure_georef_status_colummn_valid)
end
it 'casts cartodb_georef_status to boolean if needed' do
@db.run('alter table wwwwww add column cartodb_georef_status text')
@tg.send(:ensure_georef_status_colummn_valid)
sql_query = "select data_type from information_schema.columns " \
"where table_name = 'wwwwww' and column_name = 'cartodb_georef_status'"
@db.fetch(sql_query)
.first[:data_type].should eq 'boolean'
end
end
it "Geocodes a table using the batch geocoder API" do
config = YAML.load_file("#{File.dirname(__FILE__)}/../../../config/app_config.yml")["test"]["geocoder"]
pending "This is a System E2E test that can be useful for development but not suitable for CI"
pending "No Geocoder config found for test environment" unless config['app_id'] != ''
config = config.inject({}) do |memo, (k, v)|
memo[k.to_sym] = v
memo
end
config[:cache] = config[:cache].inject({}) do |memo, (k, v)|
memo[k.to_sym] = v
memo
end
t = CartoDB::TableGeocoder.new(config.merge(table_name: @table_name,
qualified_table_name: @table_name,
sequel_qualified_table_name: @table_name,
formatter: "name, ', ', iso3",
connection: @db,
schema: 'public',
log: @log,
geocoding_model: @geocoding_model,
max_rows: 1000))
t.geocoder.stubs("use_batch_process?").returns(true)
@db.fetch("select count(*) from #{@table_name} where the_geom is null").first[:count].should eq 14
t.run
until t.geocoder.status == 'completed' do
t.geocoder.update_status
puts "#{t.geocoder.status} #{t.geocoder.processed_rows}/#{t.geocoder.total_rows}"
sleep(2)
end
t.process_results
t.geocoder.status.should eq 'completed'
t.geocoder.processed_rows.to_i.should eq 0
t.cache.hits.should eq 10
@db.fetch("select count(*) from #{@table_name} where the_geom is null").first[:count].should eq 0
@db.fetch("select count(*) from #{@table_name} where cartodb_georef_status is false").first[:count].should eq 0
end
def path_to(filepath = '')
File.expand_path(
File.join(File.dirname(__FILE__), "../spec/fixtures/#{filepath}")
)
end
def load_csv(path)
@db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
@db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
end
end