Initial commit

This commit is contained in:
zhongjin
2020-06-15 10:58:47 +08:00
commit 4f1dfe7564
8590 changed files with 1516878 additions and 0 deletions
@@ -0,0 +1,44 @@
require_relative 'input_type_resolver'
require_relative '../../../importer/lib/importer/query_batcher'
module CartoDB
module InternalGeocoder
class AbstractQueryGenerator
def initialize(internal_geocoder)
@internal_geocoder = internal_geocoder
end
#TODO custom exception
def search_terms_query(page)
raise 'Not implemented'
end
def dataservices_query(search_terms)
raise 'Not implemented'
end
def copy_results_to_table_query
raise 'Not implemented'
end
def country
country = @internal_geocoder.countries
(country == %Q{'world'} || country.blank?) ? 'null' : country
end
def region
region = @internal_geocoder.regions
return 'null' if region.blank?
region
end
def dest_table
@internal_geocoder.qualified_table_name
end
end # AbstractQueryGenerator
end # InternalGeocoder
end #CartoDB
@@ -0,0 +1,35 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class Admin0TextPolygons < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS region
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
regions = search_terms.map { |row| row[:region] }.join(',')
"WITH geo_function AS (SELECT (geocode_admin0_polygons(Array[#{regions}])).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}
end
end # Admin0TextPolygons
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,40 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class Admin1ColumnPolygons < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT
trim(quote_nullable(#{@internal_geocoder.column_name})) as region,
trim(quote_nullable(#{@internal_geocoder.country_column})) as country
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
region = search_terms.map { |row| row[:region] }.join(',')
countries = search_terms.map { |row| row[:country] }.join(',')
"WITH geo_function AS (SELECT (geocode_admin1_polygons(Array[#{region}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string)
AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
AND dest.cartodb_georef_status IS NULL
}
end
end # Admin1ColumnPolygons
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,35 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class Admin1TextPolygons < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS region
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
region = search_terms.map { |row| row[:region] }.join(',')
"WITH geo_function AS (SELECT (geocode_admin1_polygons(Array[#{region}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}
end
end # Admin1TextPolygons
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,43 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class CitiesColumnColumnPoints < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT
trim(quote_nullable("#{@internal_geocoder.column_name}")) AS city,
trim(quote_nullable(#{@internal_geocoder.country_column})) AS country,
trim(quote_nullable(#{@internal_geocoder.region_column})) AS region
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
cities = search_terms.map { |row| row[:city] }.join(',')
regions = search_terms.map { |row| row[:region] }.join(',')
countries = search_terms.map { |row| row[:country] }.join(',')
"WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], Array[#{regions}], Array[#{countries}])).*) SELECT q, c, a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
AND trim(dest.#{@internal_geocoder.region_column}::text) = trim(orig.region)
AND dest.cartodb_georef_status IS NULL
}
end
end # CitiesColumnColumnPoints
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,40 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class CitiesColumnTextPoints < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT
trim(quote_nullable(#{@internal_geocoder.column_name})) as city,
trim(quote_nullable(#{@internal_geocoder.country_column})) as country
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
cities = search_terms.map { |row| row[:city] }.join(',')
countries = search_terms.map { |row| row[:country] }.join(',')
"WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], #{region}, Array[#{countries}])).*) SELECT q, c, a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string)
AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
AND dest.cartodb_georef_status IS NULL
}
end
end
end
end
@@ -0,0 +1,40 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class CitiesTextColumnPoints < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT
trim(quote_nullable("#{@internal_geocoder.column_name}")) AS city,
trim(quote_nullable(#{@internal_geocoder.region_column})) AS region
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
cities = search_terms.map { |row| row[:city] }.join(',')
regions = search_terms.map { |row| row[:region] }.join(',')
"WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], Array[#{regions}], #{country})).*) SELECT q, c, a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
AND trim(dest.#{@internal_geocoder.region_column}::text) = trim(orig.region)
AND dest.cartodb_georef_status IS NULL
}
end
end # CitiesTextColumnPoints
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,36 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class CitiesTextTextPoints < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS city
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
cities = search_terms.map { |row| row[:city] }.join(',')
"WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], #{region}, #{country})).*) SELECT q, c, a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
AND dest.cartodb_georef_status IS NULL
}
end
end # CitiesTextTextPoints
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,42 @@
module CartoDB
module InternalGeocoder
class InputTypeResolver
def initialize(internal_geocoder)
@internal_geocoder = internal_geocoder
end
def type
[kind, country_input_type, region_input_type, geometry_type]
end
def kind
@internal_geocoder.kind
end
def country_input_type
if @internal_geocoder.country_column
:column
else
:text
end
end
def region_input_type
if @internal_geocoder.region_column
:column
elsif @internal_geocoder.regions.present?
:text
end
end
def geometry_type
@internal_geocoder.geometry_type
end
end # InputTypeResolver
end # InternalGeocoder
end
@@ -0,0 +1,35 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class IpAddressTextPoint < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS ipaddress
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
ipaddress = search_terms.map { |row| row[:ipaddress] }.join(',')
"WITH geo_function AS (SELECT (geocode_ip(Array[#{ipaddress}])).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}
end
end # IpAddressTextPoint
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,55 @@
require_relative '../../../importer/lib/importer/query_batcher'
module CartoDB
module InternalGeocoder
class LatitudeLongitude
def initialize(db, logger = nil)
@db = db
@logger = logger
end
def geocode(table_schema, table_name, latitude_column, longitude_column)
qualified_table_name = "\"#{table_schema}\".\"#{table_name}\""
query_fragment_update = %Q{
UPDATE #{qualified_table_name}
SET
the_geom = ST_GeomFromText(
'POINT(' || REPLACE(TRIM(CAST("#{longitude_column}" AS text)), ',', '.') || ' ' ||
REPLACE(TRIM(CAST("#{latitude_column}" AS text)), ',', '.') || ')', 4326
)
}
query_fragment_where = %Q{
REPLACE(TRIM(CAST("#{longitude_column}" AS text)), ',', '.') ~
'^(([-+]?(([0-9]|[1-9][0-9]|1[0-7][0-9])(\.[0-9]+)?))|[-+]?180)$'
AND REPLACE(TRIM(CAST("#{latitude_column}" AS text)), ',', '.') ~
'^(([-+]?(([0-9]|[1-8][0-9])(\.[0-9]+)?))|[-+]?90)$'
}
CartoDB::Importer2::QueryBatcher.new(
@db,
@logger,
!table_has_cartodb_id(table_schema, table_name)
).execute_update(
%Q{#{query_fragment_update} where #{query_fragment_where}},
table_schema, table_name
)
end
def table_has_cartodb_id(table_schema, table_name)
result = @db.fetch(%Q{
select *
from information_schema.columns
where table_schema = '#{table_schema}'
and table_name = '#{table_name}'
and column_name = 'cartodb_id'
and data_type = 'integer';
}).all
!result[0].nil?
end
end
end
end
@@ -0,0 +1,38 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class PostalcodeColumnPoints < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT
trim(quote_nullable(#{@internal_geocoder.column_name})) as postalcode,
trim(quote_nullable(#{@internal_geocoder.country_column})) as country
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
countries = search_terms.map { |row| row[:country] }.join(',')
"WITH geo_function AS (SELECT (geocode_postalcode_points(Array[#{postalcodes}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}
end
end
end
end
@@ -0,0 +1,38 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class PostalcodeColumnPolygon < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT
trim(quote_nullable(#{@internal_geocoder.column_name})) as postalcode,
trim(quote_nullable(#{@internal_geocoder.country_column})) as country
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
countries = search_terms.map { |row| row[:country] }.join(',')
"WITH geo_function AS (SELECT (geocode_postalcode_polygons(Array[#{postalcodes}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}
end
end
end
end
@@ -0,0 +1,35 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class PostalcodeTextPoints < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS postalcode
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
"WITH geo_function AS (SELECT (geocode_postalcode_points(Array[#{postalcodes}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}
end
end # PostalcodeTextPoints
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,35 @@
require_relative 'abstract_query_generator'
module CartoDB
module InternalGeocoder
class PostalcodeTextPolygon < AbstractQueryGenerator
def search_terms_query(page)
%Q{
SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS postalcode
FROM #{@internal_geocoder.qualified_table_name}
WHERE cartodb_georef_status IS NULL
LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
}
end
def dataservices_query(search_terms)
postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
"WITH geo_function AS (SELECT (geocode_postalcode_polygons(Array[#{postalcodes}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
end
def copy_results_to_table_query
%Q{
UPDATE #{dest_table} AS dest
SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
cartodb_georef_status = orig.cartodb_georef_status
FROM #{@internal_geocoder.temp_table_name} AS orig
WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
}
end
end # PostalcodeTextPolygon
end # InternalGeocoder
end # CartoDB
@@ -0,0 +1,68 @@
require_relative 'input_type_resolver'
require_relative 'cities_text_text_points'
require_relative 'cities_column_text_points'
require_relative 'cities_text_column_points'
require_relative 'cities_column_column_points'
require_relative 'admin0_text_polygons'
require_relative 'admin1_text_polygons'
require_relative 'admin1_column_polygons'
require_relative 'postalcode_text_points'
require_relative 'postalcode_column_points'
require_relative 'postalcode_text_polygon'
require_relative 'postalcode_column_polygon'
require_relative 'ipaddress_text_point'
module CartoDB
module InternalGeocoder
class QueryGeneratorFactory
class QueryGeneratorNotImplemented < StandardError; end
class << self
private :new
def get(internal_geocoder, input_type=nil)
input_type ||= InputTypeResolver.new(internal_geocoder).type
case input_type
when [:namedplace, :text, nil, :point]
CitiesTextTextPoints.new internal_geocoder
when [:namedplace, :text, :text, :point]
CitiesTextTextPoints.new internal_geocoder
when [:namedplace, :text, :column, :point]
CitiesTextColumnPoints.new internal_geocoder
when [:namedplace, :column, nil, :point]
CitiesColumnTextPoints.new internal_geocoder
when [:namedplace, :column, :text, :point]
CitiesColumnTextPoints.new internal_geocoder
when [:namedplace, :column, :column, :point]
CitiesColumnColumnPoints.new internal_geocoder
when [:admin0, :text, nil, :polygon]
Admin0TextPolygons.new internal_geocoder
when [:admin1, :text, nil, :polygon]
Admin1TextPolygons.new internal_geocoder
when [:admin1, :column, nil, :polygon]
Admin1ColumnPolygons.new internal_geocoder
when [:postalcode, :text, nil, :point]
PostalcodeTextPoints.new internal_geocoder
when [:postalcode, :column, nil, :point]
PostalcodeColumnPoints.new internal_geocoder
when [:postalcode, :text, nil, :polygon]
PostalcodeTextPolygon.new internal_geocoder
when [:postalcode, :column, nil, :polygon]
PostalcodeColumnPolygon. new internal_geocoder
when [:ipaddress, :text, nil, :point]
IpAddressTextPoint.new internal_geocoder
else
raise QueryGeneratorNotImplemented. new "QueryGenerator not implemented for input type #{input_type}"
end
end
end
end # QueryGeneratorFactory
end # InternalGeocoder
end #CartoDB