Initial commit

services/table-geocoder/lib/abstract_table_geocoder.rb

@@ -0,0 +1,99 @@
+require_relative 'exceptions'
+require 'active_support/core_ext/numeric'
+
+module CartoDB
+  class AbstractTableGeocoder
+
+    DB_STATEMENT_TIMEOUT_MS = 5.hours.to_i * 1000
+
+    attr_reader :connection
+
+    def initialize(arguments)
+      @connection  = arguments.fetch(:connection)
+      @table_name  = arguments[:table_name]
+      @table_schema  = arguments[:table_schema]
+      @qualified_table_name = arguments[:qualified_table_name]
+      @sequel_qualified_table_name = arguments[:sequel_qualified_table_name]
+      @schema = arguments[:schema] || 'cdb'
+      @state = 'submitted'
+      @connection.run("SET statement_timeout TO #{DB_STATEMENT_TIMEOUT_MS}")
+    end
+
+    def cancel
+      raise 'Not implemented'
+    end
+
+    def run
+      raise 'Not implemented'
+    end
+
+    def remote_id
+      raise 'Not implemented'
+    end
+
+    def update_geocoding_status
+      raise 'Not implemented'
+    end
+
+    def process_results
+      raise 'Not implemented'
+    end
+
+    def name
+      raise 'Not implemented'
+    end
+
+    def used_batch_request?
+      false
+    end
+
+    def reset_cartodb_georef_status
+      ensure_georef_status_colummn_valid
+      set_georef_status_to_null
+    end
+
+    # Makes sure there's a cartodb_georef_status_column and marks all geocodifiable rows with NULL.
+    # This is important because otherwise it is hard to track what rows have been processed or not.
+    def mark_rows_to_geocode
+      ensure_georef_status_colummn_valid
+      set_georef_status_from_false_to_null
+    end
+
+
+    protected
+
+    def ensure_georef_status_colummn_valid
+      connection.run(%Q{
+          ALTER TABLE #{@qualified_table_name}
+          ADD COLUMN cartodb_georef_status BOOLEAN DEFAULT NULL
+        })
+    rescue Sequel::DatabaseError => e
+      if e.message =~ /canceling statement due to statement timeout/
+        raise Carto::GeocoderErrors::AddGeorefStatusColumnDbTimeoutError.new
+      end
+      raise unless e.message =~ /column .* of relation .* already exists/
+      cast_georef_status_column
+    end
+
+
+    private
+
+    def set_georef_status_to_null
+      connection.select.from(@sequel_qualified_table_name).update(:cartodb_georef_status => nil)
+    end
+
+    def set_georef_status_from_false_to_null
+      connection.select.from(@sequel_qualified_table_name).where(:cartodb_georef_status => false).update(:cartodb_georef_status => nil)
+    end
+
+    def cast_georef_status_column
+      connection.run(%Q{
+        ALTER TABLE #{@qualified_table_name} ALTER COLUMN cartodb_georef_status
+        TYPE boolean USING cast(cartodb_georef_status as boolean)
+      })
+    rescue => e
+      raise "Error converting cartodb_georef_status to boolean, please, convert it manually or remove it."
+    end
+
+  end
+end

services/table-geocoder/lib/exceptions.rb

@@ -0,0 +1,119 @@
+require 'active_support/core_ext/string'
+
+module Carto
+  module GeocoderErrors
+
+    GEOCODER_TIMED_OUT_TITLE = 'The geocoder timed out'
+    GEOCODER_TIMED_OUT_WHAT_ABOUT = %q{
+      Your geocoding request timed out.
+      Please <a href='mailto:support@cartob.com?subject=The geocoder timed out'>contact us</a>
+      and we'll try to fix it quickly.
+    }.squish
+
+    class AdditionalInfo
+      SOURCE_CARTODB = 'cartodb'
+      SOURCE_USER = 'user'
+
+      attr_accessor :error_code, :title, :what_about, :source
+
+      def initialize(error_code, title, what_about, source)
+        self.error_code = error_code
+        self.title = title
+        self.what_about = what_about
+        self.source = source
+      end
+    end
+
+    class GeocoderBaseError < StandardError
+      @@error_code_info_map = {}
+
+      attr_reader :original_exception
+
+      def initialize(original_exception=nil)
+        message = self.class.to_s
+        if original_exception
+          message << " " << original_exception.message
+          @original_exception = original_exception
+        end
+        super(message) # this is the only way of setting the message
+        set_backtrace(original_exception.backtrace) if original_exception # this line must appear after calling super
+      end
+
+      def original_message
+        @original_exception.message if @original_exception
+      end
+
+      def self.register_additional_info(error_code, title, what_about, source)
+        raise 'Duplicate error code' if @@error_code_info_map.has_key?(error_code)
+        @additional_info = AdditionalInfo.new(error_code, title, what_about, source)
+        @@error_code_info_map[@additional_info.error_code] = @additional_info
+      end
+
+      def self.get_info(error_code)
+        @@error_code_info_map[error_code]
+      end
+
+      class << self
+        attr_reader :additional_info
+      end
+    end
+
+    # just a convenience
+    def self.additional_info(error_code)
+      GeocoderBaseError.get_info(error_code)
+    end
+
+
+
+    class MisconfiguredGmeGeocoderError < GeocoderBaseError
+      register_additional_info(
+        1000,
+        'Google for Work account misconfigured',
+        %q{Your Google for Work account seems to be incorrectly configured.
+           Please <a href='mailto:sales@cartob.com?subject=Google for Work account misconfigured'>contact us</a>
+           and we'll try to fix it quickly.}.squish,
+        AdditionalInfo::SOURCE_USER
+        )
+    end
+
+    class GmeGeocoderTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1010,
+        'Google geocoder timed out',
+        %q{Your geocoding request timed out after several attempts.
+           Please check your quota usage in the <a href='https://console.developers.google.com/'>Google Developers Console</a>
+           and <a href='mailto:support@carto.com?subject=Google geocoder timed out'>contact us</a>
+           if you are within the usage limits.}.squish,
+        AdditionalInfo::SOURCE_USER
+        )
+    end
+
+    class AddGeorefStatusColumnDbTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1020,
+        GEOCODER_TIMED_OUT_TITLE,
+        GEOCODER_TIMED_OUT_WHAT_ABOUT,
+        AdditionalInfo::SOURCE_CARTODB
+        )
+    end
+
+    class GeocoderCacheDbTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1030,
+        GEOCODER_TIMED_OUT_TITLE,
+        GEOCODER_TIMED_OUT_WHAT_ABOUT,
+        AdditionalInfo::SOURCE_CARTODB
+        )
+    end
+
+    class TableGeocoderDbTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1040,
+        GEOCODER_TIMED_OUT_TITLE,
+        GEOCODER_TIMED_OUT_WHAT_ABOUT,
+        AdditionalInfo::SOURCE_CARTODB
+        )
+    end
+
+  end
+end

services/table-geocoder/lib/geocoder_cache.rb

@@ -0,0 +1,177 @@
+require_relative 'exceptions'
+require_relative '../../../lib/carto/http/client'
+
+module CartoDB
+  class GeocoderCache
+
+    DEFAULT_BATCH_SIZE = 5000
+    DEFAULT_MAX_ROWS   = 1000000
+    HTTP_CONNECT_TIMEOUT = 60
+    HTTP_DEFAULT_TIMEOUT = 600
+
+    attr_reader :connection, :working_dir, :table_name, :hits, :misses,
+                :max_rows, :sql_api, :formatter, :cache_results
+
+    def initialize(arguments)
+      @sql_api = arguments.fetch(:sql_api)
+      @connection = arguments.fetch(:connection)
+      @table_name = arguments.fetch(:table_name)
+      @qualified_table_name = arguments.fetch(:qualified_table_name)
+      @working_dir = arguments[:working_dir] || Dir.mktmpdir
+      `chmod 777 #{@working_dir}`
+      @formatter = arguments.fetch(:formatter)
+      @max_rows = arguments[:max_rows] || DEFAULT_MAX_ROWS
+      @cache_results = nil
+      @batch_size = arguments[:batch_size] || DEFAULT_BATCH_SIZE
+      @cache_results = File.join(working_dir, "#{temp_table_name}_results.csv")
+      @usage_metrics = arguments.fetch(:usage_metrics)
+      @log = arguments.fetch(:log)
+      init_rows_count
+    end
+
+    def run
+      @log.append_and_store "Started searching previous geocoded results in geocoder cache"
+      get_cache_results
+      create_temp_table
+      load_results_to_temp_table
+      @hits = connection.select.from(temp_table_name).where('longitude is not null and latitude is not null').count.to_i
+      copy_results_to_table
+      @log.append_and_store "Finished geocoder cache job"
+    rescue => e
+      @log.append_and_store "Error getting results from geocoder cache: #{e.inspect}"
+      handle_cache_exception e
+    ensure
+      @usage_metrics.incr(:geocoder_cache, :total_requests, @total_rows)
+      @usage_metrics.incr(:geocoder_cache, :success_responses, @hits)
+      @usage_metrics.incr(:geocoder_cache, :empty_responses, (@total_rows - @hits - @failed_rows))
+      @usage_metrics.incr(:geocoder_cache, :failed_responses, @failed_rows)
+      update_log_stats
+    end
+
+    def get_cache_results
+      begin
+        count = count + 1 rescue 0
+        limit = [@batch_size, @max_rows - (count * @batch_size)].min
+        rows = connection.fetch(%Q{
+            SELECT DISTINCT(md5(#{formatter})) AS searchtext
+            FROM #{@qualified_table_name}
+            WHERE cartodb_georef_status IS NULL
+            LIMIT #{limit} OFFSET #{count * @batch_size}
+        }).all
+        @total_rows += rows.size
+        sql   = "WITH addresses(address) AS (VALUES "
+        sql << rows.map { |r| "('#{r[:searchtext]}')" }.join(',')
+        sql << ") SELECT DISTINCT ON(geocode_string) st_x(g.the_geom) longitude, st_y(g.the_geom) latitude,g.geocode_string FROM addresses a INNER JOIN #{sql_api[:table_name]} g ON md5(g.geocode_string)=a.address"
+        response = run_query(sql, 'csv').gsub(/\A.*/, '').gsub(/^$\n/, '')
+        File.open(cache_results, 'a') { |f| f.write(response.force_encoding("UTF-8")) } unless response == "\n"
+      end while rows.size >= @batch_size && (count * @batch_size) + rows.size < @max_rows
+    end
+
+    def store
+      begin
+        count = count + 1 rescue 0
+        sql   = %Q{
+           WITH
+            -- write the new values
+           n(searchtext, the_geom) AS (
+              VALUES %%VALUES%%
+           ),
+            -- update existing rows
+           upsert AS (
+              UPDATE #{sql_api[:table_name]} o
+              SET updated_at = NOW()
+              FROM n WHERE o.geocode_string = n.searchtext
+              RETURNING o.geocode_string
+           )
+           -- insert missing rows
+           INSERT INTO #{sql_api[:table_name]} (geocode_string,the_geom)
+           SELECT n.searchtext, n.the_geom FROM n
+           WHERE n.searchtext NOT IN (
+            SELECT geocode_string FROM upsert
+           );
+        }
+        rows = connection.fetch(%Q{
+          SELECT DISTINCT(quote_nullable(#{formatter})) AS searchtext, the_geom
+          FROM #{@qualified_table_name} AS orig
+          WHERE orig.cartodb_georef_status IS TRUE AND the_geom IS NOT NULL
+          LIMIT #{@batch_size} OFFSET #{count * @batch_size}
+        }).all
+        sql.gsub! '%%VALUES%%', rows.map { |r| "(#{r[:searchtext]}, '#{r[:the_geom]}')" }.join(',')
+        run_query(sql) if rows && rows.size > 0
+      end while rows.size >= @batch_size
+    rescue => e
+      handle_cache_exception e
+    ensure
+      drop_temp_table
+    end
+
+    def create_temp_table
+      connection.run(%Q{
+        CREATE TABLE #{temp_table_name} (
+          longitude text, latitude text, geocode_string text
+        );
+      })
+    end
+
+    def load_results_to_temp_table
+      connection.copy_into(Sequel.lit(temp_table_name), data: File.read(cache_results), format: :csv)
+    end
+
+    def copy_results_to_table
+      connection.run(%Q{
+        UPDATE #{@qualified_table_name} AS dest
+        SET the_geom = ST_GeomFromText(
+              'POINT(' || orig.longitude || ' ' || orig.latitude || ')', 4326
+            ),
+            cartodb_georef_status = TRUE
+        FROM #{temp_table_name} AS orig
+        WHERE #{formatter} = orig.geocode_string
+      })
+    end
+
+    def drop_temp_table
+      connection.run("DROP TABLE IF EXISTS #{temp_table_name}")
+    end
+
+    def temp_table_name
+      @temp_table_name ||= "geocoding_cache_#{Time.now.to_i}"
+    end
+
+    def run_query(query, format = '')
+      params = { q: query, api_key: sql_api[:api_key], format: format }
+      http_client = Carto::Http::Client.get('geocoder_cache',
+                                            log_requests: true)
+      response = http_client.post(sql_api[:base_url],
+                                  body: URI.encode_www_form(params),
+                                  connecttimeout: HTTP_CONNECT_TIMEOUT,
+                                  timeout: HTTP_DEFAULT_TIMEOUT)
+      response.body
+    end
+
+    # It handles in such a way that the caching is silently stopped
+    def handle_cache_exception(exception)
+      drop_temp_table
+      if exception.class == Sequel::DatabaseError && exception.message =~ /canceling statement due to statement timeout/
+        # for the moment we just wrap the exception to get a specific error in rollbar
+        exception =  Carto::GeocoderErrors::GeocoderCacheDbTimeoutError.new(exception)
+      end
+      # In case we get some error we are going to pass all the rows as failed
+      @failed_rows = @total_rows
+      CartoDB.notify_exception(exception)
+    end
+
+    private
+
+    def init_rows_count
+      @hits = 0
+      @total_rows = 0
+      @failed_rows = 0
+    end
+
+    def update_log_stats
+      @log.append_and_store "Geocoding cache stats update. "\
+        "Total rows: #{@total_rows} "\
+        "--- Hits: #{@hits} --- Failed: #{@failed_rows}"
+    end
+  end
+end

services/table-geocoder/lib/gme/client.rb

@@ -0,0 +1,121 @@
+require 'addressable/uri'
+require 'set'
+require 'json'
+require_relative 'exceptions'
+require_relative '../exceptions'
+require_relative '../../../../lib/url_signer'
+require_relative '../../../../lib/carto/http/client'
+
+module Carto
+  module Gme
+
+    # The responsibility of this class is to perform requests to gme
+    # taking care of sigining requests, usage limits, errors, retries, etc.
+    class Client
+
+      BASE_URL = 'https://maps.googleapis.com'
+
+      DEFAULT_CONNECT_TIMEOUT = 15
+      DEFAULT_READ_TIMEOUT = 30
+      DEFAULT_RETRY_TIMEOUT = 60
+
+      OK_STATUS = 'OK'
+      ZERO_RESULTS_STATUS = 'ZERO_RESULTS'
+      OVERQUERY_LIMIT_STATUS = 'OVER_QUERY_LIMIT'
+      REQUEST_DENIED_STATUS = 'REQUEST_DENIED'
+      INVALID_REQUEST_STATUS = 'INVALID_REQUEST'
+      UNKNOWN_ERROR_STATUS = 'UNKNOWN_ERROR'
+
+      HTTP_CLIENT_TAG = 'gme_client'
+
+      RETRIABLE_STATUSES = Set.new [500, 503, 504]
+
+      # Performs requests to Google Maps API web services
+      # Based on https://github.com/googlemaps/google-maps-services-python/blob/master/googlemaps/client.py
+      def initialize(client_id, private_key, options = {})
+        @client_id = client_id
+        @url_signer = UrlSigner.new(private_key)
+        @connect_timeout = options[:connect_timeout] || DEFAULT_CONNECT_TIMEOUT
+        @read_timeout = options[:read_timeout] || DEFAULT_READ_TIMEOUT
+        @retry_timeout = options[:retry_timeout] || DEFAULT_RETRY_TIMEOUT
+        @http_client = Carto::Http::Client.get(HTTP_CLIENT_TAG)
+      end
+
+      def get(endpoint, params, first_request_time=nil, retry_counter=0)
+        first_request_time ||= Time.now
+        elapsed = Time.now - first_request_time
+        if elapsed > @retry_timeout
+          raise Carto::GeocoderErrors::GmeGeocoderTimeoutError.new('retry timeout expired')
+        end
+
+        # 0.5 * (1.5 ^ i) is an increased sleep time of 1.5x per iteration,
+        # starting at 0.5s when retry_counter=0. The first retry will occur
+        # at 1, so subtract that first.
+        if retry_counter > 0
+          delay_seconds = 0.1 * 1.5 ** (retry_counter - 1)
+          sleep delay_seconds
+        end
+
+        url = generate_auth_url(BASE_URL+endpoint, params)
+
+        resp = @http_client.get(url, timeout: @read_timeout, connecttimeout: @connect_timeout)
+        raise Timeout.new('http request timed out') if resp.timed_out?
+
+        if RETRIABLE_STATUSES.include?(resp.code)
+          return self.get(endpoint, params, first_request_time, retry_counter+1)
+        end
+
+        begin
+          get_body(resp)
+        rescue OverQueryLimit
+          CartoDB.notify_debug('Carto::Gme::Client rescuing from OverQueryLimit exception', params.symbolize_keys)
+          return self.get(endpoint, params, first_request_time, retry_counter+1)
+        end
+      end
+
+
+      private
+
+      def generate_auth_url(path, params)
+        uri = Addressable::URI.new
+        uri.path = path
+        uri.query_values = params.merge(client: @client_id)
+        @url_signer.sign_url(uri.request_uri)
+      end
+
+      # Takes a typhoeus response object and returns a hash
+      def get_body(resp)
+        if resp.code != 200
+          # Remove temporarily from rollbar because it's flooding the logs
+          if resp.code != 400
+            CartoDB::Logger.warning(message: 'Error response from GME client',
+                                    client_id: @client_id,
+                                    code: resp.code,
+                                    response_body: resp.response_body)
+          end
+          raise HttpError.new(resp.code)
+        end
+
+        body = JSON::parse(resp.body)
+
+        api_status = body['status']
+        if api_status == 'OK' || api_status == 'ZERO_RESULTS'
+          return body
+        end
+
+        if api_status == 'OVER_QUERY_LIMIT'
+          raise OverQueryLimit.new
+        end
+
+        if body.has_key?('error_message')
+          raise ApiError.new(api_status, body['error_message'])
+        else
+          raise ApiError.new(api_status)
+        end
+
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/gme/convert.rb

@@ -0,0 +1,16 @@
+module Carto
+  module Gme
+
+    module Convert
+
+      def components
+        raise 'not implemented'
+      end
+
+      def bounds
+        raise 'not implemented'
+      end
+    end
+
+  end
+end

services/table-geocoder/lib/gme/exceptions.rb

@@ -0,0 +1,23 @@
+module Carto
+  module Gme
+
+    # TODO take care of these exceptions to provide better feedback to the user
+    class ClientException < StandardError; end
+
+    class Timeout < ClientException; end
+
+    class HttpError < ClientException; end
+
+    class OverQueryLimit < ClientException; end
+
+    class ApiError < ClientException
+      attr_reader :api_status, :error_message
+      def initialize(api_status, error_message=nil)
+        super(%Q{api_status = #{api_status}, error_message = "#{error_message}"})
+        @api_status = api_status
+        @error_message = error_message
+      end
+    end
+
+  end
+end

services/table-geocoder/lib/gme/geocoder_client.rb

@@ -0,0 +1,30 @@
+require_relative 'client'
+require_relative 'convert'
+
+
+module Carto
+  module Gme
+
+    # The responsibility of this class is to geocode addresses by requesting gme.
+    class GeocoderClient
+
+      attr_reader :client
+
+      def initialize(client)
+        @client = client
+      end
+
+      def geocode(address=nil, components=nil, bounds=nil, region=nil, language=nil)
+        params = {}
+        params['address'] = address if address
+        params['components'] = Convert.components(components) if components # TODO convert
+        params['bounds'] = Convert.bounds(bounds) if bounds # TODO convert
+        params['region'] = region if region
+        params['language'] = language if language
+
+        client.get('/maps/api/geocode/json', params)
+      end
+    end
+
+  end
+end

services/table-geocoder/lib/gme/table_geocoder.rb

@@ -0,0 +1,197 @@
+require_relative '../abstract_table_geocoder'
+require_relative 'client'
+require_relative 'geocoder_client'
+
+module Carto
+  module Gme
+    class TableGeocoder < CartoDB::AbstractTableGeocoder
+
+      DEFAULT_MAX_BLOCK_SIZE = 1000
+
+      # See https://developers.google.com/maps/documentation/geocoding/#Types
+      ACCEPTED_ADDRESS_TYPES = ['street_address', 'route', 'intersection', 'neighborhood']
+
+      attr_reader :original_formatter, :processed_rows, :successful_processed_rows, :failed_processed_rows,
+                  :empty_processed_rows, :state, :max_block_size
+
+      def initialize(arguments)
+        super(arguments)
+        @original_formatter = arguments.fetch(:original_formatter)
+        client_id = arguments.fetch(:client_id)
+        private_key = arguments.fetch(:private_key)
+        @max_block_size = arguments[:max_block_size] || DEFAULT_MAX_BLOCK_SIZE
+        gme_client = Client.new(client_id, private_key)
+        @geocoder_client = GeocoderClient.new(gme_client)
+        @usage_metrics = arguments.fetch(:usage_metrics)
+        @log = arguments.fetch(:log)
+        @geocoding_model = arguments.fetch(:geocoding_model)
+      end
+
+      def cancel; end
+
+      def run
+        change_status('running')
+        init_rows_count
+        ensure_georef_status_colummn_valid
+
+        # Here's the actual stuff
+        data_input_blocks.each do |data_block|
+          geocode(data_block)
+          update_table(data_block)
+          @processed_rows += data_block.size
+        end
+
+        change_status('completed')
+      rescue => e
+        change_status('failed')
+        raise e
+      ensure
+        total_requests = @successful_processed_rows + @empty_processed_rows + @failed_processed_rows
+        @usage_metrics.incr(:geocoder_google, :success_responses, @successful_processed_rows)
+        @usage_metrics.incr(:geocoder_google, :empty_responses, @empty_processed_rows)
+        @usage_metrics.incr(:geocoder_google, :failed_responses, @failed_processed_rows)
+        @usage_metrics.incr(:geocoder_google, :total_requests, total_requests)
+        update_log_stats
+      end
+
+      # Empty methods, needed because they're triggered from geocoding.rb
+      def remote_id; end
+      def process_results; end # TODO: can be removed from here and abstract class
+
+      def update_geocoding_status
+        { processed_rows: processed_rows, state: @geocoding_model.state }
+      end
+
+      def name
+        'google'
+      end
+
+      private
+
+      # Returns a "generator"
+      def data_input_blocks
+        Enumerator.new do |enum|
+          begin
+            data_input = connection.select(:cartodb_id, searchtext_expression)
+              .from(@sequel_qualified_table_name)
+              .where(cartodb_georef_status: nil)
+              .limit(max_block_size)
+              .all
+            enum.yield data_input
+            # last iteration when data_input.length < max_block_size, no need for another query
+          end while data_input.length == max_block_size
+        end
+      end
+
+      def searchtext_expression
+        # The original_formatter has the following format:
+        #   `{street_column_name}[[, additional_free_text][, {province_column_name}][, country_free_text]]`
+        # See https://github.com/jeremyevans/sequel/blob/master/doc/security.rdoc
+        # See http://sequel.jeremyevans.net/rdoc/classes/Sequel/SQL/Builders.html
+        atoms = original_formatter.split(',').map {|s| s.strip }
+        Sequel.join(atoms.map { |atom|
+                      if match = /\A{(?<column_name>.*)}\z/.match(atom)
+                        Sequel.identifier(match[:column_name])
+                      else
+                        atom
+                      end
+                    }, ',').as(:searchtext)
+      end
+
+      def geocode(data_block)
+        data_block.each do |row|
+          response = fetch_from_gme(row[:searchtext])
+          # If we get an error we get nil so we pass to the next row
+          next if response.nil?
+          if response['status'] != Client::OK_STATUS
+            process_error_or_empty_status(response['status'])
+            row.merge!(cartodb_georef_status: false)
+          else
+            result = response['results'].select { |r| r['types'] & ACCEPTED_ADDRESS_TYPES }.first
+            if result.nil?
+              @empty_processed_rows += 1
+              row.merge!(cartodb_georef_status: false)
+            else
+              @successful_processed_rows += 1
+              location = result['geometry']['location']
+              row.merge!(location.deep_symbolize_keys.merge(cartodb_georef_status: true))
+            end
+          end
+        end
+      end
+
+      def update_table(data_block)
+        # At this point, data_block is an Array that looks like this:
+        #   [{:cartodb_id=>1, :searchtext=>"Some real street name", :lat=>19.29544, :lng=>-99.1472101, :cartodb_georef_status=>true},
+        #    {:cartodb_id=>2, :searchtext=>"foo", :cartodb_georef_status=>false}]
+        geocoded = data_block.select {|row| row[:cartodb_georef_status] == true}
+        if geocoded.count > 0
+          geocoded_to_sql = geocoded.map {|row| "(#{row[:cartodb_id]}, #{row[:lat]}, #{row[:lng]})"}.join(',')
+          query_geocoded = %Q{
+            UPDATE #{@qualified_table_name} as target SET
+              the_geom = CDB_LatLng(geocoded.lat,geocoded.lng),
+              cartodb_georef_status = TRUE
+            FROM (VALUES
+              #{geocoded_to_sql}
+            ) as geocoded(cartodb_id,lat,lng)
+            WHERE target.cartodb_id = geocoded.cartodb_id;
+          }
+          connection.run(query_geocoded)
+        end
+
+        non_geocoded = data_block.select {|row| row[:cartodb_georef_status] == false}
+        if non_geocoded.count > 0
+          non_geocoded_to_sql = non_geocoded.map {|row| "(#{row[:cartodb_id]})"}.join(',')
+          query_non_geocoded = %Q{
+            UPDATE #{@qualified_table_name} as target SET
+              cartodb_georef_status = FALSE
+            FROM (VALUES
+              #{non_geocoded_to_sql}
+            ) as nongeocoded(cartodb_id)
+            WHERE target.cartodb_id = nongeocoded.cartodb_id;
+          }
+          connection.run(query_non_geocoded)
+        end
+      end
+
+      private
+
+      def init_rows_count
+        @processed_rows = 0
+        @successful_processed_rows = 0
+        @failed_processed_rows = 0
+        @empty_processed_rows = 0
+      end
+
+      def fetch_from_gme(search_text)
+        @geocoder_client.geocode(search_text)
+      rescue => e
+        # Remove temporarily because it's flooding the logs
+        # @log.append_and_store "Error geocoding using GME for text #{search_text}: #{e.message}"
+        # CartoDB.notify_error('Error geocoding using GME', error: e.backtrace.join('\n'), search_text: search_text)
+        @failed_processed_rows += 1
+        nil
+      end
+
+      def process_error_or_empty_status(status)
+        case status
+          when Client::ZERO_RESULTS_STATUS then @empty_processed_rows += 1
+          else @failed_processed_rows += 1
+        end
+      end
+
+      def update_log_stats
+        @log.append_and_store "Geocoding using Google maps, job status update. "\
+          "Status: #{@status} --- Processed rows: #{@processed_rows} "\
+          "--- Success: #{@successful_processed_rows} --- Empty: #{@empty_processed_rows} "\
+          "--- Failed: #{@failed_processed_rows}"
+      end
+
+      def change_status(status)
+        @status = status
+        @geocoding_model.state = status
+        @geocoding_model.save
+      end
+    end
+  end
+end

services/table-geocoder/lib/internal-geocoder/abstract_query_generator.rb

@@ -0,0 +1,44 @@
+require_relative 'input_type_resolver'
+require_relative '../../../importer/lib/importer/query_batcher'
+
+module CartoDB
+  module InternalGeocoder
+
+    class AbstractQueryGenerator
+
+      def initialize(internal_geocoder)
+        @internal_geocoder = internal_geocoder
+      end
+
+      #TODO custom exception
+      def search_terms_query(page)
+        raise 'Not implemented'
+      end
+
+      def dataservices_query(search_terms)
+        raise 'Not implemented'
+      end
+
+      def copy_results_to_table_query
+        raise 'Not implemented'
+      end
+
+      def country
+        country = @internal_geocoder.countries
+        (country == %Q{'world'} || country.blank?) ? 'null' : country
+      end
+
+      def region
+        region = @internal_geocoder.regions
+        return 'null' if region.blank?
+        region
+      end
+
+      def dest_table
+        @internal_geocoder.qualified_table_name
+      end
+
+    end # AbstractQueryGenerator
+
+  end # InternalGeocoder
+end #CartoDB

services/table-geocoder/lib/internal-geocoder/admin0_text_polygons.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class Admin0TextPolygons < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        regions = search_terms.map { |row| row[:region] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_admin0_polygons(Array[#{regions}])).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # Admin0TextPolygons
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/admin1_column_polygons.rb

@@ -0,0 +1,40 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class Admin1ColumnPolygons < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as region,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        region = search_terms.map { |row| row[:region] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_admin1_polygons(Array[#{region}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # Admin1ColumnPolygons
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/admin1_text_polygons.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class Admin1TextPolygons < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        region = search_terms.map { |row| row[:region] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_admin1_polygons(Array[#{region}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # Admin1TextPolygons
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/cities_column_column_points.rb

@@ -0,0 +1,43 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesColumnColumnPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable("#{@internal_geocoder.column_name}")) AS city,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) AS country,
+            trim(quote_nullable(#{@internal_geocoder.region_column})) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        regions = search_terms.map { |row| row[:region] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], Array[#{regions}], Array[#{countries}])).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
+            AND trim(dest.#{@internal_geocoder.region_column}::text) = trim(orig.region)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # CitiesColumnColumnPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/cities_column_text_points.rb

@@ -0,0 +1,40 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesColumnTextPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as city,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], #{region}, Array[#{countries}])).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/cities_text_column_points.rb

@@ -0,0 +1,40 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesTextColumnPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable("#{@internal_geocoder.column_name}")) AS city,
+            trim(quote_nullable(#{@internal_geocoder.region_column})) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        regions = search_terms.map { |row| row[:region] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], Array[#{regions}], #{country})).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.region_column}::text) = trim(orig.region)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # CitiesTextColumnPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/cities_text_text_points.rb

@@ -0,0 +1,36 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesTextTextPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS city
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], #{region}, #{country})).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # CitiesTextTextPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/input_type_resolver.rb

@@ -0,0 +1,42 @@
+module CartoDB
+
+  module InternalGeocoder
+
+    class InputTypeResolver
+
+      def initialize(internal_geocoder)
+        @internal_geocoder = internal_geocoder
+      end
+
+      def type
+        [kind, country_input_type, region_input_type, geometry_type]
+      end
+
+      def kind
+        @internal_geocoder.kind
+      end
+
+      def country_input_type
+        if @internal_geocoder.country_column
+          :column
+        else
+          :text
+        end
+      end
+
+      def region_input_type
+        if @internal_geocoder.region_column
+          :column
+        elsif @internal_geocoder.regions.present?
+          :text
+        end
+      end
+
+      def geometry_type
+        @internal_geocoder.geometry_type
+      end
+
+    end # InputTypeResolver
+  end # InternalGeocoder
+
+end

services/table-geocoder/lib/internal-geocoder/ipaddress_text_point.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class IpAddressTextPoint < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS ipaddress
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        ipaddress = search_terms.map { |row| row[:ipaddress] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_ip(Array[#{ipaddress}])).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # IpAddressTextPoint
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/latitude_longitude.rb

@@ -0,0 +1,55 @@
+require_relative '../../../importer/lib/importer/query_batcher'
+
+module CartoDB
+  module InternalGeocoder
+
+    class LatitudeLongitude
+
+      def initialize(db, logger = nil)
+        @db = db
+        @logger = logger
+      end
+
+      def geocode(table_schema, table_name, latitude_column, longitude_column)
+        qualified_table_name = "\"#{table_schema}\".\"#{table_name}\""
+        query_fragment_update = %Q{
+          UPDATE #{qualified_table_name}
+          SET
+            the_geom = ST_GeomFromText(
+              'POINT(' || REPLACE(TRIM(CAST("#{longitude_column}" AS text)), ',', '.') || ' ' ||
+                REPLACE(TRIM(CAST("#{latitude_column}" AS text)), ',', '.') || ')', 4326
+            )
+        }
+        query_fragment_where = %Q{
+          REPLACE(TRIM(CAST("#{longitude_column}" AS text)), ',', '.') ~
+            '^(([-+]?(([0-9]|[1-9][0-9]|1[0-7][0-9])(\.[0-9]+)?))|[-+]?180)$'
+          AND REPLACE(TRIM(CAST("#{latitude_column}" AS text)), ',', '.')  ~
+            '^(([-+]?(([0-9]|[1-8][0-9])(\.[0-9]+)?))|[-+]?90)$'
+        }
+
+        CartoDB::Importer2::QueryBatcher.new(
+            @db, 
+            @logger, 
+            !table_has_cartodb_id(table_schema, table_name)
+          ).execute_update(
+              %Q{#{query_fragment_update} where #{query_fragment_where}},
+              table_schema, table_name
+          )
+      end
+
+      def table_has_cartodb_id(table_schema, table_name)
+        result = @db.fetch(%Q{
+          select *
+          from information_schema.columns
+          where table_schema = '#{table_schema}' 
+            and table_name = '#{table_name}' 
+            and column_name = 'cartodb_id'
+            and data_type = 'integer';
+        }).all
+        !result[0].nil?
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/postalcode_column_points.rb

@@ -0,0 +1,38 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeColumnPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as postalcode,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_points(Array[#{postalcodes}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/postalcode_column_polygon.rb

@@ -0,0 +1,38 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeColumnPolygon < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as postalcode,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_polygons(Array[#{postalcodes}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/postalcode_text_points.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeTextPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS postalcode
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_points(Array[#{postalcodes}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # PostalcodeTextPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/postalcode_text_polygon.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeTextPolygon < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS postalcode
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_polygons(Array[#{postalcodes}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # PostalcodeTextPolygon
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/query_generator_factory.rb

@@ -0,0 +1,68 @@
+require_relative 'input_type_resolver'
+
+require_relative 'cities_text_text_points'
+require_relative 'cities_column_text_points'
+require_relative 'cities_text_column_points'
+require_relative 'cities_column_column_points'
+require_relative 'admin0_text_polygons'
+require_relative 'admin1_text_polygons'
+require_relative 'admin1_column_polygons'
+require_relative 'postalcode_text_points'
+require_relative 'postalcode_column_points'
+require_relative 'postalcode_text_polygon'
+require_relative 'postalcode_column_polygon'
+require_relative 'ipaddress_text_point'
+
+module CartoDB
+  module InternalGeocoder
+
+    class QueryGeneratorFactory
+
+      class QueryGeneratorNotImplemented < StandardError; end
+
+      class << self
+        private :new
+
+        def get(internal_geocoder, input_type=nil)
+          input_type ||= InputTypeResolver.new(internal_geocoder).type
+
+          case input_type
+            when [:namedplace, :text, nil, :point]
+              CitiesTextTextPoints.new internal_geocoder
+            when [:namedplace, :text, :text, :point]
+              CitiesTextTextPoints.new internal_geocoder
+            when [:namedplace, :text, :column, :point]
+              CitiesTextColumnPoints.new internal_geocoder
+            when [:namedplace, :column, nil, :point]
+              CitiesColumnTextPoints.new internal_geocoder
+            when [:namedplace, :column, :text, :point]
+              CitiesColumnTextPoints.new internal_geocoder
+            when [:namedplace, :column, :column, :point]
+              CitiesColumnColumnPoints.new internal_geocoder
+            when [:admin0, :text, nil, :polygon]
+              Admin0TextPolygons.new internal_geocoder
+            when [:admin1, :text, nil, :polygon]
+              Admin1TextPolygons.new internal_geocoder
+            when [:admin1, :column, nil, :polygon]
+              Admin1ColumnPolygons.new internal_geocoder
+            when [:postalcode, :text, nil, :point]
+              PostalcodeTextPoints.new internal_geocoder
+            when [:postalcode, :column, nil, :point]
+              PostalcodeColumnPoints.new internal_geocoder
+            when [:postalcode, :text, nil, :polygon]
+              PostalcodeTextPolygon.new internal_geocoder
+            when [:postalcode, :column, nil, :polygon]
+              PostalcodeColumnPolygon. new internal_geocoder
+            when [:ipaddress, :text, nil, :point]
+              IpAddressTextPoint.new internal_geocoder
+            else
+              raise QueryGeneratorNotImplemented. new "QueryGenerator not implemented for input type #{input_type}"
+          end
+        end
+      end
+
+
+    end # QueryGeneratorFactory
+
+  end # InternalGeocoder
+end #CartoDB

services/table-geocoder/lib/internal_geocoder.rb

@@ -0,0 +1,156 @@
+require_relative '../../sql-api/sql_api'
+require_relative '../../importer/lib/importer/query_batcher'
+require_relative 'internal-geocoder/query_generator_factory'
+require_relative 'abstract_table_geocoder'
+
+module CartoDB
+  module InternalGeocoder
+    class Geocoder  < AbstractTableGeocoder
+
+      SQLAPI_CALLS_TIMEOUT = 45
+
+      attr_reader   :temp_table_name, :sql_api, :geocoding_results,
+                    :working_dir, :remote_id, :state, :processed_rows, :country_column, :region_column,
+                    :qualified_table_name, :batch_size, :countries, :regions, :kind, :geometry_type
+
+      attr_accessor :table_schema, :table_name, :column_name, :log
+
+      def initialize(arguments)
+        super(arguments)
+        @sql_api              = CartoDB::SQLApi.new(arguments.fetch(:internal)
+                                                             .merge({ timeout: SQLAPI_CALLS_TIMEOUT })
+                                                   )
+        @column_name          = arguments[:formatter]
+        @countries            = arguments[:countries].to_s
+        @country_column       = arguments[:country_column]
+        @regions              = arguments[:regions].to_s
+        @region_column        = arguments[:region_column]
+        @geometry_type        = arguments.fetch(:geometry_type, '').to_sym
+        @kind                 = arguments.fetch(:kind, '').to_sym
+        @batch_size           = (@geometry_type == :point ? 1000 : 10)
+        @working_dir          = arguments[:working_dir] || Dir.mktmpdir
+        @geocoding_results = File.join(working_dir, "#{temp_table_name}_results.csv".gsub('"', ''))
+        @query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get self
+        @log = arguments[:log]
+        @geocoding_model = arguments[:geocoding_model]
+        @usage_metrics = arguments.fetch(:usage_metrics)
+      end
+
+      def set_log(log)
+        @log = log
+      end
+
+      def run
+        log.append_and_store 'run()'
+        change_status('running')
+        ensure_georef_status_colummn_valid
+        download_results
+        create_temp_table
+        load_results_to_temp_table
+        copy_results_to_table
+        change_status('completed')
+      rescue => e
+        change_status('failed')
+        raise e
+      ensure
+        drop_temp_table
+        FileUtils.remove_entry_secure @working_dir if Dir.exists?(@working_dir)
+      end
+
+      def download_results
+        log.append_and_store 'download_results()'
+        begin
+          count = count + 1 rescue 0
+          search_terms = get_search_terms(count)
+          unless search_terms.size == 0
+            sql = @query_generator.dataservices_query(search_terms)
+
+            # Getting data from the internal geocoder is an all-or-nothing thing, so we
+            # log it as such, total_requests and failed_responses
+            begin
+              response = sql_api.fetch(sql, 'csv').gsub(/\A.*/, '').gsub(/^$\n/, '')
+            rescue CartoDB::SQLApi::SQLApiError => ex
+              @usage_metrics.incr(:geocoder_internal, :failed_responses, search_terms.length)
+              raise ex
+            ensure
+              @usage_metrics.incr(:geocoder_internal, :total_requests, search_terms.length)
+            end
+
+            # Count empty and successfully geocoded responses
+            empty_responses = 0
+            success_responses = 0
+            CSV.parse(response.chomp) do |row|
+              empty_responses += 1 if row[4] == "false"
+              success_responses += 1 if row[4] == "true"
+            end
+            @usage_metrics.incr(:geocoder_internal, :success_responses, success_responses)
+            @usage_metrics.incr(:geocoder_internal, :empty_responses, empty_responses)
+
+            log.append_and_store "Saving results to #{geocoding_results}"
+            File.open(geocoding_results, 'a') { |f| f.write(response.force_encoding("UTF-8")) } unless response.blank?
+          end
+        end while search_terms.size >= @batch_size
+        @processed_rows = `wc -l '#{geocoding_results}' 2>&1`.to_i
+        geocoding_results
+      end # download_results
+
+      def get_search_terms(page)
+        query = @query_generator.search_terms_query(page)
+        connection.fetch(query).all
+      end # get_search_terms
+
+      def create_temp_table
+        log.append_and_store 'create_temp_table()'
+        connection.run(%Q{
+          CREATE TABLE #{temp_table_name} (
+            geocode_string text, country text, region text, the_geom geometry, cartodb_georef_status boolean
+          );
+        })
+      end # create_temp_table
+
+      def update_geocoding_status
+        { processed_rows: processed_rows, state: state }
+      end # update_geocoding_status
+
+      def process_results; end
+      def cancel; end
+
+      def load_results_to_temp_table
+        log.append_and_store 'load_results_to_temp_table()'
+        connection.copy_into(Sequel.lit(temp_table_name), data: File.read(geocoding_results), format: :csv)
+      end # load_results_to_temp_table
+
+      def copy_results_to_table
+        log.append_and_store 'copy_results_to_table()'
+        # 'InternalGeocoder::copy_results_to_table'
+        CartoDB::Importer2::QueryBatcher.new(
+            connection,
+            nil,
+            create_seq_field = true,
+            batch_size
+          ).execute_update(
+            @query_generator.copy_results_to_table_query,
+            @table_schema, @table_name
+          )
+      end
+
+      def drop_temp_table
+        connection.run("DROP TABLE IF EXISTS #{temp_table_name}")
+      end # drop_temp_table
+
+      def temp_table_name
+        @temp_table_name ||= %Q{"#{@table_schema}".internal_geocoding_#{Time.now.to_i}}
+      end # temp_table_name
+
+      def name
+        'internal'
+      end
+
+      def change_status(status)
+        @status = status
+        @geocoding_model.state = status
+        @geocoding_model.save
+      end
+    end
+  end
+end

services/table-geocoder/lib/mail_geocoder.rb

@@ -0,0 +1,49 @@
+require_relative '../../../lib/resque/user_jobs'
+
+module CartoDB
+  module Geocoder
+    class MailNotifier
+
+      MIN_GEOCODER_TIME_TO_NOTIFY = 3 * 60 # seconds
+
+      def initialize(user_id, state, table_name, error_code, processable_rows, number_geocoded_rows, geocoding_time)
+        @user_id = user_id
+        @state = state
+        @table_name = table_name
+        @error_code = error_code
+        @processable_rows = processable_rows
+        @number_geocoded_rows = number_geocoded_rows
+        @geocoding_time = geocoding_time
+        @resque = ::Resque
+        @mail_sent = false
+      end
+
+      def notify_if_needed
+        send! if should_notify?
+      end
+
+      def should_notify?
+        @geocoding_time >= MIN_GEOCODER_TIME_TO_NOTIFY
+      end
+
+      def send!
+        @mail_sent = @resque.enqueue(
+          ::Resque::UserJobs::Mail::GeocoderFinished,
+          @user_id,
+          @state,
+          @table_name,
+          @error_code,
+          @processable_rows,
+          @number_geocoded_rows
+        )
+      end
+
+      def mail_sent?
+        return @mail_sent
+      end
+
+    end
+
+  end #Importer2
+end #CartoDB
+

services/table-geocoder/lib/table_geocoder.rb

@@ -0,0 +1,207 @@
+require 'uuidtools'
+require_relative '../../geocoder/lib/hires_geocoder_factory'
+require_relative '../../geocoder/lib/geocoder_config'
+require_relative 'geocoder_cache'
+require_relative 'abstract_table_geocoder'
+
+
+module CartoDB
+  class TableGeocoder < AbstractTableGeocoder
+
+    attr_reader   :working_dir, :csv_file, :result,
+                  :max_rows, :cache
+
+    attr_accessor :table_name, :formatter, :remote_id
+
+    def initialize(arguments)
+      super(arguments)
+      @working_dir = arguments[:working_dir] || Dir.mktmpdir
+      system('chmod', '777', @working_dir)
+      @formatter   = arguments[:formatter]
+      @remote_id   = arguments[:remote_id]
+      @max_rows    = arguments.fetch(:max_rows)
+      @usage_metrics = arguments.fetch(:usage_metrics)
+      @log = arguments.fetch(:log)
+      @geocoding_model = arguments.fetch(:geocoding_model)
+      @cache       = CartoDB::GeocoderCache.new(
+        connection:  connection,
+        formatter:   clean_formatter,
+        sql_api:     arguments[:cache],
+        working_dir: @working_dir,
+        table_name:  table_name,
+        qualified_table_name: @qualified_table_name,
+        max_rows:    @max_rows,
+        usage_metrics: @usage_metrics,
+        log: @log
+      )
+    end
+
+    def run
+      ensure_georef_status_colummn_valid
+      @number_of_rows_pre_cache = calculate_number_of_rows
+      cache.run unless cache_disabled?
+      @csv_file = generate_csv()
+      geocoder.run
+      # Sync state because cancel is made synchronous
+      @geocoding_model.refresh
+      if not @geocoding_model.cancelled?
+        process_results if @geocoding_model.state == 'completed'
+        cache.store unless cache_disabled?
+      end
+    ensure
+      self.remote_id = @geocoding_model.remote_id
+      update_metrics unless @geocoding_model.cancelled?
+    end
+
+    # TODO: make the geocoders update status directly in the model
+    def update_geocoding_status
+      geocoder.update_status
+      { processed_rows: geocoder.processed_rows, state: geocoder.status }
+    end
+
+    def cancel
+      # We have to be sure the cartodb_georef_status column exists
+      ensure_georef_status_colummn_valid
+      @number_of_rows_pre_cache = calculate_number_of_rows
+      geocoder.cancel
+    end
+
+    def process_results
+      download_results # TODO move to HiresBatchGeocoder
+      deflate_results # TODO move to HiresBatchGeocoder
+      create_temp_table
+      import_results_to_temp_table
+      load_results_into_original_table
+      mark_rows_not_geocoded
+    rescue Sequel::DatabaseError => e
+      if e.message =~ /canceling statement due to statement timeout/
+        # INFO: Timeouts here are not recoverable for batched geocodes, but they are for non-batched
+        # INFO: cache.store relies on having results in the target table
+        raise Carto::GeocoderErrors::TableGeocoderDbTimeoutError.new(e)
+      else
+        raise
+      end
+    ensure
+      drop_temp_table
+    end
+
+
+    def used_batch_request?
+      return geocoder.used_batch_request?
+    end
+
+    def name
+      'heremaps'
+    end
+
+    private
+
+    def geocoder
+      @geocoder ||= CartoDB::HiresGeocoderFactory.get(@csv_file, @working_dir, @log, @geocoding_model,
+                                                      @number_of_rows_pre_cache)
+    end
+
+    def cache_disabled?
+      GeocoderConfig.instance.get['disable_cache'] || false
+    end
+
+    def calculate_number_of_rows
+      rows = connection.fetch(%Q{
+        SELECT count(DISTINCT(#{clean_formatter}))
+        FROM #{@qualified_table_name}
+        WHERE cartodb_georef_status IS NULL OR cartodb_georef_status IS FALSE
+        LIMIT #{@max_rows}
+      }).first
+      rows[:count]
+    end
+
+    # Generate a csv input file from the geocodable rows
+    def generate_csv
+      csv_file = File.join(@working_dir, "wadus.csv")
+      # INFO: we exclude inputs too short and "just digits" inputs, which will remain as georef_status = false
+      query = %Q{
+        WITH geocodable AS (
+          SELECT DISTINCT(#{clean_formatter}) recId, #{clean_formatter} searchText
+          FROM #{@qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@max_rows - cache.hits}
+        )
+        SELECT * FROM geocodable
+        WHERE length(searchText) > 3 AND searchText !~ '^[\\d]*$'
+      }
+      result = connection.copy_table(connection[query], format: :csv, options: 'HEADER')
+      File.write(csv_file, result.force_encoding("UTF-8"))
+      return csv_file
+    end
+
+    def clean_formatter
+      "trim(both from regexp_replace(regexp_replace(concat(#{formatter}), E'[\\n\\r]+', ' ', 'g'), E'\"', '', 'g'))"
+    end
+
+    def download_results
+      @result = geocoder.result
+    end
+
+    def deflate_results
+      current_directory = Dir.pwd
+      Dir.chdir(@working_dir)
+      out = `unp *.zip 2>&1`
+      out = `unp #{@working_dir}/*_out.zip 2>&1`
+    ensure
+      Dir.chdir(current_directory)
+    end
+
+    def create_temp_table
+      connection.run(%Q{
+        CREATE TABLE #{temp_table_name} (
+          recId text,
+          SeqNumber int,
+          seqLength int,
+          displayLatitude float,
+          displayLongitude float
+        );}
+      )
+    end
+
+    def drop_temp_table
+      connection.run("DROP TABLE IF EXISTS #{temp_table_name}")
+    end
+
+    def import_results_to_temp_table
+      connection.copy_into(Sequel.lit(temp_table_name), data: File.read(deflated_results_path), format: :csv)
+    end
+
+    def load_results_into_original_table
+      connection.run(%Q{
+        UPDATE #{@qualified_table_name} AS dest
+        SET the_geom = ST_GeomFromText(
+            'POINT(' || orig.displayLongitude || ' ' ||
+              orig.displayLatitude || ')', 4326
+            ),
+            cartodb_georef_status = TRUE
+        FROM #{temp_table_name} AS orig
+        WHERE #{clean_formatter} = orig.recId
+      })
+    end
+
+    def mark_rows_not_geocoded
+      connection.run(%Q{UPDATE #{@qualified_table_name} SET cartodb_georef_status = FALSE WHERE cartodb_georef_status IS NULL})
+    end
+
+    def temp_table_name
+      @temp_table_name ||= "#{@schema}.geo_#{UUIDTools::UUID.timestamp_create.to_s.gsub('-', '')}"
+    end
+
+    def deflated_results_path
+      Dir[File.join(@working_dir, '*_out.txt')][0]
+    end
+
+    def update_metrics
+      total_requests = geocoder.successful_processed_rows + geocoder.empty_processed_rows + geocoder.failed_processed_rows
+      @usage_metrics.incr(:geocoder_here, :success_responses, geocoder.successful_processed_rows)
+      @usage_metrics.incr(:geocoder_here, :empty_responses, geocoder.empty_processed_rows)
+      @usage_metrics.incr(:geocoder_here, :failed_responses, geocoder.failed_processed_rows)
+      @usage_metrics.incr(:geocoder_here, :total_requests, total_requests)
+    end
+  end
+end

services/table-geocoder/lib/table_geocoder_factory.rb

@@ -0,0 +1,69 @@
+require_relative 'table_geocoder'
+require_relative 'internal_geocoder'
+require_relative 'gme/table_geocoder'
+require_relative 'exceptions'
+
+module Carto
+  class TableGeocoderFactory
+
+    def self.get(user, geocoding_model, cartodb_geocoder_config, table_service, params = {})
+      # Reset old connections to make sure changes apply.
+      # NOTE: This assumes it's being called from a Resque job
+      user.db_service.reset_pooled_connections
+      log = params.fetch(:log)
+      log.append_and_store 'TableGeocoderFactory.get()'
+      log.append_and_store "params: #{params.select{ |k| k != :log }}"
+
+      if user == table_service.owner
+        user_connection = user.in_database
+      else
+        if !table_service.table_visualization.has_permission?(user, Carto::Permission::ACCESS_READWRITE)
+          raise 'Insufficient permissions on table'
+        end
+        user_connection = table_service.owner.in_database
+      end
+
+      instance_config = cartodb_geocoder_config
+        .deep_symbolize_keys
+        .merge(
+               table_schema:  table_service.try(:database_schema),
+               table_name:    table_service.try(:name),
+               qualified_table_name: table_service.try(:qualified_table_name),
+               sequel_qualified_table_name: table_service.try(:sequel_qualified_table_name),
+               connection:    user_connection
+               )
+        .merge(params)
+
+      kind = instance_config.fetch(:kind)
+
+      if kind == 'high-resolution'
+        if user.google_maps_geocoder_enabled?
+          geocoder_class = Carto::Gme::TableGeocoder
+
+          instance_config[:client_id] = user.google_maps_client_id
+          instance_config[:private_key] = user.google_maps_private_key
+        elsif user.geocoder_provider == 'heremaps'
+          geocoder_class = CartoDB::TableGeocoder
+        else
+          raise 'Unsupported geocoder provider'
+        end
+      else
+        geocoder_class = CartoDB::InternalGeocoder::Geocoder
+      end
+
+      instance_config[:usage_metrics] = get_geocoder_metrics_instance(user)
+      instance_config[:log] = log
+      instance_config[:geocoding_model] = geocoding_model
+
+      log.append_and_store "geocoder_class = #{geocoder_class}"
+      instance = geocoder_class.new(instance_config)
+      log.append_and_store "geocoder_type = #{instance.name}"
+      instance
+    end
+
+    def self.get_geocoder_metrics_instance(user)
+      orgname = user.organization.nil? ? nil : user.organization.name
+      CartoDB::GeocoderUsageMetrics.new(user.username, orgname)
+    end
+  end
+end