Initial commit

services/table-geocoder/lib/abstract_table_geocoder.rb

@@ -0,0 +1,99 @@
+require_relative 'exceptions'
+require 'active_support/core_ext/numeric'
+
+module CartoDB
+  class AbstractTableGeocoder
+
+    DB_STATEMENT_TIMEOUT_MS = 5.hours.to_i * 1000
+
+    attr_reader :connection
+
+    def initialize(arguments)
+      @connection  = arguments.fetch(:connection)
+      @table_name  = arguments[:table_name]
+      @table_schema  = arguments[:table_schema]
+      @qualified_table_name = arguments[:qualified_table_name]
+      @sequel_qualified_table_name = arguments[:sequel_qualified_table_name]
+      @schema = arguments[:schema] || 'cdb'
+      @state = 'submitted'
+      @connection.run("SET statement_timeout TO #{DB_STATEMENT_TIMEOUT_MS}")
+    end
+
+    def cancel
+      raise 'Not implemented'
+    end
+
+    def run
+      raise 'Not implemented'
+    end
+
+    def remote_id
+      raise 'Not implemented'
+    end
+
+    def update_geocoding_status
+      raise 'Not implemented'
+    end
+
+    def process_results
+      raise 'Not implemented'
+    end
+
+    def name
+      raise 'Not implemented'
+    end
+
+    def used_batch_request?
+      false
+    end
+
+    def reset_cartodb_georef_status
+      ensure_georef_status_colummn_valid
+      set_georef_status_to_null
+    end
+
+    # Makes sure there's a cartodb_georef_status_column and marks all geocodifiable rows with NULL.
+    # This is important because otherwise it is hard to track what rows have been processed or not.
+    def mark_rows_to_geocode
+      ensure_georef_status_colummn_valid
+      set_georef_status_from_false_to_null
+    end
+
+
+    protected
+
+    def ensure_georef_status_colummn_valid
+      connection.run(%Q{
+          ALTER TABLE #{@qualified_table_name}
+          ADD COLUMN cartodb_georef_status BOOLEAN DEFAULT NULL
+        })
+    rescue Sequel::DatabaseError => e
+      if e.message =~ /canceling statement due to statement timeout/
+        raise Carto::GeocoderErrors::AddGeorefStatusColumnDbTimeoutError.new
+      end
+      raise unless e.message =~ /column .* of relation .* already exists/
+      cast_georef_status_column
+    end
+
+
+    private
+
+    def set_georef_status_to_null
+      connection.select.from(@sequel_qualified_table_name).update(:cartodb_georef_status => nil)
+    end
+
+    def set_georef_status_from_false_to_null
+      connection.select.from(@sequel_qualified_table_name).where(:cartodb_georef_status => false).update(:cartodb_georef_status => nil)
+    end
+
+    def cast_georef_status_column
+      connection.run(%Q{
+        ALTER TABLE #{@qualified_table_name} ALTER COLUMN cartodb_georef_status
+        TYPE boolean USING cast(cartodb_georef_status as boolean)
+      })
+    rescue => e
+      raise "Error converting cartodb_georef_status to boolean, please, convert it manually or remove it."
+    end
+
+  end
+end

services/table-geocoder/lib/exceptions.rb

@@ -0,0 +1,119 @@
+require 'active_support/core_ext/string'
+
+module Carto
+  module GeocoderErrors
+
+    GEOCODER_TIMED_OUT_TITLE = 'The geocoder timed out'
+    GEOCODER_TIMED_OUT_WHAT_ABOUT = %q{
+      Your geocoding request timed out.
+      Please <a href='mailto:support@cartob.com?subject=The geocoder timed out'>contact us</a>
+      and we'll try to fix it quickly.
+    }.squish
+
+    class AdditionalInfo
+      SOURCE_CARTODB = 'cartodb'
+      SOURCE_USER = 'user'
+
+      attr_accessor :error_code, :title, :what_about, :source
+
+      def initialize(error_code, title, what_about, source)
+        self.error_code = error_code
+        self.title = title
+        self.what_about = what_about
+        self.source = source
+      end
+    end
+
+    class GeocoderBaseError < StandardError
+      @@error_code_info_map = {}
+
+      attr_reader :original_exception
+
+      def initialize(original_exception=nil)
+        message = self.class.to_s
+        if original_exception
+          message << " " << original_exception.message
+          @original_exception = original_exception
+        end
+        super(message) # this is the only way of setting the message
+        set_backtrace(original_exception.backtrace) if original_exception # this line must appear after calling super
+      end
+
+      def original_message
+        @original_exception.message if @original_exception
+      end
+
+      def self.register_additional_info(error_code, title, what_about, source)
+        raise 'Duplicate error code' if @@error_code_info_map.has_key?(error_code)
+        @additional_info = AdditionalInfo.new(error_code, title, what_about, source)
+        @@error_code_info_map[@additional_info.error_code] = @additional_info
+      end
+
+      def self.get_info(error_code)
+        @@error_code_info_map[error_code]
+      end
+
+      class << self
+        attr_reader :additional_info
+      end
+    end
+
+    # just a convenience
+    def self.additional_info(error_code)
+      GeocoderBaseError.get_info(error_code)
+    end
+
+
+
+    class MisconfiguredGmeGeocoderError < GeocoderBaseError
+      register_additional_info(
+        1000,
+        'Google for Work account misconfigured',
+        %q{Your Google for Work account seems to be incorrectly configured.
+           Please <a href='mailto:sales@cartob.com?subject=Google for Work account misconfigured'>contact us</a>
+           and we'll try to fix it quickly.}.squish,
+        AdditionalInfo::SOURCE_USER
+        )
+    end
+
+    class GmeGeocoderTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1010,
+        'Google geocoder timed out',
+        %q{Your geocoding request timed out after several attempts.
+           Please check your quota usage in the <a href='https://console.developers.google.com/'>Google Developers Console</a>
+           and <a href='mailto:support@carto.com?subject=Google geocoder timed out'>contact us</a>
+           if you are within the usage limits.}.squish,
+        AdditionalInfo::SOURCE_USER
+        )
+    end
+
+    class AddGeorefStatusColumnDbTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1020,
+        GEOCODER_TIMED_OUT_TITLE,
+        GEOCODER_TIMED_OUT_WHAT_ABOUT,
+        AdditionalInfo::SOURCE_CARTODB
+        )
+    end
+
+    class GeocoderCacheDbTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1030,
+        GEOCODER_TIMED_OUT_TITLE,
+        GEOCODER_TIMED_OUT_WHAT_ABOUT,
+        AdditionalInfo::SOURCE_CARTODB
+        )
+    end
+
+    class TableGeocoderDbTimeoutError < GeocoderBaseError
+      register_additional_info(
+        1040,
+        GEOCODER_TIMED_OUT_TITLE,
+        GEOCODER_TIMED_OUT_WHAT_ABOUT,
+        AdditionalInfo::SOURCE_CARTODB
+        )
+    end
+
+  end
+end

services/table-geocoder/lib/geocoder_cache.rb

@@ -0,0 +1,177 @@
+require_relative 'exceptions'
+require_relative '../../../lib/carto/http/client'
+
+module CartoDB
+  class GeocoderCache
+
+    DEFAULT_BATCH_SIZE = 5000
+    DEFAULT_MAX_ROWS   = 1000000
+    HTTP_CONNECT_TIMEOUT = 60
+    HTTP_DEFAULT_TIMEOUT = 600
+
+    attr_reader :connection, :working_dir, :table_name, :hits, :misses,
+                :max_rows, :sql_api, :formatter, :cache_results
+
+    def initialize(arguments)
+      @sql_api = arguments.fetch(:sql_api)
+      @connection = arguments.fetch(:connection)
+      @table_name = arguments.fetch(:table_name)
+      @qualified_table_name = arguments.fetch(:qualified_table_name)
+      @working_dir = arguments[:working_dir] || Dir.mktmpdir
+      `chmod 777 #{@working_dir}`
+      @formatter = arguments.fetch(:formatter)
+      @max_rows = arguments[:max_rows] || DEFAULT_MAX_ROWS
+      @cache_results = nil
+      @batch_size = arguments[:batch_size] || DEFAULT_BATCH_SIZE
+      @cache_results = File.join(working_dir, "#{temp_table_name}_results.csv")
+      @usage_metrics = arguments.fetch(:usage_metrics)
+      @log = arguments.fetch(:log)
+      init_rows_count
+    end
+
+    def run
+      @log.append_and_store "Started searching previous geocoded results in geocoder cache"
+      get_cache_results
+      create_temp_table
+      load_results_to_temp_table
+      @hits = connection.select.from(temp_table_name).where('longitude is not null and latitude is not null').count.to_i
+      copy_results_to_table
+      @log.append_and_store "Finished geocoder cache job"
+    rescue => e
+      @log.append_and_store "Error getting results from geocoder cache: #{e.inspect}"
+      handle_cache_exception e
+    ensure
+      @usage_metrics.incr(:geocoder_cache, :total_requests, @total_rows)
+      @usage_metrics.incr(:geocoder_cache, :success_responses, @hits)
+      @usage_metrics.incr(:geocoder_cache, :empty_responses, (@total_rows - @hits - @failed_rows))
+      @usage_metrics.incr(:geocoder_cache, :failed_responses, @failed_rows)
+      update_log_stats
+    end
+
+    def get_cache_results
+      begin
+        count = count + 1 rescue 0
+        limit = [@batch_size, @max_rows - (count * @batch_size)].min
+        rows = connection.fetch(%Q{
+            SELECT DISTINCT(md5(#{formatter})) AS searchtext
+            FROM #{@qualified_table_name}
+            WHERE cartodb_georef_status IS NULL
+            LIMIT #{limit} OFFSET #{count * @batch_size}
+        }).all
+        @total_rows += rows.size
+        sql   = "WITH addresses(address) AS (VALUES "
+        sql << rows.map { |r| "('#{r[:searchtext]}')" }.join(',')
+        sql << ") SELECT DISTINCT ON(geocode_string) st_x(g.the_geom) longitude, st_y(g.the_geom) latitude,g.geocode_string FROM addresses a INNER JOIN #{sql_api[:table_name]} g ON md5(g.geocode_string)=a.address"
+        response = run_query(sql, 'csv').gsub(/\A.*/, '').gsub(/^$\n/, '')
+        File.open(cache_results, 'a') { |f| f.write(response.force_encoding("UTF-8")) } unless response == "\n"
+      end while rows.size >= @batch_size && (count * @batch_size) + rows.size < @max_rows
+    end
+
+    def store
+      begin
+        count = count + 1 rescue 0
+        sql   = %Q{
+           WITH
+            -- write the new values
+           n(searchtext, the_geom) AS (
+              VALUES %%VALUES%%
+           ),
+            -- update existing rows
+           upsert AS (
+              UPDATE #{sql_api[:table_name]} o
+              SET updated_at = NOW()
+              FROM n WHERE o.geocode_string = n.searchtext
+              RETURNING o.geocode_string
+           )
+           -- insert missing rows
+           INSERT INTO #{sql_api[:table_name]} (geocode_string,the_geom)
+           SELECT n.searchtext, n.the_geom FROM n
+           WHERE n.searchtext NOT IN (
+            SELECT geocode_string FROM upsert
+           );
+        }
+        rows = connection.fetch(%Q{
+          SELECT DISTINCT(quote_nullable(#{formatter})) AS searchtext, the_geom
+          FROM #{@qualified_table_name} AS orig
+          WHERE orig.cartodb_georef_status IS TRUE AND the_geom IS NOT NULL
+          LIMIT #{@batch_size} OFFSET #{count * @batch_size}
+        }).all
+        sql.gsub! '%%VALUES%%', rows.map { |r| "(#{r[:searchtext]}, '#{r[:the_geom]}')" }.join(',')
+        run_query(sql) if rows && rows.size > 0
+      end while rows.size >= @batch_size
+    rescue => e
+      handle_cache_exception e
+    ensure
+      drop_temp_table
+    end
+
+    def create_temp_table
+      connection.run(%Q{
+        CREATE TABLE #{temp_table_name} (
+          longitude text, latitude text, geocode_string text
+        );
+      })
+    end
+
+    def load_results_to_temp_table
+      connection.copy_into(Sequel.lit(temp_table_name), data: File.read(cache_results), format: :csv)
+    end
+
+    def copy_results_to_table
+      connection.run(%Q{
+        UPDATE #{@qualified_table_name} AS dest
+        SET the_geom = ST_GeomFromText(
+              'POINT(' || orig.longitude || ' ' || orig.latitude || ')', 4326
+            ),
+            cartodb_georef_status = TRUE
+        FROM #{temp_table_name} AS orig
+        WHERE #{formatter} = orig.geocode_string
+      })
+    end
+
+    def drop_temp_table
+      connection.run("DROP TABLE IF EXISTS #{temp_table_name}")
+    end
+
+    def temp_table_name
+      @temp_table_name ||= "geocoding_cache_#{Time.now.to_i}"
+    end
+
+    def run_query(query, format = '')
+      params = { q: query, api_key: sql_api[:api_key], format: format }
+      http_client = Carto::Http::Client.get('geocoder_cache',
+                                            log_requests: true)
+      response = http_client.post(sql_api[:base_url],
+                                  body: URI.encode_www_form(params),
+                                  connecttimeout: HTTP_CONNECT_TIMEOUT,
+                                  timeout: HTTP_DEFAULT_TIMEOUT)
+      response.body
+    end
+
+    # It handles in such a way that the caching is silently stopped
+    def handle_cache_exception(exception)
+      drop_temp_table
+      if exception.class == Sequel::DatabaseError && exception.message =~ /canceling statement due to statement timeout/
+        # for the moment we just wrap the exception to get a specific error in rollbar
+        exception =  Carto::GeocoderErrors::GeocoderCacheDbTimeoutError.new(exception)
+      end
+      # In case we get some error we are going to pass all the rows as failed
+      @failed_rows = @total_rows
+      CartoDB.notify_exception(exception)
+    end
+
+    private
+
+    def init_rows_count
+      @hits = 0
+      @total_rows = 0
+      @failed_rows = 0
+    end
+
+    def update_log_stats
+      @log.append_and_store "Geocoding cache stats update. "\
+        "Total rows: #{@total_rows} "\
+        "--- Hits: #{@hits} --- Failed: #{@failed_rows}"
+    end
+  end
+end

services/table-geocoder/lib/gme/client.rb

@@ -0,0 +1,121 @@
+require 'addressable/uri'
+require 'set'
+require 'json'
+require_relative 'exceptions'
+require_relative '../exceptions'
+require_relative '../../../../lib/url_signer'
+require_relative '../../../../lib/carto/http/client'
+
+module Carto
+  module Gme
+
+    # The responsibility of this class is to perform requests to gme
+    # taking care of sigining requests, usage limits, errors, retries, etc.
+    class Client
+
+      BASE_URL = 'https://maps.googleapis.com'
+
+      DEFAULT_CONNECT_TIMEOUT = 15
+      DEFAULT_READ_TIMEOUT = 30
+      DEFAULT_RETRY_TIMEOUT = 60
+
+      OK_STATUS = 'OK'
+      ZERO_RESULTS_STATUS = 'ZERO_RESULTS'
+      OVERQUERY_LIMIT_STATUS = 'OVER_QUERY_LIMIT'
+      REQUEST_DENIED_STATUS = 'REQUEST_DENIED'
+      INVALID_REQUEST_STATUS = 'INVALID_REQUEST'
+      UNKNOWN_ERROR_STATUS = 'UNKNOWN_ERROR'
+
+      HTTP_CLIENT_TAG = 'gme_client'
+
+      RETRIABLE_STATUSES = Set.new [500, 503, 504]
+
+      # Performs requests to Google Maps API web services
+      # Based on https://github.com/googlemaps/google-maps-services-python/blob/master/googlemaps/client.py
+      def initialize(client_id, private_key, options = {})
+        @client_id = client_id
+        @url_signer = UrlSigner.new(private_key)
+        @connect_timeout = options[:connect_timeout] || DEFAULT_CONNECT_TIMEOUT
+        @read_timeout = options[:read_timeout] || DEFAULT_READ_TIMEOUT
+        @retry_timeout = options[:retry_timeout] || DEFAULT_RETRY_TIMEOUT
+        @http_client = Carto::Http::Client.get(HTTP_CLIENT_TAG)
+      end
+
+      def get(endpoint, params, first_request_time=nil, retry_counter=0)
+        first_request_time ||= Time.now
+        elapsed = Time.now - first_request_time
+        if elapsed > @retry_timeout
+          raise Carto::GeocoderErrors::GmeGeocoderTimeoutError.new('retry timeout expired')
+        end
+
+        # 0.5 * (1.5 ^ i) is an increased sleep time of 1.5x per iteration,
+        # starting at 0.5s when retry_counter=0. The first retry will occur
+        # at 1, so subtract that first.
+        if retry_counter > 0
+          delay_seconds = 0.1 * 1.5 ** (retry_counter - 1)
+          sleep delay_seconds
+        end
+
+        url = generate_auth_url(BASE_URL+endpoint, params)
+
+        resp = @http_client.get(url, timeout: @read_timeout, connecttimeout: @connect_timeout)
+        raise Timeout.new('http request timed out') if resp.timed_out?
+
+        if RETRIABLE_STATUSES.include?(resp.code)
+          return self.get(endpoint, params, first_request_time, retry_counter+1)
+        end
+
+        begin
+          get_body(resp)
+        rescue OverQueryLimit
+          CartoDB.notify_debug('Carto::Gme::Client rescuing from OverQueryLimit exception', params.symbolize_keys)
+          return self.get(endpoint, params, first_request_time, retry_counter+1)
+        end
+      end
+
+
+      private
+
+      def generate_auth_url(path, params)
+        uri = Addressable::URI.new
+        uri.path = path
+        uri.query_values = params.merge(client: @client_id)
+        @url_signer.sign_url(uri.request_uri)
+      end
+
+      # Takes a typhoeus response object and returns a hash
+      def get_body(resp)
+        if resp.code != 200
+          # Remove temporarily from rollbar because it's flooding the logs
+          if resp.code != 400
+            CartoDB::Logger.warning(message: 'Error response from GME client',
+                                    client_id: @client_id,
+                                    code: resp.code,
+                                    response_body: resp.response_body)
+          end
+          raise HttpError.new(resp.code)
+        end
+
+        body = JSON::parse(resp.body)
+
+        api_status = body['status']
+        if api_status == 'OK' || api_status == 'ZERO_RESULTS'
+          return body
+        end
+
+        if api_status == 'OVER_QUERY_LIMIT'
+          raise OverQueryLimit.new
+        end
+
+        if body.has_key?('error_message')
+          raise ApiError.new(api_status, body['error_message'])
+        else
+          raise ApiError.new(api_status)
+        end
+
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/gme/convert.rb

@@ -0,0 +1,16 @@
+module Carto
+  module Gme
+
+    module Convert
+
+      def components
+        raise 'not implemented'
+      end
+
+      def bounds
+        raise 'not implemented'
+      end
+    end
+
+  end
+end

services/table-geocoder/lib/gme/exceptions.rb

@@ -0,0 +1,23 @@
+module Carto
+  module Gme
+
+    # TODO take care of these exceptions to provide better feedback to the user
+    class ClientException < StandardError; end
+
+    class Timeout < ClientException; end
+
+    class HttpError < ClientException; end
+
+    class OverQueryLimit < ClientException; end
+
+    class ApiError < ClientException
+      attr_reader :api_status, :error_message
+      def initialize(api_status, error_message=nil)
+        super(%Q{api_status = #{api_status}, error_message = "#{error_message}"})
+        @api_status = api_status
+        @error_message = error_message
+      end
+    end
+
+  end
+end

services/table-geocoder/lib/gme/geocoder_client.rb

@@ -0,0 +1,30 @@
+require_relative 'client'
+require_relative 'convert'
+
+
+module Carto
+  module Gme
+
+    # The responsibility of this class is to geocode addresses by requesting gme.
+    class GeocoderClient
+
+      attr_reader :client
+
+      def initialize(client)
+        @client = client
+      end
+
+      def geocode(address=nil, components=nil, bounds=nil, region=nil, language=nil)
+        params = {}
+        params['address'] = address if address
+        params['components'] = Convert.components(components) if components # TODO convert
+        params['bounds'] = Convert.bounds(bounds) if bounds # TODO convert
+        params['region'] = region if region
+        params['language'] = language if language
+
+        client.get('/maps/api/geocode/json', params)
+      end
+    end
+
+  end
+end

services/table-geocoder/lib/gme/table_geocoder.rb

@@ -0,0 +1,197 @@
+require_relative '../abstract_table_geocoder'
+require_relative 'client'
+require_relative 'geocoder_client'
+
+module Carto
+  module Gme
+    class TableGeocoder < CartoDB::AbstractTableGeocoder
+
+      DEFAULT_MAX_BLOCK_SIZE = 1000
+
+      # See https://developers.google.com/maps/documentation/geocoding/#Types
+      ACCEPTED_ADDRESS_TYPES = ['street_address', 'route', 'intersection', 'neighborhood']
+
+      attr_reader :original_formatter, :processed_rows, :successful_processed_rows, :failed_processed_rows,
+                  :empty_processed_rows, :state, :max_block_size
+
+      def initialize(arguments)
+        super(arguments)
+        @original_formatter = arguments.fetch(:original_formatter)
+        client_id = arguments.fetch(:client_id)
+        private_key = arguments.fetch(:private_key)
+        @max_block_size = arguments[:max_block_size] || DEFAULT_MAX_BLOCK_SIZE
+        gme_client = Client.new(client_id, private_key)
+        @geocoder_client = GeocoderClient.new(gme_client)
+        @usage_metrics = arguments.fetch(:usage_metrics)
+        @log = arguments.fetch(:log)
+        @geocoding_model = arguments.fetch(:geocoding_model)
+      end
+
+      def cancel; end
+
+      def run
+        change_status('running')
+        init_rows_count
+        ensure_georef_status_colummn_valid
+
+        # Here's the actual stuff
+        data_input_blocks.each do |data_block|
+          geocode(data_block)
+          update_table(data_block)
+          @processed_rows += data_block.size
+        end
+
+        change_status('completed')
+      rescue => e
+        change_status('failed')
+        raise e
+      ensure
+        total_requests = @successful_processed_rows + @empty_processed_rows + @failed_processed_rows
+        @usage_metrics.incr(:geocoder_google, :success_responses, @successful_processed_rows)
+        @usage_metrics.incr(:geocoder_google, :empty_responses, @empty_processed_rows)
+        @usage_metrics.incr(:geocoder_google, :failed_responses, @failed_processed_rows)
+        @usage_metrics.incr(:geocoder_google, :total_requests, total_requests)
+        update_log_stats
+      end
+
+      # Empty methods, needed because they're triggered from geocoding.rb
+      def remote_id; end
+      def process_results; end # TODO: can be removed from here and abstract class
+
+      def update_geocoding_status
+        { processed_rows: processed_rows, state: @geocoding_model.state }
+      end
+
+      def name
+        'google'
+      end
+
+      private
+
+      # Returns a "generator"
+      def data_input_blocks
+        Enumerator.new do |enum|
+          begin
+            data_input = connection.select(:cartodb_id, searchtext_expression)
+              .from(@sequel_qualified_table_name)
+              .where(cartodb_georef_status: nil)
+              .limit(max_block_size)
+              .all
+            enum.yield data_input
+            # last iteration when data_input.length < max_block_size, no need for another query
+          end while data_input.length == max_block_size
+        end
+      end
+
+      def searchtext_expression
+        # The original_formatter has the following format:
+        #   `{street_column_name}[[, additional_free_text][, {province_column_name}][, country_free_text]]`
+        # See https://github.com/jeremyevans/sequel/blob/master/doc/security.rdoc
+        # See http://sequel.jeremyevans.net/rdoc/classes/Sequel/SQL/Builders.html
+        atoms = original_formatter.split(',').map {|s| s.strip }
+        Sequel.join(atoms.map { |atom|
+                      if match = /\A{(?<column_name>.*)}\z/.match(atom)
+                        Sequel.identifier(match[:column_name])
+                      else
+                        atom
+                      end
+                    }, ',').as(:searchtext)
+      end
+
+      def geocode(data_block)
+        data_block.each do |row|
+          response = fetch_from_gme(row[:searchtext])
+          # If we get an error we get nil so we pass to the next row
+          next if response.nil?
+          if response['status'] != Client::OK_STATUS
+            process_error_or_empty_status(response['status'])
+            row.merge!(cartodb_georef_status: false)
+          else
+            result = response['results'].select { |r| r['types'] & ACCEPTED_ADDRESS_TYPES }.first
+            if result.nil?
+              @empty_processed_rows += 1
+              row.merge!(cartodb_georef_status: false)
+            else
+              @successful_processed_rows += 1
+              location = result['geometry']['location']
+              row.merge!(location.deep_symbolize_keys.merge(cartodb_georef_status: true))
+            end
+          end
+        end
+      end
+
+      def update_table(data_block)
+        # At this point, data_block is an Array that looks like this:
+        #   [{:cartodb_id=>1, :searchtext=>"Some real street name", :lat=>19.29544, :lng=>-99.1472101, :cartodb_georef_status=>true},
+        #    {:cartodb_id=>2, :searchtext=>"foo", :cartodb_georef_status=>false}]
+        geocoded = data_block.select {|row| row[:cartodb_georef_status] == true}
+        if geocoded.count > 0
+          geocoded_to_sql = geocoded.map {|row| "(#{row[:cartodb_id]}, #{row[:lat]}, #{row[:lng]})"}.join(',')
+          query_geocoded = %Q{
+            UPDATE #{@qualified_table_name} as target SET
+              the_geom = CDB_LatLng(geocoded.lat,geocoded.lng),
+              cartodb_georef_status = TRUE
+            FROM (VALUES
+              #{geocoded_to_sql}
+            ) as geocoded(cartodb_id,lat,lng)
+            WHERE target.cartodb_id = geocoded.cartodb_id;
+          }
+          connection.run(query_geocoded)
+        end
+
+        non_geocoded = data_block.select {|row| row[:cartodb_georef_status] == false}
+        if non_geocoded.count > 0
+          non_geocoded_to_sql = non_geocoded.map {|row| "(#{row[:cartodb_id]})"}.join(',')
+          query_non_geocoded = %Q{
+            UPDATE #{@qualified_table_name} as target SET
+              cartodb_georef_status = FALSE
+            FROM (VALUES
+              #{non_geocoded_to_sql}
+            ) as nongeocoded(cartodb_id)
+            WHERE target.cartodb_id = nongeocoded.cartodb_id;
+          }
+          connection.run(query_non_geocoded)
+        end
+      end
+
+      private
+
+      def init_rows_count
+        @processed_rows = 0
+        @successful_processed_rows = 0
+        @failed_processed_rows = 0
+        @empty_processed_rows = 0
+      end
+
+      def fetch_from_gme(search_text)
+        @geocoder_client.geocode(search_text)
+      rescue => e
+        # Remove temporarily because it's flooding the logs
+        # @log.append_and_store "Error geocoding using GME for text #{search_text}: #{e.message}"
+        # CartoDB.notify_error('Error geocoding using GME', error: e.backtrace.join('\n'), search_text: search_text)
+        @failed_processed_rows += 1
+        nil
+      end
+
+      def process_error_or_empty_status(status)
+        case status
+          when Client::ZERO_RESULTS_STATUS then @empty_processed_rows += 1
+          else @failed_processed_rows += 1
+        end
+      end
+
+      def update_log_stats
+        @log.append_and_store "Geocoding using Google maps, job status update. "\
+          "Status: #{@status} --- Processed rows: #{@processed_rows} "\
+          "--- Success: #{@successful_processed_rows} --- Empty: #{@empty_processed_rows} "\
+          "--- Failed: #{@failed_processed_rows}"
+      end
+
+      def change_status(status)
+        @status = status
+        @geocoding_model.state = status
+        @geocoding_model.save
+      end
+    end
+  end
+end

services/table-geocoder/lib/internal-geocoder/abstract_query_generator.rb

@@ -0,0 +1,44 @@
+require_relative 'input_type_resolver'
+require_relative '../../../importer/lib/importer/query_batcher'
+
+module CartoDB
+  module InternalGeocoder
+
+    class AbstractQueryGenerator
+
+      def initialize(internal_geocoder)
+        @internal_geocoder = internal_geocoder
+      end
+
+      #TODO custom exception
+      def search_terms_query(page)
+        raise 'Not implemented'
+      end
+
+      def dataservices_query(search_terms)
+        raise 'Not implemented'
+      end
+
+      def copy_results_to_table_query
+        raise 'Not implemented'
+      end
+
+      def country
+        country = @internal_geocoder.countries
+        (country == %Q{'world'} || country.blank?) ? 'null' : country
+      end
+
+      def region
+        region = @internal_geocoder.regions
+        return 'null' if region.blank?
+        region
+      end
+
+      def dest_table
+        @internal_geocoder.qualified_table_name
+      end
+
+    end # AbstractQueryGenerator
+
+  end # InternalGeocoder
+end #CartoDB

services/table-geocoder/lib/internal-geocoder/admin0_text_polygons.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class Admin0TextPolygons < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        regions = search_terms.map { |row| row[:region] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_admin0_polygons(Array[#{regions}])).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # Admin0TextPolygons
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/admin1_column_polygons.rb

@@ -0,0 +1,40 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class Admin1ColumnPolygons < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as region,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        region = search_terms.map { |row| row[:region] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_admin1_polygons(Array[#{region}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # Admin1ColumnPolygons
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/admin1_text_polygons.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class Admin1TextPolygons < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        region = search_terms.map { |row| row[:region] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_admin1_polygons(Array[#{region}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # Admin1TextPolygons
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/cities_column_column_points.rb

@@ -0,0 +1,43 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesColumnColumnPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable("#{@internal_geocoder.column_name}")) AS city,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) AS country,
+            trim(quote_nullable(#{@internal_geocoder.region_column})) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        regions = search_terms.map { |row| row[:region] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], Array[#{regions}], Array[#{countries}])).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
+            AND trim(dest.#{@internal_geocoder.region_column}::text) = trim(orig.region)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # CitiesColumnColumnPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/cities_column_text_points.rb

@@ -0,0 +1,40 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesColumnTextPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as city,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], #{region}, Array[#{countries}])).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.country_column}::text) = trim(orig.country)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/cities_text_column_points.rb

@@ -0,0 +1,40 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesTextColumnPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable("#{@internal_geocoder.column_name}")) AS city,
+            trim(quote_nullable(#{@internal_geocoder.region_column})) AS region
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        regions = search_terms.map { |row| row[:region] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], Array[#{regions}], #{country})).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
+            AND trim(dest.#{@internal_geocoder.region_column}::text) = trim(orig.region)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # CitiesTextColumnPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/cities_text_text_points.rb

@@ -0,0 +1,36 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class CitiesTextTextPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS city
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        cities = search_terms.map { |row| row[:city] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_namedplace(Array[#{cities}], #{region}, #{country})).*) SELECT q, c, a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string)
+            AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # CitiesTextTextPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/input_type_resolver.rb

@@ -0,0 +1,42 @@
+module CartoDB
+
+  module InternalGeocoder
+
+    class InputTypeResolver
+
+      def initialize(internal_geocoder)
+        @internal_geocoder = internal_geocoder
+      end
+
+      def type
+        [kind, country_input_type, region_input_type, geometry_type]
+      end
+
+      def kind
+        @internal_geocoder.kind
+      end
+
+      def country_input_type
+        if @internal_geocoder.country_column
+          :column
+        else
+          :text
+        end
+      end
+
+      def region_input_type
+        if @internal_geocoder.region_column
+          :column
+        elsif @internal_geocoder.regions.present?
+          :text
+        end
+      end
+
+      def geometry_type
+        @internal_geocoder.geometry_type
+      end
+
+    end # InputTypeResolver
+  end # InternalGeocoder
+
+end

services/table-geocoder/lib/internal-geocoder/ipaddress_text_point.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class IpAddressTextPoint < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS ipaddress
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        ipaddress = search_terms.map { |row| row[:ipaddress] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_ip(Array[#{ipaddress}])).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # IpAddressTextPoint
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/latitude_longitude.rb

@@ -0,0 +1,55 @@
+require_relative '../../../importer/lib/importer/query_batcher'
+
+module CartoDB
+  module InternalGeocoder
+
+    class LatitudeLongitude
+
+      def initialize(db, logger = nil)
+        @db = db
+        @logger = logger
+      end
+
+      def geocode(table_schema, table_name, latitude_column, longitude_column)
+        qualified_table_name = "\"#{table_schema}\".\"#{table_name}\""
+        query_fragment_update = %Q{
+          UPDATE #{qualified_table_name}
+          SET
+            the_geom = ST_GeomFromText(
+              'POINT(' || REPLACE(TRIM(CAST("#{longitude_column}" AS text)), ',', '.') || ' ' ||
+                REPLACE(TRIM(CAST("#{latitude_column}" AS text)), ',', '.') || ')', 4326
+            )
+        }
+        query_fragment_where = %Q{
+          REPLACE(TRIM(CAST("#{longitude_column}" AS text)), ',', '.') ~
+            '^(([-+]?(([0-9]|[1-9][0-9]|1[0-7][0-9])(\.[0-9]+)?))|[-+]?180)$'
+          AND REPLACE(TRIM(CAST("#{latitude_column}" AS text)), ',', '.')  ~
+            '^(([-+]?(([0-9]|[1-8][0-9])(\.[0-9]+)?))|[-+]?90)$'
+        }
+
+        CartoDB::Importer2::QueryBatcher.new(
+            @db, 
+            @logger, 
+            !table_has_cartodb_id(table_schema, table_name)
+          ).execute_update(
+              %Q{#{query_fragment_update} where #{query_fragment_where}},
+              table_schema, table_name
+          )
+      end
+
+      def table_has_cartodb_id(table_schema, table_name)
+        result = @db.fetch(%Q{
+          select *
+          from information_schema.columns
+          where table_schema = '#{table_schema}' 
+            and table_name = '#{table_name}' 
+            and column_name = 'cartodb_id'
+            and data_type = 'integer';
+        }).all
+        !result[0].nil?
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/postalcode_column_points.rb

@@ -0,0 +1,38 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeColumnPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as postalcode,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_points(Array[#{postalcodes}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/postalcode_column_polygon.rb

@@ -0,0 +1,38 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeColumnPolygon < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT
+            trim(quote_nullable(#{@internal_geocoder.column_name})) as postalcode,
+            trim(quote_nullable(#{@internal_geocoder.country_column})) as country
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        countries = search_terms.map { |row| row[:country] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_polygons(Array[#{postalcodes}], Array[#{countries}])).*) SELECT q, c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest.#{@internal_geocoder.column_name}::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end
+
+  end
+end

services/table-geocoder/lib/internal-geocoder/postalcode_text_points.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeTextPoints < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS postalcode
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_points(Array[#{postalcodes}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # PostalcodeTextPoints
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/postalcode_text_polygon.rb

@@ -0,0 +1,35 @@
+require_relative 'abstract_query_generator'
+
+module CartoDB
+  module InternalGeocoder
+
+    class PostalcodeTextPolygon < AbstractQueryGenerator
+
+      def search_terms_query(page)
+        %Q{
+          SELECT DISTINCT(trim(quote_nullable("#{@internal_geocoder.column_name}"))) AS postalcode
+          FROM #{@internal_geocoder.qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@internal_geocoder.batch_size} OFFSET #{page * @internal_geocoder.batch_size}
+        }
+      end
+
+      def dataservices_query(search_terms)
+        postalcodes = search_terms.map { |row| row[:postalcode] }.join(',')
+        "WITH geo_function AS (SELECT (geocode_postalcode_polygons(Array[#{postalcodes}], #{country})).*) SELECT q, null AS c, null AS a1, geom, success FROM geo_function"
+      end
+
+      def copy_results_to_table_query
+        %Q{
+          UPDATE #{dest_table} AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM #{@internal_geocoder.temp_table_name} AS orig
+          WHERE trim(dest."#{@internal_geocoder.column_name}"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+        }
+      end
+
+    end # PostalcodeTextPolygon
+
+  end # InternalGeocoder
+end # CartoDB

services/table-geocoder/lib/internal-geocoder/query_generator_factory.rb

@@ -0,0 +1,68 @@
+require_relative 'input_type_resolver'
+
+require_relative 'cities_text_text_points'
+require_relative 'cities_column_text_points'
+require_relative 'cities_text_column_points'
+require_relative 'cities_column_column_points'
+require_relative 'admin0_text_polygons'
+require_relative 'admin1_text_polygons'
+require_relative 'admin1_column_polygons'
+require_relative 'postalcode_text_points'
+require_relative 'postalcode_column_points'
+require_relative 'postalcode_text_polygon'
+require_relative 'postalcode_column_polygon'
+require_relative 'ipaddress_text_point'
+
+module CartoDB
+  module InternalGeocoder
+
+    class QueryGeneratorFactory
+
+      class QueryGeneratorNotImplemented < StandardError; end
+
+      class << self
+        private :new
+
+        def get(internal_geocoder, input_type=nil)
+          input_type ||= InputTypeResolver.new(internal_geocoder).type
+
+          case input_type
+            when [:namedplace, :text, nil, :point]
+              CitiesTextTextPoints.new internal_geocoder
+            when [:namedplace, :text, :text, :point]
+              CitiesTextTextPoints.new internal_geocoder
+            when [:namedplace, :text, :column, :point]
+              CitiesTextColumnPoints.new internal_geocoder
+            when [:namedplace, :column, nil, :point]
+              CitiesColumnTextPoints.new internal_geocoder
+            when [:namedplace, :column, :text, :point]
+              CitiesColumnTextPoints.new internal_geocoder
+            when [:namedplace, :column, :column, :point]
+              CitiesColumnColumnPoints.new internal_geocoder
+            when [:admin0, :text, nil, :polygon]
+              Admin0TextPolygons.new internal_geocoder
+            when [:admin1, :text, nil, :polygon]
+              Admin1TextPolygons.new internal_geocoder
+            when [:admin1, :column, nil, :polygon]
+              Admin1ColumnPolygons.new internal_geocoder
+            when [:postalcode, :text, nil, :point]
+              PostalcodeTextPoints.new internal_geocoder
+            when [:postalcode, :column, nil, :point]
+              PostalcodeColumnPoints.new internal_geocoder
+            when [:postalcode, :text, nil, :polygon]
+              PostalcodeTextPolygon.new internal_geocoder
+            when [:postalcode, :column, nil, :polygon]
+              PostalcodeColumnPolygon. new internal_geocoder
+            when [:ipaddress, :text, nil, :point]
+              IpAddressTextPoint.new internal_geocoder
+            else
+              raise QueryGeneratorNotImplemented. new "QueryGenerator not implemented for input type #{input_type}"
+          end
+        end
+      end
+
+
+    end # QueryGeneratorFactory
+
+  end # InternalGeocoder
+end #CartoDB

services/table-geocoder/lib/internal_geocoder.rb

@@ -0,0 +1,156 @@
+require_relative '../../sql-api/sql_api'
+require_relative '../../importer/lib/importer/query_batcher'
+require_relative 'internal-geocoder/query_generator_factory'
+require_relative 'abstract_table_geocoder'
+
+module CartoDB
+  module InternalGeocoder
+    class Geocoder  < AbstractTableGeocoder
+
+      SQLAPI_CALLS_TIMEOUT = 45
+
+      attr_reader   :temp_table_name, :sql_api, :geocoding_results,
+                    :working_dir, :remote_id, :state, :processed_rows, :country_column, :region_column,
+                    :qualified_table_name, :batch_size, :countries, :regions, :kind, :geometry_type
+
+      attr_accessor :table_schema, :table_name, :column_name, :log
+
+      def initialize(arguments)
+        super(arguments)
+        @sql_api              = CartoDB::SQLApi.new(arguments.fetch(:internal)
+                                                             .merge({ timeout: SQLAPI_CALLS_TIMEOUT })
+                                                   )
+        @column_name          = arguments[:formatter]
+        @countries            = arguments[:countries].to_s
+        @country_column       = arguments[:country_column]
+        @regions              = arguments[:regions].to_s
+        @region_column        = arguments[:region_column]
+        @geometry_type        = arguments.fetch(:geometry_type, '').to_sym
+        @kind                 = arguments.fetch(:kind, '').to_sym
+        @batch_size           = (@geometry_type == :point ? 1000 : 10)
+        @working_dir          = arguments[:working_dir] || Dir.mktmpdir
+        @geocoding_results = File.join(working_dir, "#{temp_table_name}_results.csv".gsub('"', ''))
+        @query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get self
+        @log = arguments[:log]
+        @geocoding_model = arguments[:geocoding_model]
+        @usage_metrics = arguments.fetch(:usage_metrics)
+      end
+
+      def set_log(log)
+        @log = log
+      end
+
+      def run
+        log.append_and_store 'run()'
+        change_status('running')
+        ensure_georef_status_colummn_valid
+        download_results
+        create_temp_table
+        load_results_to_temp_table
+        copy_results_to_table
+        change_status('completed')
+      rescue => e
+        change_status('failed')
+        raise e
+      ensure
+        drop_temp_table
+        FileUtils.remove_entry_secure @working_dir if Dir.exists?(@working_dir)
+      end
+
+      def download_results
+        log.append_and_store 'download_results()'
+        begin
+          count = count + 1 rescue 0
+          search_terms = get_search_terms(count)
+          unless search_terms.size == 0
+            sql = @query_generator.dataservices_query(search_terms)
+
+            # Getting data from the internal geocoder is an all-or-nothing thing, so we
+            # log it as such, total_requests and failed_responses
+            begin
+              response = sql_api.fetch(sql, 'csv').gsub(/\A.*/, '').gsub(/^$\n/, '')
+            rescue CartoDB::SQLApi::SQLApiError => ex
+              @usage_metrics.incr(:geocoder_internal, :failed_responses, search_terms.length)
+              raise ex
+            ensure
+              @usage_metrics.incr(:geocoder_internal, :total_requests, search_terms.length)
+            end
+
+            # Count empty and successfully geocoded responses
+            empty_responses = 0
+            success_responses = 0
+            CSV.parse(response.chomp) do |row|
+              empty_responses += 1 if row[4] == "false"
+              success_responses += 1 if row[4] == "true"
+            end
+            @usage_metrics.incr(:geocoder_internal, :success_responses, success_responses)
+            @usage_metrics.incr(:geocoder_internal, :empty_responses, empty_responses)
+
+            log.append_and_store "Saving results to #{geocoding_results}"
+            File.open(geocoding_results, 'a') { |f| f.write(response.force_encoding("UTF-8")) } unless response.blank?
+          end
+        end while search_terms.size >= @batch_size
+        @processed_rows = `wc -l '#{geocoding_results}' 2>&1`.to_i
+        geocoding_results
+      end # download_results
+
+      def get_search_terms(page)
+        query = @query_generator.search_terms_query(page)
+        connection.fetch(query).all
+      end # get_search_terms
+
+      def create_temp_table
+        log.append_and_store 'create_temp_table()'
+        connection.run(%Q{
+          CREATE TABLE #{temp_table_name} (
+            geocode_string text, country text, region text, the_geom geometry, cartodb_georef_status boolean
+          );
+        })
+      end # create_temp_table
+
+      def update_geocoding_status
+        { processed_rows: processed_rows, state: state }
+      end # update_geocoding_status
+
+      def process_results; end
+      def cancel; end
+
+      def load_results_to_temp_table
+        log.append_and_store 'load_results_to_temp_table()'
+        connection.copy_into(Sequel.lit(temp_table_name), data: File.read(geocoding_results), format: :csv)
+      end # load_results_to_temp_table
+
+      def copy_results_to_table
+        log.append_and_store 'copy_results_to_table()'
+        # 'InternalGeocoder::copy_results_to_table'
+        CartoDB::Importer2::QueryBatcher.new(
+            connection,
+            nil,
+            create_seq_field = true,
+            batch_size
+          ).execute_update(
+            @query_generator.copy_results_to_table_query,
+            @table_schema, @table_name
+          )
+      end
+
+      def drop_temp_table
+        connection.run("DROP TABLE IF EXISTS #{temp_table_name}")
+      end # drop_temp_table
+
+      def temp_table_name
+        @temp_table_name ||= %Q{"#{@table_schema}".internal_geocoding_#{Time.now.to_i}}
+      end # temp_table_name
+
+      def name
+        'internal'
+      end
+
+      def change_status(status)
+        @status = status
+        @geocoding_model.state = status
+        @geocoding_model.save
+      end
+    end
+  end
+end

services/table-geocoder/lib/mail_geocoder.rb

@@ -0,0 +1,49 @@
+require_relative '../../../lib/resque/user_jobs'
+
+module CartoDB
+  module Geocoder
+    class MailNotifier
+
+      MIN_GEOCODER_TIME_TO_NOTIFY = 3 * 60 # seconds
+
+      def initialize(user_id, state, table_name, error_code, processable_rows, number_geocoded_rows, geocoding_time)
+        @user_id = user_id
+        @state = state
+        @table_name = table_name
+        @error_code = error_code
+        @processable_rows = processable_rows
+        @number_geocoded_rows = number_geocoded_rows
+        @geocoding_time = geocoding_time
+        @resque = ::Resque
+        @mail_sent = false
+      end
+
+      def notify_if_needed
+        send! if should_notify?
+      end
+
+      def should_notify?
+        @geocoding_time >= MIN_GEOCODER_TIME_TO_NOTIFY
+      end
+
+      def send!
+        @mail_sent = @resque.enqueue(
+          ::Resque::UserJobs::Mail::GeocoderFinished,
+          @user_id,
+          @state,
+          @table_name,
+          @error_code,
+          @processable_rows,
+          @number_geocoded_rows
+        )
+      end
+
+      def mail_sent?
+        return @mail_sent
+      end
+
+    end
+
+  end #Importer2
+end #CartoDB
+

services/table-geocoder/lib/table_geocoder.rb

@@ -0,0 +1,207 @@
+require 'uuidtools'
+require_relative '../../geocoder/lib/hires_geocoder_factory'
+require_relative '../../geocoder/lib/geocoder_config'
+require_relative 'geocoder_cache'
+require_relative 'abstract_table_geocoder'
+
+
+module CartoDB
+  class TableGeocoder < AbstractTableGeocoder
+
+    attr_reader   :working_dir, :csv_file, :result,
+                  :max_rows, :cache
+
+    attr_accessor :table_name, :formatter, :remote_id
+
+    def initialize(arguments)
+      super(arguments)
+      @working_dir = arguments[:working_dir] || Dir.mktmpdir
+      system('chmod', '777', @working_dir)
+      @formatter   = arguments[:formatter]
+      @remote_id   = arguments[:remote_id]
+      @max_rows    = arguments.fetch(:max_rows)
+      @usage_metrics = arguments.fetch(:usage_metrics)
+      @log = arguments.fetch(:log)
+      @geocoding_model = arguments.fetch(:geocoding_model)
+      @cache       = CartoDB::GeocoderCache.new(
+        connection:  connection,
+        formatter:   clean_formatter,
+        sql_api:     arguments[:cache],
+        working_dir: @working_dir,
+        table_name:  table_name,
+        qualified_table_name: @qualified_table_name,
+        max_rows:    @max_rows,
+        usage_metrics: @usage_metrics,
+        log: @log
+      )
+    end
+
+    def run
+      ensure_georef_status_colummn_valid
+      @number_of_rows_pre_cache = calculate_number_of_rows
+      cache.run unless cache_disabled?
+      @csv_file = generate_csv()
+      geocoder.run
+      # Sync state because cancel is made synchronous
+      @geocoding_model.refresh
+      if not @geocoding_model.cancelled?
+        process_results if @geocoding_model.state == 'completed'
+        cache.store unless cache_disabled?
+      end
+    ensure
+      self.remote_id = @geocoding_model.remote_id
+      update_metrics unless @geocoding_model.cancelled?
+    end
+
+    # TODO: make the geocoders update status directly in the model
+    def update_geocoding_status
+      geocoder.update_status
+      { processed_rows: geocoder.processed_rows, state: geocoder.status }
+    end
+
+    def cancel
+      # We have to be sure the cartodb_georef_status column exists
+      ensure_georef_status_colummn_valid
+      @number_of_rows_pre_cache = calculate_number_of_rows
+      geocoder.cancel
+    end
+
+    def process_results
+      download_results # TODO move to HiresBatchGeocoder
+      deflate_results # TODO move to HiresBatchGeocoder
+      create_temp_table
+      import_results_to_temp_table
+      load_results_into_original_table
+      mark_rows_not_geocoded
+    rescue Sequel::DatabaseError => e
+      if e.message =~ /canceling statement due to statement timeout/
+        # INFO: Timeouts here are not recoverable for batched geocodes, but they are for non-batched
+        # INFO: cache.store relies on having results in the target table
+        raise Carto::GeocoderErrors::TableGeocoderDbTimeoutError.new(e)
+      else
+        raise
+      end
+    ensure
+      drop_temp_table
+    end
+
+
+    def used_batch_request?
+      return geocoder.used_batch_request?
+    end
+
+    def name
+      'heremaps'
+    end
+
+    private
+
+    def geocoder
+      @geocoder ||= CartoDB::HiresGeocoderFactory.get(@csv_file, @working_dir, @log, @geocoding_model,
+                                                      @number_of_rows_pre_cache)
+    end
+
+    def cache_disabled?
+      GeocoderConfig.instance.get['disable_cache'] || false
+    end
+
+    def calculate_number_of_rows
+      rows = connection.fetch(%Q{
+        SELECT count(DISTINCT(#{clean_formatter}))
+        FROM #{@qualified_table_name}
+        WHERE cartodb_georef_status IS NULL OR cartodb_georef_status IS FALSE
+        LIMIT #{@max_rows}
+      }).first
+      rows[:count]
+    end
+
+    # Generate a csv input file from the geocodable rows
+    def generate_csv
+      csv_file = File.join(@working_dir, "wadus.csv")
+      # INFO: we exclude inputs too short and "just digits" inputs, which will remain as georef_status = false
+      query = %Q{
+        WITH geocodable AS (
+          SELECT DISTINCT(#{clean_formatter}) recId, #{clean_formatter} searchText
+          FROM #{@qualified_table_name}
+          WHERE cartodb_georef_status IS NULL
+          LIMIT #{@max_rows - cache.hits}
+        )
+        SELECT * FROM geocodable
+        WHERE length(searchText) > 3 AND searchText !~ '^[\\d]*$'
+      }
+      result = connection.copy_table(connection[query], format: :csv, options: 'HEADER')
+      File.write(csv_file, result.force_encoding("UTF-8"))
+      return csv_file
+    end
+
+    def clean_formatter
+      "trim(both from regexp_replace(regexp_replace(concat(#{formatter}), E'[\\n\\r]+', ' ', 'g'), E'\"', '', 'g'))"
+    end
+
+    def download_results
+      @result = geocoder.result
+    end
+
+    def deflate_results
+      current_directory = Dir.pwd
+      Dir.chdir(@working_dir)
+      out = `unp *.zip 2>&1`
+      out = `unp #{@working_dir}/*_out.zip 2>&1`
+    ensure
+      Dir.chdir(current_directory)
+    end
+
+    def create_temp_table
+      connection.run(%Q{
+        CREATE TABLE #{temp_table_name} (
+          recId text,
+          SeqNumber int,
+          seqLength int,
+          displayLatitude float,
+          displayLongitude float
+        );}
+      )
+    end
+
+    def drop_temp_table
+      connection.run("DROP TABLE IF EXISTS #{temp_table_name}")
+    end
+
+    def import_results_to_temp_table
+      connection.copy_into(Sequel.lit(temp_table_name), data: File.read(deflated_results_path), format: :csv)
+    end
+
+    def load_results_into_original_table
+      connection.run(%Q{
+        UPDATE #{@qualified_table_name} AS dest
+        SET the_geom = ST_GeomFromText(
+            'POINT(' || orig.displayLongitude || ' ' ||
+              orig.displayLatitude || ')', 4326
+            ),
+            cartodb_georef_status = TRUE
+        FROM #{temp_table_name} AS orig
+        WHERE #{clean_formatter} = orig.recId
+      })
+    end
+
+    def mark_rows_not_geocoded
+      connection.run(%Q{UPDATE #{@qualified_table_name} SET cartodb_georef_status = FALSE WHERE cartodb_georef_status IS NULL})
+    end
+
+    def temp_table_name
+      @temp_table_name ||= "#{@schema}.geo_#{UUIDTools::UUID.timestamp_create.to_s.gsub('-', '')}"
+    end
+
+    def deflated_results_path
+      Dir[File.join(@working_dir, '*_out.txt')][0]
+    end
+
+    def update_metrics
+      total_requests = geocoder.successful_processed_rows + geocoder.empty_processed_rows + geocoder.failed_processed_rows
+      @usage_metrics.incr(:geocoder_here, :success_responses, geocoder.successful_processed_rows)
+      @usage_metrics.incr(:geocoder_here, :empty_responses, geocoder.empty_processed_rows)
+      @usage_metrics.incr(:geocoder_here, :failed_responses, geocoder.failed_processed_rows)
+      @usage_metrics.incr(:geocoder_here, :total_requests, total_requests)
+    end
+  end
+end

services/table-geocoder/lib/table_geocoder_factory.rb

@@ -0,0 +1,69 @@
+require_relative 'table_geocoder'
+require_relative 'internal_geocoder'
+require_relative 'gme/table_geocoder'
+require_relative 'exceptions'
+
+module Carto
+  class TableGeocoderFactory
+
+    def self.get(user, geocoding_model, cartodb_geocoder_config, table_service, params = {})
+      # Reset old connections to make sure changes apply.
+      # NOTE: This assumes it's being called from a Resque job
+      user.db_service.reset_pooled_connections
+      log = params.fetch(:log)
+      log.append_and_store 'TableGeocoderFactory.get()'
+      log.append_and_store "params: #{params.select{ |k| k != :log }}"
+
+      if user == table_service.owner
+        user_connection = user.in_database
+      else
+        if !table_service.table_visualization.has_permission?(user, Carto::Permission::ACCESS_READWRITE)
+          raise 'Insufficient permissions on table'
+        end
+        user_connection = table_service.owner.in_database
+      end
+
+      instance_config = cartodb_geocoder_config
+        .deep_symbolize_keys
+        .merge(
+               table_schema:  table_service.try(:database_schema),
+               table_name:    table_service.try(:name),
+               qualified_table_name: table_service.try(:qualified_table_name),
+               sequel_qualified_table_name: table_service.try(:sequel_qualified_table_name),
+               connection:    user_connection
+               )
+        .merge(params)
+
+      kind = instance_config.fetch(:kind)
+
+      if kind == 'high-resolution'
+        if user.google_maps_geocoder_enabled?
+          geocoder_class = Carto::Gme::TableGeocoder
+
+          instance_config[:client_id] = user.google_maps_client_id
+          instance_config[:private_key] = user.google_maps_private_key
+        elsif user.geocoder_provider == 'heremaps'
+          geocoder_class = CartoDB::TableGeocoder
+        else
+          raise 'Unsupported geocoder provider'
+        end
+      else
+        geocoder_class = CartoDB::InternalGeocoder::Geocoder
+      end
+
+      instance_config[:usage_metrics] = get_geocoder_metrics_instance(user)
+      instance_config[:log] = log
+      instance_config[:geocoding_model] = geocoding_model
+
+      log.append_and_store "geocoder_class = #{geocoder_class}"
+      instance = geocoder_class.new(instance_config)
+      log.append_and_store "geocoder_type = #{instance.name}"
+      instance
+    end
+
+    def self.get_geocoder_metrics_instance(user)
+      orgname = user.organization.nil? ? nil : user.organization.name
+      CartoDB::GeocoderUsageMetrics.new(user.username, orgname)
+    end
+  end
+end

services/table-geocoder/spec/factories/database.json

@@ -0,0 +1,7 @@
+{
+  "host"      : "localhost",
+  "port"      : 5432,
+  "user"      : "postgres",
+  "password"  : "",
+  "database"  : "testgeocoder"
+}

services/table-geocoder/spec/factories/database.json.sample

@@ -0,0 +1,7 @@
+{
+  "host"      : "localhost",
+  "port"      : 5432,
+  "user"      : "test",
+  "password"  : "",
+  "database"  : "test"
+}

services/table-geocoder/spec/factories/pg_connection.rb

@@ -0,0 +1,53 @@
+require 'pg'
+require 'sequel'
+require 'json'
+
+module CartoDB
+  module Importer2
+    module Factories
+      class PGConnection
+        def initialize
+          raise(
+            "Please configure your database settings " + 
+            "in spec/factories/database.json"
+          ) unless File.exists?(configuration_file)
+          @pg_options = ::JSON.parse(File.read(configuration_file))
+          create_db
+        end #initialize
+
+        def create_db
+          conn = Sequel.postgres(pg_options.reject{|k,v| k == :database})
+          begin
+            conn.run("CREATE DATABASE \"#{ pg_options[:database] }\"
+            WITH TEMPLATE = template_postgis
+            OWNER = #{ pg_options[:user] }
+            ENCODING = 'UTF8'
+            CONNECTION LIMIT=-1")
+          rescue Sequel::DatabaseError => e
+            raise unless e.message =~ /database .* already exists/
+          end
+          begin
+            conn.run("CREATE EXTENSION postgis")
+          rescue Sequel::DatabaseError => e
+            raise unless e.message =~ /extension \"postgis\" already exists/
+          end
+        end
+
+        def connection
+          Sequel.postgres(pg_options)
+        end #connection
+
+        def pg_options
+          Hash[@pg_options.map { |k, v| [k.to_sym, v] }]
+        end #pg_options
+
+        private
+
+        def configuration_file
+          File.join(File.dirname("#{__FILE__}"), 'database.json')
+        end #configuration_file
+      end # PGConnection
+    end # Factories
+  end # Importer2
+end # CartoDB
+

services/table-geocoder/spec/fixtures/adm0.csv

@@ -0,0 +1,14 @@
+1,"Netherlands"
+2,"Spain"
+3,"The Netherlands"
+4,"Russia"
+5,"Japan"
+6,"France"
+7,"Italy"
+8,"Sweden"
+9,"    Norway"
+10,"Canada"
+11,"Brazil"
+12,"Spain"
+13,"Spain"
+14,"Spain"

services/table-geocoder/spec/fixtures/gme_output_empty.json

@@ -0,0 +1,64 @@
+{
+  "results": [
+    {
+      "address_components": [
+        {
+          "long_name": "1600",
+          "short_name": "1600",
+          "types": [
+            "street_number"
+          ]
+        },
+        {
+          "long_name": "Amphitheatre Pkwy",
+          "short_name": "Amphitheatre Pkwy",
+          "types": [
+            "route"
+          ]
+        },
+        {
+          "long_name": "Mountain View",
+          "short_name": "Mountain View",
+          "types": [
+            "locality",
+            "political"
+          ]
+        },
+        {
+          "long_name": "Santa Clara County",
+          "short_name": "Santa Clara County",
+          "types": [
+            "administrative_area_level_2",
+            "political"
+          ]
+        },
+        {
+          "long_name": "California",
+          "short_name": "CA",
+          "types": [
+            "administrative_area_level_1",
+            "political"
+          ]
+        },
+        {
+          "long_name": "United States",
+          "short_name": "US",
+          "types": [
+            "country",
+            "political"
+          ]
+        },
+        {
+          "long_name": "94043",
+          "short_name": "94043",
+          "types": [
+            "postal_code"
+          ]
+        }
+      ],
+      "formatted_address": "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
+      "geometry": {}
+    }
+  ],
+  "status": "ZERO_RESULTS"
+}

services/table-geocoder/spec/fixtures/gme_output_error.json

@@ -0,0 +1,3 @@
+{
+  "status": "UNKNOWN_ERROR"
+}

services/table-geocoder/spec/fixtures/gme_output_error_with_message.json

@@ -0,0 +1,4 @@
+{
+  "error_message": "dummy error message",
+  "status": "UNKNOWN_ERROR"
+}

services/table-geocoder/spec/fixtures/gme_output_ok.json

@@ -0,0 +1,84 @@
+{
+  "results": [
+    {
+      "address_components": [
+        {
+          "long_name": "1600",
+          "short_name": "1600",
+          "types": [
+            "street_number"
+          ]
+        },
+        {
+          "long_name": "Amphitheatre Pkwy",
+          "short_name": "Amphitheatre Pkwy",
+          "types": [
+            "route"
+          ]
+        },
+        {
+          "long_name": "Mountain View",
+          "short_name": "Mountain View",
+          "types": [
+            "locality",
+            "political"
+          ]
+        },
+        {
+          "long_name": "Santa Clara County",
+          "short_name": "Santa Clara County",
+          "types": [
+            "administrative_area_level_2",
+            "political"
+          ]
+        },
+        {
+          "long_name": "California",
+          "short_name": "CA",
+          "types": [
+            "administrative_area_level_1",
+            "political"
+          ]
+        },
+        {
+          "long_name": "United States",
+          "short_name": "US",
+          "types": [
+            "country",
+            "political"
+          ]
+        },
+        {
+          "long_name": "94043",
+          "short_name": "94043",
+          "types": [
+            "postal_code"
+          ]
+        }
+      ],
+      "formatted_address": "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
+      "geometry": {
+        "location": {
+          "lat": 37.4224764,
+          "lng": -122.0842499
+        },
+        "location_type": "ROOFTOP",
+        "viewport": {
+          "northeast": {
+            "lat": 37.4238253802915,
+            "lng": -122.0829009197085
+          },
+          "southwest": {
+            "lat": 37.4211274197085,
+            "lng": -122.0855988802915
+          }
+        }
+      },
+      "place_id": "ChIJ2eUgeAK6j4ARbn5u_wAGqWA",
+      "types": [
+        "street_address"
+      ]
+    }
+  ],
+  "status": "OK"
+}

services/table-geocoder/spec/fixtures/nokia_input.csv

@@ -0,0 +1,10 @@
+recid,searchtext
+"Østerbrogade 43,  2100 KØBENHAVN, DNK","Østerbrogade 43,  2100 KØBENHAVN, DNK"
+"Amster dam, NLD","Amster dam, NLD"
+"León, ESP","León, ESP"
+"Villablino, León","Villablino, León"
+"Leganés   , Madrid","Leganés   , Madrid"
+", ESP",", ESP"
+"Groningen, Netherlands,","Groningen, Netherlands,"
+"Leganés, Madrid","Leganés, Madrid"
+"Leganes,  Madrid","Leganes,  Madrid"

services/table-geocoder/spec/fixtures/nokia_input_.csv

@@ -0,0 +1,38 @@
+recid,searchtext
+5498,"Agadez, Niger"
+6604,"Tabriz, Iran"
+206,"Ames, United States"
+1760,"Apsheronsk, Russia"
+2012,"Agapa, Russia"
+2163,"Guasave, Mexico"
+2225,"Pochutla, Mexico"
+2633,"Tongling, China"
+2985,"Anlu, China"
+3327,"Berekum, Ghana"
+3452,"Melun, French Republic"
+3666,"Shahrud, Iran"
+4081,"Sumy, Ukraine"
+4167,"Pinheiro, Brazil"
+4400,"Hinton, Canada"
+4509,"Salluit, Canada"
+5005,"Baranavichy, Belarus"
+5098,"Katoomba, Australia"
+5217,"Smithton, Australia"
+5487,"Bilbao, Kingdom of Spain"
+5826,"La Paz, Mexico"
+5886,"Wonsan, Korea, North"
+5997,"Amman, Jordan"
+6397,"Sekondi, Ghana"
+6532,"Kirkuk, Iraq"
+6841,"Dawson, Canada"
+6870,"Fredericton, Canada"
+6927,"Santa Fe, Argentina"
+7129,"Whyalla, Australia"
+7176,"Sittwe, Myanmar"
+7266,"Uummannaq, Denmark"
+887,"My Tho, Vietnam"
+3232,"Otsu, Japan"
+3602,"Pori, Finland"
+5319,"Paramaribo, Suriname"
+5693,"Tver, Russia"
+7313,"Oymyakon, Russia"

services/table-geocoder/spec/fixtures/nokia_output.txt

@@ -0,0 +1,44 @@
+"Agadez, Niger",1,1,17.0926609,7.5755601
+"Agapa, Russia",1,1,53.3041992,59.1366501
+"Ames, United States",1,5,42.02535,-93.6202
+"Ames, United States",2,5,30.05339,-94.74227
+"Ames, United States",3,5,36.24605,-98.18511
+"Ames, United States",4,5,42.83659,-74.59919
+"Ames, United States",5,5,41.45263,-96.62508
+"Amman, Jordan",1,1,31.95181,35.94042
+"Anlu, China",1,1,31.25582,113.68918
+"Apsheronsk, Russia",1,1,44.46301,39.72893
+"Baranavichy, Belarus",1,1,53.13963,26.02236
+"Berekum, Ghana",1,1,7.45498,-2.58425
+"Bilbao, Kingdom of Spain",1,1,37.2187881,-5.3498902
+"Dawson, Canada",1,2,64.06332,-139.43439
+"Dawson, Canada",2,2,48.72222,-94.48447
+"Fredericton, Canada",1,1,45.96063,-66.63911
+"Guasave, Mexico",1,1,25.5706,-108.47102
+"Hinton, Canada",1,1,53.40095,-117.57697
+"Katoomba, Australia",1,1,-33.71181,150.31056
+"Kirkuk, Iraq",1,1,35.4687309,44.3895187
+"La Paz, Mexico",1,2,19.35847,-98.95884
+"La Paz, Mexico",2,2,24.15464,-110.31023
+"My Tho, Vietnam",1,1,10.35595,106.36643
+"Otsu, Japan",1,1,42.73321,143.59012
+"Oymyakon, Russia",1,1,63.4606133,142.786026
+"Paramaribo, Suriname",1,1,5.82031,-55.16542
+"Pinheiro, Brazil",1,1,-2.52138,-45.0832
+"Pochutla, Mexico",1,1,15.7434,-96.46694
+"Pori, Finland",1,1,61.48596,21.79537
+"Salluit, Canada",1,1,62.20523,-75.64252
+"Santa Fe, Argentina",1,1,-31.63666,-60.712
+"Sekondi, Ghana",1,1,4.9340301,-1.70972
+"Shahrud, Iran",1,2,30.9944096,50.0435715
+"Shahrud, Iran",2,2,30.1367893,52.5212402
+"Sittwe, Myanmar",1,1,20.13819,92.88672
+"Smithton, Australia",1,1,-40.85546,145.12035
+"Sumy, Ukraine",1,1,50.90787,34.79759
+"Tabriz, Iran",1,1,38.0797081,46.3001518
+"Tongling, China",1,1,30.94496,117.81258
+"Tver, Russia",1,1,56.85238,35.93375
+"Whyalla, Australia",1,1,-33.03351,137.58416
+"Wonsan, Korea, North",1,3,34.5083313,126.3166122
+"Wonsan, Korea, North",2,3,36.1985207,127.0890808
+"Wonsan, Korea, North",3,3,35.188839,126.947731

services/table-geocoder/spec/fixtures/populated_places_short.csv

@@ -0,0 +1,19 @@
+1,"Amster
+dam",NLD
+2,"Groningen, Netherlands",
+3,León,ESP
+4,"Østerbrogade 43, 
+2100 KØBENHAVN",DNK
+5,León,"ESP
+
+
+"
+6,Villablino,León
+7,Villablino,León
+8,Leganés,Madrid
+9,Leganés   ,Madrid
+10,,
+11, , 
+12,,ESP
+13, ,ESP
+14,Leganes, Madrid

services/table-geocoder/spec/fixtures/result_20130919-04-55_6.2.46.1_out.txt

@@ -0,0 +1,45 @@
+206,1,5,42.02535,-93.6202
+206,2,5,30.05339,-94.74227
+206,3,5,36.24605,-98.18511
+206,4,5,42.83659,-74.59919
+206,5,5,41.45263,-96.62508
+887,1,1,10.35855,106.35897
+1760,1,1,44.46301,39.72893
+2012,1,1,44.6720581,-73.792572
+2163,1,1,25.5706,-108.47102
+2225,1,1,15.7434,-96.46694
+2633,1,1,30.94496,117.81258
+2985,1,1,31.25582,113.68918
+3232,1,1,42.73321,143.59012
+3327,1,1,7.45498,-2.58425
+3602,1,1,61.48596,21.79537
+3666,1,4,30.9944096,50.0435715
+3666,2,4,30.1367893,52.5212402
+3666,3,4,30.4089508,57.7633095
+3666,4,4,38.1983109,44.7774811
+4081,1,1,50.90787,34.79759
+4167,1,1,-2.52138,-45.0832
+4400,1,1,53.40095,-117.57697
+4509,1,1,62.20523,-75.64252
+5005,1,1,53.13963,26.02236
+5098,1,1,-33.71181,150.31056
+5217,1,1,-40.85546,145.12035
+5319,1,1,5.82031,-55.16542
+5487,1,1,37.2187881,-5.3498902
+5498,1,1,17.0926609,7.5755601
+5693,1,1,56.85238,35.93375
+5826,1,2,19.35842,-98.95862
+5826,2,2,24.15464,-110.31023
+5886,1,2,34.5083313,126.3166122
+5886,2,2,36.2474098,128.2595367
+5997,1,1,31.95181,35.94042
+6397,1,1,4.9360099,-1.72823
+6532,1,1,35.4687309,44.3895187
+6604,1,1,38.0797081,46.3001518
+6841,1,2,64.06332,-139.43439
+6841,2,2,48.72225,-94.48446
+6870,1,1,45.96063,-66.63911
+6927,1,1,-31.63666,-60.712
+7129,1,1,-33.03351,137.58416
+7176,1,1,20.13819,92.88672
+7313,1,1,63.4606133,142.786026

services/table-geocoder/spec/geocoder_cache_spec.rb

@@ -0,0 +1,73 @@
+require 'open3'
+require_relative '../lib/table_geocoder.rb'
+require_relative 'factories/pg_connection'
+require_relative '../../../spec/rspec_configuration.rb'
+
+RSpec.configure do |config|
+  config.mock_with :mocha
+end
+
+describe CartoDB::GeocoderCache do
+  before do
+    conn          = CartoDB::Importer2::Factories::PGConnection.new
+    @db           = conn.connection
+    @pg_options   = conn.pg_options
+    @table_name   = "ne_10m_populated_places_simple"
+    @usage_metrics_stub = stub
+    @log = mock
+    @log.stubs(:append)
+    @log.stubs(:append_and_store)
+
+    # Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
+    filename = 'populated_places_short.csv'
+    stdout, stderr, status =  Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
+    raise if stderr != ''
+    load_csv "/tmp/#{filename}"
+  end
+
+  after do
+    @db.drop_table @table_name
+  end
+
+  let(:default_params) { {
+      table_name: @table_name,
+      formatter: "concat(name, iso3)",
+      connection: @db, sql_api: { table_name: '' },
+      qualified_table_name: @table_name,
+      usage_metrics: @usage_metrics_stub,
+      log: @log
+    } }
+
+  describe '#get_cache_results' do
+    it "runs the query in batches" do
+      cache = CartoDB::GeocoderCache.new(default_params.merge(batch_size: 5))
+      @db.run("alter table #{@table_name} add column cartodb_georef_status BOOLEAN DEFAULT NULL")
+      cache.expects(:run_query).times(3).returns('')
+      cache.get_cache_results
+    end
+
+    it "honors max_rows" do
+      @db.run("alter table #{@table_name} add column cartodb_georef_status BOOLEAN DEFAULT NULL")
+      cache = CartoDB::GeocoderCache.new(default_params.merge(
+        batch_size: 5,
+        max_rows: 10
+      ))
+      cache.expects(:run_query).times(2).returns('')
+      cache.get_cache_results
+    end
+  end #run
+
+
+
+  def path_to(filepath = '')
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../spec/fixtures/#{filepath}")
+    )
+  end #path_to
+
+  def load_csv(path)
+    @db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
+    @db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
+  end # create_table
+
+end # CartoDB::GeocoderCache

services/table-geocoder/spec/internal-geocoder/input_type_resolver_spec.rb

@@ -0,0 +1,77 @@
+require_relative '../../lib/internal-geocoder/input_type_resolver.rb'
+require_relative '../../../../spec/rspec_configuration.rb'
+
+class String
+  # We just need this instead of adding the whole rails thing
+  def present?
+    self && !empty?
+  end
+end
+
+describe CartoDB::InternalGeocoder::InputTypeResolver do
+
+  before(:each) do
+    @internal_geocoder = mock
+    @input_type_resolver = CartoDB::InternalGeocoder::InputTypeResolver.new(@internal_geocoder)
+  end
+
+  describe '#type' do
+
+    it 'should return an array characterizing the inputs for <namedplace, country_name, region_name, point>' do
+      @internal_geocoder.stubs('kind').once.returns(:namedplace)
+      @internal_geocoder.stubs('geometry_type').once.returns(:point)
+      @internal_geocoder.stubs('country_column').once.returns(nil)
+      @internal_geocoder.stubs('region_column').once.returns(nil)
+      @internal_geocoder.stubs('regions').once.returns('region')
+
+      @input_type_resolver.type.should == [:namedplace, :text, :text, :point]
+    end
+
+  end
+
+  describe '#kind' do
+    it 'should return the type of the internal geocoding: namedplace' do
+      @internal_geocoder.stubs('kind').once.returns(:namedplace)
+      @input_type_resolver.kind.should == :namedplace
+    end
+  end
+
+  describe '#geometry_type' do
+    it 'should return the type of the geometry to be geocoded: polygon' do
+      @internal_geocoder.stubs('geometry_type').once.returns(:polygon)
+      @input_type_resolver.geometry_type.should == :polygon
+    end
+  end
+
+  describe '#country_input_type' do
+    it 'should return column if a column was passed' do
+      @internal_geocoder.stubs('country_column').once.returns('any_column_name')
+      @input_type_resolver.country_input_type.should == :column
+    end
+
+    it 'should return column if no column was passed' do
+      @internal_geocoder.stubs('country_column').once.returns(nil)
+      @input_type_resolver.country_input_type.should == :text
+    end
+  end
+
+  describe '#region_input_type' do
+    it 'should return :column if a column is present' do
+      @internal_geocoder.stubs('region_column').once.returns('wadus')
+      @input_type_resolver.region_input_type.should == :column
+    end
+
+    it 'should return :text if regions are present' do
+      @internal_geocoder.stubs('region_column').once.returns(nil)
+      @internal_geocoder.stubs('regions').once.returns('minnesota')
+      @input_type_resolver.region_input_type.should == :text
+    end
+
+    it 'should return nil if no column or text are present' do
+      @internal_geocoder.stubs('region_column').once.returns(nil)
+      @internal_geocoder.stubs('regions').once.returns('')
+      @input_type_resolver.region_input_type.should == nil
+    end
+  end
+
+end

services/table-geocoder/spec/internal-geocoder/query_generator_factory_spec.rb

@@ -0,0 +1,118 @@
+require_relative '../../lib/internal-geocoder/query_generator_factory.rb'
+require_relative '../../lib/internal-geocoder/abstract_query_generator.rb'
+require_relative '../../../../spec/rspec_configuration.rb'
+require          'active_support/core_ext' # Needed for string.blank?
+
+RSpec.configure do |config|
+  config.mock_with :mocha
+end
+
+=begin
+The class should generate queries to be used by the InternalGeocoder depending on the inputs.
+
+* Different types of inputs:
+  - kind: namedplace, ipaddress, postalcode, admin0, admin1
+  - country: column, freetext
+  - geometry_type: point, polygon
+
+* Where queries are needed:
+  - to query the data-services
+  - to import results into table
+=end
+
+describe CartoDB::InternalGeocoder::QueryGeneratorFactory do
+
+  before(:each) do
+    @internal_geocoder = mock
+  end
+
+  describe '#dataservices' do
+
+    it 'should return a query template suitable for <namedplace, country_name, region_name, point>' do
+      query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:namedplace, :text, :text, :point])
+      query_generator.should be_a_kind_of CartoDB::InternalGeocoder::AbstractQueryGenerator
+      @internal_geocoder.expects('countries').once.returns(%Q{'Spain'})
+      @internal_geocoder.expects('regions').once.returns(%Q{'Madrid'})
+      search_terms = [{city: %Q{'Madrid'}}, {city: %Q{'Granada'}}]
+
+      query = query_generator.dataservices_query(search_terms)
+
+      query.should == "WITH geo_function AS (SELECT (geocode_namedplace(Array['Madrid','Granada'], 'Madrid', 'Spain')).*) SELECT q, c, a1, geom, success FROM geo_function"
+    end
+
+  end
+
+  describe '#search_terms_query' do
+    it 'should get the search terms for <namedplace, country_name, region_name, point>' do
+      @internal_geocoder.stubs('column_name').once.returns('city')
+      @internal_geocoder.stubs('qualified_table_name').once.returns(%Q{"public"."untitled_table"})
+      @internal_geocoder.stubs('batch_size').returns(5000)
+      query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:namedplace, :text, :text, :point])
+
+      query = query_generator.search_terms_query(0)
+
+      query.squish.should == 'SELECT DISTINCT(trim(quote_nullable("city"))) AS city FROM "public"."untitled_table" WHERE cartodb_georef_status IS NULL LIMIT 5000 OFFSET 0'
+    end
+  end
+
+  describe '#copy_results_to_table_query' do
+    it 'should generate a suitable query to update geocoded table with temp table' do
+      @internal_geocoder.stubs('qualified_table_name').returns(%Q{"public"."untitled_table"})
+      @internal_geocoder.stubs('temp_table_name').once.returns('any_temp_table')
+      @internal_geocoder.stubs('column_name').once.returns('any_column_name')
+      query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:namedplace, :text, :text, :point])
+
+      query = query_generator.copy_results_to_table_query
+
+      query.squish.should == %Q{
+        UPDATE "public"."untitled_table" AS dest
+        SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+            cartodb_georef_status = orig.cartodb_georef_status
+        FROM any_temp_table AS orig
+        WHERE trim(dest."any_column_name"::text) = trim(orig.geocode_string) AND dest.cartodb_georef_status IS NULL
+      }.squish
+    end
+  end
+
+  describe 'CDB-4269' do
+    it 'should generate a suitable query generator for [:admin1, :column, nil, :polygon]' do
+      @internal_geocoder.stubs('column_name').twice.returns('region_column_name')
+      @internal_geocoder.stubs('qualified_table_name').returns('any_table_name')
+      @internal_geocoder.stubs('country_column').twice.returns('country_column_name')
+      @internal_geocoder.stubs('batch_size').twice.returns(10)
+      @internal_geocoder.stubs('temp_table_name').once.returns('any_temp_tablename')
+
+      query_generator = CartoDB::InternalGeocoder::QueryGeneratorFactory.get(@internal_geocoder, [:admin1, :column, nil, :polygon])
+
+      query_generator.search_terms_query(0).squish.should == %Q{
+        SELECT DISTINCT
+          trim(quote_nullable(region_column_name)) as region,
+          trim(quote_nullable(country_column_name)) as country
+        FROM any_table_name
+        WHERE cartodb_georef_status IS NULL
+        LIMIT 10 OFFSET 0
+      }.squish
+
+      search_terms = [
+        { region: %Q{'New York'}, country: %Q{'USA'}},
+        { region: %Q{'Sucumbios'}, country: %Q{'Ecuador'}}
+      ]
+
+      query_generator.dataservices_query(search_terms).squish.should == %Q{
+        WITH geo_function AS (SELECT (geocode_admin1_polygons(Array['New York','Sucumbios'], Array['USA','Ecuador'])).*)
+        SELECT q, c, null AS a1, geom, success FROM geo_function
+      }.squish
+
+      query_generator.copy_results_to_table_query.squish.should == %Q{
+        UPDATE any_table_name AS dest
+          SET the_geom = CASE WHEN orig.cartodb_georef_status THEN orig.the_geom ELSE dest.the_geom END,
+              cartodb_georef_status = orig.cartodb_georef_status
+          FROM any_temp_tablename AS orig
+          WHERE trim(dest.region_column_name::text) = trim(orig.geocode_string)
+            AND trim(dest.country_column_name::text) = trim(orig.country)
+            AND dest.cartodb_georef_status IS NULL
+        }.squish
+    end
+  end
+
+end

services/table-geocoder/spec/internal_Geocoder_spec.rb

@@ -0,0 +1,49 @@
+require 'open3'
+require_relative 'factories/pg_connection'
+require_relative '../lib/internal_geocoder.rb'
+require_relative '../../../spec/rspec_configuration.rb'
+
+describe CartoDB::InternalGeocoder::Geocoder do
+  before do
+    conn          = CartoDB::Importer2::Factories::PGConnection.new
+    @db           = conn.connection
+    @pg_options   = conn.pg_options
+  end
+
+  let(:default_params) { { connection: @db, internal: { username: 'geocoding' }, kind: 'admin0', geometry_type: 'polygon' } }
+
+  describe '#download_results' do
+    before do
+      # Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
+      filename = 'adm0.csv'
+      stdout, stderr, status =  Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
+      raise if stderr != ''
+      load_csv "/tmp/#{filename}", 'adm0'
+    end
+
+    after do
+      @db.drop_table 'adm0'
+    end
+
+    it "generates a csv with geocoded data" do
+      ig = CartoDB::InternalGeocoder.new(default_params.merge(table_name: 'adm0', formatter: 'geo_string'))
+      ig.ensure_georef_status_colummn_valid
+      results = ig.download_results
+      `wc -l #{results} 2>&1`.to_i.should eq 11
+      ig.processed_rows.should eq 11
+    end
+  end #run
+
+  def path_to(filepath = '')
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../spec/fixtures/#{filepath}")
+    )
+  end #path_to
+
+  def load_csv(path , table_name)
+    @db.run("DROP TABLE IF EXISTS #{table_name}")
+    @db.run("CREATE TABLE #{table_name} (the_geom geometry, cartodb_id integer, geo_string text)")
+    @db.run("COPY #{Sequel.lit(table_name)}(cartodb_id, geo_string) FROM '#{path}' DELIMITER ',' CSV")
+  end # create_table
+
+end # CartoDB::GeocoderCache

services/table-geocoder/spec/lib/abstract_table_geocoder_spec.rb

@@ -0,0 +1,82 @@
+require_relative '../factories/pg_connection'
+require_relative '../../lib/abstract_table_geocoder'
+require_relative '../../../../spec/rspec_configuration.rb'
+
+describe CartoDB::AbstractTableGeocoder do
+
+  before(:all) do
+    class ConcreteTableGeocoder < CartoDB::AbstractTableGeocoder; end
+
+    conn          = CartoDB::Importer2::Factories::PGConnection.new
+    @db           = conn.connection
+    @pg_options   = conn.pg_options
+    @table_name   = "ne_10m_populated_places_simple"
+    load_csv path_to("populated_places_short.csv")
+  end
+
+  after(:all) do
+    @db.drop_table @table_name
+  end
+
+
+  describe '#initialize' do
+    it 'sets the connection timeout to 5 hours' do
+      tg = ConcreteTableGeocoder.new({
+          connection: @db,
+          table_name: @table_name,
+          qualified_table_name: @table_name
+        })
+      timeout = @db.fetch("SHOW statement_timeout").first.fetch(:statement_timeout)
+      timeout.should == '5h'
+    end
+  end
+
+
+  describe '#ensure_georef_status_colummn_valid' do
+
+    before(:each) do
+      @tg = ConcreteTableGeocoder.new({
+          connection: @db,
+          table_name: @table_name,
+          qualified_table_name: @table_name
+        })
+    end
+
+    it 'adds a georef_status_column if it does not exists' do
+      @tg.send(:ensure_georef_status_colummn_valid)
+      assert_correctness_of_georef_status_column
+    end
+
+    it 'does nothing if the column already exists' do
+      @tg.send(:ensure_georef_status_colummn_valid)
+      @tg.send(:ensure_georef_status_colummn_valid)
+      assert_correctness_of_georef_status_column
+    end
+
+    it 'casts its type if the column exists and is not bool' do
+      @db.add_column @table_name, :georef_status_column, :text
+      @tg.send(:ensure_georef_status_colummn_valid)
+      assert_correctness_of_georef_status_column
+    end
+
+  end
+
+
+  def assert_correctness_of_georef_status_column
+    georef_status_column = @db.schema(@table_name, reload: true).select {|c| c[0] == :cartodb_georef_status}.first
+    georef_status_column.nil?.should == false
+    georef_status_column[1][:db_type].should == 'boolean'
+  end
+
+  def load_csv(path)
+    @db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
+    @db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
+  end
+
+  def path_to(filepath = '')
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../fixtures/#{filepath}")
+    )
+  end
+
+end

services/table-geocoder/spec/lib/gme/table_geocoder_spec.rb

@@ -0,0 +1,246 @@
+require 'open3'
+require_relative '../../../lib/gme/table_geocoder'
+require_relative '../../../../../lib/url_signer'
+require_relative '../../../lib/gme/exceptions'
+require_relative '../../factories/pg_connection'
+require_relative '../../../../../spec/spec_helper.rb'
+require_relative '../../../../../spec/rspec_configuration.rb'
+
+describe Carto::Gme::TableGeocoder do
+  before(:all) do
+    connection_stub = mock
+    connection_stub.stubs(:run)
+    @usage_metrics_stub = stub
+    @log = mock
+    @log.stubs(:append)
+    @log.stubs(:append_and_store)
+    @geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution', formatter: '{street}')
+
+    @mandatory_args = {
+      connection: connection_stub,
+      original_formatter: '{mock}',
+      client_id: 'my_client_id',
+      private_key: 'my_private_key',
+      usage_metrics: @usage_metrics_stub,
+      log: @log,
+      geocoding_model: @geocoding_model
+    }
+  end
+
+  describe '#initialize' do
+    it 'returns an object that responds to AbstractTableGeocoder interface' do
+      table_geocoder = Carto::Gme::TableGeocoder.new(@mandatory_args)
+      interface_methods = [:ensure_georef_status_colummn_valid,
+                           :cancel,
+                           :run,
+                           :remote_id,
+                           :update_geocoding_status,
+                           :process_results]
+      interface_methods.each do |method|
+        table_geocoder.respond_to?(method, true).should == true
+      end
+    end
+
+    it 'raises an exception if not fed with mandatory arguments' do
+      expect { Carto::Gme::TableGeocoder.new }.to raise_error(ArgumentError)
+
+      @mandatory_args.each do |arg|
+        args_missing_one = @mandatory_args.dup
+        args_missing_one.delete(arg[0])
+        lambda { Carto::Gme::TableGeocoder.new(args_missing_one) }.should raise_error(KeyError)
+      end
+    end
+
+    it 'creates a client with the provided credentials' do
+      gme_client_mock = mock
+      Carto::Gme::Client.expects(:new).with('my_client_id', 'my_private_key').once.returns(gme_client_mock)
+      Carto::Gme::GeocoderClient.expects(:new).with(gme_client_mock).once
+      Carto::Gme::TableGeocoder.new(@mandatory_args)
+    end
+  end
+
+  describe '#run' do
+    before(:each) do
+      Carto::UrlSigner.any_instance.stubs(:sign_url).returns('https://maps.googleapis.com/maps/api/geocode/json')
+      @table_geocoder = Carto::Gme::TableGeocoder.new(@mandatory_args)
+    end
+
+    it "set's the state to 'processing' when it starts" do
+      pending 'actually as a requirement this does not make much sense'
+    end
+
+    it "set's the state to 'completed' when it ends" do
+      # TODO: there's something weird that needs review here
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
+
+      @table_geocoder.stubs(:ensure_georef_status_colummn_valid)
+      @table_geocoder.stubs(:data_input_blocks).returns([])
+
+      @table_geocoder.run
+      @geocoding_model.state.should == 'completed'
+    end
+
+    it "if there's an uncontrolled exception, sets the state to 'failed' and raises it" do
+      # TODO: there's something weird that needs review here
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
+
+      @table_geocoder.stubs(:ensure_georef_status_aolummn_valid)
+      @table_geocoder.stubs(:data_input_blocks).returns([{ cartodb_id: 1, searchtext: 'dummy text' }])
+      @table_geocoder.stubs(:geocode).raises(StandardError, 'unexpected exception')
+
+      expect { @table_geocoder.run }.to raise_error('unexpected exception')
+      @geocoding_model.state.should == 'failed'
+    end
+
+    it "processes 1 block at a time, keeping track of processed rows in each block" do
+      # TODO: there's something weird that needs review here
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 4)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 4)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
+
+      @table_geocoder.stubs(:ensure_georef_status_colummn_valid)
+      mocked_input = Enumerator.new do |enum|
+        # 2 blocks of 2 rows each as input
+        enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }, { cartodb_id: 2, searchtext: 'dummy text' }]
+        enum.yield [{ cartodb_id: 3, searchtext: 'dummy text' }, { cartodb_id: 4, searchtext: 'dummy text' }]
+      end
+      @table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
+      response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_ok.json'))
+      Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
+      @table_geocoder.expects(:update_table).twice
+
+      @table_geocoder.run
+      @table_geocoder.processed_rows.should == 4
+    end
+
+    it "processes empty response" do
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 1)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 0)
+
+      @table_geocoder.stubs(:ensure_georef_status_colummn_valid)
+      mocked_input = Enumerator.new do |enum|
+        enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
+      end
+      @table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
+      response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_empty.json'))
+      Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
+      @table_geocoder.expects(:update_table).once
+
+      @table_geocoder.run
+      @table_geocoder.processed_rows.should == 1
+    end
+
+    it "processes error rows response" do
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 1)
+
+      @table_geocoder.stubs(:ensure_georef_status_colummn_valid)
+      mocked_input = Enumerator.new do |enum|
+        enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
+      end
+      @table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
+      response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_error.json'))
+      Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
+      @table_geocoder.expects(:update_table).once
+
+      @table_geocoder.run
+      @table_geocoder.processed_rows.should == 1
+    end
+
+    it "processes error with message response" do
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :total_requests, 1)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :success_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :empty_responses, 0)
+      @usage_metrics_stub.expects(:incr).with(:geocoder_google, :failed_responses, 1)
+
+      @table_geocoder.stubs(:ensure_georef_status_colummn_valid)
+      mocked_input = Enumerator.new do |enum|
+        enum.yield [{ cartodb_id: 1, searchtext: 'dummy text' }]
+      end
+      @table_geocoder.stubs(:data_input_blocks).returns(mocked_input)
+      response = Typhoeus::Response.new(code: 200, body: read_fixture_file('gme_output_error_with_message.json'))
+      Typhoeus.stub('https://maps.googleapis.com/maps/api/geocode/json', method: :get).and_return(response)
+      @table_geocoder.expects(:update_table).once
+      # CartoDB.expects(:notify_error).once
+
+      @table_geocoder.run
+      @table_geocoder.processed_rows.should == 1
+    end
+  end
+
+  describe '#data_input_blocks' do
+    before do
+      conn          = CartoDB::Importer2::Factories::PGConnection.new
+      @db           = conn.connection
+      @pg_options   = conn.pg_options
+      @table_name   = "ne_10m_populated_places_simple_#{rand.to_s[2..11]}"
+
+      # Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
+      filename = 'populated_places_short.csv'
+      _stdout, stderr, _status = Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
+      raise if stderr != ''
+      load_csv "/tmp/#{filename}"
+
+      params = {
+        connection: @db,
+        table_name: @table_name,
+        qualified_table_name: @table_name,
+        sequel_qualified_table_name: @table_name,
+        original_formatter: "{name}, {iso3}",
+        client_id: 'my_client_id',
+        private_key: 'my_private_key',
+        max_block_size: 4,
+        usage_metrics: @usage_metrics_stub,
+        log: @log,
+        geocoding_model: @geocoding_model
+      }
+
+      @table_geocoder = Carto::Gme::TableGeocoder.new(params)
+    end
+
+    after do
+      @db.drop_table @table_name
+    end
+
+    it 'performs (floor(rows / max_block_size) + 1) queries' do
+      rows = @db[@table_name.to_sym].count
+      @table_geocoder.send(:ensure_georef_status_colummn_valid)
+
+      count = 0
+      @table_geocoder.send(:data_input_blocks).each do |data_block|
+        data_block.each do |row|
+          row.merge!(cartodb_georef_status: false)
+        end
+        @table_geocoder.send(:update_table, data_block)
+        count += 1
+      end
+      count.should == (rows / @table_geocoder.max_block_size).floor + 1
+    end
+  end
+
+  def path_to(filepath = '')
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../../fixtures/#{filepath}")
+    )
+  end
+
+  def read_fixture_file(filename)
+    File.read(path_to(filename))
+  end
+
+  def load_csv(path)
+    @db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
+    @db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
+  end
+end

services/table-geocoder/spec/table_geocoder_spec.rb

@@ -0,0 +1,286 @@
+require 'open3'
+require_relative '../lib/table_geocoder'
+require_relative 'factories/pg_connection'
+require 'set'
+require_relative '../../../spec/rspec_configuration'
+require_relative '../../../spec/spec_helper'
+
+describe CartoDB::TableGeocoder do
+  let(:default_params) {{
+    app_id: '',
+    token: '',
+    mailto: '',
+    usage_metrics: mock('usage_metrics')
+    }}
+  before do
+    conn          = CartoDB::Importer2::Factories::PGConnection.new
+    @db           = conn.connection
+    @pg_options   = conn.pg_options
+    @table_name   = "ne_10m_populated_places_simple_#{rand.to_s[2..11]}"
+    @log = mock
+    @log.stubs(:append)
+    @log.stubs(:append_and_store)
+    @geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution',
+                                          formatter: '{street}', remote_id: 'dummy_request_id')
+
+    # Avoid issues on some machines if postgres system account can't read fixtures subfolder for the COPY
+    filename = 'populated_places_short.csv'
+    _stdout, stderr, _status = Open3.capture3("cp #{path_to(filename)} /tmp/#{filename}")
+    raise if stderr != ''
+    load_csv "/tmp/#{filename}"
+  end
+
+  after do
+    @db.drop_table @table_name
+  end
+
+  describe '#run' do
+    before do
+      # TODO: Note the coupling of the geocoder object and the metrics
+      success_rows = 10
+      empty_rows = 3
+      failed_rows = 4
+      total = success_rows + empty_rows + failed_rows
+      default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :success_responses, success_rows)
+      default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :empty_responses, empty_rows)
+      default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :failed_responses, failed_rows)
+      default_params[:usage_metrics].expects(:incr).with(:geocoder_here, :total_requests, total)
+
+      @tg = CartoDB::TableGeocoder.new(default_params.merge(table_name: @table_name,
+                                                            qualified_table_name: @table_name,
+                                                            sequel_qualified_table_name: @table_name,
+                                                            formatter:  "name, ', ', iso3",
+                                                            connection: @db,
+                                                            log: @log,
+                                                            geocoding_model: @geocoding_model,
+                                                            max_rows: 1000))
+      geocoder = mock
+      geocoder.stubs(:upload).returns(true)
+      geocoder.stubs(:request_id).returns('dummy_request_id')
+      geocoder.stubs(:run).returns(true)
+      geocoder.stubs(:status).returns('foo')
+
+      # TODO: Note the coupling of the geocoder object and the metrics
+      geocoder.stubs(:successful_processed_rows).returns(success_rows)
+      geocoder.stubs(:empty_processed_rows).returns(empty_rows)
+      geocoder.stubs(:failed_processed_rows).returns(failed_rows)
+
+      @tg.stubs(:geocoder).returns(geocoder)
+      @tg.stubs(:cache_disabled?).returns(true)
+      @tg.run
+    end
+
+    it "generates a csv file for uploading" do
+      expected = Set.new(File.readlines(path_to('nokia_input.csv')))
+      actual = Set.new(File.readlines("#{@tg.working_dir}/wadus.csv"))
+      actual.should == expected
+    end
+
+    it "assigns a remote_id" do
+      @tg.remote_id.should == 'dummy_request_id'
+    end
+
+    it "holds a db connection with the specified statement timeout" do
+      timeout = @tg.connection.fetch("SHOW statement_timeout").all[0][:statement_timeout]
+      timeout.should == '5h'
+    end
+  end
+
+  describe '#generate_csv' do
+    before do
+      @tg = CartoDB::TableGeocoder.new(default_params.merge(table_name: @table_name,
+                                                            qualified_table_name: @table_name,
+                                                            sequel_qualified_table_name: @table_name,
+                                                            formatter:  "name, ', ', iso3",
+                                                            connection: @db,
+                                                            log: @log,
+                                                            geocoding_model: @geocoding_model,
+                                                            max_rows: 1000))
+      @tg.send(:ensure_georef_status_colummn_valid)
+    end
+
+    it "generates a csv file with the correct format" do
+      @tg.send(:mark_rows_to_geocode)
+      @tg.send(:generate_csv)
+      File.readlines("#{@tg.working_dir}/wadus.csv").to_set.should == File.readlines(path_to('nokia_input.csv')).to_set
+    end
+
+    it "honors max_rows" do
+      max_rows = 10
+      @tg.stubs(:max_rows).returns max_rows
+      @tg.send(:mark_rows_to_geocode)
+      @tg.send(:generate_csv)
+
+      # Note there might be duplicate input strings but we send unique inputs to the geocoder api.
+      # Also note the csv file has a header.
+      File.readlines("#{@tg.working_dir}/wadus.csv").count.should <= (max_rows + 1)
+    end
+  end
+
+  describe '#download_results' do
+    it 'gets the geocoder results' do
+      tg = CartoDB::TableGeocoder.new(table_name: 'a', connection: @db, max_rows: 1000,
+                                      usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
+      geocoder = mock
+      geocoder.expects(:result).times(1).returns('a')
+      tg.stubs(:geocoder).returns(geocoder)
+      tg.send(:download_results)
+      tg.result.should == 'a'
+    end
+  end
+
+  describe '#deflate_results' do
+    it 'does not raise an error if no results file' do
+      dir = Dir.mktmpdir
+      tg = CartoDB::TableGeocoder.new(table_name: 'a',
+                                      connection: @db, working_dir: dir, max_rows: 1000,
+                                      usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
+      expect { tg.send(:deflate_results) }.to_not raise_error
+    end
+
+    it 'extracts nokia result files' do
+      dir = Dir.mktmpdir
+      `cp #{path_to('kXYkQhuDfxnUSmWFP3dmq6TzTZAzwy4x.zip')} #{dir}`
+      tg = CartoDB::TableGeocoder.new(table_name: 'a',
+                                      connection: @db, working_dir: dir, max_rows: 1000,
+                                      usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
+      tg.send(:deflate_results)
+      filename = 'result_20130919-04-55_6.2.46.1_out.txt'
+      destfile = File.open(File.join(dir, filename))
+      destfile.read.should eq File.open(path_to(filename)).read
+    end
+  end
+
+  describe '#create_temp_table' do
+    it 'raises error if no remote_id' do
+      tg = CartoDB::TableGeocoder.new(table_name: 'a', connection: @db, max_rows: 1000,
+                                      usage_metrics: nil, log: @log, geocoding_model: @geocoding_model)
+      expect { tg.send(:create_temp_table) }.to raise_error(Sequel::DatabaseError)
+    end
+
+    it 'creates a temporary table' do
+      tg = CartoDB::TableGeocoder.new(table_name: 'a',
+                                      connection: @db,
+                                      remote_id: 'geo_HvyxzttLyFhaQ7JKmnrZxdCVySd8N0Ua',
+                                      log: @log,
+                                      geocoding_model: @geocoding_model,
+                                      schema: 'public', max_rows: 1000, usage_metrics: nil)
+      tg.send(:drop_temp_table)
+      tg.send(:create_temp_table)
+      @db.fetch("select * from #{tg.send(:temp_table_name)}").all.should eq []
+    end
+  end
+
+  describe '#import_results_to_temp_table' do
+    before do
+      @tg = CartoDB::TableGeocoder.new(table_name: 'a',
+                                       connection: @db,
+                                       log: @log,
+                                       geocoding_model: @geocoding_model,
+                                       remote_id: 'temp_table', schema: 'public', max_rows: 1000, usage_metrics: nil)
+      @tg.send(:create_temp_table)
+    end
+
+    after do
+      @tg.send(:drop_temp_table)
+    end
+
+    it 'loads the Nokia output format to an existing temp table' do
+      @tg.stubs(:deflated_results_path).returns(path_to('nokia_output.txt'))
+      @tg.send(:import_results_to_temp_table)
+      @db.fetch(%{
+        SELECT count(*) FROM #{@tg.send(:temp_table_name)}
+        WHERE displayLatitude IS NOT NULL AND displayLongitude IS NOT NULL
+      }).first[:count].should eq 44
+    end
+  end
+
+  describe '#ensure_georef_status_colummn_valid' do
+    before do
+      table_name = 'wwwwww'
+      @db.run("create table #{table_name} (id integer)")
+      @tg = CartoDB::TableGeocoder.new(table_name: 'wwwwww',
+                                       qualified_table_name: table_name,
+                                       sequel_qualified_table_name: table_name,
+                                       connection: @db,
+                                       remote_id: 'wadus',
+                                       max_rows: 1000,
+                                       log: @log,
+                                       geocoding_model: @geocoding_model,
+                                       usage_metrics: nil)
+    end
+
+    after do
+      @db.run("drop table wwwwww")
+    end
+
+    it 'adds a boolean cartodb_georef_status column' do
+      @tg.send(:ensure_georef_status_colummn_valid)
+      @db.run("select cartodb_georef_status from wwwwww").should eq nil
+    end
+
+    it 'does nothing when the column already exists' do
+      @tg.expects(:cast_georef_status_column).once
+      @tg.send(:ensure_georef_status_colummn_valid)
+      @tg.send(:ensure_georef_status_colummn_valid)
+    end
+
+    it 'casts cartodb_georef_status to boolean if needed' do
+      @db.run('alter table wwwwww add column cartodb_georef_status text')
+      @tg.send(:ensure_georef_status_colummn_valid)
+      sql_query = "select data_type from information_schema.columns " \
+                  "where table_name = 'wwwwww' and column_name = 'cartodb_georef_status'"
+      @db.fetch(sql_query)
+        .first[:data_type].should eq 'boolean'
+    end
+  end
+
+  it "Geocodes a table using the batch geocoder API" do
+    config = YAML.load_file("#{File.dirname(__FILE__)}/../../../config/app_config.yml")["test"]["geocoder"]
+    pending "This is a System E2E test that can be useful for development but not suitable for CI"
+    pending "No Geocoder config found for test environment" unless config['app_id'] != ''
+    config = config.inject({}) do |memo, (k, v)|
+      memo[k.to_sym] = v
+      memo
+    end
+    config[:cache] = config[:cache].inject({}) do |memo, (k, v)|
+      memo[k.to_sym] = v
+      memo
+    end
+    t = CartoDB::TableGeocoder.new(config.merge(table_name: @table_name,
+                                                qualified_table_name: @table_name,
+                                                sequel_qualified_table_name: @table_name,
+                                                formatter:  "name, ', ', iso3",
+                                                connection: @db,
+                                                schema:     'public',
+                                                log: @log,
+                                                geocoding_model: @geocoding_model,
+                                                max_rows: 1000))
+    t.geocoder.stubs("use_batch_process?").returns(true)
+
+    @db.fetch("select count(*) from #{@table_name} where the_geom is null").first[:count].should eq 14
+    t.run
+    until t.geocoder.status == 'completed' do
+      t.geocoder.update_status
+      puts "#{t.geocoder.status} #{t.geocoder.processed_rows}/#{t.geocoder.total_rows}"
+      sleep(2)
+    end
+    t.process_results
+    t.geocoder.status.should eq 'completed'
+    t.geocoder.processed_rows.to_i.should eq 0
+    t.cache.hits.should eq 10
+    @db.fetch("select count(*) from #{@table_name} where the_geom is null").first[:count].should eq 0
+    @db.fetch("select count(*) from #{@table_name} where cartodb_georef_status is false").first[:count].should eq 0
+  end
+
+  def path_to(filepath = '')
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../spec/fixtures/#{filepath}")
+    )
+  end
+
+  def load_csv(path)
+    @db.run("CREATE TABLE #{@table_name} (the_geom geometry, cartodb_id integer, name text, iso3 text)")
+    @db.run("COPY #{Sequel.lit(@table_name)}(cartodb_id, name, iso3) FROM '#{path}' DELIMITER ',' CSV")
+  end
+end