Initial commit

services/geocoder/lib/geocoder_config.rb

@@ -0,0 +1,16 @@
+require 'singleton'
+
+module CartoDB
+  # A facility that abstracts clients from config and also allow for easy injection
+  class GeocoderConfig
+    include Singleton
+
+    def set(config = {})
+      @config = config
+    end
+
+    def get()
+      @config ||= ::Cartodb.config[:geocoder]
+    end
+  end
+end

services/geocoder/lib/hires_batch_geocoder.rb

@@ -0,0 +1,320 @@
+require 'open3'
+require 'nokogiri'
+require 'csv'
+require 'active_support/core_ext/numeric'
+require_relative '../../../lib/carto/http/client'
+require_relative 'hires_geocoder_interface'
+require_relative 'geocoder_config'
+
+module CartoDB
+  class HiresBatchGeocoder < HiresGeocoderInterface
+
+    DEFAULT_TIMEOUT = 5.hours
+    POLLING_SLEEP_TIME = 5.seconds
+    LOGGING_TIME = 5.minutes
+    DOWNLOAD_RETRIES = 5
+    DOWLOAD_RETRY_SLEEP = 5.seconds
+
+    # Generous timeouts, overriden for big files upload/download
+    HTTP_CONNECTION_TIMEOUT = 60
+    HTTP_REQUEST_TIMEOUT = 600
+
+    # Options for the csv upload endpoint of the Batch Geocoder API
+    UPLOAD_OPTIONS = {
+      action: 'run',
+      indelim: ',',
+      outdelim: ',',
+      header: false,
+      outputCombined: false,
+      outcols: "displayLatitude,displayLongitude"
+    }
+
+    # INFO: the request_id is the most important thing to care for batch requests
+    # INFO: it is called remote_id in upper layers
+    attr_reader :base_url, :request_id, :app_id, :token, :mailto,
+                :status, :processed_rows, :processed_rows, :successful_processed_rows, :failed_processed_rows,
+                :empty_processed_rows, :total_rows, :dir, :input_file
+
+    class ServiceDisabled < StandardError; end
+
+
+    def initialize(input_csv_file, working_dir, log, geocoding_model)
+      @input_file = input_csv_file
+      @dir = working_dir
+      @log = log
+      @geocoding_model = geocoding_model
+      @base_url = config.fetch('base_url')
+      @app_id = config.fetch('app_id')
+      @token = config.fetch('token')
+      @mailto = config.fetch('mailto')
+      @used_batch_request = true
+      begin
+        @batch_api_disabled = config['batch_api_disabled'] == true
+      rescue
+        @batch_api_disabled = false
+      end
+    end
+
+    def run
+      init_rows_count
+      @log.append_and_store "Started batched Here geocoding job"
+      @started_at = Time.now
+      change_status('running')
+      upload
+
+      # INFO: this loop polls for the state of the table_geocoder batch process
+      update_status
+      until ['completed', 'cancelled'].include? @geocoding_model.state do
+        if timeout?
+          begin
+            change_status('timeout')
+            cancel
+          ensure
+            @log.append_and_store "Proceding to cancel job due timeout"
+          end
+        end
+
+        break if ['failed', 'timeout'].include? @geocoding_model.state
+
+        sleep polling_sleep_time
+        # We don't want to change the status if the job has been cancelled by the user
+        update_status
+        update_log_stats
+      end
+      update_status
+      update_log_stats
+      change_status('completed')
+      @log.append_and_store "Geocoding Hires job has finished"
+    ensure
+      # Processed data at the end of the job
+      update_status
+      update_log_stats(false)
+    end
+
+    def upload
+      assert_batch_api_enabled
+      @used_batch_request = true
+      response = http_client.post(
+        api_url(UPLOAD_OPTIONS),
+        body: File.open(input_file, "r").read,
+        headers: { "Content-Type" => "text/plain" },
+        timeout: 5.hours # more than generous timeout for big file upload
+      )
+      handle_api_error(response)
+      @request_id = extract_response_field(response.body, '//Response/MetaInfo/RequestId')
+      # TODO: this is a critical error, deal with it appropriately
+      raise 'Could not get the request ID' unless @request_id
+      # Update geocodings model with needed data
+      @geocoding_model.remote_id = @request_id
+      @geocoding_model.batched = true
+      @geocoding_model.save
+      @log.append_and_store "Job sent to HERE, job id: #{@request_id}"
+
+      @request_id
+    end
+
+    def used_batch_request?
+      @used_batch_request
+    end
+
+    def cancel
+      if @geocoding_model.remote_id.nil?
+        @log.append_and_store "Can't cancel a HERE geocoder job without the request id"
+      else
+        @log.append_and_store "Trying to cancel a batch job sent to HERE"
+        assert_batch_api_enabled
+        response = http_client.put(api_url(action: 'cancel'),
+                                   connecttimeout: HTTP_CONNECTION_TIMEOUT,
+                                   timeout: HTTP_REQUEST_TIMEOUT)
+        if is_cancellable?(response)
+          @log.append_and_store "Job was already cancelled"
+        else
+          handle_api_error(response)
+          update_stats(response)
+          @log.append_and_store "Job sent to HERE has been cancelled"
+        end
+        change_status('cancelled')
+      end
+    end
+
+    def update_status
+      assert_batch_api_enabled
+      response = http_client.get(api_url(action: 'status'),
+                                 connecttimeout: HTTP_CONNECTION_TIMEOUT,
+                                 timeout: HTTP_REQUEST_TIMEOUT)
+      handle_api_error(response)
+      update_stats(response)
+    end
+
+    def assert_batch_api_enabled
+      raise ServiceDisabled if @batch_api_disabled
+    end
+
+    def result
+      return @result unless @result.nil?
+
+      raise 'No request_id provided' unless @geocoding_model.remote_id
+      results_filename = File.join(dir, "#{@geocoding_model.remote_id}.zip")
+      download_url = api_url({}, 'result')
+      download_status_code = nil
+      retries = 0
+      while true
+        if(!download_status_code.nil? && download_status_code == 200)
+          break
+        elsif !download_status_code.nil? && download_status_code == 404
+          # 404 means that the results file is not ready yet
+          sleep DOWLOAD_RETRY_SLEEP
+          retries += 1
+        elsif retries >= DOWNLOAD_RETRIES
+          raise 'Download request failed: Too many retries, should be a problem with HERE servers'
+        elsif !download_status_code.nil? && download_status_code > 200 && download_status_code != 404
+          raise "Download request failed: Http status code #{download_status_code}"
+        end
+        download_status_code = execute_results_request(download_url, results_filename)
+      end
+      @result = results_filename
+    end
+
+
+    private
+
+    def execute_results_request(download_url, results_filename)
+        download_status_code = nil
+        # generous timeout for download of results
+        request = http_client.request(download_url,
+                                    method: :get,
+                                    timeout: 5.hours)
+
+        File.open(results_filename, 'wb') do |download_file|
+          request.on_headers do |response|
+            download_status_code = response.response_code
+          end
+
+          request.on_body do |chunk|
+            if download_status_code == 200
+              download_file.write(chunk)
+            end
+          end
+
+          request.on_complete do |response|
+            download_status_code = response.response_code
+          end
+
+          request.run
+        end
+
+        return download_status_code
+    end
+
+    def config
+      GeocoderConfig.instance.get
+    end
+
+    def http_client
+      @http_client ||= Carto::Http::Client.get('hires_batch_geocoder',
+                                               log_requests: true)
+    end
+
+    def api_url(arguments, extra_components = nil)
+      arguments.merge!(app_id: app_id, token: token, mailto: mailto)
+      components = [base_url]
+      # We use the persisted remote_id because we don't have request_id
+      # in the cancel case due is an instance variable
+      components << @geocoding_model.remote_id unless @geocoding_model.remote_id.nil?
+      components << extra_components unless extra_components.nil?
+      components << '?' + URI.encode_www_form(arguments)
+      components.join('/')
+    end
+
+    def extract_response_field(response, query)
+      Nokogiri::XML(response).xpath("#{query}").first.content
+    rescue NoMethodError => e
+      CartoDB.notify_exception(e)
+      nil
+    end
+
+    def extract_numeric_response_field(response, query)
+      value = extract_response_field(response, query)
+      return nil if value.blank?
+      Integer(value)
+    rescue ArgumentError => e
+      CartoDB.notify_error("Batch geocoder value error", error: e.message, value: value)
+      nil
+    end
+
+    def handle_api_error(response)
+      if response.success? == false
+        message = extract_response_field(response.body, '//Details')
+        @failed_processed_rows = number_of_input_file_rows if not input_file.nil?
+        change_status('failed')
+        raise "Geocoding API communication failure: #{message}"
+      end
+    end
+
+    def default_timeout
+      DEFAULT_TIMEOUT
+    end
+
+    def polling_sleep_time
+      POLLING_SLEEP_TIME
+    end
+
+    def number_of_input_file_rows
+      stdout, _status = Open3.capture2('wc', '-l', input_file)
+      stdout.to_i
+    end
+
+    def update_stats(response)
+      @status = extract_response_field(response.body, '//Response/Status')
+      change_status(@status)
+      @processed_rows = extract_numeric_response_field(response.body, '//Response/ProcessedCount')
+      @successful_processed_rows = extract_numeric_response_field(response.body, '//Response/SuccessCount')
+      # addresses that could not be matched
+      @empty_processed_rows = extract_numeric_response_field(response.body, '//Response/ErrorCount')
+      # invalid input that could not be processed
+      @failed_processed_rows = extract_numeric_response_field(response.body, '//Response/InvalidCount')
+      @total_rows = extract_numeric_response_field(response.body, '//Response/TotalCount')
+    end
+
+    def init_rows_count
+      @processed_rows = 0
+      @successful_processed_rows = 0
+      @empty_processed_rows = 0
+      @failed_processed_rows = 0
+      @total_rows = 0
+    end
+
+    def update_log_stats(spaced_by_time=true)
+      @last_logging_time ||= Time.now
+      # We don't want to log every few seconds because this kind
+      # of jobs could last for hours
+      if (not spaced_by_time) || (Time.now - @last_logging_time) > LOGGING_TIME
+        @log.append_and_store "Geocoding job status update. "\
+          "Status: #{@geocoding_model.state} --- Processed rows: #{@processed_rows} "\
+          "--- Success: #{@successful_processed_rows} --- Empty: #{@empty_processed_rows} "\
+          "--- Failed: #{@failed_processed_rows}"
+        @last_logging_time = Time.now
+      end
+    end
+
+    def timeout?
+      (Time.now - @started_at) > default_timeout
+    end
+
+    def change_status(status)
+      @status = status
+      # The cancelled status should prevail to abort the job
+      @geocoding_model.refresh
+      if status != @geocoding_model.state && (not (@geocoding_model.cancelled? || @geocoding_model.timeout?))
+        @geocoding_model.state = status
+        @geocoding_model.save
+      end
+    end
+
+    def is_cancellable?(response)
+      message = extract_response_field(response.body, '//Details')
+      response.response_code == 400 && message =~ /CANNOT CANCEL THE COMPLETED, DELETED, FAILED OR ALREADY CANCELLED JOB/
+    end
+
+  end
+end

services/geocoder/lib/hires_geocoder.rb

@@ -0,0 +1,169 @@
+require 'csv'
+require 'json'
+require 'open3'
+require_relative '../../../lib/carto/http/client'
+require_relative 'hires_geocoder_interface'
+require_relative 'geocoder_config'
+
+module CartoDB
+  class HiresGeocoder < HiresGeocoderInterface
+
+    # Generous timeouts for this
+    HTTP_CONNECTION_TIMEOUT = 60
+    HTTP_REQUEST_TIMEOUT = 600
+
+    # Default options for the regular HERE Geocoding API
+    # Refer to developer.here.com for further reading
+    GEOCODER_OPTIONS = {
+      gen: 4,                # enables or disables backward incompatible behavior in the API
+      jsonattributes: 1,     # lowercase the first character of each JSON response attribute name
+      language: 'en-US',     # preferred language of address elements in the result
+      maxresults: 1
+    }
+
+    attr_reader :app_id, :token, :mailto,
+                :status, :processed_rows, :total_rows, :successful_processed_rows, :failed_processed_rows,
+                :empty_processed_rows, :dir, :non_batch_base_url
+
+    attr_accessor :input_file
+
+    def initialize(input_csv_file, working_dir, log, geocoding_model)
+      @input_file = input_csv_file
+      @dir = working_dir
+      @log = log
+      @geocoding_model = geocoding_model
+      @non_batch_base_url = config.fetch('non_batch_base_url')
+      @app_id = config.fetch('app_id')
+      @token = config.fetch('token')
+      @mailto = config.fetch('mailto')
+
+      init_rows_count
+    end
+
+    def run
+      init_rows_count
+      @log.append_and_store "Initialized non batch Here geocoding job"
+      @result = File.join(dir, 'generated_csv_out.txt')
+      change_status('running')
+      @total_rows = input_rows
+      @log.append_and_store "Total rows to be processed: #{@total_rows}"
+      ::CSV.open(@result, "wb") do |output_csv_file|
+        ::CSV.foreach(input_file, headers: true) do |input_row|
+          process_row(input_row, output_csv_file)
+        end
+      end
+      change_status('completed')
+      update_log_stats
+      @log.append_and_store "Non-batch Here geocoding job finished"
+    end
+
+    def used_batch_request?
+      false
+    end
+
+    def cancel; end
+    def update_status; end
+
+    def result
+      @result
+    end
+
+    def request_id
+      # INFO: there's no request_id for non-batch geocodings
+      nil
+    end
+
+    private
+
+    def config
+      GeocoderConfig.instance.get
+    end
+
+    def http_client
+      @http_client ||= Carto::Http::Client.get('hires_geocoder',
+                                               log_requests: true)
+    end
+
+    def input_rows
+      stdout, _stderr, _status = Open3.capture3('wc', '-l', input_file)
+      stdout.to_i
+    rescue
+      0
+    end
+
+    def process_row(input_row, output_csv_file)
+      @processed_rows += 1
+      latitude, longitude = geocode_text(input_row["searchtext"])
+      if !(latitude.nil? || latitude == "") && !(longitude.nil? || longitude == "")
+        @successful_processed_rows += 1
+        output_csv_file.add_row [input_row["searchtext"], 1, 1, latitude, longitude]
+      else
+        @empty_processed_rows += 1
+      end
+    rescue => e
+      @log.append_and_store "Error processing row with search text #{input_row['searchtext']}: #{e.message}"
+      CartoDB.notify_debug("Hires geocoding process row error",
+                           error: e.backtrace.join("\n"),
+                           searchtext: input_row["searchtext"],
+                           backtrace: e.backtrace)
+      @failed_processed_rows += 1
+    end
+
+    def geocode_text(text)
+      options = GEOCODER_OPTIONS.merge(searchtext: text, app_id: app_id, app_code: token)
+      url = "#{non_batch_base_url}?#{URI.encode_www_form(options)}"
+      http_response = http_client.get(url,
+                                      connecttimeout: HTTP_CONNECTION_TIMEOUT,
+                                      timeout: HTTP_REQUEST_TIMEOUT)
+      if http_response.success?
+        response = ::JSON.parse(http_response.body)["response"]
+        if response['view'].empty?
+          # no location info for the text input, stop here
+          return [nil, nil]
+        end
+        position = response["view"][0]["result"][0]["location"]["displayPosition"]
+        return position["latitude"], position["longitude"]
+      else
+        CartoDB.notify_debug('Non-batched geocoder failed request', http_response)
+        return [nil, nil]
+      end
+    rescue NoMethodError => e
+      if e.message == %Q(undefined method `[]' for nil:NilClass)
+        CartoDB.notify_debug("Non-batched geocoder couldn't parse response",
+          error: e.backtrace.join("\n"), backtrace: e.backtrace, text: text, response_body: http_response.body)
+        [nil, nil]
+      else
+        raise e
+      end
+    end
+
+    def api_url(arguments, extra_components = nil)
+      arguments.merge!(app_id: app_id, token: token, mailto: mailto)
+      components = [base_url]
+      components << extra_components unless extra_components.nil?
+      components << '?' + URI.encode_www_form(arguments)
+      components.join('/')
+    end
+
+    def init_rows_count
+      @processed_rows = 0
+      @successful_processed_rows = 0
+      @failed_processed_rows = 0
+      @empty_processed_rows = 0
+    end
+
+    def update_log_stats
+      @log.append_and_store "Geocoding non-batch Here job status update. "\
+        "Status: #{@status} --- Processed rows: #{@processed_rows} "\
+        "--- Success: #{@successful_processed_rows} --- Empty: #{@empty_processed_rows} "\
+        "--- Failed: #{@failed_processed_rows}"
+    end
+
+    def change_status(status)
+      @status = status
+      @geocoding_model.state = status
+      @geocoding_model.save
+    end
+
+  end
+end

services/geocoder/lib/hires_geocoder_factory.rb

@@ -0,0 +1,53 @@
+require_relative 'hires_geocoder'
+require_relative 'hires_batch_geocoder'
+require_relative 'geocoder_config'
+
+
+module CartoDB
+  class HiresGeocoderFactory
+
+    BATCH_FILES_OVER = 1100 # Use Here Batch Geocoder API with tables over x rows
+
+    def self.get(input_csv_file, working_dir, log, geocoding_model, number_of_rows = 0)
+      geocoder_class = nil
+      if use_batch_process?(input_csv_file, geocoding_model, number_of_rows)
+        geocoder_class = HiresBatchGeocoder
+      else
+        geocoder_class = HiresGeocoder
+      end
+
+      geocoder_class.new(input_csv_file, working_dir, log, geocoding_model)
+    end
+
+
+    private
+
+    def self.use_batch_process?(input_csv_file, geocoding_model, number_of_rows)
+      # Due we could check this condition to create the geocoder class and we don't
+      # have finished yet the csv file generation, and could be nil, we have to check
+      # multiples conditions. It's sorted by priority
+      if force_batch? || geocoding_model.batched
+        true
+      elsif (not input_csv_file.nil?) && (input_rows(input_csv_file) > BATCH_FILES_OVER)
+        true
+      elsif (not number_of_rows.nil?) && (number_of_rows > BATCH_FILES_OVER)
+        true
+      else
+        false
+      end
+    end
+
+    def self.force_batch?
+      GeocoderConfig.instance.get['force_batch'] || false
+    end
+
+    def self.input_rows(input_csv_file)
+      stdout, _stderr, _status = Open3.capture3('wc', '-l', input_csv_file)
+      stdout.to_i
+    rescue => e
+      CartoDB.notify_exception(e)
+      0
+    end
+
+  end
+end

services/geocoder/lib/hires_geocoder_interface.rb

@@ -0,0 +1,11 @@
+module CartoDB
+  class HiresGeocoderInterface
+    def run
+      raise 'Not implemented'
+    end
+
+    def cancel
+      raise 'Not implemented'
+    end
+  end
+end

services/geocoder/spec/fixtures/nokia_input.csv

@@ -0,0 +1,2 @@
+recid,searchtext
+"Fredericton, Canada","Fredericton, Canada"

services/geocoder/spec/fixtures/response_cancel.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?><ns2:SearchBatch xmlns:ns2="http://www.navteq.com/lbsp/Search-Batch/1"><Response><MetaInfo><RequestId>0TK8XRhsIAxMi0KNbEu58MkApMVWctFE</RequestId></MetaInfo><Status>cancelled</Status><JobStarted>2013-09-15T18:19:49.000Z</JobStarted><JobFinished>2013-09-15T18:19:54.000Z</JobFinished><TotalCount>3</TotalCount><ValidCount>2</ValidCount><InvalidCount>1</InvalidCount><ProcessedCount>2</ProcessedCount><PendingCount>0</PendingCount><SuccessCount>2</SuccessCount><ErrorCount>0</ErrorCount></Response></ns2:SearchBatch>

services/geocoder/spec/fixtures/response_example.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?><ns2:SearchBatch xmlns:ns2="http://www.navteq.com/lbsp/Search-Batch/1"><Response><MetaInfo><RequestId>K8DmCWzsZGh4gbawxOuMv2BUcZsIkt7v</RequestId></MetaInfo><Status>submitted</Status><TotalCount>0</TotalCount><ValidCount>0</ValidCount><InvalidCount>0</InvalidCount><ProcessedCount>0</ProcessedCount><PendingCount>0</PendingCount><SuccessCount>0</SuccessCount><ErrorCount>0</ErrorCount></Response></ns2:SearchBatch>

services/geocoder/spec/fixtures/response_example_non_batch.json

@@ -0,0 +1 @@
+{"response":{"metaInfo":{"timestamp":"2014-02-19T12:29:49.723+0000"},"view":[{"result":[{"relevance":1.0,"matchLevel":"country","matchQuality":{"country":1.0},"location":{"locationId":"AREA_21000001","locationType":"point","displayPosition":{"latitude":38.89037,"longitude":-77.03196},"navigationPosition":[{"latitude":38.89037,"longitude":-77.03196}],"mapView":{"topLeft":{"latitude":49.3845,"longitude":-124.749},"bottomRight":{"latitude":24.5018,"longitude":-66.9406}},"address":{"label":"United States","country":"USA","additionalData":[{"value":"United States","key":"CountryName"}]}}}],"viewId":0}]}}

services/geocoder/spec/fixtures/response_failure.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?><ns2:Error xmlns:ns2="http://www.navteq.com/lbsp/Errors/1" type="ApplicationError" subtype="InvalidInputData"><Details>Input parameter validation failed. JobId: 9rFyj7kbGMmpF50ZUFAkRnroEiOpDOEZ Email Address is missing!</Details><AdditionalData key="mailto"/></ns2:Error>

services/geocoder/spec/fixtures/response_status.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?><ns2:SearchBatch xmlns:ns2="http://www.navteq.com/lbsp/Search-Batch/1"><Response><MetaInfo><RequestId>0TK8XRhsIAxMi0KNbEu58MkApMVWctFE</RequestId></MetaInfo><Status>completed</Status><JobStarted>2013-09-15T18:19:49.000Z</JobStarted><JobFinished>2013-09-15T18:19:54.000Z</JobFinished><TotalCount>3</TotalCount><ValidCount>2</ValidCount><InvalidCount>1</InvalidCount><ProcessedCount>2</ProcessedCount><PendingCount>0</PendingCount><SuccessCount>2</SuccessCount><ErrorCount>0</ErrorCount></Response></ns2:SearchBatch>

services/geocoder/spec/fixtures/with_country.csv

@@ -0,0 +1,4 @@
+recId,searchText,country
+1,425 W Randolph St, Chicago Illinois 60606,USA
+2,31 St James Ave Boston MA 02116,USA
+3,10115 Berlin Invalidenstrasse 117,DEU

services/geocoder/spec/fixtures/without_country.csv

@@ -0,0 +1,4 @@
+recId,searchText
+1,425 W Randolph St, Chicago Illinois 60606
+2,31 St James Ave Boston MA 02116
+3,10115 Berlin Invalidenstrasse 117

services/geocoder/spec/geocoder_spec.rb

@@ -0,0 +1,171 @@
+require_relative '../../../spec/spec_helper'
+require_relative '../../../spec/rspec_configuration.rb'
+require_relative '../lib/hires_batch_geocoder'
+
+# TODO rename to hires_batch_geocoder_spec.rb or split into batch/non-batch
+
+describe CartoDB::HiresBatchGeocoder do
+
+  before(:each) do
+    @log = mock
+    @log.stubs(:append)
+    @log.stubs(:append_and_store)
+    CartoDB::HiresBatchGeocoder.any_instance.stubs(:config).returns({
+        'base_url' => 'http://wadus.nokia.com',
+        'app_id' => '',
+        'token' => '',
+        'mailto' => ''
+      })
+    @working_dir = Dir.mktmpdir
+    @geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution', formatter: '{street}',
+                                          remote_id: 'wadus')
+  end
+
+  after(:each) do
+    FileUtils.remove_entry_secure @working_dir
+  end
+
+  describe '#upload' do
+    it 'returns rec_id on success' do
+      stub_api_request 200, 'response_example.xml'
+      filepath = path_to 'without_country.csv'
+      rec_id = CartoDB::HiresBatchGeocoder.new(filepath, @working_dir, @log, @geocoding_model).upload
+      rec_id.should eq "K8DmCWzsZGh4gbawxOuMv2BUcZsIkt7v"
+    end
+
+    it 'raises error on failure' do
+      stub_api_request 400, 'response_failure.xml'
+      filepath = path_to 'without_country.csv'
+      expect {
+        CartoDB::HiresBatchGeocoder.new(filepath, @working_dir, @log, @geocoding_model).upload
+      }.to raise_error('Geocoding API communication failure: Input parameter validation failed. JobId: 9rFyj7kbGMmpF50ZUFAkRnroEiOpDOEZ Email Address is missing!')
+    end
+  end
+
+  describe '#update_status' do
+    before {
+      stub_api_request(200, 'response_status.xml')
+      CartoDB::HiresBatchGeocoder.any_instance.stubs(:request_id).returns('wadus')
+    }
+    let(:geocoder) { CartoDB::HiresBatchGeocoder.new('/tmp/dummy_input_file.csv', @working_dir, @log, @geocoding_model) }
+
+    it "updates status" do
+      expect { geocoder.update_status }.to change(geocoder, :status).from(nil).to('completed')
+    end
+    it "updates processed rows" do
+      expect { geocoder.update_status }.to change(geocoder, :processed_rows).from(nil).to(2)
+    end
+    it "updates total rows" do
+      expect { geocoder.update_status }.to change(geocoder, :total_rows).from(nil).to(3)
+    end
+  end
+
+  describe '#result' do
+    it "saves result file on working directory" do
+      pending 'move to non-batch suite' # TODO
+      filepath = path_to 'without_country.csv'
+      stub_api_request 200, 'response_example_non_batch.json'
+      geocoder = CartoDB::Geocoder.new(default_params.merge(input_file: filepath, force_batch: false))
+      geocoder.upload
+      geocoder.status.should eq 'completed'
+      result_file = geocoder.result
+      File.file?(result_file).should be true
+      File.dirname(result_file).should eq geocoder.dir
+    end
+  end
+
+  describe '#cancel' do
+    before {
+      stub_api_request(200, 'response_cancel.xml')
+      @geocoding_model.remote_id = 'wadus'
+      @geocoding_model.save
+      CartoDB::HiresBatchGeocoder.any_instance.stubs(:request_id).returns('wadus')
+    }
+    let(:geocoder) { CartoDB::HiresBatchGeocoder.new('dummy_input_file.csv', @working_dir, @log, @geocoding_model) }
+
+    it "updates the status" do
+      geocoder.cancel
+      @geocoding_model.state.should eq 'cancelled'
+    end
+  end
+
+  describe '#extract_response_field' do
+    let(:geocoder) { CartoDB::HiresBatchGeocoder.new('dummy_input.csv', @working_dir, @log, @geocoding_model) }
+    let(:response) { File.open(path_to('response_example.xml')).read }
+
+    it 'returns specified element value' do
+      geocoder.send(:extract_response_field, response, '//Response/Status').should == 'submitted'
+    end
+
+    it 'returns nil for missing elements' do
+      CartoDB.expects(:notify_exception).once
+      geocoder.send(:extract_response_field, response, 'MissingField').should == nil
+    end
+  end
+
+  describe '#api_url' do
+    # TODO move to common place for both geocoders
+    before(:each) {
+      CartoDB::HiresBatchGeocoder.any_instance.stubs(:config).returns({
+        'base_url' => '',
+        'app_id' => 'a',
+        'token' => 'b',
+        'mailto' => 'c'
+        })
+      @geocoder =  CartoDB::HiresBatchGeocoder.new('dummy_input.csv', @working_dir, @log, @geocoding_model)
+    }
+
+    it 'returns base url by default' do
+      @geocoder.send(:api_url, {}).should == "/wadus/?app_id=a&token=b&mailto=c"
+    end
+
+    it 'allows for api method specification' do
+      @geocoder.send(:api_url, {}, 'all').should == "/wadus/all/?app_id=a&token=b&mailto=c"
+    end
+
+    it 'allows for api attributes specification' do
+      @geocoder.send(:api_url, {attr: 'wadus'}, 'all').should == "/wadus/all/?attr=wadus&app_id=a&token=b&mailto=c"
+    end
+  end
+
+  describe '#geocode_text' do
+    it 'returns lat/lon on success' do
+      pending 'move to non-batched suite' # TODO
+      stub_api_request 200, 'response_example_non_batch.json'
+      g = CartoDB::Geocoder.new(default_params)
+      g.geocode_text("United States").should eq [38.89037, -77.03196]
+    end
+  end
+
+  describe '#used_batch_request?' do
+    it 'returns true if sent a request to hi-res batch api' do
+      pending 'move these to the factory tests' # TODO
+      stub_api_request 200, 'response_example.xml'
+      filepath = path_to 'without_country.csv'
+      geocoder = CartoDB::Geocoder.new(default_params.merge(input_file: filepath))
+      geocoder.used_batch_request?.should eq true
+    end
+
+    it 'returns false if sent the request was non-batched' do
+      pending 'move these to the factory tests' # TODO
+      stub_api_request 200, 'response_example_non_batch.json'
+      filepath = path_to 'without_country.csv'
+      g = CartoDB::Geocoder.new(default_params.merge(force_batch: false, input_file: filepath))
+      g.upload
+      g.used_batch_request?.should eq false
+    end
+  end
+
+  def path_to(filepath)
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../spec/fixtures/#{filepath}")
+    )
+  end #path_to
+
+  def stub_api_request(code, response_file)
+    response = File.open(path_to(response_file)).read
+    Typhoeus.stub(/.*nokia.com/).and_return(
+      Typhoeus::Response.new(code: code, body: response)
+    )
+  end
+end # CartoDB::Geocoder

services/geocoder/spec/hires_batch_geocoder_spec.rb

@@ -0,0 +1,240 @@
+require 'tmpdir'
+require 'fileutils'
+require_relative '../../../spec/rspec_configuration.rb'
+require_relative '../../../spec/spec_helper.rb'
+require_relative '../lib/hires_batch_geocoder'
+
+
+describe CartoDB::HiresBatchGeocoder do
+
+  RSpec.configure do |config|
+    config.before :each do
+      Typhoeus::Expectation.clear
+    end
+  end
+
+  before(:each) do
+    @log = mock
+    @log.stubs(:append)
+    @log.stubs(:append_and_store)
+    @working_dir = Dir.mktmpdir
+    @input_csv_file = path_to '../../table-geocoder/spec/fixtures/nokia_input.csv'
+    CartoDB::HiresBatchGeocoder.any_instance.stubs(:config).returns({
+        'base_url' => 'batch.example.com',
+        'app_id' => '',
+        'token' => '',
+        'mailto' => ''
+      })
+    @geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution', formatter: '{street}' )
+    @batch_geocoder = CartoDB::HiresBatchGeocoder.new(@input_csv_file, @working_dir, @log, @geocoding_model)
+  end
+
+  after(:each) do
+    FileUtils.rm_f @working_dir
+  end
+
+  describe '#run' do
+    it 'uploads a file to the batch server' do
+      mock_complete_response
+      @batch_geocoder.expects(:upload).once
+      @batch_geocoder.run
+      @geocoding_model.state.should == 'completed'
+    end
+
+    it 'times out if not finished before the DEFAULT_TIMEOUT' do
+      mock_complete_response('running')
+      @batch_geocoder.expects(:upload).once
+      @batch_geocoder.expects(:cancel).once
+      @batch_geocoder.stubs(:default_timeout).returns(-10) # make sure it times out
+      @batch_geocoder.run
+      @geocoding_model.state.should == 'timeout'
+    end
+  end
+
+  describe '#upload' do
+    it 'uploads a file to the batch service' do
+      url = @batch_geocoder.send(:api_url, CartoDB::HiresBatchGeocoder::UPLOAD_OPTIONS)
+      expected_request_id = 'dummy_id'
+      xml_response_body = "<Response><MetaInfo><RequestId>#{expected_request_id}</RequestId></MetaInfo></Response>"
+      response = Typhoeus::Response.new(code: 200, body: xml_response_body)
+
+      Typhoeus.stub(url, method: :post).and_return(response)
+      @batch_geocoder.upload
+
+      @batch_geocoder.request_id.should == expected_request_id
+      @batch_geocoder.used_batch_request?.should == true
+    end
+
+    it 'raises an exception if the api returns != 200' do
+      url = @batch_geocoder.send(:api_url, CartoDB::HiresBatchGeocoder::UPLOAD_OPTIONS)
+      response = Typhoeus::Response.new(code: 401)
+      Typhoeus.stub(url, method: :post).and_return(response)
+      CartoDB.expects(:notify_exception).once
+
+      expect {
+        @batch_geocoder.upload
+      }.to raise_error(RuntimeError, /Geocoding API communication failure/)
+    end
+
+    it 'raises an exception if the api does not return a RequestId' do
+      url = @batch_geocoder.send(:api_url, CartoDB::HiresBatchGeocoder::UPLOAD_OPTIONS)
+      response = Typhoeus::Response.new(code: 200)
+      Typhoeus.stub(url, method: :post).and_return(response)
+      CartoDB.expects(:notify_exception).once
+
+      expect {
+        @batch_geocoder.upload
+      }.to raise_error(RuntimeError, /Could not get the request ID/)
+    end
+
+  end
+
+  describe '#cancel' do
+    it 'sends a cancel put request and gets the status, processed and total rows' do
+      request_id = 'dummy_request_id'
+      @geocoding_model.remote_id = request_id
+      @geocoding_model.save
+      @batch_geocoder.stubs(:request_id).returns(request_id)
+      url = @batch_geocoder.send(:api_url, action: 'cancel')
+      url.should match(%r'/#{request_id}/')
+      url.should match(%r'action=cancel')
+
+      expected_status = 'cancelled'
+      expected_processed_rows = 20
+      expected_success_rows = 17
+      expected_failed_rows = 0
+      expected_empty_rows = 3
+      expected_total_rows = 30
+
+      response_body = <<END_XML
+<Response>
+  <Status>#{expected_status}</Status>
+  <ProcessedCount>#{expected_processed_rows}</ProcessedCount>
+  <SuccessCount>#{expected_success_rows}</SuccessCount>
+  <ErrorCount>#{expected_failed_rows}</ErrorCount>
+  <InvalidCount>#{expected_empty_rows}</InvalidCount>
+  <TotalCount>#{expected_total_rows}</TotalCount>
+</Response>
+END_XML
+
+      response = Typhoeus::Response.new(code: 200, body: response_body)
+      Typhoeus.stub(url, method: :put).and_return(response)
+      @batch_geocoder.cancel
+      @batch_geocoder.status.should == expected_status
+      @batch_geocoder.processed_rows.should == expected_processed_rows
+      @batch_geocoder.total_rows.should == expected_total_rows
+    end
+  end
+
+  describe '#update' do
+    it 'gets the status, processed and total rows by sending a get request' do
+      request_id = 'dummy_request_id'
+      @geocoding_model.remote_id = request_id
+      @geocoding_model.save
+      @batch_geocoder.stubs(:request_id).returns(request_id)
+      url = @batch_geocoder.send(:api_url, action: 'status')
+      url.should match(%r'/#{request_id}/')
+      url.should match(%r'action=status')
+
+      expected_status = 'running'
+      expected_processed_rows = 20
+      expected_success_rows = 17
+      expected_failed_rows = 0
+      expected_empty_rows = 3
+      expected_total_rows = 30
+
+      response_body = <<END_XML
+<Response>
+  <Status>#{expected_status}</Status>
+  <ProcessedCount>#{expected_processed_rows}</ProcessedCount>
+  <SuccessCount>#{expected_success_rows}</SuccessCount>
+  <ErrorCount>#{expected_failed_rows}</ErrorCount>
+  <InvalidCount>#{expected_empty_rows}</InvalidCount>
+  <TotalCount>#{expected_total_rows}</TotalCount>
+</Response>
+END_XML
+
+      response = Typhoeus::Response.new(code: 200, body: response_body)
+      Typhoeus.stub(url, method: :get).and_return(response)
+      @batch_geocoder.update_status
+      @batch_geocoder.status.should == expected_status
+      @batch_geocoder.processed_rows.should == expected_processed_rows
+      @batch_geocoder.total_rows.should == expected_total_rows
+    end
+  end
+
+  describe '#result' do
+    it "raises an exception if there's no request_id from a previous upload" do
+      expect {
+        @batch_geocoder.result
+      }.to raise_error(RuntimeError, /No request_id provided/)
+    end
+
+    it 'downloads the result file from the remote server' do
+      request_id = 'dummy_request_id'
+      @geocoding_model.remote_id = request_id
+      @geocoding_model.save
+      @batch_geocoder.stubs(:request_id).returns(request_id)
+      expected_response_body = 'dummy result file contents'
+      url = @batch_geocoder.send(:api_url, {}, 'result')
+      response = Typhoeus::Response.new(code: 200, body: expected_response_body)
+      Typhoeus.stub(url, method: :get).and_return(response)
+
+      result_file = @batch_geocoder.result
+      File.open(result_file).read.should == expected_response_body
+
+      # it also "memoizes" the result file and avoids further downloads
+      @batch_geocoder.expects(:http_client).never
+      @batch_geocoder.result.should == result_file
+    end
+
+    it 'raises an exception if cannot get a result file' do
+      request_id = 'dummy_request_id'
+      @geocoding_model.remote_id = request_id
+      @geocoding_model.save
+      @batch_geocoder.stubs(:request_id).returns(request_id)
+      expected_response_body = 'dummy result file contents'
+      url = @batch_geocoder.send(:api_url, {}, 'result')
+      response = Typhoeus::Response.new(code: 400)
+      Typhoeus.stub(url, method: :get).and_return(response)
+
+      expect {
+        @batch_geocoder.result
+      }.to raise_error(RuntimeError, /Download request failed/)
+    end
+  end
+
+
+  def path_to(filepath = '')
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../fixtures/#{filepath}")
+      )
+  end
+
+  def mock_complete_response(state='completed')
+    @geocoding_model.remote_id = 'dummy_id'
+    @geocoding_model.save.reload
+    url = @batch_geocoder.send(:api_url, {action: 'status'})
+    xml_response_body = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+                          <ns2:SearchBatch xmlns:ns2="http://www.navteq.com/lbsp/Search-Batch/1">
+                            <Response>
+                              <MetaInfo>
+                                <RequestId>dummy_id</RequestId>
+                              </MetaInfo>
+                              <Status>'+state+'</Status>
+                              <JobStarted>2016-04-08T08:24:05.000Z</JobStarted>
+                              <JobFinished>2016-04-08T08:24:39.000Z</JobFinished>
+                              <TotalCount>1</TotalCount>
+                              <ValidCount>1</ValidCount>
+                              <InvalidCount>0</InvalidCount>
+                              <ProcessedCount>1</ProcessedCount>
+                              <PendingCount>0</PendingCount>
+                              <SuccessCount>1</SuccessCount>
+                              <ErrorCount>0</ErrorCount>
+                            </Response>
+                          </ns2:SearchBatch>'
+    response = Typhoeus::Response.new(code: 200, body: xml_response_body)
+    Typhoeus.stub(url, method: :get).and_return(response)
+  end
+
+end

services/geocoder/spec/hires_geocoder_factory_spec.rb

@@ -0,0 +1,68 @@
+require_relative '../../../spec/rspec_configuration'
+require_relative '../../../spec/spec_helper'
+require_relative '../lib/hires_geocoder_factory'
+require_relative '../lib/geocoder_config'
+
+describe CartoDB::HiresGeocoderFactory do
+
+  after(:all) do
+    # reset config
+    CartoDB::GeocoderConfig.instance.set(nil)
+  end
+
+  before(:each) do
+    @log = mock
+    @log.stubs(:append)
+    @log.stubs(:append_and_store)
+    @geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution', formatter: '{street}')
+  end
+
+  describe '#get' do
+    it 'returns a HiresGeocoder instance if the input file has less than N rows' do
+      CartoDB::GeocoderConfig.instance.set({
+          'non_batch_base_url' => 'http://api.example.com',
+          'app_id' => 'dummy_app_id',
+          'token' => 'dummy_token',
+          'mailto' => 'dummy_mail_addr'
+        })
+      dummy_input_file = 'dummy_input_file.csv'
+      working_dir = '/tmp/any_dir'
+      input_rows = CartoDB::HiresGeocoderFactory::BATCH_FILES_OVER - 1
+      CartoDB::HiresGeocoderFactory.expects(:input_rows).once.with(dummy_input_file).returns(input_rows)
+
+      CartoDB::HiresGeocoderFactory.get(dummy_input_file, working_dir, @log, @geocoding_model).class.should == CartoDB::HiresGeocoder
+    end
+
+    it 'returns a HiresBatchGeocoder instance if the input file is above N rows' do
+      CartoDB::GeocoderConfig.instance.set({
+          'base_url' => 'http://api.example.com',
+          'app_id' => 'dummy_app_id',
+          'token' => 'dummy_token',
+          'mailto' => 'dummy_mail_addr'
+        })
+      dummy_input_file = 'dummy_input_file.csv'
+      working_dir = '/tmp/any_dir'
+      input_rows = CartoDB::HiresGeocoderFactory::BATCH_FILES_OVER + 1
+      CartoDB::HiresGeocoderFactory.expects(:input_rows).once.with(dummy_input_file).returns(input_rows)
+
+      CartoDB::HiresGeocoderFactory.get(dummy_input_file, working_dir, @log, @geocoding_model).class.should == CartoDB::HiresBatchGeocoder
+    end
+
+    it 'returns a batch geocoder if config has force_batch set to true' do
+      CartoDB::GeocoderConfig.instance.set({
+          'force_batch' => true,
+          'base_url' => 'http://api.example.com',
+          'app_id' => 'dummy_app_id',
+          'token' => 'dummy_token',
+          'mailto' => 'dummy_mail_addr'
+        })
+      dummy_input_file = 'dummy_input_file.csv'
+      working_dir = '/tmp/any_dir'
+      CartoDB::HiresGeocoderFactory.expects(:input_rows).never
+
+      CartoDB::HiresGeocoderFactory.get(dummy_input_file, working_dir, @log, @geocoding_model).class.should == CartoDB::HiresBatchGeocoder
+    end
+
+  end
+
+end

services/geocoder/spec/hires_geocoder_spec.rb

@@ -0,0 +1,135 @@
+require 'tmpdir'
+require 'fileutils'
+require 'csv'
+require_relative '../../../spec/rspec_configuration'
+require_relative '../../../spec/spec_helper'
+require_relative '../lib/hires_geocoder'
+
+
+describe CartoDB::HiresGeocoder do
+
+  MOCK_COORDINATES = [38.89037, -77.03196]
+
+  RSpec.configure do |config|
+    config.before :each do
+      Typhoeus::Expectation.clear
+    end
+  end
+
+  before(:each) do
+    @working_dir = Dir.mktmpdir
+    @input_csv_file = path_to '../../table-geocoder/spec/fixtures/nokia_input.csv'
+    @log = mock
+    @log.stubs(:append)
+    @log.stubs(:append_and_store)
+    CartoDB::HiresGeocoder.any_instance.stubs(:config).returns({
+        'non_batch_base_url' => 'batch.example.com',
+        'app_id' => '',
+        'token' => '',
+        'mailto' => ''
+      })
+    @geocoding_model = FactoryGirl.create(:geocoding, kind: 'high-resolution', formatter: '{street}')
+    @geocoder = CartoDB::HiresGeocoder.new(@input_csv_file, @working_dir, @log, @geocoding_model)
+  end
+
+  after(:each) do
+    FileUtils.rm_f @working_dir
+  end
+
+  describe '#run' do
+    it 'takes every row from input and calls geocode_text on them' do
+      rows_to_geocode = ::CSV.read(@input_csv_file, headers: true).length
+      @geocoder.expects(:geocode_text).times(rows_to_geocode).returns(MOCK_COORDINATES)
+      @geocoder.run
+      @geocoder.status.should == 'completed'
+    end
+  end
+
+  describe '#process_row' do
+    it 'increments the number of processed rows by one when called' do
+      output_csv_mock = mock
+      output_csv_mock.expects(:add_row).once
+      input_row = {'searchtext' => 'olakase'}
+      @geocoder.expects(:geocode_text).once.returns(MOCK_COORDINATES)
+      @geocoder.processed_rows.should == 0
+      @geocoder.send(:process_row, input_row, output_csv_mock)
+      @geocoder.processed_rows.should == 1
+    end
+
+    it 'adds a row with the expected format when success' do
+      output_csv_mock = mock
+      output_csv_mock.expects(:add_row).once.with ['olakase', 1, 1, MOCK_COORDINATES[0], MOCK_COORDINATES[1]]
+      input_row = {'searchtext' => 'olakase'}
+      @geocoder.expects(:geocode_text).once.returns(MOCK_COORDINATES)
+      @geocoder.send(:process_row, input_row, output_csv_mock)
+    end
+
+    it 'does not add any row when it when geolocation fails' do
+      output_csv_mock = mock
+      output_csv_mock.expects(:add_row).never
+      input_row = {'searchtext' => 'olakase'}
+      @geocoder.expects(:geocode_text).once.returns([nil, nil])
+      @geocoder.send(:process_row, input_row, output_csv_mock)
+    end
+
+  end
+
+  describe '#geocode_text' do
+    it 'sends a request to the non-batched geocoder service and gets a couple of coordinates' do
+      json_response_body = {
+        response: {
+          view: [
+            result: [
+              location: {
+                displayPosition: {
+                  latitude: MOCK_COORDINATES[0],
+                  longitude: MOCK_COORDINATES[1]
+                }
+              }
+            ]
+          ]
+        }
+      }.to_json
+      mocked_response  = Typhoeus::Response.new(code: 200, body: json_response_body)
+      Typhoeus.stub(//, method: :get).and_return(mocked_response)
+
+      @geocoder.send(:geocode_text, 'Dummy address').should == MOCK_COORDINATES
+    end
+
+    it "returns nil coordinates if the http request doesn't succeed" do
+      mocked_response  = Typhoeus::Response.new(code: 500)
+      Typhoeus.stub(//, method: :get).and_return(mocked_response)
+      CartoDB.expects(:notify_debug).with('Non-batched geocoder failed request', mocked_response).once
+
+      @geocoder.send(:geocode_text, 'Dummy address').should == [nil, nil]
+    end
+
+    it 'returns nil coordinates and log a trace if it is not able to parse the response' do
+      input_text = 'Dummy address'
+      json_response_body = {
+        unexpected: 'this response body has unexpected format for whatever reason'
+      }.to_json
+      mocked_response  = Typhoeus::Response.new(code: 200, body: json_response_body)
+      Typhoeus.stub(//, method: :get).and_return(mocked_response)
+      CartoDB.expects(:notify_debug).with("Non-batched geocoder couldn't parse response", anything()).once
+
+      @geocoder.send(:geocode_text, input_text).should == [nil, nil]
+    end
+
+    it 'returns nil coordinates and stops there if the response does not contain any location' do
+      input_text = 'Dummy address'
+      json_response_body = '{"response":{"metaInfo":{"timestamp":"2015-07-14T15:33:35.023+0000"},"view":[]}}'
+      mocked_response  = Typhoeus::Response.new(code: 200, body: json_response_body)
+      Typhoeus.stub(//, method: :get).and_return(mocked_response)
+
+      @geocoder.send(:geocode_text, input_text).should == [nil, nil]
+    end
+  end
+
+  def path_to(filepath = '')
+    File.expand_path(
+      File.join(File.dirname(__FILE__), "../fixtures/#{filepath}")
+      )
+  end
+
+end