Initial commit

services/geocoder/lib/geocoder_config.rb

@@ -0,0 +1,16 @@
+require 'singleton'
+
+module CartoDB
+  # A facility that abstracts clients from config and also allow for easy injection
+  class GeocoderConfig
+    include Singleton
+
+    def set(config = {})
+      @config = config
+    end
+
+    def get()
+      @config ||= ::Cartodb.config[:geocoder]
+    end
+  end
+end

services/geocoder/lib/hires_batch_geocoder.rb

@@ -0,0 +1,320 @@
+require 'open3'
+require 'nokogiri'
+require 'csv'
+require 'active_support/core_ext/numeric'
+require_relative '../../../lib/carto/http/client'
+require_relative 'hires_geocoder_interface'
+require_relative 'geocoder_config'
+
+module CartoDB
+  class HiresBatchGeocoder < HiresGeocoderInterface
+
+    DEFAULT_TIMEOUT = 5.hours
+    POLLING_SLEEP_TIME = 5.seconds
+    LOGGING_TIME = 5.minutes
+    DOWNLOAD_RETRIES = 5
+    DOWLOAD_RETRY_SLEEP = 5.seconds
+
+    # Generous timeouts, overriden for big files upload/download
+    HTTP_CONNECTION_TIMEOUT = 60
+    HTTP_REQUEST_TIMEOUT = 600
+
+    # Options for the csv upload endpoint of the Batch Geocoder API
+    UPLOAD_OPTIONS = {
+      action: 'run',
+      indelim: ',',
+      outdelim: ',',
+      header: false,
+      outputCombined: false,
+      outcols: "displayLatitude,displayLongitude"
+    }
+
+    # INFO: the request_id is the most important thing to care for batch requests
+    # INFO: it is called remote_id in upper layers
+    attr_reader :base_url, :request_id, :app_id, :token, :mailto,
+                :status, :processed_rows, :processed_rows, :successful_processed_rows, :failed_processed_rows,
+                :empty_processed_rows, :total_rows, :dir, :input_file
+
+    class ServiceDisabled < StandardError; end
+
+
+    def initialize(input_csv_file, working_dir, log, geocoding_model)
+      @input_file = input_csv_file
+      @dir = working_dir
+      @log = log
+      @geocoding_model = geocoding_model
+      @base_url = config.fetch('base_url')
+      @app_id = config.fetch('app_id')
+      @token = config.fetch('token')
+      @mailto = config.fetch('mailto')
+      @used_batch_request = true
+      begin
+        @batch_api_disabled = config['batch_api_disabled'] == true
+      rescue
+        @batch_api_disabled = false
+      end
+    end
+
+    def run
+      init_rows_count
+      @log.append_and_store "Started batched Here geocoding job"
+      @started_at = Time.now
+      change_status('running')
+      upload
+
+      # INFO: this loop polls for the state of the table_geocoder batch process
+      update_status
+      until ['completed', 'cancelled'].include? @geocoding_model.state do
+        if timeout?
+          begin
+            change_status('timeout')
+            cancel
+          ensure
+            @log.append_and_store "Proceding to cancel job due timeout"
+          end
+        end
+
+        break if ['failed', 'timeout'].include? @geocoding_model.state
+
+        sleep polling_sleep_time
+        # We don't want to change the status if the job has been cancelled by the user
+        update_status
+        update_log_stats
+      end
+      update_status
+      update_log_stats
+      change_status('completed')
+      @log.append_and_store "Geocoding Hires job has finished"
+    ensure
+      # Processed data at the end of the job
+      update_status
+      update_log_stats(false)
+    end
+
+    def upload
+      assert_batch_api_enabled
+      @used_batch_request = true
+      response = http_client.post(
+        api_url(UPLOAD_OPTIONS),
+        body: File.open(input_file, "r").read,
+        headers: { "Content-Type" => "text/plain" },
+        timeout: 5.hours # more than generous timeout for big file upload
+      )
+      handle_api_error(response)
+      @request_id = extract_response_field(response.body, '//Response/MetaInfo/RequestId')
+      # TODO: this is a critical error, deal with it appropriately
+      raise 'Could not get the request ID' unless @request_id
+      # Update geocodings model with needed data
+      @geocoding_model.remote_id = @request_id
+      @geocoding_model.batched = true
+      @geocoding_model.save
+      @log.append_and_store "Job sent to HERE, job id: #{@request_id}"
+
+      @request_id
+    end
+
+    def used_batch_request?
+      @used_batch_request
+    end
+
+    def cancel
+      if @geocoding_model.remote_id.nil?
+        @log.append_and_store "Can't cancel a HERE geocoder job without the request id"
+      else
+        @log.append_and_store "Trying to cancel a batch job sent to HERE"
+        assert_batch_api_enabled
+        response = http_client.put(api_url(action: 'cancel'),
+                                   connecttimeout: HTTP_CONNECTION_TIMEOUT,
+                                   timeout: HTTP_REQUEST_TIMEOUT)
+        if is_cancellable?(response)
+          @log.append_and_store "Job was already cancelled"
+        else
+          handle_api_error(response)
+          update_stats(response)
+          @log.append_and_store "Job sent to HERE has been cancelled"
+        end
+        change_status('cancelled')
+      end
+    end
+
+    def update_status
+      assert_batch_api_enabled
+      response = http_client.get(api_url(action: 'status'),
+                                 connecttimeout: HTTP_CONNECTION_TIMEOUT,
+                                 timeout: HTTP_REQUEST_TIMEOUT)
+      handle_api_error(response)
+      update_stats(response)
+    end
+
+    def assert_batch_api_enabled
+      raise ServiceDisabled if @batch_api_disabled
+    end
+
+    def result
+      return @result unless @result.nil?
+
+      raise 'No request_id provided' unless @geocoding_model.remote_id
+      results_filename = File.join(dir, "#{@geocoding_model.remote_id}.zip")
+      download_url = api_url({}, 'result')
+      download_status_code = nil
+      retries = 0
+      while true
+        if(!download_status_code.nil? && download_status_code == 200)
+          break
+        elsif !download_status_code.nil? && download_status_code == 404
+          # 404 means that the results file is not ready yet
+          sleep DOWLOAD_RETRY_SLEEP
+          retries += 1
+        elsif retries >= DOWNLOAD_RETRIES
+          raise 'Download request failed: Too many retries, should be a problem with HERE servers'
+        elsif !download_status_code.nil? && download_status_code > 200 && download_status_code != 404
+          raise "Download request failed: Http status code #{download_status_code}"
+        end
+        download_status_code = execute_results_request(download_url, results_filename)
+      end
+      @result = results_filename
+    end
+
+
+    private
+
+    def execute_results_request(download_url, results_filename)
+        download_status_code = nil
+        # generous timeout for download of results
+        request = http_client.request(download_url,
+                                    method: :get,
+                                    timeout: 5.hours)
+
+        File.open(results_filename, 'wb') do |download_file|
+          request.on_headers do |response|
+            download_status_code = response.response_code
+          end
+
+          request.on_body do |chunk|
+            if download_status_code == 200
+              download_file.write(chunk)
+            end
+          end
+
+          request.on_complete do |response|
+            download_status_code = response.response_code
+          end
+
+          request.run
+        end
+
+        return download_status_code
+    end
+
+    def config
+      GeocoderConfig.instance.get
+    end
+
+    def http_client
+      @http_client ||= Carto::Http::Client.get('hires_batch_geocoder',
+                                               log_requests: true)
+    end
+
+    def api_url(arguments, extra_components = nil)
+      arguments.merge!(app_id: app_id, token: token, mailto: mailto)
+      components = [base_url]
+      # We use the persisted remote_id because we don't have request_id
+      # in the cancel case due is an instance variable
+      components << @geocoding_model.remote_id unless @geocoding_model.remote_id.nil?
+      components << extra_components unless extra_components.nil?
+      components << '?' + URI.encode_www_form(arguments)
+      components.join('/')
+    end
+
+    def extract_response_field(response, query)
+      Nokogiri::XML(response).xpath("#{query}").first.content
+    rescue NoMethodError => e
+      CartoDB.notify_exception(e)
+      nil
+    end
+
+    def extract_numeric_response_field(response, query)
+      value = extract_response_field(response, query)
+      return nil if value.blank?
+      Integer(value)
+    rescue ArgumentError => e
+      CartoDB.notify_error("Batch geocoder value error", error: e.message, value: value)
+      nil
+    end
+
+    def handle_api_error(response)
+      if response.success? == false
+        message = extract_response_field(response.body, '//Details')
+        @failed_processed_rows = number_of_input_file_rows if not input_file.nil?
+        change_status('failed')
+        raise "Geocoding API communication failure: #{message}"
+      end
+    end
+
+    def default_timeout
+      DEFAULT_TIMEOUT
+    end
+
+    def polling_sleep_time
+      POLLING_SLEEP_TIME
+    end
+
+    def number_of_input_file_rows
+      stdout, _status = Open3.capture2('wc', '-l', input_file)
+      stdout.to_i
+    end
+
+    def update_stats(response)
+      @status = extract_response_field(response.body, '//Response/Status')
+      change_status(@status)
+      @processed_rows = extract_numeric_response_field(response.body, '//Response/ProcessedCount')
+      @successful_processed_rows = extract_numeric_response_field(response.body, '//Response/SuccessCount')
+      # addresses that could not be matched
+      @empty_processed_rows = extract_numeric_response_field(response.body, '//Response/ErrorCount')
+      # invalid input that could not be processed
+      @failed_processed_rows = extract_numeric_response_field(response.body, '//Response/InvalidCount')
+      @total_rows = extract_numeric_response_field(response.body, '//Response/TotalCount')
+    end
+
+    def init_rows_count
+      @processed_rows = 0
+      @successful_processed_rows = 0
+      @empty_processed_rows = 0
+      @failed_processed_rows = 0
+      @total_rows = 0
+    end
+
+    def update_log_stats(spaced_by_time=true)
+      @last_logging_time ||= Time.now
+      # We don't want to log every few seconds because this kind
+      # of jobs could last for hours
+      if (not spaced_by_time) || (Time.now - @last_logging_time) > LOGGING_TIME
+        @log.append_and_store "Geocoding job status update. "\
+          "Status: #{@geocoding_model.state} --- Processed rows: #{@processed_rows} "\
+          "--- Success: #{@successful_processed_rows} --- Empty: #{@empty_processed_rows} "\
+          "--- Failed: #{@failed_processed_rows}"
+        @last_logging_time = Time.now
+      end
+    end
+
+    def timeout?
+      (Time.now - @started_at) > default_timeout
+    end
+
+    def change_status(status)
+      @status = status
+      # The cancelled status should prevail to abort the job
+      @geocoding_model.refresh
+      if status != @geocoding_model.state && (not (@geocoding_model.cancelled? || @geocoding_model.timeout?))
+        @geocoding_model.state = status
+        @geocoding_model.save
+      end
+    end
+
+    def is_cancellable?(response)
+      message = extract_response_field(response.body, '//Details')
+      response.response_code == 400 && message =~ /CANNOT CANCEL THE COMPLETED, DELETED, FAILED OR ALREADY CANCELLED JOB/
+    end
+
+  end
+end

services/geocoder/lib/hires_geocoder.rb

@@ -0,0 +1,169 @@
+require 'csv'
+require 'json'
+require 'open3'
+require_relative '../../../lib/carto/http/client'
+require_relative 'hires_geocoder_interface'
+require_relative 'geocoder_config'
+
+module CartoDB
+  class HiresGeocoder < HiresGeocoderInterface
+
+    # Generous timeouts for this
+    HTTP_CONNECTION_TIMEOUT = 60
+    HTTP_REQUEST_TIMEOUT = 600
+
+    # Default options for the regular HERE Geocoding API
+    # Refer to developer.here.com for further reading
+    GEOCODER_OPTIONS = {
+      gen: 4,                # enables or disables backward incompatible behavior in the API
+      jsonattributes: 1,     # lowercase the first character of each JSON response attribute name
+      language: 'en-US',     # preferred language of address elements in the result
+      maxresults: 1
+    }
+
+    attr_reader :app_id, :token, :mailto,
+                :status, :processed_rows, :total_rows, :successful_processed_rows, :failed_processed_rows,
+                :empty_processed_rows, :dir, :non_batch_base_url
+
+    attr_accessor :input_file
+
+    def initialize(input_csv_file, working_dir, log, geocoding_model)
+      @input_file = input_csv_file
+      @dir = working_dir
+      @log = log
+      @geocoding_model = geocoding_model
+      @non_batch_base_url = config.fetch('non_batch_base_url')
+      @app_id = config.fetch('app_id')
+      @token = config.fetch('token')
+      @mailto = config.fetch('mailto')
+
+      init_rows_count
+    end
+
+    def run
+      init_rows_count
+      @log.append_and_store "Initialized non batch Here geocoding job"
+      @result = File.join(dir, 'generated_csv_out.txt')
+      change_status('running')
+      @total_rows = input_rows
+      @log.append_and_store "Total rows to be processed: #{@total_rows}"
+      ::CSV.open(@result, "wb") do |output_csv_file|
+        ::CSV.foreach(input_file, headers: true) do |input_row|
+          process_row(input_row, output_csv_file)
+        end
+      end
+      change_status('completed')
+      update_log_stats
+      @log.append_and_store "Non-batch Here geocoding job finished"
+    end
+
+    def used_batch_request?
+      false
+    end
+
+    def cancel; end
+    def update_status; end
+
+    def result
+      @result
+    end
+
+    def request_id
+      # INFO: there's no request_id for non-batch geocodings
+      nil
+    end
+
+    private
+
+    def config
+      GeocoderConfig.instance.get
+    end
+
+    def http_client
+      @http_client ||= Carto::Http::Client.get('hires_geocoder',
+                                               log_requests: true)
+    end
+
+    def input_rows
+      stdout, _stderr, _status = Open3.capture3('wc', '-l', input_file)
+      stdout.to_i
+    rescue
+      0
+    end
+
+    def process_row(input_row, output_csv_file)
+      @processed_rows += 1
+      latitude, longitude = geocode_text(input_row["searchtext"])
+      if !(latitude.nil? || latitude == "") && !(longitude.nil? || longitude == "")
+        @successful_processed_rows += 1
+        output_csv_file.add_row [input_row["searchtext"], 1, 1, latitude, longitude]
+      else
+        @empty_processed_rows += 1
+      end
+    rescue => e
+      @log.append_and_store "Error processing row with search text #{input_row['searchtext']}: #{e.message}"
+      CartoDB.notify_debug("Hires geocoding process row error",
+                           error: e.backtrace.join("\n"),
+                           searchtext: input_row["searchtext"],
+                           backtrace: e.backtrace)
+      @failed_processed_rows += 1
+    end
+
+    def geocode_text(text)
+      options = GEOCODER_OPTIONS.merge(searchtext: text, app_id: app_id, app_code: token)
+      url = "#{non_batch_base_url}?#{URI.encode_www_form(options)}"
+      http_response = http_client.get(url,
+                                      connecttimeout: HTTP_CONNECTION_TIMEOUT,
+                                      timeout: HTTP_REQUEST_TIMEOUT)
+      if http_response.success?
+        response = ::JSON.parse(http_response.body)["response"]
+        if response['view'].empty?
+          # no location info for the text input, stop here
+          return [nil, nil]
+        end
+        position = response["view"][0]["result"][0]["location"]["displayPosition"]
+        return position["latitude"], position["longitude"]
+      else
+        CartoDB.notify_debug('Non-batched geocoder failed request', http_response)
+        return [nil, nil]
+      end
+    rescue NoMethodError => e
+      if e.message == %Q(undefined method `[]' for nil:NilClass)
+        CartoDB.notify_debug("Non-batched geocoder couldn't parse response",
+          error: e.backtrace.join("\n"), backtrace: e.backtrace, text: text, response_body: http_response.body)
+        [nil, nil]
+      else
+        raise e
+      end
+    end
+
+    def api_url(arguments, extra_components = nil)
+      arguments.merge!(app_id: app_id, token: token, mailto: mailto)
+      components = [base_url]
+      components << extra_components unless extra_components.nil?
+      components << '?' + URI.encode_www_form(arguments)
+      components.join('/')
+    end
+
+    def init_rows_count
+      @processed_rows = 0
+      @successful_processed_rows = 0
+      @failed_processed_rows = 0
+      @empty_processed_rows = 0
+    end
+
+    def update_log_stats
+      @log.append_and_store "Geocoding non-batch Here job status update. "\
+        "Status: #{@status} --- Processed rows: #{@processed_rows} "\
+        "--- Success: #{@successful_processed_rows} --- Empty: #{@empty_processed_rows} "\
+        "--- Failed: #{@failed_processed_rows}"
+    end
+
+    def change_status(status)
+      @status = status
+      @geocoding_model.state = status
+      @geocoding_model.save
+    end
+
+  end
+end

services/geocoder/lib/hires_geocoder_factory.rb

@@ -0,0 +1,53 @@
+require_relative 'hires_geocoder'
+require_relative 'hires_batch_geocoder'
+require_relative 'geocoder_config'
+
+
+module CartoDB
+  class HiresGeocoderFactory
+
+    BATCH_FILES_OVER = 1100 # Use Here Batch Geocoder API with tables over x rows
+
+    def self.get(input_csv_file, working_dir, log, geocoding_model, number_of_rows = 0)
+      geocoder_class = nil
+      if use_batch_process?(input_csv_file, geocoding_model, number_of_rows)
+        geocoder_class = HiresBatchGeocoder
+      else
+        geocoder_class = HiresGeocoder
+      end
+
+      geocoder_class.new(input_csv_file, working_dir, log, geocoding_model)
+    end
+
+
+    private
+
+    def self.use_batch_process?(input_csv_file, geocoding_model, number_of_rows)
+      # Due we could check this condition to create the geocoder class and we don't
+      # have finished yet the csv file generation, and could be nil, we have to check
+      # multiples conditions. It's sorted by priority
+      if force_batch? || geocoding_model.batched
+        true
+      elsif (not input_csv_file.nil?) && (input_rows(input_csv_file) > BATCH_FILES_OVER)
+        true
+      elsif (not number_of_rows.nil?) && (number_of_rows > BATCH_FILES_OVER)
+        true
+      else
+        false
+      end
+    end
+
+    def self.force_batch?
+      GeocoderConfig.instance.get['force_batch'] || false
+    end
+
+    def self.input_rows(input_csv_file)
+      stdout, _stderr, _status = Open3.capture3('wc', '-l', input_csv_file)
+      stdout.to_i
+    rescue => e
+      CartoDB.notify_exception(e)
+      0
+    end
+
+  end
+end

services/geocoder/lib/hires_geocoder_interface.rb

@@ -0,0 +1,11 @@
+module CartoDB
+  class HiresGeocoderInterface
+    def run
+      raise 'Not implemented'
+    end
+
+    def cancel
+      raise 'Not implemented'
+    end
+  end
+end