Initial commit

lib/resque/automatic_geocoder_jobs.rb

@@ -0,0 +1,11 @@
+require_relative './base_job'
+
+module Resque
+  class AutomaticGeocoderJobs < BaseJob
+    @queue = :automatic_geocodings
+
+    def self.perform(options = {})
+      run_action(options, @queue, lambda { |options| AutomaticGeocoding[options.symbolize_keys[:job_id]].run })
+    end
+  end
+end

lib/resque/base_job.rb

@@ -0,0 +1,44 @@
+require 'sequel'
+require 'resque-metrics'
+
+module Resque
+  class BaseJob
+    extend ::Resque::Metrics
+    MAX_RETRIES = 3
+
+    @@queue = ''
+    @@retries = 0
+
+    def self.perform(options = {})
+      raise NotImplementedError("This class shouldn't be directly instantiated")
+    end
+
+    def self.run_action(options, queue_name, action)
+      @@queue = queue_name
+      begin
+        action.call(options)
+      rescue Sequel::DatabaseDisconnectError => e
+        puts "DatabaseDisconnectError: #{e.message}"
+
+        regexps = [ /server has gone away/, /decryption failed or bad record mac/, /SSL SYSCALL error: EOF detected/ ]
+        match_found = regexps.map { |regexp| regexp.match(e.message) }.any? { |matches| matches }
+
+        if (match_found)
+          @@retries += 1
+          if (@@retries < MAX_RETRIES)
+            puts 'Retrying'
+            retry
+          else
+            raise e
+          end
+        else
+          raise e
+        end
+      rescue => e
+        CartoDB.notify_exception(e)
+        raise e
+      end
+    end #self.perform
+
+  end #BaseJobs
+end

lib/resque/exporter_jobs.rb

@@ -0,0 +1,14 @@
+require_relative './base_job'
+
+module Resque
+  class ExporterJobs < BaseJob
+    @queue = :exports
+
+    def self.perform(options = {})
+      run_action(options, @queue, lambda do |options|
+        download_path = options['download_path']
+        Carto::VisualizationExport.find(options.symbolize_keys[:job_id]).run_export!(download_path: download_path)
+      end)
+    end
+  end
+end

lib/resque/geocoder_jobs.rb

@@ -0,0 +1,11 @@
+require_relative './base_job'
+
+module Resque
+  class GeocoderJobs < BaseJob
+    @queue = :geocodings
+
+    def self.perform(options = {})
+      geocoding = run_action(options, @queue, lambda { |options| geocoding = Geocoding[options.symbolize_keys[:job_id]].run! })
+    end
+  end
+end

lib/resque/importer_jobs.rb

@@ -0,0 +1,11 @@
+require_relative './base_job'
+
+module Resque
+  class ImporterJobs < BaseJob
+    @queue = :imports
+
+    def self.perform(options = {})
+      run_action(options, @queue, lambda { |options| DataImport[options.symbolize_keys[:job_id]].run_import! })
+    end
+  end
+end

lib/resque/metrics_logger.rb

@@ -0,0 +1,49 @@
+require 'carto/configuration'
+
+module CartoDB
+  module ResqueMetrics
+    def self.logger
+      @logger ||= CartoDB.unformatted_logger(Carto::Conf.new.log_file_path('resque_metrics.log'))
+    end
+  end
+end
+
+Resque::Metrics.on_job_complete do |job_class, queue, time|
+  CartoDB::ResqueMetrics.logger.info(
+    {:event => :job_complete,
+     :timestamp => Time.now.utc.iso8601,
+     :job_class => job_class.to_s,
+     :queue => queue,
+     :time => time}.to_json
+  )
+end
+
+Resque::Metrics.on_job_enqueue do |job_class, queue, time|
+  CartoDB::ResqueMetrics.logger.info(
+    {:event => :job_enqueue,
+     :timestamp => Time.now.utc.iso8601,
+     :job_class => job_class.to_s,
+     :queue => queue,
+     :time => time}.to_json
+  )
+end
+
+Resque::Metrics.on_job_fork do |job_class, queue|
+  CartoDB::ResqueMetrics.logger.info(
+    {:event => :job_fork,
+     :timestamp => Time.now.utc.iso8601,
+     :job_class => job_class.to_s,
+     :queue => queue}.to_json
+  )
+end
+
+Resque::Metrics.on_job_failure do |job_class, queue, time|
+  CartoDB::ResqueMetrics.logger.info(
+    {:event => :job_failure,
+     :timestamp => Time.now.utc.iso8601,
+     :job_class => job_class.to_s,
+     :queue => queue,
+     :time => time}.to_json
+  )
+end
+

lib/resque/reporter_jobs.rb

@@ -0,0 +1,15 @@
+require_relative './base_job'
+
+module Resque
+  module Reporter
+    module Mail
+      module TrendingMapsReport
+        @queue = :users
+
+        def self.perform(mail_to, trending_visualizations)
+          ReporterMailer.trending_maps_report(mail_to, trending_visualizations).deliver_now
+        end
+      end
+    end
+  end
+end

lib/resque/synchronization_jobs.rb

@@ -0,0 +1,16 @@
+require_relative './base_job'
+require_relative '../../app/models/synchronization/member'
+
+module Resque
+  class SynchronizationJobs < BaseJob
+    @queue = :synchronizations
+
+    def self.perform(options = {})
+      run_action(options, @queue, lambda { |options| 
+        CartoDB::Synchronization::Member.new(
+          id: options.symbolize_keys[:job_id]
+        ).fetch.run
+      })
+    end
+  end
+end

lib/resque/tracking_jobs.rb

@@ -0,0 +1,40 @@
+require 'hubspot/events_api'
+
+module Resque
+  module TrackingJobs
+    module SendSegmentEvent
+      ANONYMOUS_SEGMENT_USER_ID = '00000000-0000-0000-0000-000000000000'.freeze
+
+      @queue = :tracker
+
+      def self.perform(user_id, name, properties)
+        return unless segment_api_key = Cartodb.get_config(:segment, 'api_key')
+
+        segment = Segment::Analytics.new(write_key: segment_api_key)
+        segment.track(user_id: user_id || ANONYMOUS_SEGMENT_USER_ID, event: name, properties: properties)
+        segment.flush
+      rescue => exception
+        CartoDB::Logger.warning(message: 'Can\'t report to Segment',
+                                exception: exception,
+                                event: name,
+                                properties: properties)
+      end
+    end
+
+    module SendHubspotEvent
+      @queue = :tracker
+
+      def self.perform(id, params)
+        events_api = ::Hubspot::EventsAPI.instance
+        return unless events_api.enabled?
+
+        code, body = events_api.report(id, params: params)
+
+        unless code == '200' && body.present?
+          message = 'Carto::Tracking: Hubspot service error'
+          CartoDB::Logger.error(message: message, event_id: id, params: params)
+        end
+      end
+    end
+  end
+end

lib/resque/user_db_jobs.rb

@@ -0,0 +1,56 @@
+require_relative './base_job'
+require 'resque-metrics'
+require_relative '../cartodb/metrics'
+
+module Resque
+  module UserDBJobs
+    module UserDBMaintenance
+      module LinkGhostTables
+        extend ::Resque::Metrics
+        @queue = :user_dbs
+
+        def self.perform(user_id)
+          Carto::GhostTablesManager.new(user_id).link_ghost_tables_synchronously
+        rescue => e
+          CartoDB.notify_exception(e)
+          raise e
+        end
+      end
+
+      module LinkGhostTablesByUsername
+        extend ::Resque::Metrics
+        @queue = :user_dbs
+
+        def self.perform(username)
+          user = Carto::User.find_by_username!(username)
+          Carto::GhostTablesManager.new(user.id).link_ghost_tables_synchronously
+        rescue StandardError => e
+          CartoDB.notify_exception(e)
+          raise e
+        end
+      end
+
+      module AutoIndexTable
+        @queue = :user_dbs
+
+        def self.perform(user_table_id)
+          user_table = Carto::UserTable.where(id: user_table_id).first
+          Carto::UserTableIndexService.new(user_table).update_auto_indices if user_table
+        rescue => e
+          CartoDB::Logger.error(message: 'Error auto-indexing table', exception: e, user_table_id: user_table_id)
+        end
+      end
+    end
+
+    module CommonData
+      module LoadCommonData
+        @queue = :user_dbs
+
+        def self.perform(user_id, visualizations_api_url)
+          user = ::User.where(id: user_id).first
+          user.load_common_data(visualizations_api_url) if user.should_load_common_data?
+        end
+      end
+    end
+  end
+end

lib/resque/user_jobs.rb

@@ -0,0 +1,214 @@
+require_relative './base_job'
+require 'resque-metrics'
+require_relative '../cartodb/metrics'
+
+module Resque
+  module OrganizationJobs
+    module Mail
+      module DiskQuotaLimitReached
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(organization_id)
+          OrganizationMailer.quota_limit_reached(Organization.where(id: organization_id).first).deliver_now
+        end
+      end
+
+      module Invitation
+        @queue = :users
+
+        def self.perform(invitation_id)
+          invitation = Carto::Invitation.find(invitation_id)
+          invitation.users_emails.each do |email|
+            OrganizationMailer.invitation(invitation, email).deliver_now
+          end
+        end
+      end
+
+      module SeatLimitReached
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(organization_id)
+          OrganizationMailer.seat_limit_reached(Organization.where(id: organization_id).first).deliver_now
+        end
+      end
+    end
+  end
+
+  module UserJobs
+    module Signup
+      module NewUser
+        @queue = :users
+
+        def self.perform(user_creation_id, common_data_url = nil, organization_owner_promotion = false)
+          user_creation = Carto::UserCreation.where(id: user_creation_id).first
+          user_creation.set_common_data_url(common_data_url) unless common_data_url.nil?
+          user_creation.set_owner_promotion(organization_owner_promotion)
+          user_creation.next_creation_step! until user_creation.finished?
+        end
+      end
+    end
+
+    module RateLimitsJobs
+      module SyncRedis
+        extend ::Resque::Metrics
+        @queue = :batch_updates
+
+        def self.perform(account_type)
+          rate_limit = Carto::AccountType.find(account_type).rate_limit
+          Carto::User.where(account_type: account_type, rate_limit_id: nil).find_each do |user|
+            rate_limit.save_to_redis(user)
+          end
+        rescue StandardError => e
+          CartoDB::Logger.error(exception: e, message: 'Error syncing rate limits to redis', account_type: account_type)
+          raise e
+        end
+      end
+    end
+
+    module Notifications
+      module Send
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(user_ids, notification_id)
+          notification = Carto::Notification.find(notification_id)
+          Carto::User.where(id: user_ids).find_each do |user|
+            user.received_notifications.create!(notification: notification, received_at: DateTime.now)
+          end
+        end
+      end
+    end
+
+    module Mail
+      module NewOrganizationUser
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(user_id)
+          u = ::User.where(id: user_id).first
+          UserMailer.new_organization_user(u).deliver_now
+        end
+      end
+
+      module ShareVisualization
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(visualization_id, user_id)
+          v = Carto::Visualization.find(visualization_id)
+          u = ::User.where(id: user_id).first
+          UserMailer.share_visualization(v, u).deliver_now
+        end
+      end
+
+      module ShareTable
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(table_id, user_id)
+          t = Carto::Visualization.find(table_id)
+          u = ::User.where(id: user_id).first
+          UserMailer.share_table(t, u).deliver_now
+        end
+      end
+
+      module UnshareVisualization
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(visualization_name, visualization_owner_name, user_id)
+          u = ::User.where(id: user_id).first
+          UserMailer.unshare_visualization(visualization_name, visualization_owner_name, u).deliver_now
+        end
+      end
+
+      module UnshareTable
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(table_name, table_owner_name, user_id)
+          u = ::User.where(id: user_id).first
+          UserMailer.unshare_table(table_name, table_owner_name, u).deliver_now
+        end
+      end
+
+      module MapLiked
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(visualization_id, viewer_user_id, vis_preview_image)
+          viz = Carto::Visualization.find(visualization_id)
+          viewer_user = Carto::User.find(viewer_user_id)
+          UserMailer.map_liked(viz, viewer_user, vis_preview_image).deliver_now
+        end
+      end
+
+      module TableLiked
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(visualization_id, viewer_user_id, vis_preview_image)
+          viz = Carto::Visualization.find(visualization_id)
+          viewer_user = Carto::User.find(viewer_user_id)
+          UserMailer.table_liked(viz, viewer_user, vis_preview_image).deliver_now
+        end
+      end
+
+      module DataImportFinished
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(user_id, imported_tables, total_tables, first_imported_table, first_table, errors, filenames)
+          u = ::User.where(id: user_id).first
+          ImportMailer.data_import_finished(u, imported_tables, total_tables, first_imported_table, first_table, errors,
+                                            filenames).deliver_now
+        end
+      end
+
+      module GeocoderFinished
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(user_id, state, table_name, error_code, processable_rows, number_geocoded_rows)
+          user = ::User.where(id: user_id).first
+          GeocoderMailer.geocoding_finished(user, state, table_name, error_code, processable_rows, number_geocoded_rows)
+                        .deliver_now
+        end
+      end
+
+      module Sync
+        module MaxRetriesReached
+          extend ::Resque::Metrics
+          @queue = :users
+
+          def self.perform(user_id, visualization_id, dataset_name, error_code, error_message)
+            user = ::User.where(id: user_id).first
+            SyncMailer.max_retries_reached(user, visualization_id, dataset_name, error_code, error_message).deliver_now
+          end
+        end
+      end
+
+      module TrendingMap
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(visualization_id, mapviews, vis_preview_image)
+          visualization = Carto::Visualization.find(visualization_id)
+          UserMailer.trending_map(visualization, mapviews, vis_preview_image).deliver_now
+        end
+      end
+
+      module PasswordReset
+        extend ::Resque::Metrics
+        @queue = :users
+
+        def self.perform(user_id)
+          user = Carto::User.find(user_id)
+          UserMailer.password_reset(user).deliver_now
+        end
+      end
+    end
+  end
+end

lib/resque/user_migration_jobs.rb

@@ -0,0 +1,27 @@
+require_relative './base_job'
+
+module Resque
+  class UserMigrationJobs < BaseJob
+    module Export
+      @queue = :user_migrations
+
+      def self.perform(options = {})
+        Carto::UserMigrationExport.find(options['export_id']).run_export
+      rescue => e
+        CartoDB::Logger.error(exception: e, message: 'Error exporting user data', export_id: options['export_id'])
+        raise e
+      end
+    end
+
+    module Import
+      @queue = :user_migrations
+
+      def self.perform(options = {})
+        Carto::UserMigrationImport.find(options['import_id']).run_import
+      rescue => e
+        CartoDB::Logger.error(exception: e, message: 'Error importing user data', import_id: options['import_id'])
+        raise e
+      end
+    end
+  end
+end