Initial commit
This commit is contained in:
zhongjin
@@ -0,0 +1,153 @@
|
||||
require_dependency 'carto/uuidhelper'
|
||||
|
||||
module Carto
|
||||
|
||||
class DataImportsService
|
||||
include Carto::UUIDHelper
|
||||
|
||||
def initialize(users_metadata = $users_metadata, tables_metadata = $tables_metadata)
|
||||
@users_metadata = users_metadata
|
||||
@tables_metadata = tables_metadata
|
||||
end
|
||||
|
||||
def process_recent_user_imports(user)
|
||||
imports = DataImportQueryBuilder.new.with_user(user).with_state_not_in([Carto::DataImport::STATE_COMPLETE, Carto::DataImport::STATE_FAILURE]).with_created_at_after(Time.now - 24.hours).with_order(:created_at, :desc).build.all
|
||||
|
||||
running_ids = running_import_ids
|
||||
|
||||
imports.map { |import|
|
||||
if import.created_at < Time.now - 60.minutes && !running_ids.include?(import.id)
|
||||
# INFO: failure is handled with old model
|
||||
::DataImport[import.id].handle_failure(CartoDB::Importer2::StuckImportJobError.new)
|
||||
nil
|
||||
else
|
||||
import
|
||||
end
|
||||
}.compact
|
||||
end
|
||||
|
||||
def process_by_id(id)
|
||||
return nil if !is_uuid?(id)
|
||||
|
||||
import = Carto::DataImport.where(id: id).first
|
||||
|
||||
if stuck?(import)
|
||||
# INFO: failure because of stuck is handled with old model
|
||||
::DataImport[id].mark_as_failed_if_stuck!
|
||||
import.reload
|
||||
end
|
||||
import
|
||||
rescue RecordNotFound => e
|
||||
nil
|
||||
end
|
||||
|
||||
def validate_synchronization_oauth(user, service)
|
||||
oauth = user.oauth_for_service(service)
|
||||
return false unless oauth
|
||||
|
||||
datasource = oauth.get_service_datasource
|
||||
|
||||
begin
|
||||
valid = datasource.token_valid?
|
||||
rescue => e
|
||||
delete_oauth_if_expired_and_raise(user, e, oauth)
|
||||
valid = false
|
||||
end
|
||||
|
||||
unless valid
|
||||
delete_oauth(user, oauth)
|
||||
end
|
||||
|
||||
valid
|
||||
rescue => e
|
||||
delete_oauth_if_expired_and_raise(user, e, oauth)
|
||||
end
|
||||
|
||||
def get_service_files(user, service, filter)
|
||||
oauth = user.oauth_for_service(service)
|
||||
raise CartoDB::Datasources::AuthError.new("No oauth set for service #{service}") if oauth.nil?
|
||||
datasource = oauth.get_service_datasource
|
||||
datasource.get_resources_list(filter)
|
||||
rescue => e
|
||||
delete_oauth_if_expired_and_raise(user, e, oauth)
|
||||
end
|
||||
|
||||
def get_service_auth_url(user, service)
|
||||
oauth = user.oauth_for_service(service)
|
||||
raise CartoDB::Datasources::AuthError.new("OAuth already set for service #{service}") if oauth
|
||||
|
||||
get_datasource(user, service).get_auth_url
|
||||
end
|
||||
|
||||
def validate_service_oauth_code(user, service, code)
|
||||
oauth = user.oauth_for_service(service)
|
||||
raise CartoDB::Datasources::AuthError.new("OAuth already set for service #{service}") if oauth
|
||||
|
||||
datasource = get_datasource(user, service)
|
||||
begin
|
||||
auth_token = datasource.validate_auth_code(code)
|
||||
user.add_oauth(service, auth_token)
|
||||
return true
|
||||
rescue CartoDB::Datasources::AuthError => e
|
||||
CartoDB.notify_exception(e, { message: "Error while validating code #{code}, it won't be stored", user: user, service: service })
|
||||
return false
|
||||
end
|
||||
rescue => e
|
||||
delete_oauth_if_expired_and_raise(user, e, oauth)
|
||||
end
|
||||
|
||||
def validate_callback(user, service, params)
|
||||
oauth = user.oauth_for_service(service)
|
||||
raise CartoDB::Datasources::AuthError.new("OAuth already set for service #{service}") if oauth
|
||||
datasource = get_datasource(user, service)
|
||||
|
||||
token = datasource.validate_callback(params)
|
||||
|
||||
# TODO: workaround for https://github.com/CartoDB/cartodb/issues/4003
|
||||
#user.add_oauth(service, datasource.validate_callback(params))
|
||||
CartoDB::OAuths.new(::User.where(id: user.id).first).add(service, token)
|
||||
rescue => e
|
||||
delete_oauth_if_expired_and_raise(user, e, oauth)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def get_datasource(user, service)
|
||||
datasource = CartoDB::Datasources::DatasourcesFactory.get_datasource(service, user, {
|
||||
redis_storage: @tables_metadata,
|
||||
http_timeout: ::DataImport.http_timeout_for(user)
|
||||
})
|
||||
raise CartoDB::Datasources::AuthError.new("Couldn't fetch datasource for service #{service}") if datasource.nil?
|
||||
datasource
|
||||
end
|
||||
|
||||
def delete_oauth(user, oauth)
|
||||
# INFO: this is the straightforward way, but sometimes it fails with "ActiveRecord::StatementInvalid: PG::Error: ERROR: prepared statement "a1" does not exist" errors
|
||||
# user.synchronization_oauths.delete(oauth)
|
||||
oauth.destroy
|
||||
user.synchronization_oauths.delete(oauth)
|
||||
end
|
||||
|
||||
def delete_oauth_if_expired_and_raise(user, e, oauth = nil)
|
||||
CartoDB.notify_exception(e, { message: 'Error while processing datasource', user: user, oauth: oauth })
|
||||
if e.kind_of?(CartoDB::Datasources::TokenExpiredOrInvalidError) && oauth
|
||||
delete_oauth(user, oauth)
|
||||
end
|
||||
|
||||
raise e
|
||||
end
|
||||
|
||||
def stuck?(import)
|
||||
# TODO: this kind of method is in the service because it requires communication with external systems (resque). Anyway, should some logic (state check, for example) be inside the model?
|
||||
![Carto::DataImport::STATE_ENQUEUED, Carto::DataImport::STATE_PENDING, Carto::DataImport::STATE_COMPLETE, Carto::DataImport::STATE_FAILURE].include?(import.state) &&
|
||||
import.created_at < 5.minutes.ago &&
|
||||
!running_import_ids.include?(import.id)
|
||||
end
|
||||
|
||||
def running_import_ids
|
||||
Resque::Worker.all.map { |worker| worker.job["payload"]["args"].first["job_id"] rescue nil }.compact
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,73 @@
|
||||
require_relative '../../../services/sql-api/sql_api'
|
||||
require_relative '../../../app/controllers/carto/controller_helper'
|
||||
|
||||
module Carto
|
||||
class DataLibraryService
|
||||
# - source_api_key is used to retrieve metadata from the dataset
|
||||
# - granted_api_key is stored as the one to use for the importing
|
||||
def load_dataset!(carto_api_client,
|
||||
source_dataset:, source_username:, source_api_key:,
|
||||
target_username:, granted_api_key:, format: 'gpkg')
|
||||
target_user = Carto::User.find_by_username(target_username)
|
||||
raise Carto::LoadError.new("User not found: #{target_username}") unless target_user
|
||||
|
||||
remote_visualization = carto_api_client.get_visualization_v1(
|
||||
username: source_username, name: source_dataset, params: { api_key: source_api_key }
|
||||
)
|
||||
remote_table = remote_visualization[:table]
|
||||
privacy = target_user.default_dataset_privacy
|
||||
remote_attributes = remote_visualization.slice(:name, :description, :tags, :license, :source, :attributions)
|
||||
|
||||
base_url = "#{carto_api_client.scheme}://#{carto_api_client.base_url(source_username)}"
|
||||
sql_api_url = CartoDB::SQLApi.with_username_api_key(source_username, granted_api_key, privacy, base_url: base_url)
|
||||
.export_table_url(source_dataset, format)
|
||||
external_source = Carto::ExternalSource.new(
|
||||
import_url: sql_api_url,
|
||||
rows_counted: remote_table[:row_count],
|
||||
size: remote_table[:size],
|
||||
geometry_types: remote_table[:geometry_types],
|
||||
username: source_username
|
||||
)
|
||||
visualization = Carto::Visualization.create!(
|
||||
remote_attributes.merge(
|
||||
display_name: display_name(remote_visualization),
|
||||
user: target_user,
|
||||
type: Carto::Visualization::TYPE_REMOTE,
|
||||
privacy: privacy,
|
||||
external_source: external_source
|
||||
)
|
||||
)
|
||||
|
||||
visualization
|
||||
end
|
||||
|
||||
def load_datasets!(carto_api_client,
|
||||
source_username:, source_api_key:,
|
||||
target_username:, granted_api_key:, format: 'gpkg')
|
||||
api_keys = carto_api_client.get_api_keys_v3(username: source_username, params: { api_key: granted_api_key })
|
||||
api_key = api_keys[:result].find { |key| key[:token] == granted_api_key }
|
||||
|
||||
return unless api_key
|
||||
|
||||
database_grants = api_key[:grants].select { |g| g[:type] == 'database' }
|
||||
database_grants.each do |database_grant|
|
||||
tables = database_grant[:tables]
|
||||
tables.each do |table|
|
||||
if table[:permissions].include?('select')
|
||||
load_dataset!(carto_api_client,
|
||||
source_dataset: table[:name],
|
||||
source_username: source_username, source_api_key: source_api_key,
|
||||
target_username: target_username, granted_api_key: granted_api_key,
|
||||
format: format)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def display_name(remote_visualization)
|
||||
remote_visualization[:display_name].presence || remote_visualization[:name]
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,87 @@
|
||||
module Carto
|
||||
class DbPermissionService
|
||||
TYPE_USER = 'user'.freeze
|
||||
TYPE_ORG = 'org'.freeze
|
||||
TYPE_GROUP = 'group'.freeze
|
||||
|
||||
READ_PERMISSIONS = ['select'].freeze
|
||||
WRITE_PERMISSIONS = ['insert', 'update', 'delete'].freeze
|
||||
PERMISSIONS = {
|
||||
r: READ_PERMISSIONS,
|
||||
w: WRITE_PERMISSIONS,
|
||||
rw: READ_PERMISSIONS + WRITE_PERMISSIONS
|
||||
}.freeze
|
||||
|
||||
def self.shared_entities_revokes(old_acl, new_acl, table)
|
||||
diff = revokes_by_user_diff(old_acl, new_acl, table.owner.id)
|
||||
unless diff.blank?
|
||||
shared_apikey_revokes(table, diff)
|
||||
shared_oauth_app_user_revokes(table, diff)
|
||||
end
|
||||
end
|
||||
|
||||
def self.revokes_by_user_diff(old_acl, new_acl, table_owner_id)
|
||||
old_acl = more_permisive_by_user(old_acl)
|
||||
new_acl = more_permisive_by_user(new_acl)
|
||||
|
||||
revokes_by_user(old_acl, new_acl, table_owner_id)
|
||||
end
|
||||
|
||||
private_class_method def self.shared_apikey_revokes(table, revokes)
|
||||
Carto::ApiKey.where(user_id: revokes.keys, type: ['regular', 'oauth']).find_each do |apikey|
|
||||
apikey.revoke_permissions(table, PERMISSIONS[revokes[apikey.user_id].to_sym])
|
||||
end
|
||||
end
|
||||
|
||||
private_class_method def self.shared_oauth_app_user_revokes(table, revokes)
|
||||
Carto::OauthAppUser.where(user_id: revokes.keys).find_each do |oau|
|
||||
oau.revoke_permissions(table, PERMISSIONS[revokes[oau.user_id].to_sym])
|
||||
end
|
||||
end
|
||||
|
||||
private_class_method def self.more_permisive_by_user(acl)
|
||||
acl_hash = {}
|
||||
acl.each do |p|
|
||||
user_ids(p[:type], p[:id]).each { |uid| acl_hash[uid] = keep_more_permisive(acl_hash[uid], p[:access]) }
|
||||
end
|
||||
acl_hash
|
||||
end
|
||||
|
||||
private_class_method def self.user_ids(type, id)
|
||||
if type == TYPE_USER
|
||||
[id]
|
||||
elsif type == TYPE_ORG
|
||||
Carto::Organization.find(id).users.map(&:id)
|
||||
else
|
||||
Carto::Group.find(id).users.map(&:id)
|
||||
end
|
||||
rescue ActiveRecord::RecordNotFound => e
|
||||
CartoDB::Logger.error(exception: e, entity_type: type, entity_id: id)
|
||||
[]
|
||||
end
|
||||
|
||||
private_class_method def self.keep_more_permisive(current, new_one)
|
||||
current == 'rw' || new_one == 'rw' ? 'rw' : 'r'
|
||||
end
|
||||
|
||||
private_class_method def self.revokes_by_user(old_acl, new_acl, table_owner_id)
|
||||
diff = {}
|
||||
old_acl.each do |id, access|
|
||||
next if id == table_owner_id
|
||||
|
||||
if access == 'rw'
|
||||
if new_acl[id].nil?
|
||||
revoke = 'rw'
|
||||
elsif new_acl[id] == 'r'
|
||||
revoke = 'w'
|
||||
end
|
||||
elsif access == 'r' && new_acl[id].nil?
|
||||
revoke = 'r'
|
||||
end
|
||||
|
||||
diff[id] = revoke if revoke.present?
|
||||
end
|
||||
diff
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,50 @@
|
||||
module Carto
|
||||
class DoLicensingService
|
||||
|
||||
def initialize(username)
|
||||
@username = username
|
||||
@redis_key = "do:#{@username}:datasets"
|
||||
end
|
||||
|
||||
def subscribe(datasets)
|
||||
Cartodb::Central.new.create_do_datasets(username: @username, datasets: datasets)
|
||||
add_to_redis(datasets)
|
||||
end
|
||||
|
||||
def unsubscribe(dataset_id)
|
||||
Cartodb::Central.new.remove_do_dataset(username: @username, id: dataset_id)
|
||||
remove_from_redis(dataset_id)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def add_to_redis(datasets)
|
||||
value = ["bq", insert_redis_value(datasets, 'bq'), "spanner", insert_redis_value(datasets, 'spanner')]
|
||||
$users_metadata.hmset(@redis_key, value)
|
||||
end
|
||||
|
||||
def remove_from_redis(dataset_id)
|
||||
value = ["bq", remove_redis_value(dataset_id, 'bq'), "spanner", remove_redis_value(dataset_id, 'spanner')]
|
||||
$users_metadata.hmset(@redis_key, value)
|
||||
end
|
||||
|
||||
def insert_redis_value(datasets, storage)
|
||||
redis_value = JSON.parse($users_metadata.hget(@redis_key, storage) || '[]')
|
||||
new_datasets = filter_datasets(datasets, storage)
|
||||
(redis_value + new_datasets).uniq.to_json
|
||||
end
|
||||
|
||||
def remove_redis_value(dataset_id, storage)
|
||||
redis_value = JSON.parse($users_metadata.hget(@redis_key, storage) || '[]')
|
||||
redis_value.reject { |dataset| dataset["dataset_id"] == dataset_id }.to_json
|
||||
end
|
||||
|
||||
def filter_datasets(datasets, storage)
|
||||
filtered_datasets = datasets.select { |dataset| dataset[:available_in].include?(storage) }
|
||||
filtered_datasets.map do |dataset|
|
||||
{ "dataset_id" => dataset[:dataset_id], "expires_at" => dataset[:expires_at].to_s }
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,327 @@
|
||||
require 'json'
|
||||
require_dependency 'carto/export/layer_exporter'
|
||||
require_dependency 'carto/export/connector_configuration_exporter'
|
||||
|
||||
# Not migrated
|
||||
# invitations -> temporary by nature
|
||||
# ldap_configurations -> not enabled in SaaS
|
||||
|
||||
# Version History
|
||||
# 1.0.0: export organization metadata
|
||||
# 1.0.1: export password expiration
|
||||
# 1.0.2: export connector configurations
|
||||
# 1.0.3: export oauth_app_organizations
|
||||
module Carto
|
||||
module OrganizationMetadataExportServiceConfiguration
|
||||
CURRENT_VERSION = '1.0.3'.freeze
|
||||
EXPORTED_ORGANIZATION_ATTRIBUTES = [
|
||||
:id, :seats, :quota_in_bytes, :created_at, :updated_at, :name, :avatar_url, :owner_id, :website, :description,
|
||||
:display_name, :discus_shortname, :twitter_username, :geocoding_quota, :map_view_quota, :auth_token,
|
||||
:geocoding_block_price, :map_view_block_price, :twitter_datasource_enabled, :twitter_datasource_block_price,
|
||||
:twitter_datasource_block_size, :twitter_datasource_quota, :google_maps_key, :google_maps_private_key, :color,
|
||||
:default_quota_in_bytes, :whitelisted_email_domains, :admin_email, :auth_username_password_enabled,
|
||||
:auth_google_enabled, :location, :here_isolines_quota, :here_isolines_block_price, :strong_passwords_enabled,
|
||||
:obs_snapshot_quota, :obs_snapshot_block_price, :obs_general_quota, :obs_general_block_price,
|
||||
:salesforce_datasource_enabled, :viewer_seats, :geocoder_provider, :isolines_provider, :routing_provider,
|
||||
:auth_github_enabled, :engine_enabled, :mapzen_routing_quota, :mapzen_routing_block_price, :builder_enabled,
|
||||
:auth_saml_configuration, :no_map_logo, :password_expiration_in_d
|
||||
].freeze
|
||||
|
||||
def compatible_version?(version)
|
||||
version.to_i == CURRENT_VERSION.split('.')[0].to_i
|
||||
end
|
||||
end
|
||||
|
||||
module OrganizationMetadataExportServiceImporter
|
||||
include OrganizationMetadataExportServiceConfiguration
|
||||
include LayerImporter
|
||||
include ConnectorConfigurationImporter
|
||||
|
||||
def build_organization_from_json_export(exported_json_string)
|
||||
build_organization_from_hash_export(JSON.parse(exported_json_string, symbolize_names: true))
|
||||
end
|
||||
|
||||
def build_organization_from_hash_export(exported_hash)
|
||||
raise 'Wrong export version' unless compatible_version?(exported_hash[:version])
|
||||
|
||||
build_organization_from_hash(exported_hash[:organization])
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def save_imported_organization(organization)
|
||||
organization.save!
|
||||
::Organization[organization.id].after_save
|
||||
end
|
||||
|
||||
def build_organization_from_hash(exported_organization)
|
||||
organization = Organization.new(exported_organization.slice(*EXPORTED_ORGANIZATION_ATTRIBUTES - [:id]))
|
||||
organization.assets = exported_organization[:assets].map { |asset| build_asset_from_hash(asset.symbolize_keys) }
|
||||
organization.groups = exported_organization[:groups].map { |group| build_group_from_hash(group.symbolize_keys) }
|
||||
organization.notifications = exported_organization[:notifications].map do |notification|
|
||||
build_notification_from_hash(notification.symbolize_keys)
|
||||
end
|
||||
if exported_organization[:oauth_app_organizations]
|
||||
organization.oauth_app_organizations = exported_organization[:oauth_app_organizations].map do |oao|
|
||||
build_oauth_app_organization_from_hash(oao.symbolize_keys)
|
||||
end
|
||||
end
|
||||
|
||||
organization.connector_configurations = build_connector_configurations_from_hash(
|
||||
exported_organization[:connector_configurations]
|
||||
)
|
||||
|
||||
# Must be the last one to avoid attribute assignments to try to run SQL
|
||||
organization.id = exported_organization[:id]
|
||||
organization
|
||||
end
|
||||
|
||||
def build_asset_from_hash(exported_asset)
|
||||
Asset.new(
|
||||
public_url: exported_asset[:public_url],
|
||||
kind: exported_asset[:kind],
|
||||
storage_info: exported_asset[:storage_info]
|
||||
)
|
||||
end
|
||||
|
||||
def build_group_from_hash(exported_group)
|
||||
g = Group.new_instance_without_validation(
|
||||
name: exported_group[:name],
|
||||
display_name: exported_group[:display_name],
|
||||
database_role: exported_group[:database_role],
|
||||
auth_token: exported_group[:auth_token]
|
||||
)
|
||||
g.users_group = exported_group[:user_ids].map { |uid| UsersGroup.new(user_id: uid) }
|
||||
g.id = exported_group[:id]
|
||||
|
||||
g
|
||||
end
|
||||
|
||||
def build_notification_from_hash(notification)
|
||||
Notification.new(
|
||||
icon: notification[:icon],
|
||||
recipients: notification[:recipients],
|
||||
body: notification[:body],
|
||||
created_at: notification[:created_at],
|
||||
received_notifications: notification[:received_by].map do |received_notification|
|
||||
build_received_notification_from_hash(received_notification.symbolize_keys)
|
||||
end
|
||||
)
|
||||
end
|
||||
|
||||
def build_received_notification_from_hash(received_notification)
|
||||
ReceivedNotification.new(
|
||||
user_id: received_notification[:user_id],
|
||||
received_at: received_notification[:received_at],
|
||||
read_at: received_notification[:read_at]
|
||||
)
|
||||
end
|
||||
|
||||
def build_oauth_app_organization_from_hash(oao_hash)
|
||||
Carto::OauthAppOrganization.new(
|
||||
oauth_app_id: oao_hash[:oauth_app_id],
|
||||
organization_id: oao_hash[:organization_id],
|
||||
seats: oao_hash[:seats],
|
||||
created_at: oao_hash[:created_at],
|
||||
updated_at: oao_hash[:updated_at]
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
module OrganizationMetadataExportServiceExporter
|
||||
include OrganizationMetadataExportServiceConfiguration
|
||||
include LayerExporter
|
||||
include ConnectorConfigurationExporter
|
||||
|
||||
def export_organization_json_string(organization)
|
||||
export_organization_json_hash(organization).to_json
|
||||
end
|
||||
|
||||
def export_organization_json_hash(organization)
|
||||
{
|
||||
version: CURRENT_VERSION,
|
||||
organization: export(organization)
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def export(organization)
|
||||
organization_hash = EXPORTED_ORGANIZATION_ATTRIBUTES.map { |att| [att, organization.attributes[att.to_s]] }.to_h
|
||||
|
||||
organization_hash[:assets] = organization.assets.map { |a| export_asset(a) }
|
||||
organization_hash[:groups] = organization.groups.map { |g| export_group(g) }
|
||||
organization_hash[:notifications] = organization.notifications.map { |n| export_notification(n) }
|
||||
organization_hash[:connector_configurations] = organization.connector_configurations.map do |cc|
|
||||
export_connector_configuration(cc)
|
||||
end
|
||||
organization_hash[:oauth_app_organizations] = organization.oauth_app_organizations.map do |oao|
|
||||
export_oauth_app_organization(oao)
|
||||
end
|
||||
|
||||
organization_hash
|
||||
end
|
||||
|
||||
def export_asset(asset)
|
||||
{
|
||||
public_url: asset.public_url,
|
||||
kind: asset.kind,
|
||||
storage_info: asset.storage_info
|
||||
}
|
||||
end
|
||||
|
||||
def export_group(group)
|
||||
{
|
||||
id: group.id,
|
||||
name: group.name,
|
||||
display_name: group.display_name,
|
||||
database_role: group.database_role,
|
||||
auth_token: group.auth_token,
|
||||
user_ids: group.users.map(&:id)
|
||||
}
|
||||
end
|
||||
|
||||
def export_notification(notification)
|
||||
{
|
||||
icon: notification.icon,
|
||||
recipients: notification.recipients,
|
||||
body: notification.body,
|
||||
created_at: notification.created_at,
|
||||
received_by: notification.received_notifications.map { |rn| export_received_notification(rn) }
|
||||
}
|
||||
end
|
||||
|
||||
def export_received_notification(received_notification)
|
||||
{
|
||||
user_id: received_notification.user_id,
|
||||
received_at: received_notification.received_at,
|
||||
read_at: received_notification.read_at
|
||||
}
|
||||
end
|
||||
|
||||
def export_oauth_app_organization(oao)
|
||||
{
|
||||
oauth_app_id: oao.oauth_app_id,
|
||||
organization_id: oao.organization_id,
|
||||
seats: oao.seats,
|
||||
created_at: oao.created_at,
|
||||
updated_at: oao.updated_at
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
class OrganizationAlreadyExists < RuntimeError; end
|
||||
# Both String and Hash versions are provided because `deep_symbolize_keys` won't symbolize through arrays
|
||||
# and having separated methods make handling and testing much easier.
|
||||
class OrganizationMetadataExportService
|
||||
include OrganizationMetadataExportServiceImporter
|
||||
include OrganizationMetadataExportServiceExporter
|
||||
|
||||
def export_to_directory(organization, path)
|
||||
root_dir = Pathname.new(path)
|
||||
|
||||
# Export organization
|
||||
organization_json = export_organization_json_string(organization)
|
||||
root_dir.join("organization_#{organization.id}.json").open('w') { |file| file.write(organization_json) }
|
||||
|
||||
redis_json = Carto::RedisExportService.new.export_organization_json_string(organization)
|
||||
root_dir.join("redis_organization_#{organization.id}.json").open('w') { |file| file.write(redis_json) }
|
||||
|
||||
# Export users
|
||||
organization.users.each do |user|
|
||||
Carto::UserMetadataExportService.new.export_to_directory(user, root_dir.join("user_#{user.id}"))
|
||||
end
|
||||
end
|
||||
|
||||
def import_from_directory(meta_path)
|
||||
# Import organization
|
||||
organization = load_organization_from_directory(meta_path)
|
||||
raise OrganizationAlreadyExists.new if ::Carto::Organization.exists?(id: organization.id)
|
||||
|
||||
organization_redis_file = redis_filename(meta_path)
|
||||
Carto::RedisExportService.new.restore_redis_from_json_export(File.read(organization_redis_file))
|
||||
|
||||
# Groups and notifications must be saved after users
|
||||
groups = organization.groups.map(&:clone)
|
||||
organization.groups.clear
|
||||
notifications = organization.notifications.map(&:clone)
|
||||
organization.notifications.clear
|
||||
oauth_app_organizations = organization.oauth_app_organizations.map(&:clone)
|
||||
organization.oauth_app_organizations.clear
|
||||
|
||||
save_imported_organization(organization)
|
||||
|
||||
user_list = get_user_list(meta_path)
|
||||
|
||||
# In order to get permissions right, we first import all users, then all datasets and finally, all maps
|
||||
organization.users = user_list.map do |user_path|
|
||||
Carto::UserMetadataExportService.new.import_from_directory(user_path)
|
||||
end
|
||||
|
||||
organization.groups = groups
|
||||
organization.notifications = notifications
|
||||
organization.oauth_app_organizations = oauth_app_organizations
|
||||
organization.save
|
||||
|
||||
organization
|
||||
end
|
||||
|
||||
def rollback_import_from_directory(meta_path)
|
||||
organization_redis_file = redis_filename(meta_path)
|
||||
Carto::RedisExportService.new.remove_redis_from_json_export(File.read(organization_redis_file))
|
||||
organization = load_organization_from_directory(meta_path)
|
||||
|
||||
user_list = organization.non_owner_users + [organization.owner]
|
||||
user_list.map do |user|
|
||||
Carto::UserMetadataExportService.new.rollback_import_from_directory("#{meta_path}/user_#{user.id}")
|
||||
end
|
||||
return unless Carto::Organization.exists?(organization.id)
|
||||
|
||||
organization = Carto::Organization.find(organization.id)
|
||||
organization.groups.delete
|
||||
organization.notifications.delete
|
||||
organization.oauth_app_organizations.delete
|
||||
organization.assets.map(&:delete)
|
||||
organization.users.delete
|
||||
organization.delete
|
||||
end
|
||||
|
||||
def get_user_list(meta_path)
|
||||
Dir["#{meta_path}/user_*"]
|
||||
end
|
||||
|
||||
def redis_filename(meta_path)
|
||||
Dir["#{meta_path}/redis_organization_*.json"].first
|
||||
end
|
||||
|
||||
def load_organization_from_directory(meta_path)
|
||||
organization_file = Dir["#{meta_path}/organization_*.json"].first
|
||||
build_organization_from_json_export(File.read(organization_file))
|
||||
end
|
||||
|
||||
def import_metadata_from_directory(organization, path)
|
||||
organization.users.each do |user|
|
||||
Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
|
||||
user, Carto::Visualization::TYPE_REMOTE, "#{path}/user_#{user.id}"
|
||||
)
|
||||
|
||||
Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
|
||||
user, Carto::Visualization::TYPE_CANONICAL, "#{path}/user_#{user.id}"
|
||||
)
|
||||
end
|
||||
|
||||
# Derived must be the last because of shared canonicals
|
||||
organization.users.each do |user|
|
||||
Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
|
||||
user, Carto::Visualization::TYPE_DERIVED, "#{path}/user_#{user.id}"
|
||||
)
|
||||
end
|
||||
|
||||
organization.users.each do |user|
|
||||
Carto::UserMetadataExportService.new.import_search_tweets_from_directory(user, "#{path}/user_#{user.id}")
|
||||
end
|
||||
|
||||
organization
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,54 @@
|
||||
module Carto
|
||||
class OverquotaUsersService
|
||||
def initialize(date = Date.today)
|
||||
@date = date
|
||||
end
|
||||
|
||||
def store_overquota_users(delta)
|
||||
overquota(delta).each do |user|
|
||||
$users_metadata.hmset(date_key, user.id, user.data.to_json)
|
||||
end
|
||||
# Expire the cache after two months. Enough time to review data if needed
|
||||
$users_metadata.expire(date_key, 2.months)
|
||||
end
|
||||
|
||||
def get_stored_overquota_users
|
||||
if $users_metadata.exists(date_key)
|
||||
$users_metadata.hgetall(date_key).values.map { |v| JSON.parse(v) }
|
||||
else
|
||||
CartoDB::Logger.warning(message: "There is no overquota cached users for date #{formatted_date}")
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def date_key
|
||||
"overquota:users:#{formatted_date}"
|
||||
end
|
||||
|
||||
def formatted_date
|
||||
@date.strftime('%Y%m%d')
|
||||
end
|
||||
|
||||
##
|
||||
# SLOW! Checks redis data (geocodings and isolines) for every user
|
||||
# delta: get users who are also this percentage below their limit.
|
||||
# example: 0.20 will get all users at 80% of their map view limit
|
||||
#
|
||||
def overquota(delta)
|
||||
overquota_users = []
|
||||
::User.where(enabled: true, organization_id: nil).use_cursor.each do |u|
|
||||
overquota_users << u if services_overquota(u, delta)
|
||||
end
|
||||
overquota_users
|
||||
end
|
||||
|
||||
SERVICES = %w(geocoding twitter_datasource here_isolines obs_snapshot obs_general mapzen_routing).freeze
|
||||
def services_overquota(user, delta)
|
||||
SERVICES.any? do |service|
|
||||
user.send("get_#{service}_calls") > user.send("#{service}_quota").to_i * (1 - delta)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,47 @@
|
||||
# Create/Drop/Rename overviews...
|
||||
|
||||
require_dependency 'carto/db/sql_interface'
|
||||
|
||||
module Carto
|
||||
class OverviewsService
|
||||
|
||||
def initialize(database)
|
||||
@database = Carto::Db::SqlInterface.new(database)
|
||||
end
|
||||
|
||||
def create_overviews(table_name, tolerance_px)
|
||||
@database.run %{
|
||||
SELECT cartodb.CDB_CreateOverviewsWithToleranceInPixels('#{table_name}'::REGCLASS, '#{tolerance_px}'::float8);
|
||||
}
|
||||
end
|
||||
|
||||
def rename_overviews(table_name, new_table_name)
|
||||
# TODO: use CDB_RenameOverviews
|
||||
overview_tables(table_name).each do |overview_table|
|
||||
new_overview_table = overview_table.sub(table_name, new_table_name)
|
||||
@database.run %{
|
||||
ALTER TABLE "#{overview_table}" RENAME TO "#{new_overview_table}"
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def delete_overviews(table_name)
|
||||
if @database.fetch(%{SELECT cartodb._CDB_Table_Exists('#{table_name}')}).first[:_cdb_table_exists]
|
||||
@database.run(%{SELECT cartodb.CDB_DropOverviews('#{table_name}')})
|
||||
end
|
||||
end
|
||||
|
||||
def overview_tables(table_name)
|
||||
overviews_data = @database.fetch(%{SELECT * FROM cartodb.CDB_Overviews('#{table_name}'::REGCLASS)})
|
||||
if overviews_data
|
||||
overviews_data.map { |row| row[:overview_table] }
|
||||
else
|
||||
[]
|
||||
end
|
||||
rescue Carto::Db::SqlInterface::Error => e
|
||||
raise unless e.to_s.match /relation .does not exist/
|
||||
[]
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,157 @@
|
||||
require 'json'
|
||||
|
||||
# Version History
|
||||
# 1.0.0: export organization metadata
|
||||
module Carto
|
||||
module RedisExportServiceConfiguration
|
||||
CURRENT_VERSION = '1.0.0'.freeze
|
||||
|
||||
def compatible_version?(version)
|
||||
version.to_i == CURRENT_VERSION.split('.')[0].to_i
|
||||
end
|
||||
end
|
||||
|
||||
module RedisExportServiceImporter
|
||||
include RedisExportServiceConfiguration
|
||||
|
||||
def restore_redis_from_json_export(exported_json_string)
|
||||
restore_redis_from_hash_export(JSON.parse(exported_json_string).deep_symbolize_keys)
|
||||
end
|
||||
|
||||
def remove_redis_from_json_export(exported_json_string)
|
||||
remove_redis_from_hash_export(JSON.parse(exported_json_string).deep_symbolize_keys)
|
||||
end
|
||||
|
||||
def restore_redis_from_hash_export(exported_hash)
|
||||
raise 'Wrong export version' unless compatible_version?(exported_hash[:version])
|
||||
|
||||
restore_redis(exported_hash[:redis])
|
||||
end
|
||||
|
||||
def remove_redis_from_hash_export(exported_hash)
|
||||
raise 'Wrong export version' unless compatible_version?(exported_hash[:version])
|
||||
|
||||
remove_redis(exported_hash[:redis])
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def restore_redis(redis_export)
|
||||
restore_keys($users_metadata, redis_export[:users_metadata])
|
||||
restore_named_maps($tables_metadata, redis_export[:tables_metadata])
|
||||
end
|
||||
|
||||
def remove_redis(redis_export)
|
||||
remove_keys($users_metadata, redis_export[:users_metadata])
|
||||
remove_keys($tables_metadata, redis_export[:tables_metadata])
|
||||
end
|
||||
|
||||
def restore_keys(redis_db, redis_keys)
|
||||
redis_keys.each do |key, value|
|
||||
redis_db.restore(key, value[:ttl], Base64.decode64(value[:value]))
|
||||
end
|
||||
end
|
||||
|
||||
def restore_named_maps(redis_db, redis_keys)
|
||||
redis_keys.select { |k| k =~ /map_tpl\|/ }.each do |key, value|
|
||||
value.each do |name, named_map_config|
|
||||
redis_db.hset(key, name, Base64.decode64(named_map_config))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def remove_keys(redis_db, redis_keys)
|
||||
redis_keys.each do |key|
|
||||
redis_db.del(key)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module RedisExportServiceExporter
|
||||
include RedisExportServiceConfiguration
|
||||
|
||||
def export_organization_json_string(organization)
|
||||
export_organization_json_hash(organization).to_json
|
||||
end
|
||||
|
||||
def export_organization_json_hash(organization)
|
||||
{
|
||||
version: CURRENT_VERSION,
|
||||
redis: export_organization(organization)
|
||||
}
|
||||
end
|
||||
|
||||
def export_user_json_string(user)
|
||||
export_user_json_hash(user).to_json
|
||||
end
|
||||
|
||||
def export_user_json_hash(user)
|
||||
{
|
||||
version: CURRENT_VERSION,
|
||||
redis: export_user(user)
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def export_organization(organization)
|
||||
{
|
||||
users_metadata: export_dataservices("org:#{organization.name}"),
|
||||
tables_metadata: {}
|
||||
}
|
||||
end
|
||||
|
||||
def export_user(user)
|
||||
{
|
||||
users_metadata: export_dataservices("user:#{user.username}"),
|
||||
tables_metadata: export_named_maps(user)
|
||||
}
|
||||
end
|
||||
|
||||
def export_dataservices(prefix)
|
||||
$users_metadata_secondary.keys("#{prefix}:*").map { |key|
|
||||
export_key($users_metadata_secondary, key)
|
||||
}.reduce({}, &:merge)
|
||||
end
|
||||
|
||||
def export_named_maps(user)
|
||||
named_maps_key = "map_tpl|#{user.username}"
|
||||
named_maps_keys = $tables_metadata_secondary.hkeys(named_maps_key).reject do |named_map|
|
||||
matches_user_visualization?(named_map, user)
|
||||
end
|
||||
named_maps_hash = named_maps_keys.reduce({}) do |m, named_map|
|
||||
m.merge(named_map => Base64.encode64($tables_metadata_secondary.hget(named_maps_key, named_map)))
|
||||
end
|
||||
return {} unless named_maps_hash.any?
|
||||
{ named_maps_key => named_maps_hash }
|
||||
end
|
||||
|
||||
def matches_user_visualization?(named_map, user)
|
||||
re = /tpl_(?<viz_id>.+)/
|
||||
match = re.match(named_map)
|
||||
match && visualization_exists?(id: match[:viz_id].tr('_', '-'), user_id: user.id)
|
||||
end
|
||||
|
||||
def visualization_exists?(criteria)
|
||||
Carto::Visualization.where(criteria).exists?
|
||||
rescue
|
||||
false
|
||||
end
|
||||
|
||||
def export_key(redis_db, key)
|
||||
{
|
||||
key => {
|
||||
ttl: [0, redis_db.pttl(key)].max, # PTTL returns -1 if not set, clamp to 0
|
||||
value: Base64.encode64(redis_db.dump(key))
|
||||
}
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
# Both String and Hash versions are provided because `deep_symbolize_keys` won't symbolize through arrays
|
||||
# and having separated methods make handling and testing much easier.
|
||||
class RedisExportService
|
||||
include RedisExportServiceImporter
|
||||
include RedisExportServiceExporter
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,25 @@
|
||||
module Carto
|
||||
# A dataset may be comprised of additional tables other than the main table
|
||||
# For example, if overviews have been created for the dataset.
|
||||
# This is a utility to apply changes (e.g. privilege modifications) to all
|
||||
# the tables than comprise the dataset.
|
||||
module TableAndFriends
|
||||
def self.apply(db_connection, schema, table_name, &block)
|
||||
qualified_name = qualified_table_name(schema, table_name)
|
||||
block[schema, table_name, qualified_name]
|
||||
overviews_service = Carto::OverviewsService.new(db_connection)
|
||||
overviews_service.overview_tables(qualified_name).each do |overview_table|
|
||||
table, _ = Table.table_and_schema(overview_table)
|
||||
block[schema, table, qualified_table_name(schema, table)]
|
||||
end
|
||||
# TODO: should we apply also to raster overview tables?
|
||||
# To do so we could use SupportTables class and modify it to make #tables public.
|
||||
# Note that SupportTables handles only raster overviews and provides methods
|
||||
# to rename them, delete them and change their schema.
|
||||
end
|
||||
|
||||
def self.qualified_table_name(schema, table_name)
|
||||
%{"#{schema.gsub('"', '""')}"."#{table_name.gsub('"', '""')}"}
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,618 @@
|
||||
require 'json'
|
||||
require 'carto/export/layer_exporter'
|
||||
require 'carto/export/data_import_exporter'
|
||||
require_dependency 'carto/export/connector_configuration_exporter'
|
||||
|
||||
# Not migrated
|
||||
# snapshots -> difficult to do between clouds, not in use yet
|
||||
# tags -> regenerated from tables
|
||||
# visualization_export -> only purpose would be logging
|
||||
|
||||
# Version History
|
||||
# 1.0.0: export user metadata
|
||||
# 1.0.1: export search tweets
|
||||
# 1.0.2: export user notifications
|
||||
# 1.0.3: export rate limits
|
||||
# 1.0.4: company and phone in users table
|
||||
# 1.0.5: synchronization_oauths and connector configurations
|
||||
# 1.0.6: client_applications & friends and sql_copy rate_limits
|
||||
# 1.0.7: export password_reset_token and password_reset_sent_at user fields
|
||||
# 1.0.8: user_multifactor_auths
|
||||
# 1.0.9: oauth_apps, oauth_app_users and friends
|
||||
# 1.0.10: public_map_quota
|
||||
# 1.0.11: regular_api_key_quota
|
||||
# 1.0.12: maintenance_mode
|
||||
# 1.0.13: company_employees, use_case
|
||||
|
||||
module Carto
|
||||
module UserMetadataExportServiceConfiguration
|
||||
CURRENT_VERSION = '1.0.13'.freeze
|
||||
EXPORTED_USER_ATTRIBUTES = %i(
|
||||
email crypted_password database_name username admin enabled invite_token invite_token_date
|
||||
map_enabled quota_in_bytes table_quota public_map_quota regular_api_key_quota account_type private_tables_enabled
|
||||
period_end_date map_view_quota max_layers database_timeout user_timeout upgraded_at map_view_block_price
|
||||
geocoding_quota dashboard_viewed_at sync_tables_enabled database_host geocoding_block_price api_key
|
||||
notification organization_id created_at updated_at disqus_shortname id twitter_username website
|
||||
description name avatar_url database_schema soft_geocoding_limit auth_token
|
||||
twitter_datasource_enabled twitter_datasource_block_price twitter_datasource_block_size
|
||||
twitter_datasource_quota soft_twitter_datasource_limit available_for_hire private_maps_enabled
|
||||
google_sign_in last_password_change_date max_import_file_size max_import_table_row_count
|
||||
max_concurrent_import_count last_common_data_update_date google_maps_key google_maps_private_key
|
||||
enable_account_token location here_isolines_quota here_isolines_block_price soft_here_isolines_limit
|
||||
obs_snapshot_quota obs_snapshot_block_price soft_obs_snapshot_limit mobile_xamarin
|
||||
mobile_custom_watermark mobile_offline_maps mobile_gis_extension mobile_max_open_users
|
||||
mobile_max_private_users obs_general_quota obs_general_block_price soft_obs_general_limit viewer
|
||||
salesforce_datasource_enabled builder_enabled geocoder_provider isolines_provider routing_provider
|
||||
github_user_id engine_enabled mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit
|
||||
no_map_logo org_admin last_name user_render_timeout database_render_timeout frontend_version
|
||||
asset_host state company phone industry job_role password_reset_token password_reset_sent_at maintenance_mode
|
||||
company_employees use_case
|
||||
).freeze
|
||||
|
||||
BLANK_UUID = '00000000-0000-0000-0000-000000000000'.freeze
|
||||
|
||||
def compatible_version?(version)
|
||||
version.to_i == CURRENT_VERSION.split('.')[0].to_i
|
||||
end
|
||||
end
|
||||
|
||||
module UserMetadataExportServiceImporter
|
||||
include UserMetadataExportServiceConfiguration
|
||||
include LayerImporter
|
||||
include DataImportImporter
|
||||
include ConnectorConfigurationImporter
|
||||
|
||||
def build_user_from_json_export(exported_json_string)
|
||||
build_user_from_hash_export(parse_json(exported_json_string))
|
||||
end
|
||||
|
||||
def build_user_from_hash_export(exported_hash)
|
||||
raise 'Wrong export version' unless compatible_version?(exported_hash[:version])
|
||||
|
||||
build_user_from_hash(exported_hash[:user])
|
||||
end
|
||||
|
||||
def build_search_tweets_from_json_export(exported_json_string)
|
||||
build_search_tweets_from_hash_export(parse_json(exported_json_string))
|
||||
end
|
||||
|
||||
def build_search_tweets_from_hash_export(exported_hash)
|
||||
exported_hash[:user].fetch(:search_tweets, []).map { |st| build_search_tweet_from_hash(st) }
|
||||
end
|
||||
|
||||
def save_imported_user(user)
|
||||
user.save!
|
||||
::User[user.id].after_save
|
||||
|
||||
client_application = user.client_applications.first
|
||||
if client_application
|
||||
client_application.access_tokens.each do |t|
|
||||
# AR does not know about this, so it needs to be fixed
|
||||
t.update_column(:type, 'AccessToken')
|
||||
AccessToken[t.id].after_save
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def save_imported_search_tweet(search_tweet, user)
|
||||
if search_tweet.data_import
|
||||
persisted_import = Carto::DataImport.where(id: search_tweet.data_import.id).first
|
||||
search_tweet.data_import = persisted_import if persisted_import
|
||||
search_tweet.table_id = search_tweet.data_import.table_id
|
||||
else
|
||||
# Some search tweets can be exported without data import if the FK point to a non-existent data import.
|
||||
# However, this field is NOT NULL, so we cannot leave it empty.
|
||||
# We could skip the import of the tweet, but instead, we keep it with an invalid ID (like in the source), so
|
||||
# we can still correctly compute quota usage.
|
||||
search_tweet.data_import_id = BLANK_UUID
|
||||
end
|
||||
search_tweet.user = user
|
||||
search_tweet.save!
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse_json(exported_json_string)
|
||||
JSON.parse(exported_json_string, symbolize_names: true)
|
||||
end
|
||||
|
||||
def build_user_from_hash(exported_user)
|
||||
user = User.new(exported_user.slice(*EXPORTED_USER_ATTRIBUTES - [:id]))
|
||||
|
||||
user.feature_flags_user = exported_user[:feature_flags].map { |ff_name| build_feature_flag_from_name(ff_name) }
|
||||
.compact
|
||||
|
||||
user.assets = exported_user[:assets].map { |asset| build_asset_from_hash(asset.symbolize_keys) }
|
||||
|
||||
user.layers = build_layers_from_hash(exported_user[:layers])
|
||||
|
||||
user.rate_limit = build_rate_limit_from_hash(exported_user[:rate_limit])
|
||||
|
||||
api_keys = exported_user[:api_keys] || []
|
||||
user.api_keys += api_keys.map { |api_key| Carto::ApiKey.new_from_hash(api_key) }
|
||||
|
||||
user_multifactor_auths = exported_user[:user_multifactor_auths] || []
|
||||
user.user_multifactor_auths += user_multifactor_auths.map { |uma| Carto::UserMultifactorAuth.new_from_hash(uma) }
|
||||
|
||||
if exported_user[:notifications]
|
||||
user.static_notifications = Carto::UserNotification.create(notifications: exported_user[:notifications])
|
||||
end
|
||||
|
||||
user.synchronization_oauths = build_synchronization_oauths_from_hash(exported_user[:synchronization_oauths])
|
||||
|
||||
user.connector_configurations = build_connector_configurations_from_hash(exported_user[:connector_configurations])
|
||||
|
||||
user.client_applications = build_client_applications_from_hash(exported_user[:client_application])
|
||||
|
||||
user.oauth_app_users = build_oauth_app_users_from_hash(exported_user[:oauth_app_users])
|
||||
|
||||
# Must be the last one to avoid attribute assignments to try to run SQL
|
||||
user.id = exported_user[:id]
|
||||
user
|
||||
end
|
||||
|
||||
def build_feature_flag_from_name(ff_name)
|
||||
ff = FeatureFlag.where(name: ff_name).first
|
||||
if ff
|
||||
Carto::FeatureFlagsUser.new(feature_flag_id: ff.id)
|
||||
else
|
||||
CartoDB::Logger.warning(message: 'Feature flag not found in user import', feature_flag: ff_name)
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def build_asset_from_hash(exported_asset)
|
||||
Asset.new(
|
||||
public_url: exported_asset[:public_url],
|
||||
kind: exported_asset[:kind],
|
||||
storage_info: exported_asset[:storage_info]
|
||||
)
|
||||
end
|
||||
|
||||
def build_search_tweet_from_hash(exported_search_tweet)
|
||||
SearchTweet.new(
|
||||
data_import: build_data_import_from_hash(exported_search_tweet[:data_import]),
|
||||
service_item_id: exported_search_tweet[:service_item_id],
|
||||
retrieved_items: exported_search_tweet[:retrieved_items],
|
||||
state: exported_search_tweet[:state],
|
||||
created_at: exported_search_tweet[:created_at],
|
||||
updated_at: exported_search_tweet[:updated_at]
|
||||
)
|
||||
end
|
||||
|
||||
def build_rate_limit_from_hash(exported_hash)
|
||||
return unless exported_hash
|
||||
|
||||
rate_limit = Carto::RateLimit.from_api_attributes(exported_hash[:limits])
|
||||
rate_limit.id = exported_hash[:id]
|
||||
|
||||
rate_limit
|
||||
end
|
||||
|
||||
def build_synchronization_oauths_from_hash(exported_array)
|
||||
return [] unless exported_array.present?
|
||||
|
||||
exported_array.map { |so| build_synchronization_oauth_from_hash(so) }
|
||||
end
|
||||
|
||||
def build_synchronization_oauth_from_hash(exported_hash)
|
||||
SynchronizationOauth.new(
|
||||
service: exported_hash[:service],
|
||||
token: exported_hash[:token],
|
||||
created_at: exported_hash[:created_at],
|
||||
updated_at: exported_hash[:updated_at]
|
||||
)
|
||||
end
|
||||
|
||||
def build_oauth_token_fom_hash(exported_oauth_token)
|
||||
Carto::OauthToken.new(
|
||||
token: exported_oauth_token[:token],
|
||||
secret: exported_oauth_token[:secret],
|
||||
callback_url: exported_oauth_token[:callback_url],
|
||||
verifier: exported_oauth_token[:verifier],
|
||||
scope: exported_oauth_token[:scope],
|
||||
authorized_at: exported_oauth_token[:authorized_at],
|
||||
invalidated_at: exported_oauth_token[:invalidated_at],
|
||||
valid_to: exported_oauth_token[:valid_to],
|
||||
created_at: exported_oauth_token[:created_at],
|
||||
updated_at: exported_oauth_token[:updated_at]
|
||||
)
|
||||
end
|
||||
|
||||
def build_client_applications_from_hash(client_app_hash)
|
||||
return [] unless client_app_hash
|
||||
|
||||
client_application = Carto::ClientApplication.new(
|
||||
name: client_app_hash[:name],
|
||||
url: client_app_hash[:url],
|
||||
support_url: client_app_hash[:support_url],
|
||||
callback_url: client_app_hash[:callback_url],
|
||||
key: client_app_hash[:key],
|
||||
secret: client_app_hash[:secret],
|
||||
created_at: client_app_hash[:created_at],
|
||||
updated_at: client_app_hash[:updated_at],
|
||||
oauth_tokens: client_app_hash[:oauth_tokens].map { |t| build_oauth_token_fom_hash(t) },
|
||||
access_tokens: client_app_hash[:access_tokens].map { |t| build_oauth_token_fom_hash(t) }
|
||||
)
|
||||
|
||||
[client_application]
|
||||
end
|
||||
|
||||
def build_oauth_app_users_from_hash(oauth_app_users)
|
||||
return [] unless oauth_app_users
|
||||
oauth_app_users.map { |oau| build_oauth_app_user_from_hash(oau) }
|
||||
end
|
||||
|
||||
def build_oauth_app_user_from_hash(oau_hash)
|
||||
oau = Carto::OauthAppUser.new(
|
||||
id: oau_hash[:id],
|
||||
oauth_app_id: oau_hash[:oauth_app_id],
|
||||
scopes: oau_hash[:scopes],
|
||||
created_at: oau_hash[:created_at],
|
||||
updated_at: oau_hash[:updated_at],
|
||||
skip_role_setup: true
|
||||
)
|
||||
|
||||
if oau_hash[:oauth_authorization_codes]
|
||||
oau.oauth_authorization_codes = oau_hash[:oauth_authorization_codes].map do |oac_hash|
|
||||
build_oauth_authorization_code_from_hash(oac_hash)
|
||||
end
|
||||
end
|
||||
|
||||
if oau_hash[:oauth_access_tokens]
|
||||
oau.oauth_access_tokens = oau_hash[:oauth_access_tokens].map do |oat_hash|
|
||||
build_oauth_access_token_from_hash(oat_hash)
|
||||
end
|
||||
end
|
||||
|
||||
if oau_hash[:oauth_refresh_tokens]
|
||||
oau.oauth_refresh_tokens = oau_hash[:oauth_refresh_tokens].map do |ort_hash|
|
||||
build_oauth_refresh_token_from_hash(ort_hash)
|
||||
end
|
||||
end
|
||||
|
||||
oau
|
||||
end
|
||||
|
||||
def build_oauth_authorization_code_from_hash(oac_hash)
|
||||
Carto::OauthAuthorizationCode.new(
|
||||
scopes: oac_hash[:scopes],
|
||||
code: oac_hash[:code],
|
||||
redirect_uri: oac_hash[:redirect_uri],
|
||||
created_at: oac_hash[:created_at]
|
||||
)
|
||||
end
|
||||
|
||||
def build_oauth_access_token_from_hash(oat_hash)
|
||||
Carto::OauthAccessToken.new(
|
||||
api_key_id: oat_hash[:api_key_id],
|
||||
scopes: oat_hash[:scopes],
|
||||
created_at: oat_hash[:created_at],
|
||||
skip_api_key_creation: true
|
||||
)
|
||||
end
|
||||
|
||||
def build_oauth_refresh_token_from_hash(ort_hash)
|
||||
Carto::OauthRefreshToken.new(
|
||||
token: ort_hash[:token],
|
||||
scopes: ort_hash[:scopes],
|
||||
created_at: ort_hash[:created_at],
|
||||
updated_at: ort_hash[:updated_at],
|
||||
skip_token_regeneration: true
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
module UserMetadataExportServiceExporter
|
||||
include UserMetadataExportServiceConfiguration
|
||||
include LayerExporter
|
||||
include DataImportExporter
|
||||
include ConnectorConfigurationExporter
|
||||
|
||||
def export_user_json_string(user)
|
||||
export_user_json_hash(user).to_json
|
||||
end
|
||||
|
||||
def export_user_json_hash(user)
|
||||
{
|
||||
version: CURRENT_VERSION,
|
||||
user: export(user)
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def export(user)
|
||||
user_hash = EXPORTED_USER_ATTRIBUTES.map { |att| [att, user.attributes[att.to_s]] }.to_h
|
||||
|
||||
user_hash[:feature_flags] = user.feature_flags_user.map(&:feature_flag).map(&:name)
|
||||
|
||||
user_hash[:assets] = user.assets.map { |a| export_asset(a) }
|
||||
|
||||
user_hash[:layers] = user.layers.map { |l| export_layer(l) }
|
||||
|
||||
user_hash[:search_tweets] = user.search_tweets.map { |st| export_search_tweet(st) }
|
||||
|
||||
user_hash[:api_keys] = user.api_keys.map { |api_key| export_api_key(api_key) }
|
||||
|
||||
user_hash[:user_multifactor_auths] = user.user_multifactor_auths.map { |uma| export_user_multifactor_auth(uma) }
|
||||
|
||||
user_hash[:rate_limit] = export_rate_limit(user.rate_limit)
|
||||
|
||||
user_hash[:notifications] = user.static_notifications.notifications
|
||||
|
||||
user_hash[:synchronization_oauths] = user.synchronization_oauths.map { |so| export_synchronization_oauth(so) }
|
||||
|
||||
user_hash[:connector_configurations] = user.connector_configurations.map do |cc|
|
||||
export_connector_configuration(cc)
|
||||
end
|
||||
|
||||
# Use Sequel models to export. Single table inheritance causes AR to try and create Sequel models -> fail.
|
||||
user_hash[:client_application] = export_client_application(::User[user.id].client_application)
|
||||
|
||||
user_hash[:oauth_app_users] = user.oauth_app_users.map { |oau| export_oauth_app_user(oau) }
|
||||
|
||||
user_hash
|
||||
end
|
||||
|
||||
def export_client_application(app)
|
||||
return nil unless app
|
||||
a_t_tokens = app.access_tokens.map(&:token)
|
||||
{
|
||||
name: app.name,
|
||||
url: app.url,
|
||||
support_url: app.support_url,
|
||||
callback_url: app.callback_url,
|
||||
key: app.key,
|
||||
secret: app.secret,
|
||||
created_at: app.created_at,
|
||||
updated_at: app.updated_at,
|
||||
oauth_tokens: app.oauth_tokens.reject { |t| a_t_tokens.include?(t.token) }.map { |ot| export_oauth_token(ot) },
|
||||
access_tokens: app.access_tokens.map { |ot| export_oauth_token(ot) }
|
||||
}
|
||||
end
|
||||
|
||||
def export_oauth_token(oauth_token)
|
||||
{
|
||||
token: oauth_token.token,
|
||||
secret: oauth_token.secret,
|
||||
callback_url: oauth_token.callback_url,
|
||||
verifier: oauth_token.verifier,
|
||||
scope: oauth_token.scope,
|
||||
authorized_at: oauth_token.authorized_at,
|
||||
invalidated_at: oauth_token.invalidated_at,
|
||||
valid_to: oauth_token.valid_to,
|
||||
created_at: oauth_token.created_at,
|
||||
updated_at: oauth_token.updated_at
|
||||
}
|
||||
end
|
||||
|
||||
def export_asset(asset)
|
||||
{
|
||||
public_url: asset.public_url,
|
||||
kind: asset.kind,
|
||||
storage_info: asset.storage_info
|
||||
}
|
||||
end
|
||||
|
||||
def export_search_tweet(search_tweet)
|
||||
{
|
||||
data_import: export_data_import(search_tweet.data_import),
|
||||
service_item_id: search_tweet.service_item_id,
|
||||
retrieved_items: search_tweet.retrieved_items,
|
||||
state: search_tweet.state,
|
||||
created_at: search_tweet.created_at,
|
||||
updated_at: search_tweet.updated_at
|
||||
}
|
||||
end
|
||||
|
||||
def export_api_key(api_key)
|
||||
{
|
||||
id: api_key.id,
|
||||
created_at: api_key.created_at,
|
||||
db_password: api_key.db_password,
|
||||
db_role: api_key.db_role,
|
||||
name: api_key.name,
|
||||
token: api_key.token,
|
||||
type: api_key.type,
|
||||
updated_at: api_key.updated_at,
|
||||
grants: api_key.grants,
|
||||
user_id: api_key.user_id
|
||||
}
|
||||
end
|
||||
|
||||
def export_user_multifactor_auth(user_multifactor_auth)
|
||||
exported_keys = %i(user_id type shared_secret enabled last_login created_at updated_at)
|
||||
user_multifactor_auth.to_h.slice(*exported_keys)
|
||||
end
|
||||
|
||||
def export_rate_limit(rate_limit)
|
||||
return unless rate_limit
|
||||
|
||||
{
|
||||
id: rate_limit.id,
|
||||
limits: rate_limit.api_attributes
|
||||
}
|
||||
end
|
||||
|
||||
def export_synchronization_oauth(sync_oauth)
|
||||
{
|
||||
service: sync_oauth.service,
|
||||
token: sync_oauth.token,
|
||||
created_at: sync_oauth.created_at,
|
||||
updated_at: sync_oauth.updated_at
|
||||
}
|
||||
end
|
||||
|
||||
def export_oauth_app_user(oau)
|
||||
oauth_authorization_codes = oau.oauth_authorization_codes.map { |oac| export_oauth_authorization_code(oac) }
|
||||
oauth_access_tokens = oau.oauth_access_tokens.map { |oat| export_oauth_access_token(oat) }
|
||||
oauth_refresh_tokens = oau.oauth_refresh_tokens.map { |ort| export_oauth_refresh_token(ort) }
|
||||
|
||||
{
|
||||
id: oau.id,
|
||||
oauth_app_id: oau.oauth_app_id,
|
||||
scopes: oau.scopes,
|
||||
created_at: oau.created_at,
|
||||
updated_at: oau.updated_at,
|
||||
oauth_authorization_codes: oauth_authorization_codes,
|
||||
oauth_access_tokens: oauth_access_tokens,
|
||||
oauth_refresh_tokens: oauth_refresh_tokens
|
||||
}
|
||||
end
|
||||
|
||||
def export_oauth_authorization_code(oac)
|
||||
{
|
||||
scopes: oac.scopes,
|
||||
code: oac.code,
|
||||
redirect_uri: oac.redirect_uri,
|
||||
created_at: oac.created_at
|
||||
}
|
||||
end
|
||||
|
||||
def export_oauth_access_token(oat)
|
||||
{
|
||||
api_key_id: oat.api_key_id,
|
||||
scopes: oat.scopes,
|
||||
created_at: oat.created_at
|
||||
}
|
||||
end
|
||||
|
||||
def export_oauth_refresh_token(ort)
|
||||
{
|
||||
token: ort.token,
|
||||
scopes: ort.scopes,
|
||||
created_at: ort.created_at,
|
||||
updated_at: ort.updated_at
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
class UserAlreadyExists < RuntimeError; end
|
||||
|
||||
# Both String and Hash versions are provided because `deep_symbolize_keys` won't symbolize through arrays
|
||||
# and having separated methods make handling and testing much easier.
|
||||
class UserMetadataExportService
|
||||
include UserMetadataExportServiceImporter
|
||||
include UserMetadataExportServiceExporter
|
||||
|
||||
def export_to_directory(user, path)
|
||||
root_dir = Pathname.new(path)
|
||||
root_dir.mkpath
|
||||
|
||||
user_json = export_user_json_string(user)
|
||||
root_dir.join("user_#{user.id}.json").open('w') { |file| file.write(user_json) }
|
||||
|
||||
redis_json = Carto::RedisExportService.new.export_user_json_string(user)
|
||||
root_dir.join("redis_user_#{user.id}.json").open('w') { |file| file.write(redis_json) }
|
||||
|
||||
export_user_visualizations_to_directory(user, Carto::Visualization::TYPE_REMOTE, path)
|
||||
export_user_visualizations_to_directory(user, Carto::Visualization::TYPE_CANONICAL, path)
|
||||
export_user_visualizations_to_directory(user, Carto::Visualization::TYPE_DERIVED, path)
|
||||
end
|
||||
|
||||
def import_from_directory(path)
|
||||
user = user_from_file(path)
|
||||
|
||||
raise UserAlreadyExists.new if ::Carto::User.exists?(id: user.id)
|
||||
save_imported_user(user)
|
||||
|
||||
Carto::RedisExportService.new.restore_redis_from_json_export(redis_user_file(path))
|
||||
|
||||
user
|
||||
end
|
||||
|
||||
def rollback_import_from_directory(path)
|
||||
user = user_from_file(path)
|
||||
return unless user
|
||||
|
||||
user = ::User[user.id]
|
||||
return unless user
|
||||
|
||||
carto_user = Carto::User.find(user.id)
|
||||
carto_user.assets.each(&:delete)
|
||||
carto_user.destroy
|
||||
user.before_destroy(skip_table_drop: true)
|
||||
|
||||
Carto::RedisExportService.new.remove_redis_from_json_export(redis_user_file(path))
|
||||
rescue ActiveRecord::RecordNotFound
|
||||
# User was not created so not found and no redis removal needed
|
||||
end
|
||||
|
||||
def import_user_visualizations_from_directory(user, type, meta_path)
|
||||
with_non_viewer_user(user) do
|
||||
Dir["#{meta_path}/#{type}_*#{Carto::VisualizationExporter::EXPORT_EXTENSION}"].each do |fname|
|
||||
imported_vis = Carto::VisualizationsExportService2.new.build_visualization_from_json_export(File.read(fname))
|
||||
Carto::VisualizationsExportPersistenceService.new.save_import(user, imported_vis, full_restore: true)
|
||||
if Carto::VisualizationsExportService2.new.marked_as_vizjson2_from_json_export?(File.read(fname))
|
||||
imported_vis.mark_as_vizjson2
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def import_metadata_from_directory(user, meta_path)
|
||||
import_user_visualizations_from_directory(user, Carto::Visualization::TYPE_REMOTE, meta_path)
|
||||
import_user_visualizations_from_directory(user, Carto::Visualization::TYPE_CANONICAL, meta_path)
|
||||
import_user_visualizations_from_directory(user, Carto::Visualization::TYPE_DERIVED, meta_path)
|
||||
|
||||
import_search_tweets_from_directory(user, meta_path)
|
||||
end
|
||||
|
||||
def import_search_tweets_from_directory(user, meta_path)
|
||||
user_file = user_file_dir(meta_path)
|
||||
search_tweets = build_search_tweets_from_json_export(File.read(user_file))
|
||||
search_tweets.each { |st| save_imported_search_tweet(st, user) }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def user_from_file(path)
|
||||
build_user_from_json_export(File.read(user_file_dir(path)))
|
||||
end
|
||||
|
||||
def user_file_dir(path)
|
||||
Dir["#{path}/user_*.json"].first
|
||||
end
|
||||
|
||||
def redis_user_file(path)
|
||||
File.read(Dir["#{path}/redis_user_*.json"].first)
|
||||
end
|
||||
|
||||
def export_user_visualizations_to_directory(user, type, path)
|
||||
root_dir = Pathname.new(path)
|
||||
user.visualizations.where(type: type).each do |visualization|
|
||||
next if visualization.canonical? && should_skip_canonical_viz_export(visualization)
|
||||
next if !visualization.remote? && visualization.map.nil?
|
||||
|
||||
visualization_export = Carto::VisualizationsExportService2.new.export_visualization_json_string(
|
||||
visualization.id, user, with_password: true
|
||||
)
|
||||
filename = "#{visualization.type}_#{visualization.id}#{Carto::VisualizationExporter::EXPORT_EXTENSION}"
|
||||
root_dir.join(filename).open('w') { |file| file.write(visualization_export) }
|
||||
end
|
||||
end
|
||||
|
||||
def should_skip_canonical_viz_export(viz)
|
||||
return true if viz.table.nil?
|
||||
|
||||
viz.user.visualizations.where(type: viz.type,
|
||||
name: viz.name).all.sort_by(&:updated_at).last.id != viz.id
|
||||
end
|
||||
|
||||
def with_non_viewer_user(user)
|
||||
was_viewer = user.viewer
|
||||
if user.viewer
|
||||
user.update_attributes(viewer: false)
|
||||
::User[user.id].reload
|
||||
end
|
||||
|
||||
yield
|
||||
ensure
|
||||
if was_viewer
|
||||
user.update_attributes(viewer: true)
|
||||
::User[user.id].reload
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,31 @@
|
||||
module Carto
|
||||
class UserMultifactorAuthUpdateService
|
||||
|
||||
def initialize(user_id:)
|
||||
@user_id = user_id
|
||||
end
|
||||
|
||||
def update(enabled:, type: Carto::UserMultifactorAuth::TYPE_TOTP)
|
||||
enabled ? create_user_multifactor_auth(type) : destroy_user_multifactor_auth(type)
|
||||
end
|
||||
|
||||
def exists?(type: Carto::UserMultifactorAuth::TYPE_TOTP)
|
||||
user_multifactor_auth(type).present?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def create_user_multifactor_auth(type)
|
||||
Carto::UserMultifactorAuth.create!(user_id: @user_id, type: type) unless exists?
|
||||
end
|
||||
|
||||
def destroy_user_multifactor_auth(type)
|
||||
user_multifactor_auth(type).each(&:destroy!)
|
||||
end
|
||||
|
||||
def user_multifactor_auth(type)
|
||||
Carto::UserMultifactorAuth.where(user_id: @user_id, type: type)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,132 @@
|
||||
module Carto
|
||||
class UserTableIndexService
|
||||
AUTO_INDEX_TTL_MS = 600000
|
||||
AUTO_INDEX_PREFIX = '_auto_idx_'.freeze
|
||||
INDEXABLE_WIDGET_TYPES = %w(histogram category time-series).freeze
|
||||
|
||||
# Not worth it to create indices on small tables, where a full scan is cheap.
|
||||
# Maybe better to use number of pages instead of rows instead.
|
||||
MINIMUM_ROW_COUNT_TO_INDEX = 10000
|
||||
# Number of categories (most common) shown by default in widgets, used to determine the number
|
||||
# of categories likely to be used for filtering.
|
||||
CATEGORIES_SHOWN_IN_WIDGET = 5
|
||||
# Number of different values in a column to consider indexing it, to avoid indexing boolean columns.
|
||||
# Note: Also used to index columns with less values but an unbalanced probability distribution.
|
||||
MINIMUM_COLUMN_VALUES_TO_INDEX = 5
|
||||
# Regardless of anything, columns with high clustering factor (sorted in physical order) are also indexed.
|
||||
MINIMUM_CORRELATION_TO_INDEX = 0.9
|
||||
|
||||
def initialize(user_table)
|
||||
@user_table = user_table
|
||||
@table = user_table.service
|
||||
end
|
||||
|
||||
def update_auto_indices
|
||||
bolt = Carto::Bolt.new("user_table:#{@user_table.id}:auto_index", ttl_ms: AUTO_INDEX_TTL_MS)
|
||||
|
||||
bolt.run_locked { generate_indices }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate_indices
|
||||
auto_indices(valid: false).each do |idx|
|
||||
CartoDB::Logger.debug(message: 'Auto index', action: 'drop invalid', table: @user_table, column: idx[:column])
|
||||
@table.drop_index(idx[:column], AUTO_INDEX_PREFIX, concurrent: true)
|
||||
end
|
||||
|
||||
widget_columns = (@table.estimated_row_count > MINIMUM_ROW_COUNT_TO_INDEX) ? columns_with_widgets : []
|
||||
columns_to_index = widget_columns.select { |c| indexable_column?(c) }
|
||||
|
||||
indexed_columns = valid_indices.map { |i| i[:column] }
|
||||
create_index_on = columns_to_index - indexed_columns
|
||||
create_index_on.each do |col|
|
||||
CartoDB::Logger.debug(message: 'Auto index', action: 'create', table: @user_table, column: col)
|
||||
@table.create_index(col, AUTO_INDEX_PREFIX, concurrent: true)
|
||||
end
|
||||
|
||||
auto_indexed_columns = auto_indices.map { |i| i[:column] }
|
||||
drop_index_on = auto_indexed_columns - columns_to_index
|
||||
drop_index_on.each do |col|
|
||||
@table.drop_index(col, AUTO_INDEX_PREFIX, concurrent: true)
|
||||
end
|
||||
rescue => e
|
||||
CartoDB::Logger.error(exception: e, message: 'Error auto-indexing table', table: @user_table)
|
||||
end
|
||||
|
||||
def indexable_column?(column)
|
||||
stats = pg_stats_by_column[column]
|
||||
return false unless stats
|
||||
|
||||
# Accept columns with several different values
|
||||
# Checks common values and accepts cases with several value
|
||||
# or with few but unbalanced values (e.g: boolean with 80/20 distribution)
|
||||
common_freqs = stats[:most_common_freqs] || stats[:most_common_elem_freqs]
|
||||
if common_freqs.present?
|
||||
check_common_freq = common_freqs[CATEGORIES_SHOWN_IN_WIDGET - 1] || common_freqs.last
|
||||
if check_common_freq < (1.0 / MINIMUM_COLUMN_VALUES_TO_INDEX)
|
||||
return true
|
||||
end
|
||||
else
|
||||
# No histogram, rely on distinct values. Note: Values < 0 represent a proportion over the number of rows.
|
||||
# They are generated when the analyzer gives up counting or identifies the value as a continuous magnitude
|
||||
distinct = stats[:n_distinct]
|
||||
return true if distinct < 0 || distinct > MINIMUM_COLUMN_VALUES_TO_INDEX
|
||||
end
|
||||
|
||||
# Accept columns with high correlation (values related to physical row order)
|
||||
if stats[:correlation].abs > MINIMUM_CORRELATION_TO_INDEX
|
||||
return true
|
||||
end
|
||||
|
||||
# Default
|
||||
false
|
||||
end
|
||||
|
||||
def auto_indices(valid: true)
|
||||
indices.select { |i| i[:name].starts_with?(AUTO_INDEX_PREFIX) && i[:valid] == valid }
|
||||
end
|
||||
|
||||
def valid_indices
|
||||
indices.select { |i| i[:valid] }
|
||||
end
|
||||
|
||||
def indices
|
||||
@indices ||= @table.pg_indexes
|
||||
end
|
||||
|
||||
def columns_with_widgets
|
||||
columns = Set.new
|
||||
table_widgets.select { |w| INDEXABLE_WIDGET_TYPES.include?(w.type) }.each do |w|
|
||||
columns.add(w.column)
|
||||
end
|
||||
columns
|
||||
end
|
||||
|
||||
def table_widgets
|
||||
widgets.select do |w|
|
||||
node = w.analysis_node
|
||||
node && node.table_source?(@user_table.name)
|
||||
end
|
||||
end
|
||||
|
||||
def widgets
|
||||
@user_table.layers.map(&:widgets).flatten
|
||||
end
|
||||
|
||||
def pg_stats_by_column
|
||||
@stats ||= get_pg_stats_by_column
|
||||
end
|
||||
|
||||
def get_pg_stats_by_column
|
||||
@table.update_table_pg_stats
|
||||
stats = @table.pg_stats
|
||||
if stats && !stats.empty?
|
||||
stats.map { |s| { s[:attname] => s } }.reduce(:merge)
|
||||
else
|
||||
CartoDB::Logger.warning(message: 'Error retrieving stats for table', table: @user_table)
|
||||
{}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,41 @@
|
||||
require_dependency 'carto/visualizations_export_service_2'
|
||||
|
||||
module Carto
|
||||
module VisualizationBackupService
|
||||
include Carto::VisualizationsExportService2Exporter
|
||||
include Carto::VisualizationsExportService2Importer
|
||||
|
||||
def create_visualization_backup(visualization:, category:, with_mapcaps: true, with_password: true)
|
||||
export_json = export_visualization_json_hash(
|
||||
visualization.id,
|
||||
visualization.user,
|
||||
with_mapcaps: with_mapcaps,
|
||||
with_password: with_password
|
||||
)
|
||||
Carto::VisualizationBackup.create!(
|
||||
user_id: visualization.user.id,
|
||||
visualization_id: visualization.id,
|
||||
category: category,
|
||||
export: export_json
|
||||
)
|
||||
rescue StandardError => exception
|
||||
# The backup should not break the flow
|
||||
CartoDB::Logger.error(
|
||||
message: 'Error creating a visualization backup',
|
||||
exception: exception,
|
||||
visualization: visualization
|
||||
)
|
||||
end
|
||||
|
||||
def restore_visualization_backup(visualization_backup_id)
|
||||
backup = Carto::VisualizationBackup.find(visualization_backup_id)
|
||||
if Carto::Visualization.find_by_id(backup.visualization_id)
|
||||
raise 'A visualization with the same id as the backup one already exists'
|
||||
end
|
||||
|
||||
user = Carto::User.find(backup.user_id)
|
||||
visualization_json = build_visualization_from_hash_export(backup.export)
|
||||
Carto::VisualizationsExportPersistenceService.new.save_import(user, visualization_json, full_restore: true)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,172 @@
|
||||
require 'uuidtools'
|
||||
require_dependency 'carto/uuidhelper'
|
||||
require_dependency 'carto/query_rewriter'
|
||||
|
||||
module Carto
|
||||
# Export/import is versioned, but importing should generate a model tree that can be persisted by this class
|
||||
class VisualizationsExportPersistenceService
|
||||
include Carto::UUIDHelper
|
||||
include Carto::QueryRewriter
|
||||
|
||||
# `full_restore = true` means keeping the visualization id and permission. Its intended use is to restore an exact
|
||||
# copy of a visualization (e.g: user migration). When it's `false` it will skip restoring ids and permissions. This
|
||||
# is the default, and how it's used by the Import API to allow to import a visualization into a different user
|
||||
def save_import(user, visualization, renamed_tables: {}, full_restore: false)
|
||||
old_username = visualization.user.username if visualization.user
|
||||
apply_user_limits(user, visualization)
|
||||
ActiveRecord::Base.transaction do
|
||||
visualization.id = random_uuid unless visualization.id && full_restore
|
||||
visualization.user = user
|
||||
|
||||
ensure_unique_name(user, visualization)
|
||||
|
||||
visualization.layers.each { |layer| layer.fix_layer_user_information(old_username, user, renamed_tables) }
|
||||
visualization.analyses.each do |analysis|
|
||||
analysis.analysis_node.fix_analysis_node_queries(old_username, user, renamed_tables)
|
||||
end
|
||||
|
||||
saved_acl = visualization.permission.access_control_list if full_restore
|
||||
visualization.permission = Carto::Permission.new(owner: user, owner_username: user.username)
|
||||
|
||||
map = visualization.map
|
||||
map.user = user if map
|
||||
|
||||
visualization.analyses.each do |analysis|
|
||||
analysis.user_id = user.id
|
||||
end
|
||||
|
||||
sync = visualization.synchronization
|
||||
if sync
|
||||
sync.id = random_uuid unless sync.id && full_restore
|
||||
sync.user = user
|
||||
sync.log.user_id = user.id if sync.log
|
||||
end
|
||||
|
||||
user_table = visualization.map.user_table if map
|
||||
if user_table
|
||||
user_table.user = user
|
||||
user_table.service.register_table_only = true
|
||||
raise 'Cannot import a dataset without physical table' unless user_table.service.real_table_exists?
|
||||
|
||||
data_import = user_table.data_import
|
||||
if data_import
|
||||
existing_data_import = Carto::DataImport.where(id: data_import.id).first
|
||||
user_table.data_import = existing_data_import if existing_data_import
|
||||
data_import.synchronization_id = sync.id if sync
|
||||
data_import.user_id = user.id
|
||||
end
|
||||
end
|
||||
|
||||
unless full_restore
|
||||
visualization.mapcaps.clear
|
||||
visualization.created_at = DateTime.now
|
||||
visualization.updated_at = DateTime.now
|
||||
visualization.locked = false
|
||||
end
|
||||
|
||||
unless visualization.save
|
||||
error_message = "Errors saving imported visualization: #{visualization.errors.full_messages}"
|
||||
raise Carto::UnauthorizedError.new(error_message) if visualization.errors.include?(:privacy)
|
||||
raise error_message
|
||||
end
|
||||
|
||||
# Save permissions after visualization, in order to be able to regenerate shared_entities
|
||||
if saved_acl
|
||||
visualization.permission.access_control_list = saved_acl
|
||||
visualization.permission.save!
|
||||
end
|
||||
|
||||
visualization.layers.map do |layer|
|
||||
# Flag needed because `.changed?` won't realize about options hash changes
|
||||
changed = false
|
||||
if layer.options.has_key?(:id)
|
||||
layer.options[:id] = layer.id
|
||||
changed = true
|
||||
end
|
||||
if layer.options.has_key?(:stat_tag)
|
||||
layer.options[:stat_tag] = visualization.id
|
||||
changed = true
|
||||
end
|
||||
layer.save if changed
|
||||
end
|
||||
|
||||
if map
|
||||
map.data_layers.each(&:register_table_dependencies)
|
||||
|
||||
new_user_layers = map.base_layers.select(&:custom?).select { |l| !contains_equivalent_base_layer?(user.layers, l) }
|
||||
new_user_layers.map(&:dup).map { |l| user.layers << l }
|
||||
data_import = map.user_table.try(:data_import)
|
||||
if data_import
|
||||
data_import.table_id = map.user_table.id
|
||||
data_import.save!
|
||||
data_import.external_data_imports.each do |edi|
|
||||
edi.synchronization_id = sync.id if sync
|
||||
edi.save!
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
visualization.save!
|
||||
visualization
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def contains_equivalent_base_layer?(layers, layer)
|
||||
layers.any? { |l| equivalent_base_layer?(l, layer) }
|
||||
end
|
||||
|
||||
def equivalent_base_layer?(layer_a, layer_b)
|
||||
layer_a.kind == 'tiled' && layer_a.kind == layer_b.kind && layer_a.options == layer_b.options
|
||||
end
|
||||
|
||||
def ensure_unique_name(user, visualization)
|
||||
existing_names = Carto::Visualization.uniq
|
||||
.where(user_id: user.id)
|
||||
.where("name ~ ?", "#{Regexp.escape(visualization.name)}( Import)?( \d*)?$")
|
||||
.where(type: visualization.type)
|
||||
.pluck(:name)
|
||||
if existing_names.include?(visualization.name)
|
||||
raise 'Cannot rename a dataset during import' if visualization.canonical?
|
||||
import_name = "#{visualization.name} Import"
|
||||
i = 1
|
||||
while existing_names.include?(import_name)
|
||||
import_name = "#{visualization.name} Import #{i += 1}"
|
||||
end
|
||||
visualization.name = import_name
|
||||
end
|
||||
end
|
||||
|
||||
def apply_user_limits(user, visualization)
|
||||
can_be_private = visualization.derived? ? user.private_maps_enabled : user.private_tables_enabled
|
||||
unless can_be_private
|
||||
visualization.privacy = Carto::Visualization::PRIVACY_PUBLIC
|
||||
visualization.user_table.privacy = Carto::UserTable::PRIVACY_PUBLIC if visualization.canonical?
|
||||
end
|
||||
# If password is not exported we must fallback to private
|
||||
if visualization.password_protected? && !visualization.has_password?
|
||||
visualization.privacy = Carto::Visualization::PRIVACY_PRIVATE
|
||||
end
|
||||
|
||||
layers = []
|
||||
data_layer_count = 0
|
||||
|
||||
if visualization.map
|
||||
visualization.map.layers.each do |layer|
|
||||
if layer.data_layer?
|
||||
if data_layer_count < user.max_layers
|
||||
layers.push(layer)
|
||||
data_layer_count += 1
|
||||
end
|
||||
else
|
||||
layers.push(layer)
|
||||
end
|
||||
end
|
||||
visualization.map.layers.clear
|
||||
visualization.map.layers_maps.clear
|
||||
layers.each { |l| visualization.map.layers << l }
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,447 @@
|
||||
require 'json'
|
||||
require_dependency 'carto/export/layer_exporter'
|
||||
require_dependency 'carto/export/data_import_exporter'
|
||||
|
||||
# Version History
|
||||
# TODO: documentation at http://cartodb.readthedocs.org/en/latest/operations/exporting_importing_visualizations.html
|
||||
# 2: export full visualization. Limitations:
|
||||
# - No Odyssey support: export fails if any of parent_id / prev_id / next_id / slide_transition_options are set.
|
||||
# - Privacy is exported, but permissions are not.
|
||||
# 2.0.1: export Widget.source_id
|
||||
# 2.0.2: export username
|
||||
# 2.0.3: export state (Carto::State)
|
||||
# 2.0.4: export legends (Carto::Legend)
|
||||
# 2.0.5: export explicit widget order
|
||||
# 2.0.6: export version
|
||||
# 2.0.7: export map options
|
||||
# 2.0.8: export widget style
|
||||
# 2.0.9: export visualization id
|
||||
# 2.1.0: export datasets: permissions, user_tables and syncs
|
||||
# 2.1.1: export vizjson2 mark
|
||||
# 2.1.2: export locked and password
|
||||
# 2.1.3: export synchronization id
|
||||
module Carto
|
||||
module VisualizationsExportService2Configuration
|
||||
CURRENT_VERSION = '2.1.3'.freeze
|
||||
|
||||
def compatible_version?(version)
|
||||
version.to_i == CURRENT_VERSION.split('.')[0].to_i
|
||||
end
|
||||
end
|
||||
|
||||
module VisualizationsExportService2Validator
|
||||
def check_valid_visualization(visualization)
|
||||
raise 'Only derived or canonical visualizations can be exported' unless visualization.derived? ||
|
||||
visualization.canonical? ||
|
||||
visualization.remote?
|
||||
end
|
||||
end
|
||||
|
||||
module VisualizationsExportService2Importer
|
||||
include VisualizationsExportService2Configuration
|
||||
include LayerImporter
|
||||
include DataImportImporter
|
||||
|
||||
def build_visualization_from_json_export(exported_json_string)
|
||||
build_visualization_from_hash_export(parse_json(exported_json_string))
|
||||
end
|
||||
|
||||
def build_visualization_from_hash_export(exported_hash)
|
||||
raise 'Wrong export version' unless compatible_version?(exported_hash[:version])
|
||||
|
||||
build_visualization_from_hash(exported_hash[:visualization])
|
||||
end
|
||||
|
||||
def marked_as_vizjson2_from_json_export?(exported_json_string)
|
||||
marked_as_vizjson2_from_hash_export?(parse_json(exported_json_string))
|
||||
end
|
||||
|
||||
def marked_as_vizjson2_from_hash_export?(exported_hash)
|
||||
exported_hash[:visualization][:uses_vizjson2]
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse_json(exported_json_string)
|
||||
JSON.parse(exported_json_string, symbolize_names: true)
|
||||
end
|
||||
|
||||
def build_visualization_from_hash(exported_visualization)
|
||||
exported_layers = exported_visualization[:layers]
|
||||
exported_overlays = exported_visualization[:overlays]
|
||||
|
||||
visualization = Carto::Visualization.new(
|
||||
name: exported_visualization[:name],
|
||||
description: exported_visualization[:description],
|
||||
version: exported_visualization[:version] || 2,
|
||||
type: exported_visualization[:type],
|
||||
tags: exported_visualization[:tags],
|
||||
privacy: exported_visualization[:privacy],
|
||||
source: exported_visualization[:source],
|
||||
license: exported_visualization[:license],
|
||||
title: exported_visualization[:title],
|
||||
kind: exported_visualization[:kind],
|
||||
attributions: exported_visualization[:attributions],
|
||||
bbox: exported_visualization[:bbox],
|
||||
display_name: exported_visualization[:display_name],
|
||||
map: build_map_from_hash(
|
||||
exported_visualization[:map],
|
||||
layers: build_layers_from_hash(exported_layers)),
|
||||
overlays: build_overlays_from_hash(exported_overlays),
|
||||
analyses: exported_visualization[:analyses].map { |a| build_analysis_from_hash(a) },
|
||||
permission: build_permission_from_hash(exported_visualization[:permission]),
|
||||
mapcaps: [build_mapcap_from_hash(exported_visualization[:mapcap])].compact,
|
||||
external_source: build_external_source_from_hash(exported_visualization[:external_source]),
|
||||
created_at: exported_visualization[:created_at],
|
||||
updated_at: exported_visualization[:updated_at],
|
||||
locked: exported_visualization[:locked] || false,
|
||||
encrypted_password: exported_visualization[:encrypted_password],
|
||||
password_salt: exported_visualization[:password_salt]
|
||||
)
|
||||
|
||||
# This is optional as it was added in version 2.0.2
|
||||
exported_user = exported_visualization[:user]
|
||||
if exported_user
|
||||
visualization.user = Carto::User.new(username: exported_user[:username])
|
||||
end
|
||||
|
||||
# Added in version 2.0.3
|
||||
visualization.state = build_state_from_hash(exported_visualization[:state])
|
||||
|
||||
active_layer_order = exported_layers.index { |l| l[:active_layer] }
|
||||
if active_layer_order
|
||||
visualization.active_layer = visualization.layers.find { |l| l.order == active_layer_order }
|
||||
end
|
||||
|
||||
# Dataset-specific
|
||||
user_table = build_user_table_from_hash(exported_visualization[:user_table])
|
||||
visualization.map.user_table = user_table if user_table
|
||||
visualization.synchronization = build_synchronization_from_hash(exported_visualization[:synchronization])
|
||||
|
||||
visualization.id = exported_visualization[:id] if exported_visualization[:id]
|
||||
visualization
|
||||
end
|
||||
|
||||
def build_map_from_hash(exported_map, layers:)
|
||||
return nil unless exported_map
|
||||
|
||||
Carto::Map.new(
|
||||
provider: exported_map[:provider],
|
||||
bounding_box_sw: exported_map[:bounding_box_sw],
|
||||
bounding_box_ne: exported_map[:bounding_box_ne],
|
||||
center: exported_map[:center],
|
||||
zoom: exported_map[:zoom],
|
||||
view_bounds_sw: exported_map[:view_bounds_sw],
|
||||
view_bounds_ne: exported_map[:view_bounds_ne],
|
||||
scrollwheel: exported_map[:scrollwheel],
|
||||
legends: exported_map[:legends],
|
||||
layers: layers,
|
||||
options: exported_map[:options]
|
||||
)
|
||||
end
|
||||
|
||||
def build_overlays_from_hash(exported_overlays)
|
||||
return [] unless exported_overlays
|
||||
|
||||
exported_overlays.map.with_index.map do |overlay, i|
|
||||
build_overlay_from_hash(overlay, order: (i + 1))
|
||||
end
|
||||
end
|
||||
|
||||
def build_overlay_from_hash(exported_overlay, order:)
|
||||
Carto::Overlay.new(
|
||||
order: order,
|
||||
options: exported_overlay[:options],
|
||||
type: exported_overlay[:type],
|
||||
template: exported_overlay[:template]
|
||||
)
|
||||
end
|
||||
|
||||
def build_analysis_from_hash(exported_analysis)
|
||||
return nil unless exported_analysis
|
||||
|
||||
Carto::Analysis.new(analysis_definition: exported_analysis[:analysis_definition])
|
||||
end
|
||||
|
||||
def build_state_from_hash(exported_state)
|
||||
Carto::State.new(json: exported_state ? exported_state[:json] : nil)
|
||||
end
|
||||
|
||||
def build_permission_from_hash(exported_permission)
|
||||
return nil unless exported_permission
|
||||
|
||||
Carto::Permission.new(access_control_list: JSON.dump(exported_permission[:access_control_list]))
|
||||
end
|
||||
|
||||
def build_synchronization_from_hash(exported_synchronization)
|
||||
return nil unless exported_synchronization
|
||||
|
||||
sync = Carto::Synchronization.new(
|
||||
name: exported_synchronization[:name],
|
||||
interval: exported_synchronization[:interval],
|
||||
url: exported_synchronization[:url],
|
||||
state: exported_synchronization[:state],
|
||||
created_at: exported_synchronization[:created_at],
|
||||
updated_at: exported_synchronization[:updated_at],
|
||||
run_at: exported_synchronization[:run_at],
|
||||
retried_times: exported_synchronization[:retried_times],
|
||||
log: build_log_from_hash(exported_synchronization[:log]),
|
||||
error_code: exported_synchronization[:error_code],
|
||||
error_message: exported_synchronization[:error_message],
|
||||
ran_at: exported_synchronization[:ran_at],
|
||||
modified_at: exported_synchronization[:modified_at],
|
||||
etag: exported_synchronization[:etag],
|
||||
checksum: exported_synchronization[:checksum],
|
||||
service_name: exported_synchronization[:service_name],
|
||||
service_item_id: exported_synchronization[:service_item_id],
|
||||
type_guessing: exported_synchronization[:type_guessing],
|
||||
quoted_fields_guessing: exported_synchronization[:quoted_fields_guessing],
|
||||
content_guessing: exported_synchronization[:content_guessing]
|
||||
)
|
||||
|
||||
sync.id = exported_synchronization[:id]
|
||||
sync
|
||||
end
|
||||
|
||||
def build_user_table_from_hash(exported_user_table)
|
||||
return nil unless exported_user_table
|
||||
|
||||
user_table = Carto::UserTable.new
|
||||
user_table.name = exported_user_table[:name]
|
||||
user_table.privacy = exported_user_table[:privacy]
|
||||
user_table.tags = exported_user_table[:tags]
|
||||
user_table.geometry_columns = exported_user_table[:geometry_columns]
|
||||
user_table.rows_counted = exported_user_table[:rows_counted]
|
||||
user_table.rows_estimated = exported_user_table[:rows_estimated]
|
||||
user_table.indexes = exported_user_table[:indexes]
|
||||
user_table.database_name = exported_user_table[:database_name]
|
||||
user_table.description = exported_user_table[:description]
|
||||
user_table.table_id = exported_user_table[:table_id]
|
||||
user_table.data_import = build_data_import_from_hash(exported_user_table[:data_import])
|
||||
|
||||
user_table
|
||||
end
|
||||
|
||||
def build_mapcap_from_hash(exported_mapcap)
|
||||
return nil unless exported_mapcap
|
||||
|
||||
Carto::Mapcap.new(
|
||||
ids_json: exported_mapcap[:ids_json],
|
||||
export_json: exported_mapcap[:export_json],
|
||||
created_at: exported_mapcap[:created_at]
|
||||
)
|
||||
end
|
||||
|
||||
def build_external_source_from_hash(exported_external_source)
|
||||
return nil unless exported_external_source
|
||||
|
||||
es = Carto::ExternalSource.new(
|
||||
import_url: exported_external_source[:import_url],
|
||||
rows_counted: exported_external_source[:rows_counted],
|
||||
size: exported_external_source[:size],
|
||||
username: exported_external_source[:username],
|
||||
geometry_types: exported_external_source[:geometry_types]
|
||||
)
|
||||
es.id = exported_external_source[:id]
|
||||
|
||||
es
|
||||
end
|
||||
end
|
||||
|
||||
module VisualizationsExportService2Exporter
|
||||
include VisualizationsExportService2Configuration
|
||||
include VisualizationsExportService2Validator
|
||||
include LayerExporter
|
||||
include DataImportExporter
|
||||
|
||||
def export_visualization_json_string(visualization_id, user, with_password: false)
|
||||
export_visualization_json_hash(visualization_id, user, with_password: with_password).to_json
|
||||
end
|
||||
|
||||
def export_visualization_json_hash(visualization_id, user, with_mapcaps: true, with_password: false)
|
||||
{
|
||||
version: CURRENT_VERSION,
|
||||
visualization: export(Carto::Visualization.find(visualization_id), user,
|
||||
with_mapcaps: with_mapcaps, with_password: with_password)
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def export(visualization, user, with_mapcaps: true, with_password: false)
|
||||
check_valid_visualization(visualization)
|
||||
export_visualization(visualization, user, with_mapcaps: with_mapcaps, with_password: with_password)
|
||||
end
|
||||
|
||||
def export_visualization(visualization, user, with_mapcaps: true, with_password: false)
|
||||
layers = visualization.layers_with_data_readable_by(user)
|
||||
active_layer_id = visualization.active_layer_id
|
||||
layer_exports = layers.map do |layer|
|
||||
export_layer(layer, active_layer: active_layer_id == layer.id)
|
||||
end
|
||||
|
||||
export = {
|
||||
id: visualization.id,
|
||||
name: visualization.name,
|
||||
description: visualization.description,
|
||||
version: visualization.version,
|
||||
type: visualization.type,
|
||||
tags: visualization.tags,
|
||||
privacy: visualization.privacy,
|
||||
source: visualization.source,
|
||||
license: visualization.license,
|
||||
title: visualization.title,
|
||||
kind: visualization.kind,
|
||||
attributions: visualization.attributions,
|
||||
bbox: visualization.bbox,
|
||||
display_name: visualization.display_name,
|
||||
map: export_map(visualization.map),
|
||||
layers: layer_exports,
|
||||
overlays: visualization.overlays.map { |o| export_overlay(o) },
|
||||
analyses: visualization.analyses.map { |a| exported_analysis(a) },
|
||||
user: export_user(visualization.user),
|
||||
state: export_state(visualization.state),
|
||||
permission: export_permission(visualization.permission),
|
||||
synchronization: export_syncronization(visualization.synchronization),
|
||||
user_table: export_user_table(visualization.map.try(:user_table)),
|
||||
uses_vizjson2: visualization.uses_vizjson2?,
|
||||
mapcap: with_mapcaps ? export_mapcap(visualization.latest_mapcap) : nil,
|
||||
external_source: export_external_source(visualization.external_source),
|
||||
created_at: visualization.created_at,
|
||||
updated_at: visualization.updated_at,
|
||||
locked: visualization.locked
|
||||
}
|
||||
|
||||
if with_password
|
||||
export[:encrypted_password] = visualization.encrypted_password
|
||||
export[:password_salt] = visualization.password_salt
|
||||
end
|
||||
|
||||
export
|
||||
end
|
||||
|
||||
def export_user(user)
|
||||
{
|
||||
username: user.username
|
||||
}
|
||||
end
|
||||
|
||||
def export_map(map)
|
||||
return nil unless map
|
||||
|
||||
{
|
||||
provider: map.provider,
|
||||
bounding_box_sw: map.bounding_box_sw,
|
||||
bounding_box_ne: map.bounding_box_ne,
|
||||
center: map.center,
|
||||
zoom: map.zoom,
|
||||
view_bounds_sw: map.view_bounds_sw,
|
||||
view_bounds_ne: map.view_bounds_ne,
|
||||
scrollwheel: map.scrollwheel,
|
||||
legends: map.legends,
|
||||
options: map.options
|
||||
}
|
||||
end
|
||||
|
||||
def export_overlay(overlay)
|
||||
{
|
||||
options: overlay.options,
|
||||
type: overlay.type,
|
||||
template: overlay.template
|
||||
}
|
||||
end
|
||||
|
||||
def exported_analysis(analysis)
|
||||
{
|
||||
analysis_definition: analysis.analysis_definition
|
||||
}
|
||||
end
|
||||
|
||||
def export_state(state)
|
||||
{
|
||||
json: state.json
|
||||
}
|
||||
end
|
||||
|
||||
def export_permission(permission)
|
||||
{
|
||||
access_control_list: JSON.parse(permission.access_control_list, symbolize_names: true)
|
||||
}
|
||||
end
|
||||
|
||||
def export_syncronization(synchronization)
|
||||
return nil unless synchronization
|
||||
{
|
||||
id: synchronization.id,
|
||||
name: synchronization.name,
|
||||
interval: synchronization.interval,
|
||||
url: synchronization.url,
|
||||
state: synchronization.state,
|
||||
created_at: synchronization.created_at,
|
||||
updated_at: synchronization.updated_at,
|
||||
run_at: synchronization.run_at,
|
||||
retried_times: synchronization.retried_times,
|
||||
log: export_log(synchronization.log),
|
||||
error_code: synchronization.error_code,
|
||||
error_message: synchronization.error_message,
|
||||
ran_at: synchronization.ran_at,
|
||||
modified_at: synchronization.modified_at,
|
||||
etag: synchronization.etag,
|
||||
checksum: synchronization.checksum,
|
||||
service_name: synchronization.service_name,
|
||||
service_item_id: synchronization.service_item_id,
|
||||
type_guessing: synchronization.type_guessing,
|
||||
quoted_fields_guessing: synchronization.quoted_fields_guessing,
|
||||
content_guessing: synchronization.content_guessing
|
||||
}
|
||||
end
|
||||
|
||||
def export_user_table(user_table)
|
||||
return nil unless user_table
|
||||
|
||||
{
|
||||
name: user_table.name,
|
||||
privacy: user_table.privacy,
|
||||
tags: user_table.tags,
|
||||
geometry_columns: user_table.geometry_columns,
|
||||
rows_counted: user_table.rows_counted,
|
||||
rows_estimated: user_table.rows_estimated,
|
||||
indexes: user_table.indexes,
|
||||
database_name: user_table.database_name,
|
||||
description: user_table.description,
|
||||
table_id: user_table.table_id,
|
||||
data_import: export_data_import(user_table.data_import)
|
||||
}
|
||||
end
|
||||
|
||||
def export_external_source(external_source)
|
||||
return nil unless external_source
|
||||
|
||||
{
|
||||
id: external_source.id,
|
||||
import_url: external_source.import_url,
|
||||
rows_counted: external_source.rows_counted,
|
||||
size: external_source.size,
|
||||
username: external_source.username,
|
||||
geometry_types: external_source.geometry_types
|
||||
}
|
||||
end
|
||||
|
||||
def export_mapcap(mapcap)
|
||||
return nil unless mapcap
|
||||
|
||||
{
|
||||
ids_json: mapcap.ids_json,
|
||||
export_json: mapcap.export_json,
|
||||
created_at: mapcap.created_at
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
# Both String and Hash versions are provided because `deep_symbolize_keys` won't symbolize through arrays
|
||||
# and having separated methods make handling and testing much easier.
|
||||
class VisualizationsExportService2
|
||||
include VisualizationsExportService2Importer
|
||||
include VisualizationsExportService2Exporter
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,211 @@
|
||||
require 'rollbar'
|
||||
require_relative '../../models/carto/visualization'
|
||||
require_relative '../../models/carto/external_source'
|
||||
require_relative '../../models/common_data/singleton'
|
||||
|
||||
module CartoDB
|
||||
|
||||
module Visualization
|
||||
|
||||
class CommonDataService
|
||||
|
||||
def initialize(datasets = nil)
|
||||
@datasets = datasets
|
||||
end
|
||||
|
||||
def self.load_common_data(user, controller)
|
||||
if self.configured?
|
||||
common_data_url = CartoDB::Visualization::CommonDataService.build_url(controller)
|
||||
user.load_common_data(common_data_url)
|
||||
end
|
||||
end
|
||||
|
||||
def self.configured?
|
||||
Cartodb.get_config(:common_data, 'base_url').present?
|
||||
end
|
||||
|
||||
def self.build_url(controller)
|
||||
common_data_config = Cartodb.config[:common_data]
|
||||
return nil unless common_data_config
|
||||
|
||||
common_data_base_url = common_data_config['base_url']
|
||||
common_data_username = common_data_config['username']
|
||||
common_data_user = Carto::User.where(username: common_data_username).first
|
||||
if !common_data_base_url.nil?
|
||||
# We set user_domain to nil to avoid duplication in the url for subdomainfull urls. Ie. user.carto.com/u/cartodb/...
|
||||
common_data_base_url + CartoDB.path(controller, 'api_v1_visualizations_index', {type: 'table', privacy: 'public', user_domain: nil})
|
||||
elsif !common_data_user.nil?
|
||||
CartoDB.url(controller, 'api_v1_visualizations_index', params: { type: 'table', privacy: 'public' },
|
||||
user: common_data_user)
|
||||
else
|
||||
CartoDB.notify_error(
|
||||
'cant create common-data url. User doesn\'t exist and base_url is nil',
|
||||
username: common_data_username
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def load_common_data_for_user(user, visualizations_api_url)
|
||||
update_user_date_flag(user)
|
||||
|
||||
datasets = begin
|
||||
get_datasets(visualizations_api_url)
|
||||
rescue StandardError => e
|
||||
CartoDB::Logger.error(message: "Loading common data failed", exception: e, user: user)
|
||||
nil
|
||||
end
|
||||
# As deletion would delete all user syncs, if the endpoint fails or return nothing, just do nothing.
|
||||
# The user date flag is updated to avoid DDoS-ing.
|
||||
return nil unless datasets.present?
|
||||
|
||||
added = 0
|
||||
updated = 0
|
||||
not_modified = 0
|
||||
deleted = 0
|
||||
failed = 0
|
||||
|
||||
carto_user = Carto::User.find(user.id)
|
||||
|
||||
common_data_config = Cartodb.config[:common_data]
|
||||
common_data_username = common_data_config['username']
|
||||
common_data_remotes_by_name = Carto::Visualization.remotes
|
||||
.where(user_id: user.id)
|
||||
.joins(:external_source)
|
||||
.readonly(false) # joins causes readonly
|
||||
.where(external_sources: { username: common_data_username })
|
||||
.map { |v| [v.name, v] }.to_h
|
||||
ActiveRecord::Base.transaction do
|
||||
datasets.each do |dataset|
|
||||
begin
|
||||
visualization = common_data_remotes_by_name.delete(dataset['name'])
|
||||
if visualization
|
||||
visualization.attributes = dataset_visualization_attributes(dataset)
|
||||
if visualization.changed?
|
||||
visualization.save!
|
||||
updated += 1
|
||||
else
|
||||
not_modified += 1
|
||||
end
|
||||
else
|
||||
visualization = Carto::Visualization.new(
|
||||
dataset_visualization_attributes(dataset).merge(
|
||||
name: dataset['name'],
|
||||
user: carto_user,
|
||||
type: Carto::Visualization::TYPE_REMOTE
|
||||
)
|
||||
)
|
||||
visualization.save!
|
||||
added += 1
|
||||
end
|
||||
|
||||
external_source = Carto::ExternalSource.where(visualization_id: visualization.id).first
|
||||
if external_source
|
||||
if external_source.update_data(
|
||||
dataset['url'],
|
||||
dataset['geometry_types'],
|
||||
dataset['rows'],
|
||||
dataset['size'],
|
||||
'common-data'
|
||||
).changed?
|
||||
external_source.save!
|
||||
end
|
||||
else
|
||||
external_source = Carto::ExternalSource.create(
|
||||
visualization_id: visualization.id,
|
||||
import_url: dataset['url'],
|
||||
rows_counted: dataset['rows'],
|
||||
size: dataset['size'],
|
||||
geometry_types: dataset['geometry_types'],
|
||||
username: 'common-data'
|
||||
)
|
||||
end
|
||||
rescue => e
|
||||
CartoDB.notify_exception(e, {
|
||||
name: dataset.fetch('name', 'ERR: name'),
|
||||
source: dataset.fetch('source', 'ERR: source'),
|
||||
rows: dataset.fetch('rows', 'ERR: rows'),
|
||||
updated_at: dataset.fetch('updated_at', 'ERR: updated_at'),
|
||||
url: dataset.fetch('url', 'ERR: url')
|
||||
})
|
||||
failed += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
common_data_remotes_by_name.each_value do |remote|
|
||||
deleted += 1 if delete_remote_visualization(remote)
|
||||
end
|
||||
|
||||
return added, updated, not_modified, deleted, failed
|
||||
end
|
||||
|
||||
def update_user_date_flag(user)
|
||||
begin
|
||||
user.last_common_data_update_date = Time.now
|
||||
if user.valid?
|
||||
user.save(raise_on_failure: true)
|
||||
elsif user.errors[:quota_in_bytes]
|
||||
# This happens for the organization quota validation in the user model so we bypass this
|
||||
user.save(:validate => false, raise_on_failure: true)
|
||||
end
|
||||
rescue => e
|
||||
CartoDB.notify_exception(e, {user: user})
|
||||
end
|
||||
end
|
||||
|
||||
def delete_common_data_for_user(user)
|
||||
deleted = 0
|
||||
vqb = Carto::VisualizationQueryBuilder.new
|
||||
vqb.with_type(Carto::Visualization::TYPE_REMOTE).with_user_id(user.id).build.each do |v|
|
||||
delete_remote_visualization(v)
|
||||
deleted += 1
|
||||
end
|
||||
|
||||
deleted
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def get_datasets(visualizations_api_url)
|
||||
@datasets ||= CommonDataSingleton.instance.datasets(visualizations_api_url)
|
||||
end
|
||||
|
||||
def delete_remote_visualization(visualization)
|
||||
begin
|
||||
visualization.destroy
|
||||
true
|
||||
rescue => e
|
||||
match = e.message =~ /violates foreign key constraint "external_data_imports_external_source_id_fkey"/
|
||||
if match.present? && match >= 0
|
||||
# After #13667 this should no longer happen: deleting remote visualizations is propagated, and external
|
||||
# sources, external data imports and syncs are deleted
|
||||
CartoDB::Logger.error(message: "Couldn't delete, already imported", visualization_id: visualization.id)
|
||||
false
|
||||
else
|
||||
CartoDB.notify_error(
|
||||
"Couldn't delete remote visualization",
|
||||
visualization: visualization.id,
|
||||
error: e.inspect
|
||||
)
|
||||
raise e
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def dataset_visualization_attributes(dataset)
|
||||
{
|
||||
privacy: Carto::Visualization::PRIVACY_PUBLIC,
|
||||
description: dataset['description'],
|
||||
tags: dataset['tags'],
|
||||
license: dataset['license'],
|
||||
source: dataset['source'],
|
||||
attributions: dataset['attributions'],
|
||||
display_name: dataset['display_name']
|
||||
}
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
Reference in New Issue
Block a user