Initial commit

This commit is contained in:
zhongjin
2020-06-15 10:58:47 +08:00
commit 4f1dfe7564
8590 changed files with 1516878 additions and 0 deletions
+241
View File
@@ -0,0 +1,241 @@
module CartoDB
class ColumnTypecaster
CONVERSION_MAP = {
'number' => {
'boolean' => 'number_to_boolean',
'date' => 'number_to_datetime',
'datetime' => 'number_to_datetime',
'timestamp' => 'number_to_datetime',
'timestamptz' => 'number_to_datetime'
},
'boolean' => {
'double precision' => 'boolean_to_number',
'date' => 'boolean_to_datetime',
'datetime' => 'boolean_to_datetime',
'timestamp' => 'boolean_to_datetime',
'timestamptz' => 'boolean_to_datetime'
},
'string' => {
'date' => 'string_to_datetime',
'datetime' => 'string_to_datetime',
'timestamp' => 'string_to_datetime',
'timestamptz' => 'string_to_datetime',
'double precision' => 'string_to_number',
'boolean' => 'string_to_boolean'
},
'date' => {
'double precision' => 'date_to_number',
'boolean' => 'date_to_boolean'
}
}
def initialize(arguments)
@user_database = arguments.fetch(:user_database)
@schema = arguments.fetch(:schema)
@table_name = arguments.fetch(:table_name)
@column_name = arguments.fetch(:column_name)
@new_type = arguments.fetch(:new_type)
end
def run
return if nothing_to_do
user_database.transaction do straight_cast(@new_type.convert_to_db_type) end
rescue
# attempt various lossy conversions by regex nullifying
# unmatching data and retrying conversion.
#
# conversions ok by default:
# * number => string
# * boolean => string
user_database.transaction do
old_type = column_type(column_name).convert_to_cartodb_type
self.send conversion_method_for(old_type, new_type)
straight_cast(@new_type.convert_to_db_type)
end
end #run
protected
attr_reader :new_type, :column_name
private
attr_reader :user_database, :table_name, :schema, :old_type
def qualified_table
"\"#{schema}\".\"#{table_name}\""
end
def nothing_to_do
@new_type.blank? || @new_type == column_type(@column_name).convert_to_cartodb_type
end
def conversion_method_for(old_type, new_type)
CONVERSION_MAP.fetch(old_type.to_s).fetch(new_type.to_s.convert_to_db_type)
end #conversion_method_for
def column_type(column_name)
user_database.schema(table_name, schema: @schema, reload: true).select { |c|
c[0] == column_name.to_sym
}.flatten.last.fetch(:db_type).to_s
end #column_type
def straight_cast(new_type=self.new_type, options = {})
cast = options.fetch(:cast, "cast(#{column_name} as #{new_type})")
user_database.run(%Q{
ALTER TABLE #{qualified_table}
ALTER COLUMN #{column_name}
TYPE #{new_type}
USING #{cast}
})
end #straight_cast
def string_to_number
thousand_separator, decimal_separator = get_digit_separators_for(column_name)
normalize_number(thousand_separator, decimal_separator)
nullify_if_non_convertible
end
def string_to_datetime
straight_cast('timestamptz', cast: "cartodb.CDB_StringToDate(#{column_name})")
end #string_to_datetime
def string_to_boolean
falsy = '0|f|false'
normalize_empty_string_to_null
# normalise truthy (anything not false and NULL is true...)
user_database.run(%Q{
UPDATE #{qualified_table}
SET #{column_name}='t'
WHERE trim(\"#{column_name}\") !~* '^(#{falsy})$'
AND #{column_name} IS NOT NULL
})
# normalise falsy
user_database.run(%Q{
UPDATE #{qualified_table}
SET #{column_name}='f'
WHERE trim(\"#{column_name}\") ~* '^(#{falsy})$'
})
end #string_to_boolean
def boolean_to_number
# first to string
straight_cast('text')
# normalise truthy to 1
user_database.run(%Q{
UPDATE #{qualified_table}
SET #{column_name}='1'
WHERE #{column_name} = 'true'
AND #{column_name} IS NOT NULL
})
# normalise falsy to 0
user_database.run(%Q{
UPDATE #{qualified_table}
SET #{column_name}='0'
WHERE #{column_name} = 'false'
AND #{column_name} IS NOT NULL
})
end #boolean_to_number
def boolean_to_datetime
straight_cast('text')
nullify(column_name)
end #boolean_to_datetime
alias_method :number_to_datetime, :boolean_to_datetime
def number_to_boolean
straight_cast('text')
# normalise 0 to falsy else truthy
# normalise truthy
user_database.run(%Q{
UPDATE #{qualified_table}
SET #{column_name}='t'
WHERE #{column_name} !~* '^0$'
AND #{column_name} IS NOT NULL
})
# normalise falsy
user_database.run(%Q{
UPDATE #{qualified_table}
SET #{column_name}='f'
WHERE #{column_name} ~* '^0$'
})
end #number_to_boolean
def date_to_number
straight_cast('double precision', cast: "cartodb.CDB_DateToNumber(#{column_name})")
end
def date_to_boolean
nullify
end
def normalize_empty_string_to_null
user_database.run(%Q{
UPDATE #{qualified_table}
SET "#{column_name}" = NULL
WHERE \"#{column_name}\" = ''
})
end #normalize_empty_string_to_null
def nullify(column_name=self.column_name)
user_database.run(%Q{
UPDATE "#{table_name}"
SET #{column_name} = NULL
})
end #nullify
def nullify_if_non_convertible
user_database.run(%Q{
UPDATE "#{table_name}"
SET #{column_name}=NULL
WHERE trim(\"#{column_name}\") !~* '^([-+]?[0-9]+(\.[0-9]+)?)$'
})
end #nullify_if_non_convertible
def get_digit_separators_for(column_name)
user_database.execute(%Q{
SELECT t,d FROM cartodb.CDB_DigitSeparator('#{table_name}', '#{column_name}')
}, &:to_a).first.values
end #get_digit_separators
def normalize_number(thousand_separator=nil, decimal_separator=nil)
return unless thousand_separator && decimal_separator
replacements = [
[thousand_separator, ''],
[decimal_separator, '.'],
['$', ''],
['€', '']
]
user_database.execute(%Q{
UPDATE #{qualified_table}
SET "#{column_name}" = trim(#{pg_replace_expression(column_name, replacements)})
})
end
# Takes a column_name and a mapping of replacements and returns an expression such as:
# replace(replace(replace(column_name, k1, v1), k2, v2), k3, v3)
def pg_replace_expression(column_name, replacement_map=[])
# base case
k1, v1 = replacement_map.shift
expression = %Q{replace("#{column_name}", '#{k1}', '#{v1}')}
# rest
replacement_map.each do |k, v|
expression = %Q{replace(#{expression}, '#{k}', '#{v}')}
end
return expression
end
end
end
+171
View File
@@ -0,0 +1,171 @@
require_relative '../../../lib/varnish/lib/cartodb-varnish'
require_relative '../user'
require_relative 'user_table'
module CartoDB
class TablePrivacyManager
def initialize(table)
@table = table
end
def apply_privacy_change(metadata_table, old_privacy, privacy_changed = false)
if @table.table_visualization
affected_entities = [@table.table_visualization] + metadata_table.affected_visualizations
@table.map.save
# Separate on purpose as if fails here no need to revert visualizations' privacy
revertable_privacy_change(metadata_table, old_privacy) do
# Map saving actually doesn't changes privacy, but to keep on same reverting logic
@table.map.save
set_from_table_privacy(@table.privacy)
end
revertable_privacy_change(metadata_table, old_privacy, affected_entities) do
propagate_to([@table.table_visualization])
notify_privacy_affected_entities(metadata_table) if privacy_changed
end
end
rescue NoMethodError => exception
CartoDB::Logger.debug(
exception: exception,
message: "#{exception.message} #{exception.backtrace}",
user: Carto::User.find(@table.user_id),
table_id: @table.id,
table_name: @table.name,
data_import_id: @table.data_import_id
)
raise exception
end
def set_from_visualization(visualization)
set_public if visualization.public?
set_public_with_link_only if visualization.public_with_link?
set_private if visualization.private? || visualization.organization?
table.update(privacy: privacy)
self
end
def update_cdb_tablemetadata
table.update_cdb_tablemetadata
end
private
attr_reader :table
attr_accessor :privacy
def set_from_table_privacy(table_privacy)
case table_privacy
when Carto::UserTable::PRIVACY_PUBLIC
set_public
when Carto::UserTable::PRIVACY_LINK
set_public_with_link_only
else
set_private
end
end
# Propagation flow: Table -> Table PrivacyManager -> Visualization -> Visualization NamedMap
def propagate_to(visualizations, table_privacy_changed = false)
visualizations.each do |visualization|
if visualization.type == CartoDB::Visualization::Member::TYPE_CANONICAL
# Each table has a canonical visualization which must have privacy synced
visualization.privacy = Carto::UserTable::PRIVACY_VALUES_TO_TEXTS[privacy]
visualization.store_using_table(table_privacy_changed)
else
visualization.invalidate_cache
end
end
self
end
def set_public
self.privacy = Carto::UserTable::PRIVACY_PUBLIC
Carto::TableAndFriends.apply(owner.in_database, owner.database_schema, table.name) do |schema, table_name|
set_database_permissions(grant_query(schema, table_name))
end
self
end
def set_private
self.privacy = Carto::UserTable::PRIVACY_PRIVATE
Carto::TableAndFriends.apply(owner.in_database, owner.database_schema, table.name) do |schema, table_name|
set_database_permissions(revoke_query(schema, table_name))
end
self
end
def set_public_with_link_only
self.privacy = Carto::UserTable::PRIVACY_LINK
Carto::TableAndFriends.apply(owner.in_database, owner.database_schema, table.name) do |schema, table_name|
set_database_permissions(grant_query(schema, table_name))
end
self
end
def owner
@owner ||= ::User.where(id: table.user_id).first
end
def set_database_permissions(query)
owner.in_database(as: :superuser).run(query)
end
def revoke_query(schema, table_name)
%{
REVOKE SELECT ON #{fully_qualified_table_name(schema, table_name)}
FROM #{CartoDB::PUBLIC_DB_USER}
}
end
def grant_query(schema, table_name)
%{
GRANT SELECT ON #{fully_qualified_table_name(schema, table_name)}
TO #{CartoDB::PUBLIC_DB_USER};
}
end
def revertable_privacy_change(metadata_table, old_privacy = nil, entities = [])
yield
rescue => exception
CartoDB.notify_exception(exception, user_id: metadata_table.user_id, table_id: metadata_table.id)
revert_to_previous_privacy(metadata_table, old_privacy, entities)
raise exception
end
def revert_to_previous_privacy(metadata_table, old_privacy, additional_revert_targets = [])
return unless old_privacy
set_from_table_privacy(old_privacy)
errors = []
additional_revert_targets.each do |visualization|
begin
propagate_to([visualization])
rescue => exception
# Can't do much more, just let remaining ones finish
errors << exception
end
end
if !errors.empty?
CartoDB.notify_error("Errors reverting Table privacy", user_id: metadata_table.user_id,
table_id: metadata_table.id,
errors: errors)
end
end
def notify_privacy_affected_entities(metadata_table)
propagate_to(metadata_table.affected_visualizations, true)
update_cdb_tablemetadata
end
def fully_qualified_table_name(schema, table_name)
%{"#{schema}"."#{table_name}"}
end
end
end
+90
View File
@@ -0,0 +1,90 @@
require_relative '../visualization/collection'
require_relative '../visualization/member'
module CartoDB
class TableRelator
INTERFACE = %w{
synchronization
row_count_and_size
related_templates
}
def initialize(db, table)
@db = db
@table = table
end
def dependent_visualizations
affected_visualizations.select { |v| v.dependent_on?(table) }
end
def affected_visualizations
affected_visualization_records.to_a
.uniq { |attributes| attributes.fetch(:id) }
.map { |attributes| Visualization::Member.new(attributes) }
end
def preview_for(visualization)
data = {
id: visualization.id,
name: visualization.name,
updated_at: visualization.updated_at
}
if visualization[:permission_id].present? && !visualization.permission.nil?
data[:permission] = CartoDB::PermissionPresenter.new(visualization.permission).to_poro.select do |key, _val|
[:id, :owner].include?(key)
end
end
data[:auth_tokens] = if visualization.password_protected?
visualization.get_auth_tokens
elsif visualization.is_privacy_private?
visualization.user.get_auth_tokens
else
[]
end
data
end
def synchronization
return nil unless synchronization_record && !synchronization_record.empty?
CartoDB::Synchronization::Member.new(synchronization_record.first)
end
def row_count_and_size
@table.row_count_and_size
end
def related_templates
Carto::Template.all.select { |template| template.relates_to_table?(@table) }
end
private
attr_reader :db, :table
def affected_visualization_records
db[:visualizations].with_sql(%Q{
SELECT *
FROM layers_user_tables, layers_maps, visualizations
WHERE layers_user_tables.user_table_id = '#{table.id}'
AND layers_user_tables.layer_id = layers_maps.layer_id
AND layers_maps.map_id = visualizations.map_id
})
end
def synchronization_record
@syncronization_record ||= db[:synchronizations].with_sql(%Q{
SELECT *
FROM synchronizations
WHERE synchronizations.user_id = '#{table.user_id}'
AND synchronizations.name = '#{table.name}'
LIMIT 1
}).to_a
rescue => exception
puts exception.to_s
puts exception.backtrace
nil
end
end
end
+395
View File
@@ -0,0 +1,395 @@
require 'cartodb/per_request_sequel_cache'
require 'forwardable'
# This class is intended to deal exclusively with storage
class UserTable < Sequel::Model
extend Forwardable
INTERFACE = %w{
pk
associations
id
id=
name
description
description=
user_id
user_id=
[]
[]=
new_record?
save
save_changes
map_id
map_id=
valid?
table_id
table_id=
map
tags
tags=
set_tag_array
data_import_id
data_import_id=
user_id
user_id=
updated_at
automatic_geocoding
private?
public?
public_with_link_only?
privacy
privacy=
privacy_changed?
privacy_text
destroy
errors
set_except
update_updated_at
values
affected_visualizations
fully_dependent_visualizations
partially_dependent_visualizations
dependent_visualizations
reload
}
PRIVACY_PRIVATE = 0
PRIVACY_PUBLIC = 1
PRIVACY_LINK = 2
PRIVACY_VALUES_TO_TEXTS = {
PRIVACY_PRIVATE => 'private',
PRIVACY_PUBLIC => 'public',
PRIVACY_LINK => 'link'
}
# For compatibility with AR model
def new_record?
new?
end
# Associations
many_to_one :map
many_to_many :layers, join_table: :layers_user_tables,
left_key: :user_table_id,
right_key: :layer_id,
reciprocal: :user_tables
one_to_one :automatic_geocoding, key: :table_id
one_to_many :geocodings, key: :table_id
many_to_one :data_import, key: :data_import_id
many_to_one :user
plugin :association_dependencies, map: :destroy,
layers: :nullify,
automatic_geocoding: :destroy
plugin :dirty
def_delegators :relator, :affected_visualizations, :synchronization
# Ignore mass-asigment on not allowed columns
self.strict_param_setting = false
# Allowed columns
set_allowed_columns(:privacy, :tags, :description)
# The service should take care of all hooks
def set_service(table_obj)
@service = table_obj
end
# Lazy initialization of service if not present
def service
@service ||= ::Table.new(user_table: self)
end
def sync_table_id
self.table_id = service.get_table_id
end
def self.find_by_identifier(user_id, identifier)
col = 'name'
table = fetch(%Q{
SELECT *
FROM user_tables
WHERE user_tables.user_id = ?
AND user_tables.#{col} = ?},
user_id, identifier
).first
raise RecordNotFound if table.nil?
table
end
def self.from_map_id_key(map_id)
"UserTable:from_map_id:#{map_id}"
end
def self.from_map_id(map_id)
key = self.from_map_id_key(map_id)
user_table = PerRequestSequelCache.get(key)
if user_table.nil?
user_table = self[map_id: map_id]
PerRequestSequelCache.set(key, user_table, nil) unless user_table.nil?
end
user_table
end
# Hooks definition -----------------------------------------------------------
def validate
super
# userid and table name tuple must be unique
validates_unique [:name, :user_id], message: 'is already taken'
# tables must have a user
errors.add(:user_id, "can't be blank") if user_id.blank?
errors.add(:user, "Viewer users can't create tables") if user && user.viewer
if Carto::DB::Sanitize::RESERVED_TABLE_NAMES.include?(name)
errors.add(:name, 'is a reserved keyword, please choose a different one')
end
# TODO this kind of check should be moved to the DB
# privacy setting must be a sane value
if privacy != PRIVACY_PRIVATE && privacy != PRIVACY_PUBLIC && privacy != PRIVACY_LINK
errors.add(:privacy, "has an invalid value '#{privacy}'")
end
unless user.try(:private_tables_enabled)
# If it's a new table and the user is trying to make it private
if new? && privacy == PRIVACY_PRIVATE
errors.add(:privacy, 'unauthorized to create private tables')
end
# if the table exists, is private, but the owner no longer has private privileges
if !new? && privacy == PRIVACY_PRIVATE && changed_columns.include?(:privacy)
errors.add(:privacy, 'unauthorized to modify privacy status to private')
end
# cannot change any existing table to 'with link'
if !new? && privacy == PRIVACY_LINK && changed_columns.include?(:privacy)
errors.add(:privacy, 'unauthorized to modify privacy status to public with link')
end
end
end
def before_validation
set_default_table_privacy
super
end
def before_create
super
update_updated_at # TODO move to a DB trigger
service.before_create
end
def after_create
super
create_default_map_and_layers
create_default_visualization
set_default_table_privacy
save
service.after_create
end
def before_save
super
service.before_save
end
def after_save
super
service.after_save
end
def before_destroy
raise CartoDB::InvalidMember.new(user: "Viewer users can't destroy tables") if user && user.viewer
@table_visualization = table_visualization
@fully_dependent_visualizations_cache = fully_dependent_visualizations.to_a
@partially_dependent_visualizations_cache = partially_dependent_visualizations.to_a
super
end
def after_destroy
@table_visualization.delete_from_table if @table_visualization
@fully_dependent_visualizations_cache.each(&:delete)
@partially_dependent_visualizations_cache.each do |visualization|
visualization.unlink_from(self)
end
synchronization.delete if synchronization
service.after_destroy
super
end
def before_update
PerRequestSequelCache.delete(self.class.from_map_id_key(self.map_id))
super
end
def delete
PerRequestSequelCache.delete(self.class.from_map_id_key(self.map_id))
super
end
# --------------------------------------------------------------------------------
def update_cdb_tablemetadata
service.update_cdb_tablemetadata
end
def privacy_text
PRIVACY_VALUES_TO_TEXTS[self.privacy].upcase
end
# TODO move privacy to value object
# enforce standard format for this field
def privacy=(value)
case value
when 'PUBLIC', PRIVACY_PUBLIC, PRIVACY_PUBLIC.to_s
self[:privacy] = PRIVACY_PUBLIC
when 'LINK', PRIVACY_LINK, PRIVACY_LINK.to_s
self[:privacy] = PRIVACY_LINK
when 'PRIVATE', PRIVACY_PRIVATE, PRIVACY_PRIVATE.to_s
self[:privacy] = PRIVACY_PRIVATE
else
raise "Invalid privacy value '#{value}'"
end
end
def private?
self.privacy == PRIVACY_PRIVATE
end #private?
def public?
self.privacy == PRIVACY_PUBLIC
end #public?
def public_with_link_only?
self.privacy == PRIVACY_LINK
end #public_with_link_only?
# TODO move tags to value object. A set is more appropriate
def tags=(value)
return unless value
set_tag_array(value.split(','))
end
def set_tag_array(tag_array)
return unless tag_array
self[:tags] = tag_array.map{ |t| t.strip }.compact.delete_if{ |t| t.blank? }.uniq.join(',')
end
# Needed by syncs
def update_updated_at
self.updated_at = Time.now
end
def estimated_row_count
service.estimated_row_count
end
def actual_row_count
service.actual_row_count
end
def external_source_visualization
data_import.try(:external_data_imports).try(:first).try(:external_source).try(:visualization)
end
def table_visualization
@table_visualization ||= map_id && CartoDB::Visualization::Collection.new.fetch(
map_id: map_id,
type: CartoDB::Visualization::Member::TYPE_CANONICAL
).first
end
def privacy_changed?
previous_changes && previous_changes.keys.include?(:privacy)
end
def privacy_was
previous_changes[:privacy].first
end
def fully_dependent_visualizations
affected_visualizations.select { |v| v.fully_dependent_on?(self) }
end
def partially_dependent_visualizations
affected_visualizations.select { |v| v.partially_dependent_on?(self) }
end
def dependent_visualizations
affected_visualizations.select { |v| v.dependent_on?(self) }
end
def is_owner?(user)
return false unless user
user_id == user.id
end
private
def default_privacy_value
user.try(:private_tables_enabled) ? PRIVACY_PRIVATE : PRIVACY_PUBLIC
end
def set_default_table_privacy
self.privacy ||= default_privacy_value
end
def create_default_map_and_layers
base_layer = ::ModelFactories::LayerFactory.get_default_base_layer(user)
self.map = ::ModelFactories::MapFactory.get_map(base_layer, user.id, id)
map.add_layer(base_layer)
geometry_type = service.the_geom_type || 'geometry'
data_layer = ::ModelFactories::LayerFactory.get_default_data_layer(name, user, geometry_type)
map.add_layer(data_layer)
if base_layer.supports_labels_layer?
labels_layer = ::ModelFactories::LayerFactory.get_default_labels_layer(base_layer)
map.add_layer(labels_layer)
end
end
def create_default_visualization
kind = service.is_raster? ? CartoDB::Visualization::Member::KIND_RASTER : CartoDB::Visualization::Member::KIND_GEOM
esv = external_source_visualization
member = CartoDB::Visualization::Member.new(
name: name,
map_id: map.id,
type: CartoDB::Visualization::Member::TYPE_CANONICAL,
description: description,
attributions: esv.try(:attributions),
source: esv.try(:source),
tags: (tags.split(',') if tags),
privacy: UserTable::PRIVACY_VALUES_TO_TEXTS[default_privacy_value],
user_id: user.id,
kind: kind
)
member.store
member.map.set_default_boundaries!
map.reload
CartoDB::Visualization::Overlays.new(member).create_default_overlays
end
def relator
@relator ||= CartoDB::TableRelator.new(SequelRails.connection, self)
end
end