Initial commit
This commit is contained in:
zhongjin
@@ -0,0 +1,46 @@
|
||||
require 'active_record'
|
||||
|
||||
class Carto::DashboardPreviewSearcher
|
||||
|
||||
delegate :url_helpers, to: 'Rails.application.routes'
|
||||
|
||||
DEFAULT_TYPES = %w(table derived remote tag).freeze
|
||||
|
||||
def initialize(user:, pattern:, types: DEFAULT_TYPES, limit:)
|
||||
@user = user
|
||||
@pattern = pattern
|
||||
types = DEFAULT_TYPES unless types.present?
|
||||
@visualization_types = types - ["tag"]
|
||||
@include_tags = types.include?("tag")
|
||||
@limit = limit
|
||||
end
|
||||
|
||||
def search
|
||||
result = Carto::DashboardSearchResult.new
|
||||
|
||||
if @include_tags
|
||||
tag_query_builder = initialize_builder(Carto::TagQueryBuilder)
|
||||
tag_results = tag_query_builder.build_paged(1, @limit)
|
||||
result.tags = tag_results.map { |hash| hash[:tag] }
|
||||
result.total_count += tag_query_builder.total_count
|
||||
end
|
||||
|
||||
if @visualization_types.any?
|
||||
visualization_query_builder = initialize_builder(Carto::VisualizationQueryBuilder)
|
||||
result.visualizations = visualization_query_builder.build_paged(1, @limit).to_a
|
||||
result.total_count += visualization_query_builder.build.count
|
||||
end
|
||||
|
||||
result
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def initialize_builder(builder_class)
|
||||
builder_class.new
|
||||
.with_owned_by_or_shared_with_user_id(@user.id)
|
||||
.with_types(@visualization_types)
|
||||
.with_partial_match(@pattern)
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,52 @@
|
||||
require 'active_record'
|
||||
|
||||
class Carto::DataImportQueryBuilder
|
||||
|
||||
def initialize
|
||||
@order = {}
|
||||
end
|
||||
|
||||
def with_user(user)
|
||||
@user_id = user.id
|
||||
self
|
||||
end
|
||||
|
||||
def with_state_not_in(states)
|
||||
@excluded_states = states
|
||||
self
|
||||
end
|
||||
|
||||
def with_created_at_after(date)
|
||||
@created_at_after = date
|
||||
self
|
||||
end
|
||||
|
||||
def with_order(order, asc_desc = :asc)
|
||||
@order[order] = asc_desc
|
||||
self
|
||||
end
|
||||
|
||||
def build
|
||||
query = Carto::DataImport.all
|
||||
|
||||
if @user_id
|
||||
query = query.where(user_id: @user_id)
|
||||
end
|
||||
|
||||
if @excluded_states
|
||||
query = query.where('state not in (?)', @excluded_states)
|
||||
end
|
||||
|
||||
if @created_at_after
|
||||
query = query.where('created_at >= ?', @created_at_after)
|
||||
end
|
||||
|
||||
@order.each { |k, v|
|
||||
query = query.order(k)
|
||||
query = query.reverse_order if v == :desc
|
||||
}
|
||||
|
||||
query
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,71 @@
|
||||
# Class that allows "Grantable" searches.
|
||||
# Note a Grantable is not an actual ActiveRecord model but an aggregation of User and Group.
|
||||
# Since ActiveRecord doesn't support polymorphic queries or has_many associations
|
||||
# with table per class inheritance, we do it manually here
|
||||
class Carto::GrantableQueryBuilder
|
||||
|
||||
def initialize(organization)
|
||||
@filter = nil
|
||||
@organization = organization
|
||||
end
|
||||
|
||||
def with_filter(filter)
|
||||
return self unless filter
|
||||
|
||||
clean_filter = filter.gsub('%', '\\%').gsub('_', '\\_')
|
||||
@filter = "%#{clean_filter}%"
|
||||
self
|
||||
end
|
||||
|
||||
def run(page = 1, per_page = 200, order = 'name')
|
||||
query = ActiveRecord::Base.send(:sanitize_sql_array, paged_query_array(page, per_page, order))
|
||||
ActiveRecord::Base.connection.execute(query).map { |r| Carto::Grantable.new(r) }
|
||||
end
|
||||
|
||||
def count
|
||||
query = ActiveRecord::Base.send(:sanitize_sql_array, count_query_array)
|
||||
ActiveRecord::Base.connection.execute(query).first['count'].to_i
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def compose_query
|
||||
query = <<-SQL
|
||||
select * from
|
||||
(select id, display_name as name, 'group' as type, '' as avatar_url,
|
||||
created_at, updated_at,
|
||||
organization_id
|
||||
from groups
|
||||
union
|
||||
select id, username as name, 'user' as type, avatar_url,
|
||||
created_at, updated_at,
|
||||
organization_id
|
||||
from users) grantables
|
||||
where grantables.organization_id = ?
|
||||
SQL
|
||||
@filter.nil? ? query : "#{query} and name ilike ?"
|
||||
end
|
||||
|
||||
def paged_query_array(page, per_page, order)
|
||||
offset = (page - 1) * per_page
|
||||
query = "#{compose_query} order by #{safe_order(order)} limit ? offset ?"
|
||||
if @filter.nil?
|
||||
[query, @organization.id, per_page, offset]
|
||||
else
|
||||
[query, @organization.id, @filter, per_page, offset]
|
||||
end
|
||||
end
|
||||
|
||||
def count_query_array
|
||||
query = "select count(1) from (#{compose_query}) c"
|
||||
if @filter.nil?
|
||||
[query, @organization.id]
|
||||
else
|
||||
[query, @organization.id, @filter]
|
||||
end
|
||||
end
|
||||
|
||||
def safe_order(order)
|
||||
order && %w{ id name type avatar_url created_at updated_at organization_id }.include?(order.to_s) ? order.to_s : 'name'
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,73 @@
|
||||
# NOTE: Implement further methods from ActiveRecord::Relation here as needed
|
||||
module Carto
|
||||
class OffdatabaseQueryAdapter
|
||||
|
||||
# @param query ActiveRecord::Relation
|
||||
# @param order_by_asc_or_desc_by_attribute Hash { key: value, key: value } where
|
||||
# key = { String => { submodel: String|nil, attribute: String }, ... }
|
||||
# value = "asc" | "desc"
|
||||
def initialize(query, order_by_asc_or_desc_by_attribute)
|
||||
@query = query
|
||||
@order_by_asc_or_desc_by_attribute = order_by_asc_or_desc_by_attribute
|
||||
@offset = 0
|
||||
@limit = nil
|
||||
end
|
||||
|
||||
def offset(offset)
|
||||
@offset = offset
|
||||
self
|
||||
end
|
||||
|
||||
def limit(limit)
|
||||
@limit = limit
|
||||
self
|
||||
end
|
||||
|
||||
def all
|
||||
results
|
||||
end
|
||||
|
||||
def map
|
||||
results.map { |a|
|
||||
yield(a)
|
||||
}
|
||||
end
|
||||
|
||||
def count
|
||||
results.count
|
||||
end
|
||||
|
||||
def size
|
||||
results.size
|
||||
end
|
||||
|
||||
def first
|
||||
results.first
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def results
|
||||
@results ||= get_results
|
||||
end
|
||||
|
||||
# NOTE: Sorting for model associations (which return arrays) or arrays will be done based on length
|
||||
# e.g. visualization.analyses
|
||||
def get_results
|
||||
all = @query.all
|
||||
@order_by_asc_or_desc_by_attribute.each do |attribute, asc_or_desc|
|
||||
# Cache attribute type
|
||||
is_array = all.present? && all.first.send(attribute).respond_to?(:each)
|
||||
all = all.sort do |x, y|
|
||||
x_attribute = is_array ? x.send(attribute).count : x.send(attribute)
|
||||
y_attribute = is_array ? y.send(attribute).count : y.send(attribute)
|
||||
x_attribute = 0 if x_attribute.nil?
|
||||
y_attribute = 0 if y_attribute.nil?
|
||||
asc_or_desc == "asc" ? x_attribute <=> y_attribute : y_attribute <=> x_attribute
|
||||
end
|
||||
end
|
||||
all[@offset, @limit.nil? ? all.count : @limit]
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,114 @@
|
||||
require 'active_record'
|
||||
|
||||
class Carto::TagQueryBuilder
|
||||
|
||||
PATTERN_ESCAPE_CHARS = ['_', '%'].freeze
|
||||
|
||||
DEFAULT_TYPES = %w(table derived remote).freeze
|
||||
TYPE_TRANSLATIONS = {
|
||||
"table" => :datasets,
|
||||
"derived" => :maps,
|
||||
"remote" => :data_library
|
||||
}.freeze
|
||||
|
||||
def initialize
|
||||
@types = DEFAULT_TYPES
|
||||
end
|
||||
|
||||
def with_owned_by_user_id(user_id)
|
||||
@owner_id = user_id
|
||||
@user_id = nil
|
||||
self
|
||||
end
|
||||
|
||||
def with_owned_by_or_shared_with_user_id(user_id)
|
||||
@user_id = user_id
|
||||
@owner_id = nil
|
||||
self
|
||||
end
|
||||
|
||||
def with_types(types)
|
||||
@types = types if types.present?
|
||||
self
|
||||
end
|
||||
|
||||
def with_partial_match(pattern)
|
||||
return self unless pattern.present?
|
||||
clean_pattern = escape_characters_from_pattern(pattern)
|
||||
@pattern = clean_pattern.split(' ').map { |word| "%#{word}%" }
|
||||
self
|
||||
end
|
||||
|
||||
def build
|
||||
build_base_visualization_query
|
||||
.select(select_sql)
|
||||
.where('array_length(tags, 1) > 0')
|
||||
.group('tag')
|
||||
.order('total DESC')
|
||||
end
|
||||
|
||||
def build_paged(page, per_page)
|
||||
offset = (page.to_i - 1) * per_page.to_i
|
||||
limit = per_page.to_i
|
||||
query = filter_query(query: build, offset: offset, limit: limit)
|
||||
result = run_query(query)
|
||||
format_response(result)
|
||||
end
|
||||
|
||||
def total_count
|
||||
query = filter_query(query: build, count: true)
|
||||
result = run_query(query)
|
||||
result.first["count"].to_i
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def build_base_visualization_query
|
||||
query = Carto::VisualizationQueryBuilder.new
|
||||
query.with_user_id(@owner_id) if @owner_id
|
||||
query.with_owned_by_or_shared_with_user_id(@user_id) if @user_id
|
||||
query.with_types(@types) if @types
|
||||
query.build
|
||||
end
|
||||
|
||||
def select_sql
|
||||
"lower(unnest(tags)) AS tag, #{counts_sql}, count(*) as total"
|
||||
end
|
||||
|
||||
def counts_sql
|
||||
@types.map { |type|
|
||||
"sum(CASE type WHEN '#{type}' THEN 1 ELSE 0 END) AS #{type}_count"
|
||||
}.join(', ')
|
||||
end
|
||||
|
||||
def filter_query(query:, count: false, limit: nil, offset: nil)
|
||||
select_clause = count ? "COUNT(tag)" : "*"
|
||||
where_clause = "WHERE tag ILIKE ANY (array[?])" if @pattern
|
||||
limit_clause = "LIMIT ?" if limit
|
||||
offset_clause = "OFFSET ?" if offset
|
||||
filter_sql = %{
|
||||
SELECT #{select_clause} FROM (#{query.to_sql}) AS tags #{where_clause} #{limit_clause} #{offset_clause}
|
||||
}.squish
|
||||
|
||||
ActiveRecord::Base.send(:sanitize_sql_array, [filter_sql, @pattern, limit, offset].compact)
|
||||
end
|
||||
|
||||
def run_query(query)
|
||||
connection = ActiveRecord::Base.connection
|
||||
connection.exec_query(query)
|
||||
end
|
||||
|
||||
def format_response(result)
|
||||
result.map do |row|
|
||||
types_count = @types.map { |type|
|
||||
{ TYPE_TRANSLATIONS.fetch(type, type) => row["#{type}_count"].to_i }
|
||||
}.inject(:merge)
|
||||
{ tag: row['tag'] }.merge(types_count)
|
||||
end
|
||||
end
|
||||
|
||||
def escape_characters_from_pattern(pattern)
|
||||
pattern.chars.map { |c| PATTERN_ESCAPE_CHARS.include?(c) ? "\\" + c : c }.join
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,270 @@
|
||||
require 'active_record'
|
||||
|
||||
require_relative '../../models/carto/shared_entity'
|
||||
require_dependency 'carto/bounding_box_utils'
|
||||
require_dependency 'carto/uuidhelper'
|
||||
|
||||
# TODO: consider moving some of this to model scopes if convenient
|
||||
class Carto::VisualizationQueryBuilder
|
||||
include Carto::UUIDHelper
|
||||
|
||||
def self.user_public_tables(user)
|
||||
user_public(user).with_type(Carto::Visualization::TYPE_CANONICAL)
|
||||
end
|
||||
|
||||
def self.user_public_visualizations(user)
|
||||
user_public_privacy_visualizations(user).with_published
|
||||
end
|
||||
|
||||
def self.user_public_privacy_visualizations(user)
|
||||
user_public(user).with_type(Carto::Visualization::TYPE_DERIVED)
|
||||
end
|
||||
|
||||
def self.user_link_privacy_visualizations(user)
|
||||
new.with_user_id(user.id)
|
||||
.with_type(Carto::Visualization::TYPE_DERIVED)
|
||||
.with_privacy(Carto::Visualization::PRIVACY_LINK)
|
||||
end
|
||||
|
||||
def self.user_password_privacy_visualizations(user)
|
||||
new.with_user_id(user.id)
|
||||
.with_type(Carto::Visualization::TYPE_DERIVED)
|
||||
.with_privacy(Carto::Visualization::PRIVACY_PROTECTED)
|
||||
end
|
||||
|
||||
def self.user_all_visualizations(user)
|
||||
new.with_user_id(user ? user.id : nil).with_type(Carto::Visualization::TYPE_DERIVED)
|
||||
end
|
||||
|
||||
def self.user_public(user)
|
||||
new.with_user_id(user ? user.id : nil).with_privacy(Carto::Visualization::PRIVACY_PUBLIC)
|
||||
end
|
||||
|
||||
def initialize
|
||||
@include_associations = []
|
||||
@eager_load_associations = []
|
||||
@filtering_params = {}
|
||||
end
|
||||
|
||||
def with_id_or_name(id_or_name)
|
||||
raise 'VisualizationQueryBuilder: id or name supplied is nil' if id_or_name.nil?
|
||||
|
||||
if is_uuid?(id_or_name)
|
||||
with_id(id_or_name)
|
||||
else
|
||||
with_name(id_or_name)
|
||||
end
|
||||
end
|
||||
|
||||
def with_id(id)
|
||||
@filtering_params[:id] = id
|
||||
self
|
||||
end
|
||||
|
||||
def with_excluded_ids(ids)
|
||||
@filtering_params[:excluded_ids] = ids
|
||||
self
|
||||
end
|
||||
|
||||
def without_synced_external_sources
|
||||
@filtering_params[:exclude_synced_external_sources] = true
|
||||
self
|
||||
end
|
||||
|
||||
def without_imported_remote_visualizations
|
||||
@filtering_params[:exclude_imported_remote_visualizations] = true
|
||||
self
|
||||
end
|
||||
|
||||
def without_raster
|
||||
@filtering_params[:excluded_kinds] ||= []
|
||||
@filtering_params[:excluded_kinds] << Carto::Visualization::KIND_RASTER
|
||||
self
|
||||
end
|
||||
|
||||
def with_name(name)
|
||||
@filtering_params[:name] = name
|
||||
self
|
||||
end
|
||||
|
||||
def with_user_id(user_id)
|
||||
@filtering_params[:user_id] = user_id
|
||||
self
|
||||
end
|
||||
|
||||
def with_user_id_not(user_id)
|
||||
@filtering_params[:user_id_not] = user_id
|
||||
self
|
||||
end
|
||||
|
||||
def with_privacy(privacy)
|
||||
@filtering_params[:privacy] = privacy
|
||||
self
|
||||
end
|
||||
|
||||
def with_liked_by_user_id(user_id)
|
||||
@filtering_params[:liked_by_user_id] = user_id
|
||||
self
|
||||
end
|
||||
|
||||
def with_shared_with_user_id(user_id)
|
||||
@filtering_params[:shared_with_user_id] = user_id
|
||||
self
|
||||
end
|
||||
|
||||
def with_owned_by_or_shared_with_user_id(user_id)
|
||||
@filtering_params[:owned_by_or_shared_with_user_id] = user_id
|
||||
self
|
||||
end
|
||||
|
||||
def with_prefetch_user(force_join = false)
|
||||
if force_join
|
||||
with_eager_load_of(:user)
|
||||
else
|
||||
with_include_of(:user)
|
||||
end
|
||||
end
|
||||
|
||||
def with_prefetch_table
|
||||
nested_association = { map: :user_table }
|
||||
with_eager_load_of(nested_association)
|
||||
end
|
||||
|
||||
def with_prefetch_dependent_visualizations
|
||||
inner_visualization = { visualization: { map: { layers: :layers_user_tables }, permission: :owner } }
|
||||
nested_association = { map: { user_table: { layers: { maps: inner_visualization } } } }
|
||||
with_eager_load_of(nested_association)
|
||||
end
|
||||
|
||||
def with_prefetch_permission
|
||||
nested_association = { permission: :owner }
|
||||
with_eager_load_of(nested_association)
|
||||
end
|
||||
|
||||
def with_prefetch_external_source
|
||||
with_eager_load_of(:external_source)
|
||||
end
|
||||
|
||||
def with_prefetch_synchronization
|
||||
with_eager_load_of(:synchronization)
|
||||
self
|
||||
end
|
||||
|
||||
def with_type(type)
|
||||
@filtering_params[:type] = type
|
||||
self
|
||||
end
|
||||
|
||||
alias with_types with_type
|
||||
|
||||
def with_locked(locked)
|
||||
@filtering_params[:locked] = locked
|
||||
self
|
||||
end
|
||||
|
||||
def with_current_user_id(user_id)
|
||||
@current_user_id = user_id
|
||||
end
|
||||
|
||||
def with_order(order, direction = 'asc')
|
||||
@order = order.to_s
|
||||
@direction = direction.to_s
|
||||
self
|
||||
end
|
||||
|
||||
def with_partial_match(tainted_search_pattern)
|
||||
@filtering_params[:tainted_search_pattern] = tainted_search_pattern
|
||||
self
|
||||
end
|
||||
|
||||
def with_tags(tags)
|
||||
@filtering_params[:tags] = tags
|
||||
self
|
||||
end
|
||||
|
||||
def with_bounding_box(bounding_box)
|
||||
@filtering_params[:bounding_box] = bounding_box
|
||||
self
|
||||
end
|
||||
|
||||
def with_display_name
|
||||
@filtering_params[:only_with_display_name] = true
|
||||
self
|
||||
end
|
||||
|
||||
def with_organization_id(organization_id)
|
||||
@filtering_params[:organization_id] = organization_id
|
||||
self
|
||||
end
|
||||
|
||||
# Published: see `Carto::Visualization#published?`
|
||||
def with_published
|
||||
@filtering_params[:only_published] = true
|
||||
self
|
||||
end
|
||||
|
||||
def with_version(version)
|
||||
@filtering_params[:version] = version
|
||||
self
|
||||
end
|
||||
|
||||
def build
|
||||
query = Carto::Visualization.all
|
||||
query = Carto::VisualizationQueryFilterer.new(query).filter(@filtering_params)
|
||||
query = with_associations(query)
|
||||
order_query(query)
|
||||
end
|
||||
|
||||
def build_paged(page = 1, per_page = 20)
|
||||
build.offset((page.to_i - 1) * per_page.to_i).limit(per_page.to_i)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def order_query(query)
|
||||
# Search has its own ordering criteria
|
||||
return query if @tainted_search_pattern
|
||||
|
||||
orderer = Carto::VisualizationQueryOrderer.new(query)
|
||||
orderer.order(@order, @direction)
|
||||
end
|
||||
|
||||
def with_include_of(association)
|
||||
@include_associations << association
|
||||
self
|
||||
end
|
||||
|
||||
def with_eager_load_of(association)
|
||||
@eager_load_associations << association
|
||||
self
|
||||
end
|
||||
|
||||
def with_associations(query)
|
||||
query = query.includes(@include_associations) unless @include_associations.empty?
|
||||
query = query.eager_load(@eager_load_associations) unless @eager_load_associations.empty?
|
||||
# We have to include favorites if we're not filtering by them
|
||||
# Why? Both of them include a join with the likes table: favorited uses
|
||||
# a left-join one and the filter will use an inner-join.
|
||||
# So what is the problem? It'll fail because is not possible to include two
|
||||
# joins for the same table
|
||||
# And what is the difference?
|
||||
# - Filtering leaves only the favorited/liked visualizations by the user
|
||||
# - With favorited we add the like/favorite data to the visualization information
|
||||
query = with_favorited(query) unless @filtering_params[:liked_by_user_id]
|
||||
with_dependent_visualization_count(query)
|
||||
end
|
||||
|
||||
def with_favorited(query)
|
||||
return query unless @order && @order.include?("favorited")
|
||||
raise 'Cannot order by favorited if no user is provided' unless @current_user_id
|
||||
|
||||
Carto::VisualizationQueryIncluder.new(query).include_favorited(@current_user_id)
|
||||
end
|
||||
|
||||
def with_dependent_visualization_count(query)
|
||||
return query unless @order && @order.include?("dependent_visualizations")
|
||||
|
||||
Carto::VisualizationQueryIncluder.new(query).include_dependent_visualization_count(@filtering_params)
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,172 @@
|
||||
require 'active_record'
|
||||
|
||||
class Carto::VisualizationQueryFilterer
|
||||
|
||||
REGULAR_FILTERS = %i(id name user_id privacy type version locked).freeze
|
||||
|
||||
def initialize(query)
|
||||
@query = query
|
||||
end
|
||||
|
||||
def filter(params = {})
|
||||
query = @query
|
||||
|
||||
if params[:name] &&
|
||||
!(params[:id] || params[:user_id] || params[:organization_id] || params[:owned_by_or_shared_with_user_id] ||
|
||||
params[:shared_with_user_id])
|
||||
CartoDB::Logger.error(message: "VQB query by name without user_id nor org_id")
|
||||
raise 'VQB query by name without user_id nor org_id'
|
||||
end
|
||||
|
||||
query = query.where(params.slice(*REGULAR_FILTERS).compact)
|
||||
|
||||
if params[:excluded_ids]
|
||||
query = query.where('visualizations.id not in (?)', params[:excluded_ids])
|
||||
end
|
||||
|
||||
if params[:user_id_not]
|
||||
query = query.where('visualizations.user_id != ?', params[:user_id_not])
|
||||
end
|
||||
|
||||
if params[:liked_by_user_id]
|
||||
query = query
|
||||
.joins(:likes)
|
||||
.where(likes: { actor: params[:liked_by_user_id] })
|
||||
end
|
||||
|
||||
if params[:shared_with_user_id]
|
||||
# The problem here is to manage to generate a query that Postgres will correctly optimize. The problem is that the
|
||||
# optimizer seems to have problem determining the best plan when there are many JOINS, such as when VQB is called
|
||||
# with many prefetch options, e.g: from visualizations index.
|
||||
# This is hacky but works, without a performance hit. Other approaches:
|
||||
# - Use a WHERE visualization.id IN (SELECT entity_id...).
|
||||
# psql does a very bad query plan on this, but only when adding the `LEFT OUTER JOIN synchronizations`
|
||||
# - Use a CTE (WITH shared_vizs AS (SELECT entity_id ...) SELECT FROM visualizations JOIN shared_vizs)
|
||||
# This generates a nice query plan, but I was unable to generate this with ActiveRecord
|
||||
# - Create a view for shared_entities grouped by entity_id, and then create a fake model to join to the
|
||||
# view instead of the table. Should work, but adds a view just to cover for a failure in Rails
|
||||
# - Use `GROUP BY visualizations.id, synchronizations.id ...`
|
||||
# For some reason, Rails generates a wrong result when combining `group` with `count`
|
||||
# - Use a JOIN `query.joins("JOIN (#{shared_vizs.to_sql} ...)")`
|
||||
# Rails insists in putting custom SQL joins at the end, and psql fails at optimizing. This would work
|
||||
# if this JOIN was written as the first JOIN in the query. psql uses order to inform the optimizer.
|
||||
# This is precisely what this hacks achieves, by tricking the `FROM` part of the query
|
||||
# Context: https://github.com/CartoDB/cartodb/issues/13970
|
||||
user = Carto::User.where(id: params[:shared_with_user_id]).first
|
||||
shared_vizs = Carto::SharedEntity.where(recipient_id: recipient_ids(user)).select(:entity_id).uniq
|
||||
query = query.from([Arel::Nodes::SqlLiteral.new("
|
||||
visualizations
|
||||
JOIN
|
||||
(#{shared_vizs.to_sql}) shared_ent
|
||||
ON
|
||||
visualizations.id = shared_ent.entity_id")])
|
||||
end
|
||||
|
||||
if params[:owned_by_or_shared_with_user_id]
|
||||
# TODO: sql strings are suboptimal and compromise compositability, but
|
||||
# I haven't found a better way to do this OR in Rails
|
||||
shared_with_viz_ids = ::Carto::VisualizationQueryBuilder.new.with_shared_with_user_id(
|
||||
params[:owned_by_or_shared_with_user_id]).build.uniq.pluck('visualizations.id')
|
||||
query = if shared_with_viz_ids.empty?
|
||||
query.where(' "visualizations"."user_id" = (?)', params[:owned_by_or_shared_with_user_id])
|
||||
else
|
||||
query = query.where(' ("visualizations"."user_id" = (?) or "visualizations"."id" in (?))',
|
||||
params[:owned_by_or_shared_with_user_id], shared_with_viz_ids)
|
||||
end
|
||||
end
|
||||
|
||||
if params[:exclude_synced_external_sources]
|
||||
query = query.joins(%{
|
||||
LEFT JOIN external_sources es
|
||||
ON es.visualization_id = visualizations.id
|
||||
})
|
||||
.joins(%{
|
||||
LEFT JOIN external_data_imports edi
|
||||
ON edi.external_source_id = es.id
|
||||
AND (SELECT state FROM data_imports WHERE id = edi.data_import_id) <> 'failure'
|
||||
#{exclude_only_synchronized(params[:exclude_imported_remote_visualizations])}
|
||||
})
|
||||
.where("edi.id IS NULL")
|
||||
end
|
||||
|
||||
if params[:exclude_imported_remote_visualizations]
|
||||
# Right now only common-data public visualizations have display name setted so
|
||||
# the data-library visualizations have it too. So if we want to filter legacy remote
|
||||
# visualizations without display_name, we have to to this way.
|
||||
# We take into account other types and exclude from the display_name because the search
|
||||
# of datasets, for example, make a query with multiples types (table, remote) and we don't
|
||||
# want to filter the table ones
|
||||
query = query.where(
|
||||
%{
|
||||
"visualizations"."type" <> 'remote' OR
|
||||
("visualizations"."type" = 'remote' AND "visualizations"."display_name" IS NOT NULL)
|
||||
}.squish
|
||||
)
|
||||
end
|
||||
|
||||
if params[:excluded_kinds]
|
||||
params[:excluded_kinds].each do |kind|
|
||||
query = query.where("\"visualizations\".\"kind\" != '#{kind}'")
|
||||
end
|
||||
end
|
||||
|
||||
if params[:tainted_search_pattern]
|
||||
query = Carto::VisualizationQuerySearcher.new(query).search(params[:tainted_search_pattern])
|
||||
end
|
||||
|
||||
if params[:tags]
|
||||
params[:tags].each(&:downcase!)
|
||||
query = query.where(
|
||||
"array_to_string(visualizations.tags, ', ') ILIKE '%' || array_to_string(ARRAY[?]::text[], ', ') || '%'",
|
||||
params[:tags]
|
||||
)
|
||||
end
|
||||
|
||||
if params[:bounding_box]
|
||||
bbox_sql = Carto::BoundingBoxUtils.to_polygon(
|
||||
params[:bounding_box][:minx], params[:bounding_box][:miny],
|
||||
params[:bounding_box][:maxx], params[:bounding_box][:maxy]
|
||||
)
|
||||
query = query.where("visualizations.bbox IS NOT NULL AND visualizations.bbox && #{bbox_sql}")
|
||||
end
|
||||
|
||||
if params[:only_with_display_name]
|
||||
query = query.where("display_name IS NOT NULL")
|
||||
end
|
||||
|
||||
if params[:organization_id]
|
||||
query = query.joins(:user).where(users: { organization_id: params[:organization_id] })
|
||||
end
|
||||
|
||||
if params[:only_published]
|
||||
# "Published" is only required for builder maps
|
||||
# This SQL check should match Ruby `Carto::Visualization#published?` definition
|
||||
query = query.where(
|
||||
%{
|
||||
visualizations.privacy <> '#{Carto::Visualization::PRIVACY_PRIVATE}'
|
||||
AND ((visualizations.version <> #{Carto::Visualization::VERSION_BUILDER} OR visualizations.version IS NULL)
|
||||
OR visualizations.type <> '#{Carto::Visualization::TYPE_DERIVED}'
|
||||
OR EXISTS (SELECT 1 FROM mapcaps mc_pub WHERE visualizations.id = mc_pub.visualization_id LIMIT 1)
|
||||
)
|
||||
}.squish
|
||||
)
|
||||
end
|
||||
|
||||
query
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def exclude_only_synchronized(exclude_imported_remote_visualizations)
|
||||
"AND edi.synchronization_id IS NOT NULL" unless exclude_imported_remote_visualizations
|
||||
end
|
||||
|
||||
def recipient_ids(user)
|
||||
[user.id, user.organization_id].compact + groups_ids(user)
|
||||
end
|
||||
|
||||
def groups_ids(user)
|
||||
user.groups.nil? ? [] : user.groups.map(&:id)
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,52 @@
|
||||
require 'active_record'
|
||||
|
||||
class Carto::VisualizationQueryIncluder
|
||||
|
||||
def initialize(query)
|
||||
@query = query
|
||||
end
|
||||
|
||||
def include_dependent_visualization_count(params = {})
|
||||
join_sql = "LEFT OUTER JOIN (#{dependencies_query(params)}) AS dependencies ON dependencies.id = visualizations.id"
|
||||
@query.joins(join_sql)
|
||||
end
|
||||
|
||||
def include_favorited(user_id)
|
||||
@query.joins(
|
||||
%{
|
||||
LEFT JOIN likes
|
||||
ON "likes"."subject" = "visualizations"."id"
|
||||
AND "likes"."actor" = #{ActiveRecord::Base::sanitize(user_id)}
|
||||
}.squish
|
||||
)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def dependencies_query(params)
|
||||
select_count = 'count(distinct(visualizations.id, dependency_visualizations.id)) as dependent_visualization_count'
|
||||
query = Carto::Visualization.select(:id, select_count)
|
||||
.joins(dependency_joins)
|
||||
.where(dependency_visualizations: { type: 'derived' })
|
||||
.group(:id)
|
||||
filtered_query = Carto::VisualizationQueryFilterer.new(query).filter(params)
|
||||
filtered_query.to_sql
|
||||
end
|
||||
|
||||
def dependency_joins
|
||||
%{
|
||||
INNER JOIN "maps" "dependency_maps" ON "dependency_maps"."id" = "visualizations"."map_id"
|
||||
INNER JOIN "user_tables" "dependency_user_tables" ON "dependency_user_tables"."map_id" = "dependency_maps"."id"
|
||||
INNER JOIN "layers_user_tables" "dependency_layers_user_tables"
|
||||
ON "dependency_layers_user_tables"."user_table_id" = "dependency_user_tables"."id"
|
||||
INNER JOIN "layers" "dependency_layers"
|
||||
ON "dependency_layers"."id" = "dependency_layers_user_tables"."layer_id"
|
||||
INNER JOIN "layers_maps" "dependency_layers_maps"
|
||||
ON "dependency_layers_maps"."layer_id" = "dependency_layers"."id"
|
||||
INNER JOIN "maps" "dependency_maps_2" ON "dependency_maps_2"."id" = "dependency_layers_maps"."map_id"
|
||||
INNER JOIN "visualizations" "dependency_visualizations"
|
||||
ON "dependency_visualizations"."map_id" = "dependency_maps_2"."id"
|
||||
}.squish
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,63 @@
|
||||
require 'active_record'
|
||||
|
||||
class Carto::VisualizationQueryOrderer
|
||||
|
||||
DEFAULT_ORDER_DIRECTION = 'asc'.freeze
|
||||
SUPPORTED_OFFDATABASE_ORDERS = %w(size mapviews likes estimated_row_count).freeze
|
||||
VISUALIZATION_TABLE_ORDERS = %w(name updated_at privacy).freeze
|
||||
|
||||
DEPENDENT_VISUALIZATIONS_ORDER_CLAUSE = "coalesce(dependent_visualization_count, 0)".freeze
|
||||
FAVORITED_ORDER_CLAUSE = "(likes.actor IS NOT NULL)".freeze
|
||||
|
||||
def initialize(query)
|
||||
@query = query
|
||||
end
|
||||
|
||||
def order(order, direction = "")
|
||||
return @query unless order
|
||||
|
||||
prepare_order_params(order, direction)
|
||||
|
||||
if offdatabase_orders.empty?
|
||||
@query.order(database_orders)
|
||||
else
|
||||
Carto::OffdatabaseQueryAdapter.new(@query, offdatabase_orders)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def prepare_order_params(order_string, direction_string)
|
||||
directions = direction_string.split(',')
|
||||
orders = order_string.split(',')
|
||||
|
||||
@order_hash = orders.zip(directions).map { |order, direction|
|
||||
order = custom_order_clause(order)
|
||||
direction ||= directions[0] || DEFAULT_ORDER_DIRECTION
|
||||
[order, direction]
|
||||
}.to_h
|
||||
end
|
||||
|
||||
def custom_order_clause(order)
|
||||
case order
|
||||
when 'favorited'
|
||||
FAVORITED_ORDER_CLAUSE
|
||||
when 'dependent_visualizations'
|
||||
DEPENDENT_VISUALIZATIONS_ORDER_CLAUSE
|
||||
when *VISUALIZATION_TABLE_ORDERS
|
||||
"visualizations.#{order}"
|
||||
else
|
||||
order
|
||||
end
|
||||
end
|
||||
|
||||
def database_orders
|
||||
db_order_hash = @order_hash.except(*SUPPORTED_OFFDATABASE_ORDERS)
|
||||
db_order_hash.map { |key, value| "#{key} #{value}" }.join(",")
|
||||
end
|
||||
|
||||
def offdatabase_orders
|
||||
@order_hash.slice(*SUPPORTED_OFFDATABASE_ORDERS)
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,48 @@
|
||||
require 'active_record'
|
||||
|
||||
class Carto::VisualizationQuerySearcher
|
||||
|
||||
PATTERN_ESCAPE_CHARS = ['_', '%'].freeze
|
||||
|
||||
def initialize(query)
|
||||
@query = query
|
||||
end
|
||||
|
||||
def search(tainted_search_pattern)
|
||||
search_pattern = escape_characters_from_pattern(tainted_search_pattern)
|
||||
@query.where(partial_match_sql, search_pattern, "%#{search_pattern}%")
|
||||
.order("#{rank_sql(search_pattern)} DESC, visualizations.updated_at DESC")
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def escape_characters_from_pattern(pattern)
|
||||
pattern.chars.map { |c| PATTERN_ESCAPE_CHARS.include?(c) ? "\\" + c : c }.join
|
||||
end
|
||||
|
||||
def tsvector
|
||||
%{
|
||||
setweight(to_tsvector('english', coalesce("visualizations"."name",'')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(array_to_string(visualizations.tags, ''),'')), 'B') ||
|
||||
setweight(to_tsvector('english', coalesce("visualizations"."description",'')), 'C')
|
||||
}
|
||||
end
|
||||
|
||||
def rank_sql(search_pattern)
|
||||
%{
|
||||
ts_rank(
|
||||
#{tsvector},
|
||||
plainto_tsquery('english', #{ActiveRecord::Base.sanitize(search_pattern)})
|
||||
)
|
||||
}.squish
|
||||
end
|
||||
|
||||
def partial_match_sql
|
||||
%{
|
||||
#{tsvector} @@ plainto_tsquery('english', ?)
|
||||
OR CONCAT("visualizations"."name", array_to_string(visualizations.tags, ''), "visualizations"."description")
|
||||
ILIKE ?
|
||||
}.squish
|
||||
end
|
||||
|
||||
end
|
||||
Reference in New Issue
Block a user