Initial commit
This commit is contained in:
zhongjin
@@ -0,0 +1,47 @@
|
||||
require_relative '../../../lib/cartodb/stats/editor_apis'
|
||||
|
||||
class Api::ApplicationController < ApplicationController
|
||||
protect_from_forgery with: :null_session
|
||||
|
||||
# Don't force org urls
|
||||
skip_before_filter :ensure_org_url_if_org_user, :browser_is_html5_compliant?
|
||||
skip_before_filter :verify_authenticity_token, if: :json_formatted_request?
|
||||
|
||||
before_filter :api_authorization_required
|
||||
before_filter :ensure_account_has_been_activated
|
||||
|
||||
before_filter :setup_stats_instance
|
||||
|
||||
protected
|
||||
|
||||
def set_start_time
|
||||
@time_start = Time.now
|
||||
end
|
||||
|
||||
# dry up the jsonp output
|
||||
def render_jsonp(obj, status = 200, options = {})
|
||||
if callback_valid?
|
||||
options.reverse_merge! :json => obj, :status => status, :callback => params[:callback]
|
||||
else
|
||||
options.reverse_merge! :json => { errors: { callback: "Invalid callback format" } }, :status => 400
|
||||
end
|
||||
render options
|
||||
end
|
||||
|
||||
def setup_stats_instance
|
||||
@stats_aggregator = CartoDB::Stats::EditorAPIs.instance
|
||||
end
|
||||
|
||||
def valid_password_confirmation
|
||||
unless current_user.valid_password_confirmation(params[:password_confirmation])
|
||||
raise Carto::PasswordConfirmationError.new
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def callback_valid?
|
||||
# While only checks basic characters, represents most common use of JS function names
|
||||
params[:callback].nil? || !!(params[:callback] =~ /\A[$a-z_][0-9a-z_$]*\z/i)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,42 @@
|
||||
class Api::Json::AssetsController < Api::ApplicationController
|
||||
|
||||
ssl_required :create, :destroy
|
||||
|
||||
def create
|
||||
@stats_aggregator.timing('assets.create') do
|
||||
begin
|
||||
@asset = Asset.new
|
||||
@asset.raise_on_save_failure = true
|
||||
@asset.user_id = current_user.id
|
||||
@asset.asset_file = params[:filename]
|
||||
@asset.url = params[:url]
|
||||
@asset.kind = params[:kind]
|
||||
|
||||
@stats_aggregator.timing('save') do
|
||||
@asset.save
|
||||
end
|
||||
|
||||
render_jsonp(Carto::Api::AssetPresenter.new(@asset).to_hash)
|
||||
rescue Sequel::ValidationFailed => e
|
||||
CartoDB::Logger.warning(exception: e, message: 'Validation error creating asset')
|
||||
render json: { error: @asset.errors.full_messages }, status: 400
|
||||
rescue => e
|
||||
CartoDB::Logger.error(exception: e, message: 'Error creating asset')
|
||||
render json: { error: [e.message] }, status: 400
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
@stats_aggregator.timing('assets.destroy.delete') do
|
||||
begin
|
||||
Asset[params[:id]].destroy
|
||||
head :ok
|
||||
rescue => e
|
||||
CartoDB::Logger.error(exception: e, message: 'Error destroying asset')
|
||||
render json: { error: [e.message] }, status: 400
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,90 @@
|
||||
require Rails.root.join('services', 'sql-api', 'sql_api')
|
||||
|
||||
class Api::Json::GeocodingsController < Api::ApplicationController
|
||||
ssl_required :create, :update
|
||||
|
||||
before_filter :load_table, only: [:create, :estimation_for]
|
||||
|
||||
# In seconds
|
||||
GEOCODING_SQLAPI_CALLS_TIMEOUT = 45
|
||||
|
||||
def update
|
||||
@stats_aggregator.timing('geocodings.update') do
|
||||
|
||||
begin
|
||||
geocoding = current_user.geocodings_dataset.where(id: params[:id]).first
|
||||
return head(401) unless geocoding && params[:state] == 'cancelled'
|
||||
@stats_aggregator.timing('save') do
|
||||
geocoding.cancel
|
||||
end
|
||||
render_jsonp(geocoding.reload)
|
||||
rescue => e
|
||||
render_jsonp({ errors: e.message }, 400)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
def create
|
||||
@stats_aggregator.timing('geocodings.create') do
|
||||
|
||||
begin
|
||||
geocoding = Geocoding.new params.slice(:kind, :geometry_type, :formatter, :country_code, :region_code)
|
||||
geocoding.user = current_user
|
||||
geocoding.table_id = @table.try(:id)
|
||||
geocoding.table_name = params[:table_name] ? params[:table_name] : @table.try(:name)
|
||||
geocoding.raise_on_save_failure = true
|
||||
geocoding.force_all_rows = (params[:force_all_rows].to_s == 'true')
|
||||
|
||||
geocoding.formatter = "{#{ params[:column_name] }}" if params[:column_name].present?
|
||||
|
||||
geocoding = @stats_aggregator.timing('special-params') do
|
||||
# TODO api should be more regular
|
||||
unless ['high-resolution', 'ipaddress'].include? params[:kind] then
|
||||
if params[:text]
|
||||
countries = [params[:location]]
|
||||
else
|
||||
countries = @table.sequel.distinct.select_map(params[:location].to_sym)
|
||||
geocoding.country_column = params[:location]
|
||||
end
|
||||
geocoding.country_code = countries.map{|c| "'#{ c }'"}.join(',')
|
||||
|
||||
if params[:region]
|
||||
if params[:region_text]
|
||||
regions = [params[:region]]
|
||||
else
|
||||
regions = @table.sequel.distinct.select_map(params[:region].to_sym)
|
||||
geocoding.region_column = params[:region]
|
||||
end
|
||||
geocoding.region_code = regions.map{|r| "'#{ r }'"}.join(',')
|
||||
end
|
||||
end
|
||||
geocoding
|
||||
end
|
||||
|
||||
geocoding = @stats_aggregator.timing('save') do
|
||||
geocoding.save
|
||||
geocoding
|
||||
end
|
||||
|
||||
@table.automatic_geocoding.destroy if @table.automatic_geocoding.present?
|
||||
Resque.enqueue(Resque::GeocoderJobs, job_id: geocoding.id)
|
||||
|
||||
render_jsonp(geocoding.to_json)
|
||||
rescue Sequel::ValidationFailed => e
|
||||
CartoDB.notify_exception(e)
|
||||
render_jsonp( { description: e.message }, 422)
|
||||
rescue => e
|
||||
CartoDB.notify_exception(e)
|
||||
render_jsonp( { description: e.message }, 500)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def load_table
|
||||
@table = Carto::Helpers::TableLocator.new.get_by_id_or_name(params.fetch('table_name'), current_user).try(:service)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,233 @@
|
||||
require_relative '../../../helpers/file_upload'
|
||||
require_relative '../../../../services/datasources/lib/datasources'
|
||||
require_relative '../../../models/visualization/external_source'
|
||||
require_relative '../../../../services/platform-limits/platform_limits'
|
||||
require_relative '../../../../services/importer/lib/importer/exceptions'
|
||||
require_dependency 'carto/uuidhelper'
|
||||
require_dependency 'carto/url_validator'
|
||||
|
||||
class Api::Json::ImportsController < Api::ApplicationController
|
||||
include Carto::UUIDHelper
|
||||
include Carto::UrlValidator
|
||||
|
||||
ssl_required :create
|
||||
ssl_allowed :invalidate_service_token
|
||||
|
||||
# NOTE: When/If OAuth tokens management is built into the UI, remove this to send and check CSRF
|
||||
skip_before_filter :verify_authenticity_token, only: [:invalidate_service_token]
|
||||
|
||||
INVALID_TOKEN_MESSAGE = 'OAuth token invalid or expired'
|
||||
|
||||
# -------- Import process -------
|
||||
|
||||
def create
|
||||
@stats_aggregator.timing('imports.create') do
|
||||
|
||||
begin
|
||||
file_upload_helper = CartoDB::FileUpload.new(Cartodb.config[:importer].fetch("uploads_path", nil))
|
||||
|
||||
external_source = nil
|
||||
concurrent_import_limit =
|
||||
CartoDB::PlatformLimits::Importer::UserConcurrentImportsAmount.new({
|
||||
user: current_user,
|
||||
redis: { db: $users_metadata }
|
||||
})
|
||||
raise CartoDB::Importer2::UserConcurrentImportsLimitError.new if concurrent_import_limit.is_over_limit!
|
||||
|
||||
options = default_creation_options
|
||||
|
||||
if params[:url].present?
|
||||
validate_url!(params.fetch(:url)) unless Rails.env.development? || Rails.env.test?
|
||||
options[:data_source] = params.fetch(:url)
|
||||
elsif params[:connector].present?
|
||||
options[:service_name] = 'connector'
|
||||
options[:service_item_id] = params[:connector].to_json
|
||||
elsif params[:remote_visualization_id].present?
|
||||
external_source = external_source(params[:remote_visualization_id])
|
||||
options[:data_source] = external_source.import_url.presence
|
||||
else
|
||||
options = @stats_aggregator.timing('upload-or-enqueue') do
|
||||
results = file_upload_helper.upload_file_to_storage(
|
||||
filename_param: params[:filename],
|
||||
file_param: params[:file],
|
||||
request_body: request.body,
|
||||
s3_config: Cartodb.config[:importer]['s3'])
|
||||
|
||||
# In Rack < 1.6 / Rails < 4, tempfiles are not inmediately cleaned (https://github.com/rack/rack/pull/671).
|
||||
# Instead they stay around until a GC cycle which can take a while in instances with low traffic.
|
||||
# This forces the tempfile to be removed right away, just after we have saved it to our storage.
|
||||
[:filename, :file].each do |param_name|
|
||||
file = params[param_name]
|
||||
file.tempfile.close! if file
|
||||
end
|
||||
|
||||
# Not queued import is set by skipping pending state and setting directly as already enqueued
|
||||
options.merge({
|
||||
data_source: results[:file_uri].presence,
|
||||
state: results[:enqueue] ? DataImport::STATE_PENDING : DataImport::STATE_ENQUEUED
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
# override param to store as string
|
||||
user_limits = ::JSON.dump(options[:user_defined_limits])
|
||||
data_import = @stats_aggregator.timing('save') do
|
||||
DataImport.create(options.merge!({ user_defined_limits: user_limits }))
|
||||
end
|
||||
|
||||
if external_source.present?
|
||||
@stats_aggregator.timing('external-data-import.save') do
|
||||
ExternalDataImport.new(data_import.id, external_source.id).save
|
||||
end
|
||||
end
|
||||
|
||||
Resque.enqueue(Resque::ImporterJobs, job_id: data_import.id) if options[:state] == DataImport::STATE_PENDING
|
||||
|
||||
render_jsonp({ item_queue_id: data_import.id, success: true })
|
||||
rescue CartoDB::Importer2::UserConcurrentImportsLimitError
|
||||
rl_value = decrement_concurrent_imports_rate_limit
|
||||
render_jsonp({
|
||||
errors: { imports: "We're sorry but you're already using your allowed #{rl_value} import slots" }
|
||||
}, 429)
|
||||
rescue => ex
|
||||
decrement_concurrent_imports_rate_limit
|
||||
CartoDB::StdoutLogger.info('Error: create', "#{ex.message} #{ex.backtrace.inspect}")
|
||||
render_jsonp({ errors: { imports: ex.message } }, 400)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
# ----------- Import OAuths Management -----------
|
||||
|
||||
def invalidate_service_token
|
||||
@stats_aggregator.timing('imports.invalidate-service-token') do
|
||||
|
||||
begin
|
||||
oauth = current_user.oauths.select(params[:id])
|
||||
raise CartoDB::Datasources::AuthError.new("No oauth set for service #{params[:id]}") if oauth.nil?
|
||||
|
||||
datasource = oauth.get_service_datasource
|
||||
raise CartoDB::Datasources::AuthError.new("Couldn't fetch datasource for service #{params[:id]}") if datasource.nil?
|
||||
unless datasource.kind_of? CartoDB::Datasources::BaseOAuth
|
||||
raise CartoDB::Datasources::InvalidServiceError.new("Datasource #{params[:id]} does not support OAuth")
|
||||
end
|
||||
|
||||
result = @stats_aggregator.timing('revoke') do
|
||||
datasource.revoke_token
|
||||
end
|
||||
|
||||
if result
|
||||
@stats_aggregator.timing('remove') do
|
||||
current_user.oauths.remove(oauth.service)
|
||||
end
|
||||
end
|
||||
|
||||
render_jsonp({ success: true })
|
||||
rescue => ex
|
||||
CartoDB::StdoutLogger.info('Error: invalidate_service_token', "#{ex.message} #{ex.backtrace.inspect}")
|
||||
render_jsonp({ errors: { imports: ex.message } }, 400)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def default_creation_options
|
||||
user_defined_limits = params.fetch(:user_defined_limits, {})
|
||||
# Sanitize
|
||||
user_defined_limits[:twitter_credits_limit] =
|
||||
user_defined_limits[:twitter_credits_limit].presence.nil? ? 0 : user_defined_limits[:twitter_credits_limit].to_i
|
||||
|
||||
# Already had an internal issue due to forgetting to send always a string (e.g. for Twitter is an stringified JSON)
|
||||
raise "service_item_id field should be empty or a string" unless (params[:service_item_id].is_a?(String) ||
|
||||
params[:service_item_id].is_a?(NilClass))
|
||||
|
||||
# Keep in sync with https://docs.carto.com/cartodb-platform/import-api.html#params
|
||||
{
|
||||
user_id: current_user.id,
|
||||
table_name: params[:table_name].presence,
|
||||
# Careful as this field has rules (@see DataImport data_source=)
|
||||
data_source: nil,
|
||||
table_id: params[:table_id].presence,
|
||||
append: (params[:append].presence == 'true'),
|
||||
table_copy: params[:table_copy].presence,
|
||||
from_query: params[:sql].presence,
|
||||
service_name: params[:service_name].present? ? params[:service_name] : CartoDB::Datasources::Url::PublicUrl::DATASOURCE_NAME,
|
||||
service_item_id: params[:service_item_id].present? ? params[:service_item_id] : params[:url].presence,
|
||||
type_guessing: !["false", false].include?(params[:type_guessing]),
|
||||
quoted_fields_guessing: !["false", false].include?(params[:quoted_fields_guessing]),
|
||||
content_guessing: ["true", true].include?(params[:content_guessing]),
|
||||
state: DataImport::STATE_PENDING, # Pending == enqueue the task
|
||||
upload_host: Socket.gethostname,
|
||||
create_visualization: ["true", true].include?(params[:create_vis]),
|
||||
user_defined_limits: user_defined_limits,
|
||||
privacy: privacy,
|
||||
collision_strategy: params[:collision_strategy]
|
||||
}
|
||||
end
|
||||
|
||||
def decorate_twitter_import_data!(data, data_import)
|
||||
return if data_import.service_name != CartoDB::Datasources::Search::Twitter::DATASOURCE_NAME
|
||||
|
||||
audit_entry = ::SearchTweet.where(data_import_id: data_import.id).first
|
||||
data[:tweets_georeferenced] = audit_entry.retrieved_items
|
||||
data[:tweets_cost] = audit_entry.price
|
||||
data[:tweets_overquota] = audit_entry.user.remaining_twitter_quota == 0
|
||||
end
|
||||
|
||||
def decorate_default_visualization_data!(data, data_import)
|
||||
derived_vis_id = nil
|
||||
|
||||
if data_import.create_visualization && !data_import.visualization_id.nil?
|
||||
derived_vis_id = data_import.visualization_id
|
||||
end
|
||||
|
||||
data[:derived_visualization_id] = derived_vis_id
|
||||
end
|
||||
|
||||
def external_source(remote_visualization_id)
|
||||
external_source = Carto::ExternalSource.where(visualization_id: remote_visualization_id).first
|
||||
unless remote_visualization_id.present? && external_source.importable_by?(current_user)
|
||||
raise CartoDB::Datasources::AuthError.new('Illegal external load')
|
||||
end
|
||||
external_source
|
||||
end
|
||||
|
||||
def decrement_concurrent_imports_rate_limit
|
||||
begin
|
||||
concurrent_import_limit =
|
||||
CartoDB::PlatformLimits::Importer::UserConcurrentImportsAmount.new({
|
||||
user: current_user,
|
||||
redis: {
|
||||
db: $users_metadata
|
||||
}
|
||||
})
|
||||
# It's ok to decrease always as if over limit, will get just at limit and next try again go overlimit
|
||||
concurrent_import_limit.decrement!
|
||||
concurrent_import_limit.peek # return limit value
|
||||
rescue => sub_exception
|
||||
CartoDB::StdoutLogger.info('Error decreasing concurrent import limit',
|
||||
"#{sub_exception.message} #{sub_exception.backtrace.inspect}")
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def privacy
|
||||
if params[:privacy].present?
|
||||
privacy = Carto::UserTable::PRIVACY_VALUES_TO_TEXTS.invert[params[:privacy].downcase]
|
||||
if privacy.nil?
|
||||
valid_privacies = [
|
||||
Carto::UserTable::PRIVACY_VALUES_TO_TEXTS.values[0..-2].join(', '),
|
||||
Carto::UserTable::PRIVACY_VALUES_TO_TEXTS.values[-1]
|
||||
].join(' and ')
|
||||
raise "Unknown value '#{params[:privacy]}' for 'privacy'. Allowed values are: #{valid_privacies}"
|
||||
elsif !current_user.valid_privacy?(privacy)
|
||||
raise "Your account type (#{current_user.account_type.tr('[]', '')}) does not allow to create private "\
|
||||
"datasets. Check https://carto.com/pricing for more info."
|
||||
end
|
||||
privacy
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,20 @@
|
||||
class Api::Json::OrganizationsController < Api::ApplicationController
|
||||
include CartoDB
|
||||
|
||||
ssl_required :show, :users
|
||||
|
||||
# Fetch info from the current user orgranization
|
||||
def show
|
||||
render json: {}.to_json if current_user.organization.nil?
|
||||
|
||||
render json: current_user.organization.to_poro
|
||||
end
|
||||
|
||||
# Return user list of current user organization
|
||||
def users
|
||||
render json: {}.to_json if current_user.organization.nil?
|
||||
|
||||
render json: current_user.organization.to_poro[:users]
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,243 @@
|
||||
require 'json'
|
||||
require_relative '../../../models/synchronization/member'
|
||||
require_relative '../../../models/synchronization/collection'
|
||||
require_relative '../../../../services/datasources/lib/datasources'
|
||||
require_relative '../../../../services/platform-limits/platform_limits'
|
||||
require_dependency 'carto/url_validator'
|
||||
|
||||
class Api::Json::SynchronizationsController < Api::ApplicationController
|
||||
include CartoDB
|
||||
include Carto::UrlValidator
|
||||
|
||||
ssl_required :create, :update, :destroy, :sync, :sync_now
|
||||
|
||||
before_filter :set_external_source, only: [ :create ]
|
||||
|
||||
# Upon creation, no rate limit checks
|
||||
def create
|
||||
return head(401) unless current_user.sync_tables_enabled || @external_source
|
||||
|
||||
@stats_aggregator.timing('synchronizations.create') do
|
||||
|
||||
begin
|
||||
member_attributes = setup_member_attributes
|
||||
member = Synchronization::Member.new(member_attributes)
|
||||
member = @stats_aggregator.timing('member.save') do
|
||||
member.store
|
||||
end
|
||||
|
||||
options = setup_data_import_options(member_attributes, member.id)
|
||||
data_import = @stats_aggregator.timing('save') do
|
||||
DataImport.create(options)
|
||||
end
|
||||
|
||||
if @external_source
|
||||
@stats_aggregator.timing('external-data-import.save') do
|
||||
ExternalDataImport.new(data_import.id, @external_source.id, member.id).save
|
||||
end
|
||||
end
|
||||
|
||||
::Resque.enqueue(::Resque::ImporterJobs, job_id: data_import.id)
|
||||
|
||||
# Need to mark the synchronization job as queued state.
|
||||
# If this is missed there is an error state that can be
|
||||
# achieved where the synchronization job can never be
|
||||
# manually kicked off ever again. This state will occur if the
|
||||
# resque job fails to mark the synchronization state to success or
|
||||
# failure (ie: resque never runs, or bug in ImporterJobs code)
|
||||
member.state = Synchronization::Member::STATE_QUEUED
|
||||
member.store
|
||||
|
||||
response = {
|
||||
data_import: {
|
||||
endpoint: '/api/v1/imports',
|
||||
item_queue_id: data_import.id
|
||||
}
|
||||
}.merge(member.to_hash)
|
||||
|
||||
render_jsonp(response)
|
||||
rescue CartoDB::InvalidMember => exception
|
||||
render_jsonp({ errors: member.full_errors }, 400)
|
||||
puts exception.to_s
|
||||
puts exception.backtrace
|
||||
rescue CartoDB::InvalidInterval => exception
|
||||
render_jsonp({ errors: "#{exception.detail['message']}: #{exception.detail['hint']}" }, 400)
|
||||
rescue InvalidUrlError => exception
|
||||
CartoDB::StdoutLogger.info('Error: create', "#{exception.message} #{exception.backtrace.inspect}")
|
||||
render_jsonp({ errors: exception.message }, 400)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
def sync(from_sync_now=false)
|
||||
@stats_aggregator.timing('synchronizations.sync') do
|
||||
|
||||
begin
|
||||
enqueued = false
|
||||
member = Synchronization::Member.new(id: params[:id]).fetch
|
||||
return head(401) unless member.authorize?(current_user)
|
||||
|
||||
# @see /services/synchronizer/lib/synchronizer/collection.rb -> enqueue_rate_limited()
|
||||
if ( member.should_auto_sync? || (from_sync_now && member.can_manually_sync?) )
|
||||
platform_limit = CartoDB::PlatformLimits::Importer::UserConcurrentSyncsAmount.new({
|
||||
user: current_user, redis: { db: $users_metadata }
|
||||
})
|
||||
if platform_limit.is_within_limit?
|
||||
@stats_aggregator.timing('enqueue') do
|
||||
member.enqueue
|
||||
end
|
||||
enqueued = true
|
||||
platform_limit.increment!
|
||||
end
|
||||
end
|
||||
|
||||
render_jsonp( { enqueued: enqueued, synchronization_id: member.id})
|
||||
rescue => exception
|
||||
CartoDB.notify_exception(exception)
|
||||
head(404)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
def sync_now
|
||||
sync(true)
|
||||
end
|
||||
|
||||
def update
|
||||
@stats_aggregator.timing('synchronizations.update') do
|
||||
|
||||
begin
|
||||
member = Synchronization::Member.new(id: params.fetch('id')).fetch
|
||||
return head(401) unless member.authorize?(current_user)
|
||||
|
||||
member.attributes = payload
|
||||
member = @stats_aggregator.timing('save') do
|
||||
member.store.fetch
|
||||
end
|
||||
render_jsonp(member)
|
||||
rescue KeyError
|
||||
head(404)
|
||||
rescue CartoDB::InvalidMember
|
||||
render_jsonp({ errors: member.full_errors }, 400)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
@stats_aggregator.timing('synchronizations.destroy') do
|
||||
|
||||
begin
|
||||
member = Synchronization::Member.new(id: params.fetch('id')).fetch
|
||||
return(head 401) unless member.authorize?(current_user)
|
||||
|
||||
@stats_aggregator.timing('delete') do
|
||||
member.delete
|
||||
end
|
||||
|
||||
return head 204
|
||||
rescue KeyError
|
||||
head(404)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def set_external_source
|
||||
@external_source =
|
||||
if params[:remote_visualization_id].present?
|
||||
get_external_source(params[:remote_visualization_id])
|
||||
end
|
||||
end
|
||||
|
||||
def setup_member_attributes
|
||||
member_attributes = payload.merge(
|
||||
name: params[:table_name],
|
||||
user_id: current_user.id,
|
||||
state: Synchronization::Member::STATE_CREATED,
|
||||
# Keep in sync with https://carto.com/developers/import-api/guides/sync-tables/#params-1
|
||||
type_guessing: !["false", false].include?(params[:type_guessing]),
|
||||
quoted_fields_guessing: !["false", false].include?(params[:quoted_fields_guessing]),
|
||||
content_guessing: ["true", true].include?(params[:content_guessing])
|
||||
)
|
||||
|
||||
if from_sync_file_provider?
|
||||
member_attributes = member_attributes.merge({
|
||||
service_name: params[:service_name],
|
||||
service_item_id: params[:service_item_id]
|
||||
})
|
||||
end
|
||||
|
||||
if params[:remote_visualization_id].present?
|
||||
member_attributes[:interval] = Carto::ExternalSource::REFRESH_INTERVAL
|
||||
external_source = @external_source
|
||||
member_attributes[:url] = external_source.import_url.presence
|
||||
member_attributes[:service_item_id] = external_source.import_url.presence
|
||||
end
|
||||
|
||||
if params[:connector].present?
|
||||
member_attributes[:service_name] = 'connector'
|
||||
member_attributes[:service_item_id] = params[:connector].to_json
|
||||
end
|
||||
|
||||
member_attributes
|
||||
end
|
||||
|
||||
def setup_data_import_options(member_attributes, member_id)
|
||||
if from_sync_file_provider?
|
||||
service_name = params[:service_name]
|
||||
service_item_id = params[:service_item_id]
|
||||
else
|
||||
service_name = CartoDB::Datasources::Url::PublicUrl::DATASOURCE_NAME
|
||||
service_item_id = params[:url].presence
|
||||
end
|
||||
|
||||
options = {
|
||||
user_id: current_user.id,
|
||||
table_name: params[:table_name].presence,
|
||||
service_name: service_name,
|
||||
service_item_id: service_item_id,
|
||||
type_guessing: member_attributes[:type_guessing],
|
||||
quoted_fields_guessing: member_attributes[:quoted_fields_guessing],
|
||||
content_guessing: member_attributes[:content_guessing],
|
||||
create_visualization: ["true", true].include?(params[:create_vis])
|
||||
}
|
||||
|
||||
if params[:remote_visualization_id].present?
|
||||
external_source = get_external_source(params[:remote_visualization_id])
|
||||
options.merge!(data_source: external_source.import_url.presence)
|
||||
elsif params[:connector].present?
|
||||
options[:service_name] = 'connector'
|
||||
options[:service_item_id] = params[:connector].to_json
|
||||
else
|
||||
url = params[:url]
|
||||
validate_url!(url) unless Rails.env.development? || Rails.env.test? || url.nil? || url.empty?
|
||||
options.merge!(data_source: url)
|
||||
end
|
||||
|
||||
options.merge!({ synchronization_id: member_id })
|
||||
|
||||
options
|
||||
end
|
||||
|
||||
def from_sync_file_provider?
|
||||
params.include?(:service_name) && params.include?(:service_item_id)
|
||||
end
|
||||
|
||||
def payload
|
||||
request.body.rewind
|
||||
::JSON.parse(request.body.read.to_s || String.new)
|
||||
end
|
||||
|
||||
def get_external_source(remote_visualization_id)
|
||||
external_source = Carto::ExternalSource.where(visualization_id: remote_visualization_id).first
|
||||
unless remote_visualization_id.present? && external_source.importable_by?(current_user)
|
||||
raise CartoDB::Datasources::AuthError.new('Illegal external load')
|
||||
end
|
||||
external_source
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,50 @@
|
||||
class Api::Json::UploadsController < Api::ApplicationController
|
||||
|
||||
ssl_required :create
|
||||
|
||||
skip_before_filter :verify_authenticity_token
|
||||
before_filter :api_or_user_authorization_required
|
||||
|
||||
def create
|
||||
@stats_aggregator.timing('uploads.create') do
|
||||
|
||||
begin
|
||||
temp_file = filename = filedata = nil
|
||||
|
||||
case
|
||||
when params[:filename].present? && request.body.present?
|
||||
filename = params[:filename]
|
||||
filedata = request.body.read.force_encoding('utf-8')
|
||||
when params[:file].present?
|
||||
filename = params[:file].original_filename
|
||||
filedata = params[:file].read.force_encoding('utf-8')
|
||||
end
|
||||
|
||||
random_token = Digest::SHA2.hexdigest("#{Time.now.utc}--#{filename.object_id.to_s}").first(20)
|
||||
|
||||
file_upload_helper = CartoDB::FileUpload.new(Cartodb.config[:importer].fetch("uploads_path", nil))
|
||||
file_upload_helper.get_uploads_path
|
||||
|
||||
@stats_aggregator.timing('save') do
|
||||
FileUtils.mkdir_p(file_upload_helper.get_uploads_path.join(random_token))
|
||||
file = File.new(file_upload_helper.get_uploads_path.join(random_token).join(File.basename(filename)), 'w')
|
||||
file.write filedata
|
||||
file.close
|
||||
end
|
||||
|
||||
render :json => {:file_uri => file.path[/(\/uploads\/.*)/, 1], :success => true}
|
||||
rescue => e
|
||||
logger.error e
|
||||
logger.error e.backtrace
|
||||
head(400)
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
def api_or_user_authorization_required
|
||||
api_authorization_required || login_required
|
||||
end
|
||||
private :api_or_user_authorization_required
|
||||
|
||||
end
|
||||
@@ -0,0 +1,16 @@
|
||||
require 'uri'
|
||||
require 'json'
|
||||
require_relative '../../../../services/wms/proxy'
|
||||
|
||||
class Api::Json::WmsController < Api::ApplicationController
|
||||
ssl_required :index
|
||||
ssl_allowed :proxy
|
||||
|
||||
def proxy
|
||||
proxy = CartoDB::WMS::Proxy.new(params.fetch(:url))
|
||||
render_jsonp(proxy.serialize)
|
||||
rescue URI::InvalidURIError => exception
|
||||
render_jsonp({ errors: "Couldn't load URL" }, 400)
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user