Initial commit

This commit is contained in:
zhongjin
2020-06-15 10:58:47 +08:00
commit 4f1dfe7564
8590 changed files with 1516878 additions and 0 deletions
@@ -0,0 +1,204 @@
module CartoDB
module PlatformLimits
# Abstract limit class, providing scaffolding and common logic.
#
# Each limit should specify here the usage and expected contents of 'context' parameter used in most methods.
class AbstractLimit
TYPE_USER = 'u'
TYPE_IP = 'i'
def self.classname
self.name.gsub(/.*::/, '').to_sym
end
# Class constructor
# @param options Hash
# {
# :user => ::User|nil
# :ip => Integer|nil
# :initial_value => mixed|nil
# :max_value => mixed|nil
# :ttl => Integer|nil
# }
# @throws ArgumentError
def initialize(options={})
@user = options.fetch(:user, nil)
@ip = options.fetch(:ip, nil)
raise ArgumentError.new('options must be a Hash') unless options.is_a?(Hash)
raise ArgumentError.new('Must supply at least a user or an IP address') if @user.nil? && @ip.nil?
unless @user.nil?
raise ArgumentError.new('Supplied user object must have :username') unless @user.respond_to?(:username)
end
@initial_value = options.fetch(:initial_value, nil)
@max_value = options.fetch(:max_value, nil)
@time_frame = options.fetch(:ttl, nil)
end
# Checks if user is over limit, increasing the internal counter (where proceeds)
# @param context mixed|nil
# @return bool
def is_over_limit!(context=nil)
increase(context)
is_over_limit(context)
end
# Checks if user is over limit without increasing the internal counter
# @param context mixed|nil
# @return bool
def is_over_limit?(context=nil)
is_over_limit(context)
end
# Checks if user is within the allowed limit, increasing the internal counter (where proceeds)
# @param context mixed|nil
# @return bool
def is_within_limit!(context=nil)
!is_over_limit!(context)
end
# Checks if user is within the allowed limit without increasing the internal counter
# @param context mixed|nil
# @return bool
def is_within_limit?(context=nil)
!is_over_limit?(context)
end
# Peeks the current value of the limit without increasing the internal counter
# @param context mixed|nil
# @return mixed
def peek(context=nil)
get(context)
end
# Increments the limit
# @param context mixed|nil
# @param amount integer
def increment!(context=nil, amount=1)
increase(context, amount)
end
# Decrements the limit
# @param context mixed|nil
# @param amount integer
def decrement!(context=nil, amount=1)
decrease(context, amount)
end
# Gets (where proceeds) the remaining count for the limit
# Useful for a "X-Rate-Limit-Remaining" header
# @param context mixed|nil
# @return mixed|nil
def remaining_limit?(context=nil)
max = get_maximum(context)
current = get(context)
(max.nil? || current.nil?) ? nil : max - current
end
# Gets (where proceeds) the maximum value until hitting the limit
# Useful for a "X-Rate-Limit-Limit" header
# @param context mixed|nil
# @return mixed|nil
def maximum_limit?(context=nil)
get_maximum(context)
end
# Gets (where proceeds) the timestamp when the limit will expire/reset
# Useful for a "X-Rate-Limit-Reset" header
# @param context mixed|nil
# @return integer|nil
def time_period?(context=nil)
get_time_period(context)
end
# Returns the key for this limit instance
# e.g. limits:Importer:InputFileSize:u:cartodb_user
# e.g. limits:Security::LoginAttempts:ui:cartodb_user127.0.0.1
# @return string
def key
"limits:#{subkey}:#{user_ip_key_fragment}"
end
# TODO: Revisit this methods, might not be needed hidden and just provided with default logic and overriden
protected
attr_accessor :user, :ip, :initial_value, :max_value, :time_frame, :current_value
# Builds the subkey part that identifies current limit
# Must have format 'xxx:yyy' Where xxx is the limit group and yyy the limit name.
# e.g. Importer:InputFileSize
def subkey
raise "Implement at child classes"
end
# @param context mixed
# @return bool
def is_over_limit(context)
raise "Implement at child classes"
end
# Gets current value of the limit
# @param context mixed
# @return mixed
def get(context)
raise "Implement at child classes"
end
# Gets the maximum limit value
# @param context mixed
# @return mixed
def get_maximum(context)
raise "Implement at child classes"
end
# Gets when the limit expires
# @param context mixed
# @return integer|nil Timestamp
def get_time_period(context)
raise "Implement at child classes"
end
# Increases the limit
# @param context mixed
# @param amount integer
def increase(context, amount=1)
raise "Implement at child classes"
end
# Decreases the limit
# @param context mixed
# @param amount integer
def decrease(context, amount=1)
raise "Implement at child classes"
end
# Resets the limit
def expire
raise "Implement at child classes"
end
private
# Builds the key fragment that indicates if limit is based on user, ip or both (and their values)
# @return string
def user_ip_key_fragment
type_fragment = ''
value_fragment = ''
unless @user.nil?
type_fragment << TYPE_USER
value_fragment << @user.username
end
unless @ip.nil?
type_fragment << TYPE_IP
value_fragment << @ip.to_s
end
"#{type_fragment}:#{value_fragment}"
end
end
end
end
@@ -0,0 +1,86 @@
module CartoDB
module PlatformLimits
module Importer
# This limit uses User.max_import_file_size attribute to limit file sizes upon imports.
# Has no storage as there's no need of it, and serves as a really simple Limit example.
#
# 'context' is expected to be an integer containing the file size in bytes
class InputFileSize < AbstractLimit
# This limit needs additional fields present at options Hash:
# :user (already defined, but mandatory)
# @see CartoDB::PlatformLimits::AbstractLimit initialize()
# @throws ArgumentError
def initialize(options={})
super(options)
raise ArgumentError.new('Must supply a user object') if user.nil?
unless user.respond_to?(:max_import_file_size)
raise ArgumentError.new('Supplied user object must have :max_import_file_size')
end
unless user.max_import_file_size.is_a?(Integer) && user.max_import_file_size > 0
raise ArgumentError.new('invalid user max_import_file_size (must be positive integer)')
end
self.max_value = user.max_import_file_size
end
protected
def subkey
'Importer:InputFileSize'
end
# @param context mixed
# @return bool
def is_over_limit(context)
get(context) > max_value
end
# Gets current value of the limit
# @param context mixed
# @return mixed
# @throws ArgumentError
def get(context)
raise ArgumentError.new('context must be an integer') unless context.is_a?(Integer)
context
end
# Gets the maximum limit value
# @param context mixed
# @return mixed
def get_maximum(context)
max_value
end
# Gets when the limit expires
# @param context mixed
# @return integer|nil Timestamp
def get_time_period(context)
nil
end
# Increases the limit
# @param context mixed
# @param amount integer
def increase(context, amount=1)
# Not useful here
end
# Decreases the limit
# @param context mixed
# @param amount integer
def decrease(context, amount=1)
# Not useful here
end
# Resets the limit
def expire
# Does nothing
end
end
end
end
end
@@ -0,0 +1,112 @@
module CartoDB
module PlatformLimits
module Importer
# This limit uses User.max_import_table_row_count attribute to limit table size regarding row count.
# Has no storage.
#
# 'context' is expected to be:
# Hash {
# :table_name String
# :tables_schema String|nil (Optional, defaults to import schema 'cdb_importer')
# }
class TableRowCount < AbstractLimit
# Where to search for the table if no schema specified at context
DEFAULT_SCHEMA = 'cdb_importer'
# This limit needs additional fields present at options Hash:
# :db
# :user (already defined, but mandatory)
# @see CartoDB::PlatformLimits::AbstractLimit initialize()
# @throws ArgumentError
def initialize(options={})
super(options)
raise ArgumentError.new('Must supply a user object') if user.nil?
unless user.respond_to?(:max_import_table_row_count)
raise ArgumentError.new('Supplied user object must have :max_import_table_row_count')
end
unless user.max_import_table_row_count.is_a?(Integer) && user.max_import_file_size > 0
raise ArgumentError.new('invalid user max_import_table_row_count (must be positive integer)')
end
self.max_value = user.max_import_table_row_count
@db = options.fetch(:db, nil)
raise ArgumentError.new('Must supply db connection object') if @db.nil?
end
protected
attr_accessor :db
def subkey
'Importer:TableRowCount'
end
# @param context mixed
# @return bool
def is_over_limit(context)
get(context) > max_value
end
# Gets current value of the limit
# @param context mixed
# Hash {
# :table_name String
# :tables_schema String|nil (Optional, defaults to import schema)
# }
# @return mixed
# @throws ArgumentError
def get(context)
raise ArgumentError.new('context must be a hash') unless context.is_a?(Hash)
table_name = context.fetch(:table_name, nil)
raise ArgumentError.new('context must be a hash') if table_name.nil?
schema_name = context.fetch(:tables_schema, DEFAULT_SCHEMA)
db.fetch(%Q{
SELECT reltuples::bigint AS row_count
FROM pg_class
WHERE oid='#{schema_name}.#{table_name}'::regclass;
})
.first[:row_count]
end
# Gets the maximum limit value
# @param context mixed
# @return mixed
def get_maximum(context)
max_value
end
# Gets when the limit expires
# @param context mixed
# @return integer|nil Timestamp
def get_time_period(context)
nil
end
# Increases the limit
# @param context mixed
# @param amount integer
def increase(context, amount=1)
# Not useful here
end
# Decreases the limit
# @param context mixed
# @param amount integer
def decrease(context, amount=1)
# Not useful here
end
# Resets the limit
def expire
# Does nothing
end
end
end
end
end
@@ -0,0 +1,139 @@
module CartoDB
module PlatformLimits
module Importer
UNLIMITED_STATEMENT_TIMEOUT_TTL = 30*24*60*60*1000
# If unspecified at constructor, will default to this value to add a threshold to the key TTL
DEFAULT_EXPIRE_TTL_THRESHOLD_PERCENT = 0.2
# This limit uses User.max_concurrent_import_count attribute to limit how many imports a user can have at once.
# It is not perfect as a safety measure the import will be considered imported after a given time
# (user statement timeout *2) to avoid stale keys in case of errors
# Uses Redis storage.
#
# 'context' is unused in this limit
class UserConcurrentImportsAmount < AbstractLimit
# This limit needs additional fields present at options Hash:
# :redis Hash {
# :db Redis
# :expire_ttl_threshold_percent Integer|nil
# }
# :user (already defined, but mandatory)
# @see CartoDB::PlatformLimits::AbstractLimit initialize()
# @throws ArgumentError
def initialize(options={})
super(options)
raise ArgumentError.new('Must supply a user object') if user.nil?
unless user.respond_to?(:max_concurrent_import_count)
raise ArgumentError.new('Supplied user object must have :max_concurrent_import_count')
end
unless user.max_concurrent_import_count.is_a?(Integer) && user.max_concurrent_import_count > 0
raise ArgumentError.new('invalid user max_concurrent_import_count (must be positive integer)')
end
self.max_value = user.max_concurrent_import_count
unless user.respond_to?(:database_timeout)
raise ArgumentError.new('Supplied user object must have :database_timeout')
end
unless user.database_timeout.is_a?(Integer)
raise ArgumentError.new('invalid user database_timeout (must be integer)')
end
timeout_to_use = user.database_timeout == 0 ? UNLIMITED_STATEMENT_TIMEOUT_TTL : user.database_timeout
redis_options = options.fetch(:redis, {})
@redis = redis_options.fetch(:db, nil)
raise ArgumentError.new('Must supply redis connection object') if @redis.nil?
@expire_ttl = ( timeout_to_use / 1000 *
(1.0 + redis_options.fetch(:expire_ttl_threshold_percent, DEFAULT_EXPIRE_TTL_THRESHOLD_PERCENT)) ).to_i
end
protected
attr_accessor :redis, :expire_ttl
def subkey
'Importer:UserConcurrentImportsAmount'
end
# @param context mixed
# @return bool
def is_over_limit(context)
get(context) > max_value
end
# Gets current value of the limit
# @param context mixed
# @return mixed
# @throws ArgumentError
def get(context)
redis.llen(key)
end
# Gets the maximum limit value
# @param context mixed
# @return mixed
def get_maximum(context)
max_value
end
# Gets when the limit expires
# @param context mixed
# @return integer|nil Timestamp
def get_time_period(context)
remaining_secs = redis.ttl(key)
remaining_secs.seconds.from_now
end
# Increases the limit
# @remark Will always increment by 1, no matter the value of the parameter
# @param context mixed
# @param amount integer
def increase(context, amount=1)
if redis.exists(key)
redis.rpushx(key, user.username)
redis.multi do
(amount-1).times do
redis.rpushx(key, user.username)
end
end if amount > 1
else
redis.multi do
redis.rpush(key, user.username)
redis.expire(key, expire_ttl)
end
redis.multi do
(amount-1).times do
redis.rpushx(key, user.username)
end
end if amount > 1
end
end
# Decreases the limit
# @param context mixed
# @param amount integer
def decrease(context, amount=1)
amount.times do
if redis.exists(key)
redis.rpop(key)
end
end
end
# Resets the limit
def expire
redis.del(key)
end
private
end
end
end
end
@@ -0,0 +1,129 @@
module CartoDB
module PlatformLimits
module Importer
# This limit controls how many synchronizations a user can have at once.
# Uses Redis storage.
#
# 'context' is unused in this limit
class UserConcurrentSyncsAmount < AbstractLimit
# In seconds
KEY_TTL = 2*60*60
MAX_SYNCS_PER_USER = 3
# This limit needs additional fields present at options Hash:
# :redis Hash {
# :db Redis
# }
# :user (already defined, but mandatory)
# @see CartoDB::PlatformLimits::AbstractLimit initialize()
# @throws ArgumentError
def initialize(options={})
super(options)
raise ArgumentError.new('Must supply a user object') if user.nil?
self.max_value = MAX_SYNCS_PER_USER
redis_options = options.fetch(:redis, {})
@redis = redis_options.fetch(:db, nil)
raise ArgumentError.new('Must supply redis connection object') if @redis.nil?
@expire_ttl = KEY_TTL
end
protected
attr_accessor :redis, :expire_ttl
def subkey
'Importer:UserConcurrentSyncsAmount'
end
# @param context mixed
# @return bool
def is_over_limit(context)
get(context) >= max_value
end
# Gets current value of the limit
# @param context mixed
# @return mixed
# @throws ArgumentError
def get(context)
redis.llen(key)
end
# Gets the maximum limit value
# @param context mixed
# @return mixed
def get_maximum(context)
max_value
end
# Gets when the limit expires
# @param context mixed
# @return integer|nil Timestamp
def get_time_period(context)
remaining_secs = redis.ttl(key)
remaining_secs.seconds.from_now
end
# Increases the limit
# @param context mixed
# @param amount integer
def increase(context, amount=1)
actual_amount = [max_value - get(context), amount].min
return if actual_amount <= 0
if redis.exists(key)
redis.rpushx(key, user.username)
redis.multi do
(actual_amount-1).times do
redis.rpushx(key, user.username)
end
end if actual_amount > 1
else
redis.multi do
redis.rpush(key, user.username)
redis.expire(key, expire_ttl)
end
redis.multi do
(actual_amount-1).times do
redis.rpushx(key, user.username)
end
end if actual_amount > 1
end
end
# Decreases the limit
# @param context mixed
# @param amount integer
def decrease(context, amount=1)
current_amount = get(context)
actual_amount = [current_amount, amount].min
return if actual_amount <= 0
actual_amount.times do
if redis.exists(key)
redis.rpop(key)
end
end
end
# Resets the limit
def expire
redis.del(key)
end
private
end
end
end
end
@@ -0,0 +1,5 @@
require_relative './lib/abstract_limit'
require_relative './lib/importer/input_file_size'
require_relative './lib/importer/table_row_count'
require_relative './lib/importer/user_concurrent_imports_amount'
require_relative './lib/importer/user_concurrent_syncs_amount'
@@ -0,0 +1,14 @@
module CartoDB
module PlatformLimits
module Doubles
class User
def initialize(attributes={})
@username = attributes.fetch(:username, 'wadus')
@max_import_file_size = attributes.fetch(:max_import_file_size, nil)
end
attr_reader :username, :max_import_file_size
end
end
end
end
@@ -0,0 +1,128 @@
require_relative '../../platform_limits'
require_relative '../doubles/user'
require 'uuidtools'
include CartoDB::PlatformLimits
class EmptyClass; end
describe Importer::InputFileSize do
describe "#methods" do
it "new(), maximum_limit?(), key()" do
max_value = 50*1024*1024
fake_user_1 = CartoDB::PlatformLimits::Doubles::User.new({username: 'test_1', max_import_file_size: max_value })
fake_user_2 = CartoDB::PlatformLimits::Doubles::User.new({username: 'test_1', max_import_file_size: max_value })
options_1 = {
user: fake_user_1
}
options_2 = {
user: fake_user_2,
ip: 12345,
initial_value: 1,
max_value: 5, # Should get overriden
ttl: 10
}
expect {
Importer::InputFileSize.new({})
}.to raise_exception ArgumentError
expect {
Importer::InputFileSize.new({ user: EmptyClass.new })
}.to raise_exception ArgumentError
expect {
Importer::InputFileSize.new({ user: CartoDB::PlatformLimits::Doubles::User.new({username: 'invalid', max_import_file_size: -1}) })
}.to raise_exception ArgumentError
expect {
Importer::InputFileSize.new({ user: CartoDB::PlatformLimits::Doubles::User.new({username: 'invalid', max_import_file_size: 0}) })
}.to raise_exception ArgumentError
expect {
Importer::InputFileSize.new({ user: CartoDB::PlatformLimits::Doubles::User.new({username: 'invalid', max_import_file_size: nil}) })
}.to raise_exception ArgumentError
instance = Importer::InputFileSize.new(options_1)
instance.maximum_limit?.should eq max_value
instance.send(:ip).should eq nil
instance.send(:user).should eq options_1[:user]
instance.send(:initial_value).should eq nil
instance.send(:max_value).should eq max_value
instance.send(:time_frame).should eq nil
instance.send(:current_value).should eq nil
instance.key.should eq "limits:Importer:InputFileSize:u:#{options_1[:user].username}"
instance = Importer::InputFileSize.new(options_2)
instance.maximum_limit?.should eq max_value
instance.send(:ip).should eq options_2[:ip]
instance.send(:user).should eq options_2[:user]
instance.send(:initial_value).should eq options_2[:initial_value]
instance.send(:max_value).should eq max_value
instance.send(:time_frame).should eq options_2[:ttl]
instance.send(:current_value).should eq nil
instance.key.should eq "limits:Importer:InputFileSize:ui:#{options_2[:user].username}#{options_2[:ip]}"
end
it "is_over_limit?/! and is_within_limit?/!" do
max_value = 50*1024*1024
options = {
user: CartoDB::PlatformLimits::Doubles::User.new({username: 'test_1', max_import_file_size: max_value })
}
instance = Importer::InputFileSize.new(options)
ok_value = 10
limit_value = instance.maximum_limit?
exceeded_value = max_value + 1
expect {
instance.is_over_limit?('a')
}.to raise_exception ArgumentError
expect {
instance.is_over_limit?('1')
}.to raise_exception ArgumentError
expect {
instance.is_over_limit?(2.0)
}.to raise_exception ArgumentError
expect {
instance.is_over_limit?(nil)
}.to raise_exception ArgumentError
expect {
instance.is_over_limit?([2])
}.to raise_exception ArgumentError
expect {
instance.is_over_limit?({ 1 => 2})
}.to raise_exception ArgumentError
instance.is_over_limit?(0).should eq false
instance.is_over_limit!(0).should eq false
instance.is_within_limit?(0).should eq true
instance.is_within_limit!(0).should eq true
instance.is_over_limit?(ok_value).should eq false
instance.is_over_limit!(ok_value).should eq false
instance.is_within_limit?(ok_value).should eq true
instance.is_within_limit!(ok_value).should eq true
instance.is_over_limit?(limit_value).should eq false
instance.is_over_limit!(limit_value).should eq false
instance.is_within_limit?(limit_value).should eq true
instance.is_within_limit!(limit_value).should eq true
instance.is_over_limit?(exceeded_value).should eq true
instance.is_over_limit!(exceeded_value).should eq true
instance.is_within_limit?(exceeded_value).should eq false
instance.is_within_limit!(exceeded_value).should eq false
end
end
end
@@ -0,0 +1,2 @@
Other testing can be located at /cartodb/spec/ subfolders (/requests/ , /models/platform-limits , ...), as they cover
end to end scenarios or at least don't require mocking way too many things.