cartodb4/cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
zhongjin
2020-06-15 10:58:47 +08:00
commit 4f1dfe7564
8590 changed files with 1516878 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
script/cdb_import.sh Executable file
View File

@@ -0,0 +1,80 @@
#!/bin/bash
# Disclaimer:
# We are not actively maintaining this script. We can't assure it will work, but we'll do our best to keep it updated.
# Credits:
# Original author: https://gist.github.com/lbosque/5876697
# Contributors:
# https://gist.github.com/andrewxhill/5884845 Mac-compatible version
# https://gist.github.com/kentr / http://maplight.org/ new uuid format bugfix + email notification
CDB_USER=$1
API_KEY=$2
IMPORT_FILE=$3
NOTIFICATION_EMAIL=$4
PROTOCOL=https
DEBUG=true
ITEM_ID_REGEX='\"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\"'
if [[ -z $CDB_USER ]]
then
echo "Missing user"
exit 1
fi
if [[ -z $API_KEY ]]
then
echo "Missing api key"
exit 1
fi
if [[ -z $IMPORT_FILE ]]
then
echo "Missing file"
exit 1
fi
function log {
if [[ ${DEBUG} == true ]]
then
echo $1
fi
}
v1=$(uname)
log "Sending file '${IMPORT_FILE}'"
if [[ "$v1" = Darwin ]];
then
job_id=`curl -s -F file=@${IMPORT_FILE} "${PROTOCOL}://${CDB_USER}.carto.com/api/v1/imports/?api_key=${API_KEY}" | sed -E "s/\{\"item_queue_id\":${ITEM_ID_REGEX}.*/\1/"`
else
job_id=`curl -s -F file=@${IMPORT_FILE} "${PROTOCOL}://${CDB_USER}.carto.com/api/v1/imports/?api_key=${API_KEY}" | sed -r "s/\{\"item_queue_id\":${ITEM_ID_REGEX}.*/\1/"`
fi
log "Waiting for job '${job_id}' to be completed"
while true
do
if [[ "$v1" = Darwin ]];
then
status=`curl -s "${PROTOCOL}://${CDB_USER}.carto.com/api/v1/imports/${job_id}?api_key=${API_KEY}" | sed -E 's/(.*)\"state\":\"([a-z]+)\"(.*)/\2/'`
else
status=`curl -s "${PROTOCOL}://${CDB_USER}.carto.com/api/v1/imports/${job_id}?api_key=${API_KEY}" | sed -r 's/(.*)\"state\":\"([a-z]+)\"(.*)/\2/'`
fi
log "JOB '${job_id}' STATE: ${status}"
if [[ -n $NOTIFICATION_EMAIL ]]
then
log "${PROTOCOL}://${CDB_USER}.carto.com" | mail -s "CartoDB import finished: ${IMPORT_FILE}" "${NOTIFICATION_EMAIL}"
fi
if [[ $status == 'complete' ]]
then
log "Import successful"
exit 0
elif [[ $status == 'failure' ]]
then
log "Failed import"
exit 1
fi
sleep 2
done

51
script/ci/cleaner.sh Executable file
View File

@@ -0,0 +1,51 @@
#!/bin/bash
# Kill redis
killall redis-server
# Drop all databases
databases=$(psql -U postgres -t -c "select datname from pg_database where datname like 'carto_db_test_%'")
touch parallel_tests/databases.log
echo $databases >> parallel_tests/databases.log
touch parallel_tests/databases_new.log
sed -e 's/\s\+/\n/g' parallel_tests/databases.log > parallel_tests/databases_new.log
while read -r line
do
psql -U postgres -t -c "drop database $line" >> parallel_tests/cleaner.log
done < parallel_tests/databases_new.log
# Drop all user databases
databases=$(psql -U postgres -t -c "select datname from pg_database where datname like 'cartodb_test_user_%'")
touch parallel_tests/user_databases.log
echo $databases >> parallel_tests/user_databases.log
touch parallel_tests/user_databases_new.log
sed -e 's/\s\+/\n/g' parallel_tests/user_databases.log > parallel_tests/user_databases_new.log
while read -r line
do
psql -U postgres -t -c "drop database \"$line\"" >> parallel_tests/cleaner.log
done < parallel_tests/user_databases_new.log
rm parallel_tests/user_databases.log
rm parallel_tests/users_databases_new.log
# Drop all testing databases
databases=$(psql -U postgres -t -c "select datname from pg_database where datname like 'cartodb_user_%'")
touch parallel_tests/user_databases.log
echo $databases >> parallel_tests/user_databases.log
touch parallel_tests/user_databases_new.log
sed -e 's/\s\+/\n/g' parallel_tests/user_databases.log > parallel_tests/user_databases_new.log
while read -r line
do
psql -U postgres -t -c "drop database \"$line\"" >> parallel_tests/cleaner.log
done < parallel_tests/user_databases_new.log
# Cleanup
rm parallel_tests/databases.log
rm parallel_tests/databases_new.log
rm parallel_tests/user_databases.log
rm parallel_tests/users_databases_new.log
echo "# Cleaner finished"

24
script/ci/executor.sh Executable file
View File

@@ -0,0 +1,24 @@
#!/bin/bash
main() {
port=$((6000 + $2))
# Run the rspec
start=$SECONDS
ZEUSSOCK=".zeus$port.sock" bundle exec zeus rspec -J#$3 $1 >> parallel_tests/$port.log 2>&1;
exitCode=$?
taken=$(($SECONDS - $start))
formatted_time=$(date -u -d @$taken +'%-0M:%-0S')
# Give some feedback
if [ $exitCode -eq 0 ]; then
echo "[$formatted_time] Finished: $1 Port: $port";
echo "$1" >> parallel_tests/specsuccess.log;
else
echo "[$formatted_time] Finished (FAILED): $1 Port: $port";
echo "$1" >> parallel_tests/specfailed.log;
fi
}
# Init
main $1 $2 $3;
exit 0;

47
script/ci/generateSpecFull.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/bin/bash
# Jesus Vazquez
rm -r parallel_tests
mkdir parallel_tests
# The following tests are disabled in a parallel environment and are run afterwards, sequentially
DISABLED_TESTS=(
'spec/models/asset_spec.rb' # Hangs sometimes when serving files
'services/user-mover/spec/user_mover_spec.rb' # Database recreation fails in parallel
)
# This is a file that contains a list of specs in the order you want them executed
# A good way to obtain it is by taking the output of this script and sorting them based on time taken
# so that longest tests run first. You can do it with: `sort result.txt -k 2 -r | grep -o '[^ ]*rb'`
# This script will try to follow that order, but any unlisted tests will be put at the beginning
ORDERED_TESTS='script/ci/ordered_tests.txt'
# Disabled tests get put into specfailed.txt for later execution and are omitted from
# specfull.txt by builiding and OR regex (spec\|spec\|spec)
first=1
DISABLED_TEST_REGEX=''
for spec in ${DISABLED_TESTS[@]}
do
echo $spec >> parallel_tests/specfailed.log
if [[ $first -eq 0 ]]
then
DISABLED_TEST_REGEX="$DISABLED_TEST_REGEX\\|$spec"
else
DISABLED_TEST_REGEX="$DISABLED_TEST_REGEX$spec"
fi
first=0
done
cat Makefile | grep -v $DISABLED_TEST_REGEX | \
grep -v 'require ./spec/rspec_configuration.rb'| \
grep 'rb'| sed -e 's/^\s*//' -e '/^$/d' | sed '/^#/ d' | sed 's/\\//' | sed 's/\s.*$//' > parallel_tests/specfull.txt
# Sort the specfull
# 1. New tests (specfull - ordered)
grep -Fxv -f $ORDERED_TESTS parallel_tests/specfull.txt > parallel_tests/ordered.txt
# 2. Ordered tests, excluding deleted ones: (ordered & specfull)
grep -Fx -f parallel_tests/specfull.txt $ORDERED_TESTS >> parallel_tests/ordered.txt
# Overwrite
mv parallel_tests/ordered.txt parallel_tests/specfull.txt
echo "# Speclist has been created"

293
script/ci/ordered_tests.txt Normal file
View File

@@ -0,0 +1,293 @@
spec/models/user_spec.rb
spec/models/carto/user_migration_spec.rb
spec/requests/carto/api/visualizations_controller_spec.rb
spec/models/data_import_spec.rb
spec/models/carto/user_spec.rb
spec/models/map_spec.rb
spec/requests/superadmin/users_spec.rb
spec/models/table_spec.rb
spec/models/organization_spec.rb
spec/services/carto/visualizations_export_service_2_spec.rb
spec/requests/carto/api/users_controller_spec.rb
spec/requests/carto/builder/visualizations_controller_spec.rb
spec/connectors/importer_spec.rb
spec/services/carto/user_metadata_export_service_spec.rb
spec/models/carto/layer_spec.rb
spec/models/carto/user_creation_spec.rb
spec/models/permission_spec.rb
spec/models/carto/permission_spec.rb
spec/requests/carto/api/layers_controller_spec.rb
spec/models/carto/rate_limit_spec.rb
spec/requests/api/imports_spec.rb
spec/requests/carto/api/geocodings_controller_spec.rb
spec/requests/superadmin/organizations_spec.rb
spec/models/user_organization_spec.rb
spec/requests/sessions_controller_spec.rb
spec/requests/admin/organization_users_controller_spec.rb
services/importer/spec/acceptance/shp_spec.rb
spec/requests/carto/api/api_keys_controller_spec.rb
spec/requests/signup_controller_spec.rb
spec/requests/sessions_spec.rb
spec/connectors/importer_overviews_spec.rb
spec/models/visualization/collection_spec.rb
spec/requests/admin/visualizations_spec.rb
spec/requests/carto/api/organization_users_controller_spec.rb
spec/models/visualization/member_spec.rb
spec/models/user_presenter_spec.rb
spec/models/layer_spec.rb
services/importer/spec/unit/json2csv_spec.rb
spec/lib/cartodb/connection_pool_spec.rb
spec/requests/warden_spec.rb
spec/requests/carto/api/visualization_exports_controller_spec.rb
spec/requests/carto/api/layer_presenter_spec.rb
spec/requests/api/assets_spec.rb
spec/requests/carto/api/maps_controller_spec.rb
spec/models/visualization/table_blender_spec.rb
spec/requests/user_state_spec.rb
spec/requests/carto/superadmin/organizations_controller_spec.rb
spec/requests/password_change_controller_spec.rb
spec/requests/admin/pages_controller_spec.rb
spec/queries/carto/visualization_query_builder_spec.rb
spec/lib/explore_api_spec.rb
spec/requests/carto/api/imports_controller_spec.rb
spec/requests/carto/api/permissions_controller_spec.rb
spec/models/table_registrar_spec.rb
spec/services/visualization/common_data_service_spec.rb
spec/requests/admin/organizations_controller_spec.rb
spec/models/carto/user_db_service_spec.rb
spec/requests/carto/api/tables_controller_spec.rb
spec/requests/api/json/layer_presenter_spec.rb
spec/models/user_table_spec.rb
spec/models/table_privacy_manager_spec.rb
services/importer/spec/acceptance/kml_spec.rb
spec/services/carto/overquota_users_service_spec.rb
spec/requests/carto/builder/public/embeds_controller_spec.rb
spec/models/carto/visualization_spec.rb
spec/models/carto/visualization_export_spec.rb
spec/requests/carto/builder/datasets_controller_spec.rb
spec/requests/carto/api/layer_presenter_spec.rb
spec/requests/application_controller_spec.rb
spec/models/carto/organization_spec.rb
services/importer/spec/acceptance/csv_spec.rb
spec/requests/carto/api/analyses_controller_spec.rb
spec/models/carto/template_spec.rb
spec/services/carto/organization_metadata_export_service_spec.rb
spec/requests/admin/users_controller_spec.rb
spec/models/carto/user_migration_import_spec.rb
spec/lib/tasks/layers_rake_spec.rb
spec/requests/carto/api/widgets_controller_spec.rb
spec/requests/carto/admin/mobile_apps_controller_spec.rb
spec/requests/account_tokens_controller_spec.rb
spec/models/carto/username_proposer_spec.rb
spec/requests/superadmin/platform_controller_spec.rb
spec/requests/carto/api/user_creations_controller_spec.rb
spec/requests/carto/api/templates_controller_spec.rb
spec/requests/carto/api/overlays_controller_spec.rb
spec/models/synchronization/member_overviews_spec.rb
spec/models/carto/user_table_spec.rb
spec/requests/carto/superadmin/users_controller_spec.rb
spec/requests/carto/api/vizjson3_presenter_spec.rb
spec/requests/carto/api/legends_controller_spec.rb
spec/models/carto/api_key_spec.rb
spec/lib/tasks/fix_unique_legends_spec.rb
spec/lib/carto/tracking/events_spec.rb
services/importer/spec/acceptance/gpx_spec.rb
spec/services/carto/redis_export_service_spec.rb
spec/requests/carto/api/database_groups_controller_spec.rb
spec/requests/api/visualizations_spec.rb
spec/models/geocoding_spec.rb
spec/requests/carto/api/snapshots_controller_specs.rb
spec/requests/carto/api/groups_controller_spec.rb
spec/requests/api/synchronizations_spec.rb
spec/models/table/column_typecaster_spec.rb
spec/models/layer/presenter_spec.rb
spec/models/carto/widget_spec.rb
spec/models/carto/analysis_spec.rb
spec/requests/visualizations_controller_helper_spec.rb
spec/requests/carto/api/organization_assets_controller_spec.rb
spec/requests/carto/api/mapcaps_controller_spec.rb
spec/requests/api/json/synchronizations_controller_spec.rb
spec/models/carto/mapcap_spec.rb
services/importer/spec/acceptance/zip_spec.rb
services/importer/spec/acceptance/osm_spec.rb
services/importer/spec/acceptance/mapinfo_spec.rb
spec/requests/carto/superadmin/user_migration_imports_spec.rb
spec/requests/carto/api/records_controller_spec.rb
spec/requests/carto/api/connectors_controller_spec.rb
spec/requests/carto/api/columns_controller_spec.rb
spec/lib/carto/visualization_migrator_spec.rb
services/importer/spec/acceptance/raster2pgsql_spec.rb
spec/services/carto/user_table_index_service_spec.rb
spec/requests/carto/api/table_presenter_spec.rb
spec/models/visualization/organization_visualization_spec.rb
spec/models/visualization/locator_spec.rb
spec/models/synchronization/member_spec.rb
spec/models/carto/data_import_spec.rb
spec/requests/carto/api/states_controller_spec.rb
spec/requests/carto/api/received_notifications_controller_spec.rb
spec/requests/carto/api/organizations_controller_spec.rb
spec/requests/carto/api/metrics_controller_spec.rb
spec/requests/api/json/geocodings_controller_spec.rb
spec/lib/carto/visualization_invalidation_service_spec.rb
services/importer/spec/unit/ogr2ogr_spec.rb
spec/requests/api/geocodings_spec.rb
spec/models/carto/ldap/configuration_spec.rb
services/importer/spec/unit/runner_spec.rb
services/importer/spec/unit/connector_spec.rb
services/importer/spec/acceptance/rar_spec.rb
spec/services/carto/data_library_service_spec.rb
spec/requests/carto/api/invitations_controller_spec.rb
spec/models/visualization/name_checker_spec.rb
spec/models/carto/invitation_spec.rb
spec/lib/carto/named_maps/template_spec.rb
services/twitter-search/spec/unit/json_to_csv_converter_spec.rb
services/importer/spec/unit/downloader_spec.rb
services/importer/spec/unit/connector_runner_spec.rb
spec/services/carto/visualizations_export_service_spec.rb
spec/requests/carto/superadmin/user_migration_exports_spec.rb
spec/requests/carto/api/grantables_controller_spec.rb
spec/models/carto/map_spec.rb
spec/models/carto/asset_spec.rb
spec/lib/carto/ghost_tables_manager_spec.rb
services/importer/spec/unit/unp_spec.rb
services/importer/spec/unit/shp_normalizer_spec.rb
services/importer/spec/acceptance/gz_tgz_spec.rb
spec/requests/api/json/imports_controller_spec.rb
spec/models/carto/visualization/watcher_spec.rb
spec/models/carto/snapshot_spec.rb
spec/models/carto/group_spec.rb
services/table-geocoder/spec/lib/abstract_table_geocoder_spec.rb
services/importer/spec/unit/url_translator/github_spec.rb
spec/requests/carto/api/organization_notifications_controller_spec.rb
spec/requests/admin/tables_spec.rb
spec/models/visualization/relator_spec.rb
spec/models/carto/overlay_spec.rb
spec/models/carto/legend_spec.rb
spec/lib/carto/styles/geometry_spec.rb
spec/lib/carto/mapcapped_visualization_updater_spec.rb
spec/lib/cartodb/stats/platform_spec.rb
services/table-geocoder/spec/table_geocoder_spec.rb
services/importer/spec/unit/url_translator/google_maps_spec.rb
services/importer/spec/unit/url_translator/google_docs_spec.rb
services/importer/spec/acceptance/geojson_spec.rb
spec/requests/carto/api/synchronizations_controller_spec.rb
spec/requests/carto/api/static_notifications_controller_spec.rb
spec/models/shared_entity_spec.rb
spec/models/platform-limits/user_concurrent_syncs_amount_spec.rb
spec/models/map/copier_spec.rb
spec/models/carto/user_service_spec.rb
spec/models/carto/notification_spec.rb
services/importer/spec/unit/column_spec.rb
spec/services/carto/user_authenticator_spec.rb
spec/requests/superadmin/feature_flag_spec.rb
spec/requests/carto/api/data_import_presenter_spec.rb
spec/requests/carto/api/assets_controller_spec.rb
spec/models/synchronization/synchronization_oauth_spec.rb
spec/models/synchronization/collection_spec.rb
spec/models/platform-limits/user_concurrent_imports_amount_spec.rb
spec/models/carto/shared_entity_spec.rb
spec/models/carto/received_notification_spec.rb
spec/models/access_token_spec.rb
spec/lib/carto/google_maps_api_spec.rb
spec/lib/carto/db/user_schema_spec.rb
spec/helpers/carto/html_safe_spec.rb
services/wms/spec/unit/wms_spec.rb
services/importer/spec/unit/georeferencer_spec.rb
services/importer/spec/unit/csv_normalizer_spec.rb
services/datasources/spec/acceptance/public_url_spec.rb
spec/lib/carto/assets_service_spec.rb
services/sql-api/spec/sql_api_spec.rb
services/importer/spec/unit/gpx_splitter_spec.rb
services/importer/spec/unit/excel2csv_spec.rb
services/geocoder/spec/hires_geocoder_spec.rb
services/geocoder/spec/geocoder_spec.rb
services/datasources/spec/unit/arcgis_spec.rb
services/datasources/spec/integration/csv_file_dumper_spec.rb
spec/lib/carto/legend_definition_validator_spec.rb
spec/lib/cartodb/redis_vizjson_cache_spec.rb
spec/helpers/carto_db_spec.rb
services/importer/spec/unit/namedplaces_guesser_spec.rb
services/importer/spec/acceptance/sql_spec.rb
services/datasources/spec/acceptance/gdrive_spec.rb
services/dataservices-metrics/spec/unit/service_usage_metrics_spec.rb
spec/requests/superadmin/account_types_spec.rb
spec/requests/carto/saml_controller_spec.rb
spec/requests/carto/api/overlay_presenter_spec.rb
spec/requests/carto/api/oembed_controller_spec.rb
spec/models/visualization/presenter_spec.rb
spec/models/visualization/overlays_spec.rb
spec/models/common_data_spec.rb
spec/models/carto/mobile_app_presenter_spec.rb
spec/lib/user_account_creator_spec.rb
spec/lib/url_signer_spec.rb
spec/lib/tasks/fix_unique_overlays_spec.rb
spec/lib/initializers/carto_db_spec.rb
spec/lib/errors_spec.rb
spec/lib/carto/table_utils_spec.rb
spec/lib/carto/storage_options/local_spec.rb
spec/lib/carto/legend_migrator_spec.rb
spec/lib/carto/form_spec.rb
spec/lib/carto/definition_spec.rb
spec/lib/carto/db/sanitize_spec.rb
spec/lib/carto/bolt_spec.rb
spec/helpers/application_helper_spec.rb
services/table-geocoder/spec/internal-geocoder/query_generator_factory_spec.rb
services/table-geocoder/spec/internal-geocoder/input_type_resolver_spec.rb
services/table-geocoder/spec/geocoder_cache_spec.rb
services/importer/spec/unit/url_translator/fusion_tables_spec.rb
services/importer/spec/unit/loader_spec.rb
services/importer/spec/unit/kml_splitter_spec.rb
services/importer/spec/unit/content_guesser_spec.rb
services/datasources/spec/integration/twitter_spec.rb
services/datasources/spec/acceptance/dropbox_spec.rb
services/datasources/spec/acceptance/datasources_factory_spec.rb
spec/requests/carto/api/presenter_cache_spec.rb
spec/requests/carto/api/infowindow_migrator_spec.rb
spec/models/visualization/tags_spec.rb
spec/models/carto/helpers/billing_cycle_spec.rb
spec/models/carto/analysis_node_spec.rb
spec/models/carto/account_type_spec.rb
spec/lib/trending_maps_spec.rb
spec/lib/string_spec.rb
spec/lib/initializers/zz_patch_reconnect_spec.rb
spec/lib/image_metadata_spec.rb
spec/lib/central_spec.rb
spec/lib/carto/valid_table_name_proposer_spec.rb
spec/lib/carto/users_metadata_redis_cache_spec.rb
spec/lib/carto/styles/presenters/cartocss_spec.rb
spec/lib/carto/styles/polygon_spec.rb
spec/lib/carto/styles/point_spec.rb
spec/lib/carto/styles/line_spec.rb
spec/lib/carto/styles/cartography_spec.rb
spec/lib/carto/strong_password_validator_spec.rb
spec/lib/carto/saml_service_spec.rb
spec/lib/carto/organization_assets_service_spec.rb
spec/lib/carto/http_header_authentication_spec.rb
spec/lib/carto/http/client_spec.rb
spec/lib/carto/forms_definition_spec.rb
spec/lib/carto/file_system/sanitize_spec.rb
spec/lib/carto/filename_generator_spec.rb
spec/lib/cartodb/stats/importer_spec.rb
spec/lib/carto/db/sql_interface_spec.rb
spec/lib/api_calls_spec.rb
spec/helpers/uuidhelper_spec.rb
spec/helpers/url_validator_spec.rb
services/twitter-search/spec/unit/search_api_spec.rb
services/table-geocoder/spec/lib/gme/table_geocoder_spec.rb
services/platform-limits/spec/unit/input_file_size_spec.rb
services/importer/spec/unit/url_translator/osm_spec.rb
services/importer/spec/unit/url_translator/osm2_spec.rb
services/importer/spec/unit/table_sampler_spec.rb
services/importer/spec/unit/sql_loader_spec.rb
services/importer/spec/unit/source_file_spec.rb
services/importer/spec/unit/shp_helper_spec.rb
services/importer/spec/unit/post_import_handler_spec.rb
services/importer/spec/unit/mail_notifier_spec.rb
services/importer/spec/regression/query_batcher_spec.rb
services/geocoder/spec/hires_geocoder_factory_spec.rb
services/geocoder/spec/hires_batch_geocoder_spec.rb
services/datasources/spec/unit/twitter_spec.rb
services/datasources/spec/unit/gdrive_spec.rb
services/datasources/spec/unit/dropbox_spec.rb
services/datasources/spec/unit/box_spec.rb
spec/models/visualization/name_generator_spec.rb

32
script/ci/reporter.sh Executable file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
# Jesus Vazquez
# reporter.sh: This script is the exit point for the tests execution. It reads from specfailed.log
# the amount of tests that have failed. If there are none it sends a success status but if there are
# 1 or more it sends a failure status to warn the developer
filename="parallel_tests/specfailed.log"
lines=$(cat $filename | wc -l)
if [ "$lines" -eq "0" ];
then
echo "Tests were OK";
# TODO
# gsu "Backend tests were OK" "Backend" success
exit 0; #OK
else
while read line;
do
# For each error cat its log file
logfile=$(echo $line | grep -o '[0-9][0-9][0-9][0-9].log')
# cat $logfile;
# Give feedback to github
# spec=$(echo $line | sed 's/\s.*$//')
# echo "GSU with spec $spec" TODO
# gsu "$spec failed" "$spec" failure TODO
done < $filename
# TODO
# gsu "Backend tests failed" "Backend" failure
exit 1; # ERROR
fi

19
script/ci/runParallelTests.sh Executable file
View File

@@ -0,0 +1,19 @@
#!/bin/bash
# BACKEND PARALLEL
script/ci/generateSpecFull.sh || exit 1
# CLEANER
script/ci/cleaner.sh || exit 1
# WRAPPER
script/ci/wrapper.sh $1 || exit 1
# TESTS
time parallel -j $1 -a parallel_tests/specfull.txt 'script/ci/executor.sh {} {%} {#}' || exit 1
# SECOND TRY
script/ci/secondTry.sh || exit 1
# REPORTER
script/ci/reporter.sh || exit 1

31
script/ci/secondTry.sh Executable file
View File

@@ -0,0 +1,31 @@
#!/bin/bash
# Jesus Vazquez
# secondTry
# This is a hack for those specs that failed in the parallel execution.
# Here we can check if they failed because they can't run in parallel
# or because the PR code is wrong
# Requisites
cp config/database.yml.sample config/database.yml
# Start
failedSpecs=$(cat parallel_tests/specfailed.log | wc -l)
if [ "$failedSpecs" -eq "0" ];
then
exit 0;
else
specs=$(cat parallel_tests/specfailed.log | sed ':a;N;$!ba;s/\n/ /g')
fi
echo "Giving a second try to the next specs"
cat parallel_tests/specfailed.log
RAILS_ENV=test bundle exec rspec $specs
if [ $? -eq 0 ]; then
truncate -s 0 parallel_tests/specfailed.log # Here is where the hack takes place. If im the second try we dont have errors then we're OK
else
exit 0; # The reporter script will output the failed specs
fi

33
script/ci/wrapper.sh Executable file
View File

@@ -0,0 +1,33 @@
#!/bin/sh
rm config/database_*
threads=$1
databaseName="carto_db_test"
dbAdmin="postgres"
lastPort=$((threads + 6000))
startPort=6001
# Iterate and create one database per spec
for i in $(seq $startPort $lastPort)
do
# Get database owner
owner=$(psql -U $dbAdmin -t -c "select r.rolname from pg_database d, pg_roles r where d.datname='carto_db_test' and d.datdba = r.oid")
newDatabase="${databaseName}_${i}";
# Create the database with specific owner and template
$(psql -U $dbAdmin -t -c "create database $newDatabase with owner $owner template $databaseName;") >> parallel_tests/wrapper.log 2>&1
# Create the database.yml file
echo "# Creating database_$i.yml file" >> parallel_tests/wrapper.log 2>&1
sed -e s/carto_db_test/carto_db_test_$i/g config/database.yml.sample > config/database_$i.yml
done
for i in $(seq $startPort $lastPort)
do
# Start Zeus server
TURBO=1 ZEUSSOCK=".zeus$i.sock" RAILS_DATABASE_FILE=database_$i.yml REDIS_PORT=$i bundle exec zeus start >/dev/null 2>/dev/null &
done
# Wait for a few seconds for Zeus servers to startup.
# A better way to do it would be to wait for all `.zeus*.sock` files to be created.
sleep 5
touch parallel_tests/specfailed.log
touch parallel_tests/specsuccess.log
echo "# Wrapper finished"

5
script/clean_assets Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
echo "--- Cleaning assets"
PACKAGE_VERSION=$(grep -m1 version package.json | awk -F: '{ print $2 }' | sed 's/[", ]//g')
find public/assets/* -type d ! -name $PACKAGE_VERSION -depth 0 | xargs rm -rf

102
script/compare_metadata.rb Normal file
View File

@@ -0,0 +1,102 @@
#!/usr/bin/env ruby
# compare_metadata.rb: this is a Nagios check to ensure information on the
# metadata PostgreSQL database matches the one on Redis database.
require 'yaml'
require 'pg'
require 'redis'
require_dependency 'carto/configuration'
script_name = "script/compare_metadata.rb"
data = `ps aux | grep "#{script_name}"`
.split("\n")
.select { |item| item =~ /ruby #{script_name}/i }
.select { |item| (item =~ / #{Process.pid} /i).nil? }
if data.length > 0
puts "compare_metadata script already running, exiting"
exit 0
end
carto_conf = Carto::Conf.new
config = carto_conf.app_config
database = carto_conf.db_config
RAILS_ENV = ENV['RAILS_ENV'] || 'production'
DBNAME = ENV['DB_NAME'] || database[RAILS_ENV]['database']
DBUSER = ENV['DB_USER'] || database[RAILS_ENV]['username']
DBPASS = ENV['DB_PASS'] || database[RAILS_ENV]['password']
DBHOST = ENV['DB_HOST'] || database[RAILS_ENV]['host']
DBPORT = ENV['DB_PORT'] || database[RAILS_ENV]['port']
REDISPORT = ENV['REDIS_PORT'] || config[RAILS_ENV]['redis']['port']
REDISHOST = ENV['REDIS_HOST'] || config[RAILS_ENV]['redis']['host']
redis = Redis.new(host: REDISHOST, port: REDISPORT)
pg = PGconn.connect(user: DBUSER, dbname: DBNAME, port: DBPORT, host: DBHOST)
pg_users = {}
pg.query('SELECT * FROM users') do |result|
result.each do |row|
pg_users[row['username']] = row
end
end
redis_users = {}
redis.select(5)
redis.keys('rails:users:*').each do |user|
if user.split(":").count == 3
user_info = redis.hgetall user
redis_users[user.split(':')[2]] = user_info
end
end
redis_not_postgres = (redis_users.keys - pg_users.keys)
postgres_not_redis = (pg_users.keys - redis_users.keys)
COMPARE = [
'database_host',
'database_name',
['api_key', 'map_key']
]
mismatched = ""
pg_users.each do |user_id, data|
if redis_users[user_id] != nil
COMPARE.each do |key_to_compare|
if key_to_compare.is_a? String
redis_key = key_to_compare; pg_key = key_to_compare
elsif key_to_compare.is_a? Array
pg_key, redis_key = key_to_compare
end
if data[pg_key] != redis_users[user_id][redis_key]
mismatched << "\n#{user_id}: #{pg_key}"
mismatched << "\n - PostgreSQL:\t#{data[pg_key]}"
mismatched << "\n - Redis:\t#{redis_users[user_id][redis_key]}"
end
end
end
end
if redis_not_postgres.size > 0
puts "ERROR: #{redis_not_postgres.size} users in Redis, not on Postgres"
p redis_not_postgres
exit 2
end
if postgres_not_redis.size > 0
puts "ERROR: #{postgres_not_redis.size} users in Postgres, not on Redis:"
p postgres_not_redis
exit 2
end
if mismatched != ""
puts "ERROR: Mismatched data between PostgreSQL and Redis"
puts mismatched
exit 2
end
puts "OK"
exit 0

42
script/create_dev_user Executable file
View File

@@ -0,0 +1,42 @@
#!/bin/sh
set -e
SUBDOMAIN="dev"
PASSWORD="pass1234"
ADMIN_PASSWORD="pass1234"
EMAIL="dev@contoso.com"
echo "--- Creating databases"
bundle exec rake cartodb:db:setup
echo "--- Create '${SUBDOMAIN}' user"
bundle exec rake cartodb:db:create_user --trace SUBDOMAIN="${SUBDOMAIN}" \
PASSWORD="${PASSWORD}" ADMIN_PASSWORD="${ADMIN_PASSWORD}" \
EMAIL="${EMAIL}"
# # Update your quota to 100GB
echo "--- Updating quota to 100GB"
bundle exec rake cartodb:db:set_user_quota["${SUBDOMAIN}",102400]
# # Allow unlimited tables to be created
echo "--- Allowing unlimited tables creation"
bundle exec rake cartodb:db:set_unlimited_table_quota["${SUBDOMAIN}"]
# # Allow user to create private tables in addition to public
echo "--- Allowing private tables creation"
bundle exec rake cartodb:db:set_user_private_tables_enabled["${SUBDOMAIN}",'true']
# # Set the account type
echo "--- Setting cartodb account type"
bundle exec rake cartodb:db:set_user_account_type["${SUBDOMAIN}",'[DEDICATED]']
# Set dataservices server
bundle exec rake cartodb:db:configure_geocoder_extension_for_non_org_users[$SUBDOMAIN]
# Set import limits
bundle exec rake cartodb:set_custom_limits_for_user["${SUBDOMAIN}",10240000000,100000000,1]
# Enable sync tables
echo "UPDATE users SET sync_tables_enabled=true WHERE username='${SUBDOMAIN}'" | psql -U postgres -t carto_db_development

13
script/fill_geocoder.sh Executable file
View File

@@ -0,0 +1,13 @@
#!/bin/bash
# See https://github.com/CartoDB/data-services/issues/228#issuecomment-280037353
# Not run during Docker build phase as it would make the image too big
cd /data-services/geocoder
./geocoder_download_dumps
GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
./geocoder_restore_dump postgres $GEOCODER_DB db_dumps/*.sql
rm -r db_dumps
chmod +x geocoder_download_patches.sh geocoder_apply_patches.sh
./geocoder_download_patches.sh
./geocoder_apply_patches.sh postgres $GEOCODER_DB data_patches/*.sql
rm -r data_patches

40
script/geocoder.sh Normal file
View File

@@ -0,0 +1,40 @@
cd /cartodb
bundle exec rake cartodb:db:create_user --trace SUBDOMAIN="geocoder" \
PASSWORD="pass1234" ADMIN_PASSWORD="pass1234" \
EMAIL="geocoder@contoso.com"
# # Update your quota to 100GB
echo "--- Updating quota to 100GB"
bundle exec rake cartodb:db:set_user_quota[geocoder,102400]
# # Allow unlimited tables to be created
echo "--- Allowing unlimited tables creation"
bundle exec rake cartodb:db:set_unlimited_table_quota[geocoder]
GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
psql -U postgres $GEOCODER_DB < /cartodb/script/geocoder_server.sql
# Import observatory test dataset
psql -U postgres -d $GEOCODER_DB -f /observatory-extension/src/pg/test/fixtures/load_fixtures.sql
# Setup permissions for observatory
psql -U postgres -d $GEOCODER_DB -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory VERSION 'dev'; COMMIT" -e
psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO geocoder; COMMIT" -e
psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder; COMMIT" -e
psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO geocoder; COMMIT" -e
psql -U postgres -d $GEOCODER_DB -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA observatory TO geocoder; COMMIT" -e
# Setup dataservices client
# dev user
USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB# } user=postgres\"}');" | psql -U postgres $USER_DB
bundle exec rake cartodb:services:set_user_quota['dev',geocoding,100000]
# example organization
ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB# } user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB
bundle exec rake cartodb:services:set_org_quota['example',geocoding,100000]

36
script/geocoder_server.sql Normal file
View File

@@ -0,0 +1,36 @@
create extension cdb_geocoder;
create extension plproxy;
create extension observatory;
create extension cdb_dataservices_server;
create extension cdb_dataservices_client;
SELECT CDB_Conf_SetConf(
'redis_metadata_config',
'{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'
);
SELECT CDB_Conf_SetConf(
'redis_metrics_config',
'{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'
);
SELECT CDB_Conf_SetConf(
'user_config',
'{"is_organization": false, "entity_name": "geocoder"}'
);
SELECT CDB_Conf_SetConf(
'server_conf',
'{"environment": "development"}'
);
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}');
-- dummy conf from https://github.com/CartoDB/dataservices-api/blob/master/server/extension/test/sql/00_install_test.sql
SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');

52
script/geocoder_test Executable file
View File

@@ -0,0 +1,52 @@
#!/usr/bin/env ruby
require 'yaml'
require 'tmpdir'
require_relative '../services/geocoder/lib/hires_geocoder_factory'
TIMEOUT = 300 # seconds before raising a timeout error
def usage
abort "usage: script/geocoder_test environment <non-batch>"
end
environment = ARGV[0]
non_batch = (ARGV[1] == 'non-batch')
usage unless environment
def load_config(environment)
config_file_hash = YAML.load_file(File.expand_path('../../config/app_config.yml', __FILE__))
config_file_hash[environment]["geocoder"]
rescue => e
raise "Missing or inaccessible config for environment #{environment} in config/app_config.yml: #{e.message}"
end
config = load_config(environment)
CartoDB::GeocoderConfig.instance.set config.merge('force_batch' => !non_batch)
input_file = File.expand_path('../../services/geocoder/spec/fixtures/nokia_input.csv', __FILE__)
working_dir = Dir.mktmpdir
geocoder = CartoDB::HiresGeocoderFactory.get(input_file, working_dir)
puts "Runing #{(non_batch ? 'non batch' : 'batch')} geocoder..."
start = Time.now
geocoder.run
finish = Time.now
until geocoder.status == 'completed' do
finish = Time.now
raise "Geocoder FAILURE: Timeout" if (finish - start) > TIMEOUT
geocoder.update_status
sleep(2)
end
raise "Geocoder FAILURE" unless geocoder.status == 'completed'
Dir.chdir geocoder.dir
`unp #{geocoder.result} 2>&1`
file_path = (non_batch ? geocoder.result : Dir[File.join(geocoder.dir, '*_out.txt')][0])
response = File.read(file_path).split(',')
expected_position = [45.96, -66.60]
position = [response[4].to_f, response[5].to_f]
distance = Math.sqrt(expected_position.zip(position).map { |x| (x[1] - x[0])**2 }.reduce(:+))
raise "Geocoder FAILURE: wrong geocoded position #{position}, expected #{expected_position}, distance: #{distance}" if distance > 0.04
FileUtils.rm_f working_dir
puts "\e[32mGeocoder OK #{(finish - start)} secs\e[0m"

8
script/import Executable file
View File

@@ -0,0 +1,8 @@
#!/bin/sh
if [ -z "$2" ]; then
echo "Usage: script/import <username> <path to file>"
exit 1
fi
bundle exec rake cartodb:import[$1,$2]
if test $? -ne 0; then exit 1; fi

39
script/metadata-goodies.sql Normal file
View File

@@ -0,0 +1,39 @@
-- This is a set of functions not intended to be used in production
-- but useful to explore the database without rails ORM.
-- They are thought to be loosely coupled to the models and just rely on FK relations.
--
-- To install:
-- \i metadata-goodies.psql
-- To uninstall:
-- DROP SCHEMA goodies CASCADE;
CREATE SCHEMA IF NOT EXISTS goodies AUTHORIZATION postgres;
CREATE OR REPLACE FUNCTION goodies.related_layer_ids(table_id uuid)
RETURNS SETOF uuid AS $$
BEGIN
RETURN QUERY SELECT DISTINCT(layer_id) FROM layers_user_tables WHERE layers_user_tables.user_table_id = table_id;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION goodies.related_map_ids(table_id uuid)
RETURNS SETOF uuid AS $$
BEGIN
RETURN QUERY SELECT DISTINCT(map_id) FROM layers_maps WHERE layers_maps.layer_id
IN (SELECT goodies.related_layer_ids(table_id));
END;
$$ LANGUAGE plpgsql;
-- E.g: SELECT id, name, type FROM goodies.related_vizs('6eb62d84-c4bc-446b-89dd-5435f6cc9346');
CREATE OR REPLACE FUNCTION goodies.related_vizs(table_id uuid)
RETURNS SETOF visualizations AS $$
BEGIN
RETURN QUERY SELECT * FROM visualizations WHERE visualizations.map_id
IN (SELECT goodies.related_map_ids(table_id));
END;
$$ LANGUAGE plpgsql;

460
script/migrate_to_uuid.rb Normal file
View File

@@ -0,0 +1,460 @@
#!/usr/bin/env ruby
require 'pg'
require 'redis'
# RAILS_ENV=development DBNAME=carto_db_development DBHOST=127.0.0.1 DBUSER=postgres REDIS_HOST=127.0.0.1
def execution_summary()
wait_time = 30
puts <<-EOH
###
#
# You are running the action '#{ACTION}' which performs the next actions:
#
# #{@actions[ACTION]}
#
# It's highly recommended to have a database backup of the PostgreSQL databases, mainly the metadata one and Redis.
#
#
# Params used within the execution of the script
#
# DB Connection
# -------------
# Database host: #{DBHOST}
# Database port: #{DBPORT}
# Database name: #{DBNAME}
# Database user: #{DBUSER}
#
# Redis Connection
# ----------------
# Redis host: #{REDIS_HOST}
#
###
EOH
puts "###"
puts "If there is anything wrong or you are not sure about what you are doing, you have #{wait_time} secs to cancel before the process starts"
counter(wait_time)
end
def usage(message = nil)
if !message.nil?
puts ""
puts "ERROR: #{message}"
puts ""
end
puts "Usage:"
puts " export RAILS_ENV=<rails_env>"
puts " export DBNAME=<your_postgresql_database_name>"
puts " export DBHOST=<your_postgresql_database_host>"
puts " export DBPORT=<your_postgresql_database_port>"
puts " export DBUSER=<your_postgresql_database_user>"
puts " export REDIS_HOST=<your_redis_host>"
puts ""
puts "# Notice that you probably want to use the same values found in your database.yml and app_config.yml"
puts ""
puts " #{__FILE__} <action>"
puts ""
puts "Actions:"
@actions.each {|k,v| puts " %10s %s" % [k, v]; puts "" }
exit 1
end
def counter(max)
(0..max).each do |n|
print n
sleep 1
print "\r"
end
end
## MAIN
#
@actions = {
'schema' => 'Creates a UUID column in every table with a id. Also creates a\nll the UUID dependency columns between tables. You can still rollback after this step',
'meta' => 'Update every dependency UUID column with the proper one based on the id integer relations. This is the last step that you can rollback',
'rollback' => 'Try to rollback previous steps',
'data' => 'Migrate all postgresql database and users names to UUID format. Also update the user model database_name attribute. Update all redis info with UUIDs. IMPORTANT: You cannot rollback this step',
'clean' => 'Drop old id columns. Rename new uuid colums to id. Rename all uuid dependency columns to id. Create new primary keys from UUID attributes. IMPORTANT: You cannot rollback this step'
}
usage "You need to initialize a environment with RAILS_ENV" if (ENV['RAILS_ENV'].nil? || ENV['RAILS_ENV'].empty?)
usage "You need to set a DBNAME env" if (ENV['DBNAME'].nil? || ENV['DBNAME'].empty?)
usage "You need to set a DBHOST env" if (ENV['DBHOST'].nil? || ENV['DBHOST'].empty?)
usage "You need to set a DBPORT env" if (ENV['DBPORT'].nil? || ENV['DBPORT'].empty?)
usage "You need to set a DBUSER env" if (ENV['DBUSER'].nil? || ENV['DBUSER'].empty?)
usage "You need to set a REDIS_HOST env" if (ENV['REDIS_HOST'].nil? || ENV['REDIS_HOST'].empty?)
ENVIRONMENT = ENV['RAILS_ENV']
DBHOST = ENV['DBHOST']
DBPORT = ENV['DBPORT']
DBUSER = ENV['DBUSER']
DBNAME = ENV['DBNAME']
REDIS_HOST = ENV['REDIS_HOST']
ACTION = ARGV[0]
if ACTION.nil? || !@actions.keys.include?(ACTION)
usage "Missing action"
end
@logs = Hash.new
tables = {
:assets => {
:related => [],
:singular => 'asset'
},
:automatic_geocodings => {
:related => ['geocodings'],
:singular => 'automatic_geocoding'
},
:client_applications => {
:related => ['oauth_tokens'],
:singular => 'client_application'
},
:data_imports => {
:related => ['user_tables'],
:singular => 'data_import'
},
:geocodings => {
:related => [],
:singular => 'geocoding'
},
:layers => {
:related => ['layers_maps', 'layers_users', 'layers_user_tables', 'visualizations'],
:singular => 'layer',
:relation_for => {'visualizations' => 'active_layer'}
},
:layers_maps => {
:related => [],
:singular => 'layer_map'
},
:layers_user_tables => {
:related => [],
:singular => 'layer_user_table'
},
:layers_users => {
:related => [],
:singular => 'layer_user'
},
:maps => {
:related => ['user_tables', 'layers_maps', 'visualizations'],
:singular => 'map'
},
:oauth_nonces => {
:related => [],
:singular => 'oauth_nonce'
},
:oauth_tokens => {
:related => [],
:singular => 'oauth_token'
},
:overlays => {
:related => [],
:singular => 'overlay'
},
:tags => {
:related => [],
:singular => 'tag'
},
:user_tables => {
:related => ['data_imports', 'layers_user_tables', 'tags', 'automatic_geocodings', 'geocodings'],
:singular => 'table',
:relation_for => {'layers_user_tables' => 'user_table'}
},
:users => {
:related => ['user_tables', 'maps', 'layers_users', 'assets', 'client_applications', 'oauth_tokens', 'tags', 'data_imports', 'synchronizations', 'geocodings'],
:singular => 'user'
},
#:visualizations => {
# :related => ['overlays'],
# :singular => 'visualization'
#}
}
redis_keys = {
:map_style => {
:template => "map_style|USERDB|*",
:var_position => 1,
:separator => '|',
:db => 0,
:type => 'string'
},
:table => {
:template => "rails:USERDB:*",
:var_position => 1,
:separator => ':',
:db => 0,
:type => 'hash',
:attributes => {
:user_id => 'USERID'
}
},
:user => {
:template => "rails:users:USERNAME",
:no_clone => true,
:var_position => 2,
:separator => ':',
:db => 5,
:type => 'hash',
:attributes => {
:database_name => 'USERDB',
:id => 'USERID'
}
}
}
def redis_replace_from_template(template, id, username)
if template.include?('USERDB')
user_database(id)
elsif template.include?('DBUSER')
database_username(id)
elsif template.include?('USERNAME')
username
elsif template.include?('USERID')
id
else
''
end
end
def redis_template_user_gsub(template, id, username)
replacement = redis_replace_from_template(template, id, username)
if template.include?('USERDB')
template.gsub('USERDB', replacement)
elsif template.include?('DBUSER')
template.gsub('DBUSER', replacement)
elsif template.include?('USERNAME')
template.gsub('USERNAME', replacement)
else
''
end
end
def copy_redis_keys(redis_keys, id, uuid, username)
redis = Redis.new(:host => REDIS_HOST)
redis_keys.each do |k,v|
redis.select(v[:db])
these_redis_keys = redis.keys(redis_template_user_gsub(v[:template], id, username))
these_redis_keys.each do |trd|
original_value = redis.dump(trd)
new_array = trd.split(v[:separator])
new_array[v[:var_position]] = redis_replace_from_template(v[:template], uuid, username)
new_key = new_array.join(v[:separator])
unless v[:no_clone]
redis.restore(new_key, 0, original_value)
end
if v[:type] == 'hash'
v[:attributes].each do |a,av|
redis.hset(new_key, a.to_s, redis_replace_from_template(av, uuid, username))
end
end
end
end
redis.quit
end
def alter_redis_hash(redis_key, redis_attribute, redis_value, options = {})
redis_db = options['db'].nil? ? 0 : options['db']
redis = Redis.new(:host => REDIS_HOST)
redis.select(redis_db)
redis.hset(redis_key, user_id, redis_value)
redis.quit
end
def relation_column_name_for(tables, table, related)
if tables[table][:related].include?(related) && tables[table][:relation_for] && tables[table][:relation_for][related]
tables[table][:relation_for][related]
else
tables[table][:singular]
end
end
def log(severity, type, msg)
puts " #{msg}. Ignoring.."
@logs.merge({:severity => severity, :type => type, :msg => msg})
end
def database_username(user_id)
"#{db_username_prefix}#{user_id}"
end #database_username
def user_database(user_id)
"#{database_name_prefix}#{user_id}_db"
end #user_database
def db_username_prefix
return "cartodb_user_" if ENVIRONMENT == 'production'
return "development_cartodb_user_" if ENVIRONMENT == 'development'
"cartodb_user_#{ENVIRONMENT}_"
end #username_prefix
def database_name_prefix
return "cartodb_user_" if ENVIRONMENT == 'production'
return "cartodb_dev_user_" if ENVIRONMENT == 'development'
"cartodb_#{ENVIRONMENT}_user_"
end #database_prefix
def alter_schema(tables)
tables.each do |tname, tinfo|
# Create main uuid column in every table
puts "Creating uuid column in #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} ADD uuid uuid UNIQUE NOT NULL DEFAULT uuid_generate_v4()")
rescue => e
log('C', "Creating uuid column in #{tname}", e.error.strip)
end
tinfo[:related].each do |rtable|
# Create relation uuid column in a dependent table
puts "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} ADD #{relation_column_name_for(tables, tname, rtable)}_uuid uuid")
rescue => e
log('C', "Creating #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
end
end
end
end
def rollback_schema(tables)
tables.each do |tname, tinfo|
tinfo[:related].each do |rtable|
# Create relation uuid column in a dependent table
puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_uuid")
rescue => e
log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_uuid column in related table #{rtable}", e.error.strip)
end
end
# Destroy main uuid column in every table
puts "Dropping uuid column in #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} DROP IF EXISTS uuid")
rescue => e
log('C', "Dropping uuid column in #{tname}", e.error.strip)
end
end
end
def migrate_meta(tables)
tables.each do |tname, tinfo|
@conn.exec("SELECT id,uuid FROM #{tname}") do |result|
result.each do |row|
tinfo[:related].each do |rtable|
puts "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}"
begin
@conn.exec("UPDATE #{rtable} SET #{relation_column_name_for(tables, tname, rtable)}_uuid='#{row['uuid']}' WHERE #{relation_column_name_for(tables, tname, rtable)}_id='#{row['id']}'")
rescue => e
log('C', "Setting #{relation_column_name_for(tables, tname, rtable)}_uuid in #{rtable}", e.error.strip)
end
end
end
end
end
end
def migrate_data(redis_keys)
sconn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: 'postgres' )
@conn.exec("SELECT id,uuid,database_name,username FROM users") do |result|
result.each do |row|
puts "Renaming pg user and db for id #{row['id']}"
begin
sconn.exec("ALTER DATABASE \"#{row['database_name']}\" RENAME TO \"#{user_database(row['uuid'])}\"")
sconn.exec("ALTER ROLE \"#{database_username(row['id'])}\" RENAME TO \"#{database_username(row['uuid'])}\"")
@conn.exec("UPDATE users SET database_name='#{user_database(row['uuid'])}' WHERE id=#{row['id']} AND uuid='#{row['uuid']}'")
@conn.exec("UPDATE user_tables SET database_name='#{user_database(row['uuid'])}' WHERE user_id='#{row['id']}' AND user_uuid='#{row['uuid']}'")
rescue => e
log('C', "Renaming pg user and db for id #{row['id']}", e.error.strip)
end
puts "Copying redis keys with uuid for id #{row['id']}"
#begin
copy_redis_keys(redis_keys, row['id'], row['uuid'], row['username'])
#rescue => e
# log('C', "Copying redis keys with uuid for id #{row['id']}", e.error.strip)
#end
end
end
@conn.exec("SELECT token,user_id FROM oauth_tokens WHERE type='AccessToken'") do |result|
result.each do |row|
puts "Chaing user_id for oauth token '#{row['token']}'"
begin
alter_redis_hash("rails:oauth_access_tokens:#{row['token']}", 'user_id', row['user_id'], {'db' => 3})
rescue => e
log('C', "Changing user id to uuid in oauth token #{row['token']}", e.error.strip)
end
end
end
end
def clean_db(tables)
tables.each do |tname, tinfo|
tinfo[:related].each do |rtable|
# Drop old id relation column in every table
puts "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} DROP IF EXISTS #{relation_column_name_for(tables, tname, rtable)}_id")
rescue => e
log('C', "Dropping #{relation_column_name_for(tables, tname, rtable)}_id from #{rtable}", e.error.strip)
end
# Rename new uuid relation column to id
puts "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}"
begin
@conn.exec("ALTER TABLE #{rtable} RENAME #{relation_column_name_for(tables, tname, rtable)}_uuid TO #{relation_column_name_for(tables, tname, rtable)}_id")
rescue => e
log('C', "Renaming #{relation_column_name_for(tables, tname, rtable)}_uuid to #{relation_column_name_for(tables, tname, rtable)}_id in #{rtable}", e.error.strip)
end
end
# Drop old id column in every table
puts "Dropping old id from #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} DROP IF EXISTS id")
rescue => e
log('C', "Dropping old id from #{rtable}", e.error.strip)
end
# Rename new uuid relation column to id
puts "Renaming uuid to id in #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} RENAME uuid TO id")
rescue => e
log('C', "Renaming uuid to id in #{tname}", e.error.strip)
end
# Set new id as primary key
puts "Setting new id as primary key on #{tname}"
begin
@conn.exec("ALTER TABLE #{tname} ADD PRIMARY KEY (id)")
rescue => e
log('C', "Setting new id as primary key on #{tname}", e.error.strip)
end
end
end
execution_summary
@conn = PGconn.connect( host: DBHOST, port: DBPORT, user: DBUSER, dbname: DBNAME )
@conn.exec("CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"")
if ACTION == 'schema'
alter_schema(tables)
elsif ACTION == 'rollback'
rollback_schema(tables)
elsif ACTION == 'meta'
migrate_meta(tables)
elsif ACTION == 'data'
migrate_data(redis_keys)
elsif ACTION == 'clean'
clean_db(tables)
end
puts ""
puts "#############"
puts "#{@logs.length} errors"
puts "#############"

6
script/rails Executable file
View File

@@ -0,0 +1,6 @@
#!/usr/bin/env ruby
# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
APP_PATH = File.expand_path('../../config/application', __FILE__)
require File.expand_path('../../config/boot', __FILE__)
require 'rails/commands'

2
script/resque Executable file
View File

@@ -0,0 +1,2 @@
#!/bin/sh
VVERBOSE=true QUEUE=imports,exports,users,user_dbs,geocodings,synchronizations,tracker,user_migrations,batch_updates,gears rake environment resque:work

5
script/restore_redis Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
echo "--- Restoring user metadata"
bundle exec rake cartodb:redis:user_metadata
if test $? -ne 0; then exit 1; fi

21
script/server_load.rb Executable file
View File

@@ -0,0 +1,21 @@
#!/usr/bin/env ruby
require_relative '../lib/cartodb/threads_machine'
require_relative '../lib/cartodb/scripts/server_load'
class ServerLoad < ThreadsMachine
def execute
MAX_THREADS.times do
queue.enq(ServerLoadScript)
end
async do |load_script|
load_script.new
end
end
end
ServerLoad.new.execute

19
script/setup_organization.sh Normal file
View File

@@ -0,0 +1,19 @@
ORGANIZATION_NAME="example"
USERNAME="admin4example"
EMAIL="admin@contoso.com"
PASSWORD="pass1234"
bundle exec rake cartodb:db:create_user EMAIL="${EMAIL}" PASSWORD="${PASSWORD}" SUBDOMAIN="${USERNAME}"
bundle exec rake cartodb:db:set_unlimited_table_quota["${USERNAME}"]
bundle exec rake cartodb:db:set_user_quota["${USERNAME}",102400]
bundle exec rake cartodb:db:create_new_organization_with_owner ORGANIZATION_NAME="${ORGANIZATION_NAME}" USERNAME="${USERNAME}" ORGANIZATION_SEATS=100 ORGANIZATION_QUOTA=102400 ORGANIZATION_DISPLAY_NAME="${ORGANIZATION_NAME}"
bundle exec rake cartodb:db:set_organization_quota[$ORGANIZATION_NAME,5000]
bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations[$ORGANIZATION_NAME]
bundle exec rake cartodb:set_custom_limits_for_user["${USERNAME}",10240000000,100000000,1]
# Enable sync tables
echo "UPDATE users SET sync_tables_enabled=true WHERE username='${USERNAME}'" | psql -U postgres -t carto_db_development
# Enable private maps
echo "UPDATE users SET private_maps_enabled = 't'" | psql -U postgres -t carto_db_development
bundle exec rake cartodb:features:enable_feature_for_all_users["new_dashboard"]

32
script/start_stack Executable file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
echo "--- CartoDB stack starter for testing ---"
echo "---> Killing Rails.."
kill -INT $(lsof -i:3000) &> /dev/null
echo "---> Killing Tiler.."
kill -INT $(lsof -i:8080) &> /dev/null
echo "---> Killing SQL API.."
kill -INT $(lsof -i:8181) &> /dev/null
echo "---> Killing our Redis server..."
kill -INT $(lsof -i:6379) &> /dev/null
echo "---> Restarting PostgreSQL.."
sudo /etc/init.d/postgresql restart
echo "---> Stopping system Redis server (if any)..."
sudo /etc/init.d/redis-server stop
echo "Starting Redis..."
redis-server &
echo "---> Starting Rails..."
bundle exec rails server -p 3000 -d
echo "---> Starting Tiler.."
node ../CartoDB-SQL-API/app.js development &
echo "---> Starting SQL API..."
node ../Windshaft-cartodb/app.js development &

8
script/sync_tables_trigger.sh Executable file
View File

@@ -0,0 +1,8 @@
#!/bin/bash
while :
do
sleep $SYNC_TABLES_INTERVAL
cd /cartodb
bundle exec rake cartodb:sync_tables[true]
done

24
script/xls2csv.rb Normal file
View File

@@ -0,0 +1,24 @@
require 'rubygems'
require 'roo'
if ARGV[0].nil?
puts "You should indicate a file to import"
return -1
else
unless File.file?(ARGV[0])
puts "File #{ARGV[0]} not found"
return -1
end
ext = File.extname(ARGV[0])
csv_name = File.basename(ARGV[0], ext)
s = case ext
when '.odt'
Openoffice.new(ARGV[0])
when '.xls'
Roo::Excel.new(ARGV[0])
when '.xlsx'
Roo::Excelx.new(ARGV[0])
end
s.to_csv("/tmp/#{csv_name}.csv")
puts "/tmp/#{csv_name}.csv"
end