first commit

app/services/cached-query-tables.js

@@ -0,0 +1,44 @@
+'use strict';
+
+var QueryTables = require('cartodb-query-tables');
+
+var generateMD5 = require('../utils/md5');
+
+function CachedQueryTables(tableCache) {
+    this.tableCache = tableCache;
+}
+
+module.exports = CachedQueryTables;
+
+CachedQueryTables.prototype.getAffectedTablesFromQuery = function(pg, sql, skipCache, callback) {
+    var self = this;
+
+    var cacheKey = sqlCacheKey(pg.username(), sql);
+
+    var cachedResult;
+    if (!skipCache) {
+        cachedResult = this.tableCache.peek(cacheKey);
+    }
+
+    if (cachedResult) {
+        cachedResult.hits++;
+        return callback(null, cachedResult.result);
+    } else {
+        QueryTables.getAffectedTablesFromQuery(pg, sql, function(err, result) {
+            if (err) {
+                return callback(err);
+            }
+
+            self.tableCache.set(cacheKey, {
+                result: result,
+                hits: 0
+            });
+
+            return callback(null, result);
+        });
+    }
+};
+
+function sqlCacheKey(user, sql) {
+    return user + ':' + generateMD5(sql);
+}

app/services/error_handler.js

@@ -0,0 +1,36 @@
+'use strict';
+
+class ErrorHandler extends Error {
+    constructor({ message, context, detail, hint, http_status, name }) {
+        super(message);
+
+        this.http_status = this.getHttpStatus(http_status);
+        this.context = context;
+        this.detail = detail;
+        this.hint = hint;
+
+        if (name) {
+            this.name = name;
+        }
+    }
+
+    getResponse() {
+        return {
+            error: [this.message],
+            context: this.context,
+            detail: this.detail,
+            hint: this.hint
+        };
+    }
+
+    getHttpStatus(http_status = 400) {
+        if (this.message.includes('permission denied')) {
+            return 403;
+        }
+
+        return http_status;
+    }
+
+}
+
+module.exports = ErrorHandler;

app/services/error_handler_factory.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const ErrorHandler = require('./error_handler');
+const { codeToCondition } = require('../postgresql/error_codes');
+
+module.exports = function ErrorHandlerFactory (err) {
+    if (isTimeoutError(err)) {
+        return createTimeoutError();
+    } else {
+        return createGenericError(err);
+    }
+};
+
+function isTimeoutError(err) {
+    return err.message && (
+        err.message.indexOf('statement timeout') > -1 ||
+        err.message.indexOf('RuntimeError: Execution of function interrupted by signal') > -1 ||
+        err.message.indexOf('canceling statement due to user request') > -1
+    );
+}
+
+function createTimeoutError() {
+    return new ErrorHandler({
+        message: 'You are over platform\'s limits: SQL query timeout error.' +
+                 ' Refactor your query before running again or contact CARTO support for more details.',
+        context: 'limit',
+        detail: 'datasource',
+        http_status: 429
+    });
+}
+
+function createGenericError(err) {
+    return new ErrorHandler({
+        message: err.message,
+        context: err.context,
+        detail: err.detail,
+        hint: err.hint,
+        http_status: err.http_status,
+        name: codeToCondition[err.code] || err.name
+    });
+}

app/services/logger.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const bunyan = require('bunyan');
+
+class Logger {
+    constructor (path, name) {
+        const stream = {
+            level: process.env.NODE_ENV === 'test' ? 'fatal' : 'info'
+        };
+
+        if (path) {
+            stream.path = path;
+        } else {
+            stream.stream = process.stdout;
+        }
+
+        this.path = path;
+        this.logger = bunyan.createLogger({
+            name,
+            streams: [stream]
+        });
+    }
+
+    info (log, message) {
+        this.logger.info(log, message);
+    }
+
+    warn (log, message) {
+        this.logger.warn(log, message);
+    }
+
+    reopenFileStreams () {
+        console.log('Reloading log file', this.path);
+        this.logger.reopenFileStreams();
+    }
+}
+ 
+module.exports = Logger;

app/services/pg-entities-access-validator.js

@@ -0,0 +1,63 @@
+'use strict';
+
+const FORBIDDEN_ENTITIES = {
+    carto: ['*'],
+    cartodb: [
+        'cdb_analysis_catalog',
+        'cdb_conf',
+        'cdb_tablemetadata'
+    ],
+    pg_catalog: ['*'],
+    information_schema: ['*'],
+    public: ['spatial_ref_sys'],
+    topology: [
+        'layer',
+        'topology'
+    ]
+};
+
+const Validator = {
+    validate(affectedTables, authorizationLevel) {
+        let hardValidationResult = true;
+        let softValidationResult = true;
+
+        if (!!affectedTables && affectedTables.tables) {
+            if (global.settings.validatePGEntitiesAccess) {
+                hardValidationResult = this.hardValidation(affectedTables.tables);
+            }
+
+            if (authorizationLevel !== 'master') {
+                softValidationResult = this.softValidation(affectedTables.tables);
+            }
+        }
+
+        return hardValidationResult && softValidationResult;
+    },
+
+    hardValidation(tables) {
+        for (let table of tables) {
+            if (FORBIDDEN_ENTITIES[table.schema_name] && FORBIDDEN_ENTITIES[table.schema_name].length &&
+                (
+                    FORBIDDEN_ENTITIES[table.schema_name][0] === '*' ||
+                    FORBIDDEN_ENTITIES[table.schema_name].includes(table.table_name)
+                )
+            ) {
+                return false;
+            }
+        }
+
+        return true;
+    },
+
+    softValidation(tables) {
+        for (let table of tables) {
+            if (table.table_name.match(/\bpg_/)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+};
+
+module.exports = Validator;

app/services/stream_copy.js

@@ -0,0 +1,77 @@
+'use strict';
+
+const PSQL = require('cartodb-psql');
+const copyTo = require('pg-copy-streams').to;
+const copyFrom = require('pg-copy-streams').from;
+const { Client } = require('pg');
+
+const ACTION_TO = 'to';
+const ACTION_FROM = 'from';
+const DEFAULT_TIMEOUT = "'5h'";
+
+module.exports = class StreamCopy {
+
+    constructor(sql, userDbParams) {
+        const dbParams = Object.assign({}, userDbParams, {
+            port: global.settings.db_batch_port || userDbParams.port
+        });
+        this.pg = new PSQL(dbParams);
+        this.sql = sql;
+        this.stream = null;
+        this.timeout = global.settings.copy_timeout || DEFAULT_TIMEOUT;
+    }
+
+    static get ACTION_TO() {
+        return ACTION_TO;
+    }
+
+    static get ACTION_FROM() {
+        return ACTION_FROM;
+    }
+
+    getPGStream(action, cb) {
+        this.pg.connect((err, client, done) => {
+            if (err) {
+                return cb(err);
+            }
+
+            client.query('SET statement_timeout=' + this.timeout, (err) => {
+
+                if (err) {
+                    return cb(err);
+                }
+
+                const streamMaker = action === ACTION_TO ? copyTo : copyFrom;
+                this.stream = streamMaker(this.sql);
+                const pgstream = client.query(this.stream);
+
+                pgstream
+                    .on('end', () => {
+                        if(action === ACTION_TO) {
+                            pgstream.connection.stream.resume();
+                        }
+                        done();
+                    })
+                    .on('error', err => done(err))
+                    .on('cancelQuery', err => {
+                        if(action === ACTION_TO) {
+                            // See https://www.postgresql.org/docs/9.5/static/protocol-flow.html#PROTOCOL-COPY
+                            const cancelingClient = new Client(client.connectionParameters);
+                            cancelingClient.cancel(client, pgstream);
+
+                            // see https://node-postgres.com/api/pool#releasecallback
+                            return done(err);
+                        } else if (action === ACTION_FROM) {
+                            client.connection.sendCopyFail('CARTO SQL API: Connection closed by client');
+                        }
+                    });
+
+                cb(null, pgstream);
+            });
+        });
+    }
+
+    getRowCount() {
+        return this.stream.rowCount;
+    }
+};

app/services/stream_copy_metrics.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const { getFormatFromCopyQuery } = require('../utils/query_info');
+
+module.exports = class StreamCopyMetrics {
+    constructor(logger, type, sql, user, isGzip = false) {
+        this.logger = logger;
+
+        this.type = type;
+        this.format = getFormatFromCopyQuery(sql);
+        this.isGzip = isGzip;
+        this.username = user;
+        this.size = 0;
+        this.gzipSize = 0;
+        this.rows = 0;
+
+        this.startTime = new Date();
+        this.endTime = null;
+        this.time = null;
+
+        this.success = true;
+        this.error = null;
+
+        this.ended = false;
+    }
+
+    addSize(size) {
+        this.size += size;
+    }
+
+    addGzipSize(size) {
+        this.gzipSize += size;
+    }
+
+    end(rows = null, error = null) {
+        if (this.ended) {
+            return;
+        }
+
+        this.ended = true;
+
+        if (Number.isInteger(rows)) {
+            this.rows = rows;
+        }
+
+        if (error instanceof Error) {
+            this.error = error;
+        }
+
+        this.endTime = new Date();
+        this.time = (this.endTime.getTime() - this.startTime.getTime()) / 1000;
+
+        this._log(
+            this.startTime.toISOString(),
+            this.isGzip && this.gzipSize ? this.gzipSize : null,
+            this.error ? this.error.message : null
+        );
+    }
+
+    _log(timestamp, gzipSize = null, errorMessage = null) {
+        let logData = {
+            type: this.type,
+            format: this.format,
+            size: this.size,
+            rows: this.rows,
+            gzip: this.isGzip,
+            'cdb-user': this.username,
+            time: this.time,
+            timestamp
+        };
+
+        if (gzipSize) {
+            logData.gzipSize = gzipSize;
+        }
+
+        if (errorMessage) {
+            logData.error = errorMessage;
+            this.success = false;
+        }
+
+        logData.success = this.success;
+
+        this.logger.info(logData);
+    }
+};

app/services/user_database_service.js

@@ -0,0 +1,107 @@
+'use strict';
+
+function isApiKeyFound(apikey) {
+    return apikey.type !== null &&
+        apikey.user !== null &&
+        apikey.databasePassword !== null &&
+        apikey.databaseRole !== null;
+}
+
+function UserDatabaseService(metadataBackend) {
+    this.metadataBackend = metadataBackend;
+}
+
+function errorUserNotFoundMessageTemplate (user) {
+    return `Sorry, we can't find CARTO user '${user}'. Please check that you have entered the correct domain.`;
+}
+
+function isOauthAuthorization({ apikeyToken, authorizationLevel }) {
+    return (authorizationLevel === 'master') && !apikeyToken;
+}
+
+/**
+ * Callback is invoked with `dbParams` and `authDbParams`.
+ * `dbParams` depends on AuthApi verification so it might return a public user with just SELECT permission, where
+ * `authDbParams` will always return connection params as AuthApi had authorized the connection.
+ * That might be useful when you have to run a query with and without permissions.
+ *
+ * @param {AuthApi} authApi
+ * @param {String} cdbUsername
+ * @param {Function} callback (err, dbParams, authDbParams)
+ */
+UserDatabaseService.prototype.getConnectionParams = function (username, apikeyToken, authorizationLevel, callback) {
+    this.metadataBackend.getAllUserDBParams(username, (err, dbParams) => {
+        if (err) {
+            err.http_status = 404;
+            err.message = errorUserNotFoundMessageTemplate(username);
+
+            return callback(err);
+        }
+
+        const commonDBConfiguration = {
+            port: global.settings.db_port,
+            host: dbParams.dbhost,
+            dbname: dbParams.dbname,
+        };
+
+        this.metadataBackend.getMasterApikey(username, (err, masterApikey) => {
+
+            if (err) {
+                err.http_status = 404;
+                err.message = errorUserNotFoundMessageTemplate(username);
+
+                return callback(err);
+            }
+
+            if (!isApiKeyFound(masterApikey)) {
+                const apiKeyNotFoundError = new Error('Unauthorized');
+                apiKeyNotFoundError.type = 'auth';
+                apiKeyNotFoundError.subtype = 'api-key-not-found';
+                apiKeyNotFoundError.http_status = 401;
+
+                return callback(apiKeyNotFoundError);
+            }
+
+            const masterDBConfiguration = Object.assign({
+                user: masterApikey.databaseRole,
+                pass: masterApikey.databasePassword
+            },
+                commonDBConfiguration);
+
+            if (isOauthAuthorization({ apikeyToken, authorizationLevel})) {
+                return callback(null, masterDBConfiguration, masterDBConfiguration);
+            }
+
+            // Default Api key fallback
+            apikeyToken = apikeyToken || 'default_public';
+
+            this.metadataBackend.getApikey(username, apikeyToken, (err, apikey) => {
+                if (err) {
+                    err.http_status = 404;
+                    err.message = errorUserNotFoundMessageTemplate(username);
+
+                    return callback(err);
+                }
+
+                if (!isApiKeyFound(apikey)) {
+                    const apiKeyNotFoundError = new Error('Unauthorized');
+                    apiKeyNotFoundError.type = 'auth';
+                    apiKeyNotFoundError.subtype = 'api-key-not-found';
+                    apiKeyNotFoundError.http_status = 401;
+
+                    return callback(apiKeyNotFoundError);
+                }
+
+                const DBConfiguration = Object.assign({
+                    user: apikey.databaseRole,
+                    pass: apikey.databasePassword
+                },
+                    commonDBConfiguration);
+
+                callback(null, DBConfiguration, masterDBConfiguration);
+            });
+        });
+    });
+};
+
+module.exports = UserDatabaseService;

app/services/user_limits.js

@@ -0,0 +1,27 @@
+'use strict';
+
+/**
+ * UserLimits
+ * @param {cartodb-redis} metadataBackend
+ * @param {object} options
+ */
+class UserLimits {
+    constructor(metadataBackend, options = {}) {
+        this.metadataBackend = metadataBackend;
+        this.options = options;
+
+        this.preprareRateLimit();
+    }
+
+    preprareRateLimit() {
+        if (this.options.limits.rateLimitsEnabled) {
+            this.metadataBackend.loadRateLimitsScript();
+        }
+    }
+
+    getRateLimit(user, endpointGroup, callback) {
+        this.metadataBackend.getRateLimit(user, 'sql', endpointGroup, callback);
+    }
+}
+
+module.exports = UserLimits;