Files
yunkong2.js-controller/lib/setup/setupBackup.js
2018-09-17 20:32:19 +08:00

603 lines
27 KiB
JavaScript

'use strict';
function BackupRestore(options) {
var fs = require('fs');
var tools = require(__dirname + '/../tools.js');
var pathLib = require('path');
var tmpDir = pathLib.normalize(__dirname + '/../../tmp');
var bkpDir = pathLib.normalize(__dirname + '/../../backups');
var hostname = tools.getHostName();
// allow use without new operator
if (!(this instanceof BackupRestore)) return new BackupRestore(options);
options = options || {};
if (!options.states) throw 'Invalid arguments: states is missing';
if (!options.objects) throw 'Invalid arguments: objects is missing';
if (!options.processExit) throw 'Invalid arguments: processExit is missing';
if (!options.cleanDatabase) throw 'Invalid arguments: cleanDatabase is missing';
if (!options.restartController) throw 'Invalid arguments: restartController is missing';
var objects = options.objects;
var states = options.states;
var processExit = options.processExit;
var cleanDatabase = options.cleanDatabase;
var restartController = options.restartController;
var mime;
var Upload = require(__dirname + '/setupUpload.js');
var upload = new Upload(options);
// --------------------------------------- BACKUP ---------------------------------------------------
function _copyFile(id, srcPath, destPath, callback) {
objects.readFile(id, srcPath, "", function (err, data) {
if (data) fs.writeFileSync(destPath, data);
callback();
});
}
function copyDir(id, srcPath, destPath, callback) {
var count = 0;
if (!fs.existsSync(destPath)) fs.mkdirSync(destPath);
objects.readDir(id, srcPath, function (err, res) {
if (res) {
for (var t = 0; t < res.length; t++) {
if (res[t].isDir) {
count++;
copyDir(id, srcPath + '/' + res[t].file, destPath + '/' + res[t].file, function () {
count--;
if (!count) callback();
});
} else {
if (!fs.existsSync(destPath)) fs.mkdirSync(destPath);
count++;
_copyFile(id, srcPath + '/' + res[t].file, destPath + '/' + res[t].file, function () {
count--;
if (!count) callback();
});
}
}
}
if (!count) callback();
});
}
function getBackupDir() {
var dataDir = tools.getDefaultDataDir();
// All pathes are returned always relative to /node_modules/appName.js-controller
if (dataDir) {
if (dataDir[0] === '.' && dataDir[1] === '.') {
dataDir = __dirname + '/../../' + dataDir;
} else if (dataDir[0] === '.' && dataDir[1] === '/') {
dataDir = __dirname + '/../../' + dataDir.substring(2);
}
}
dataDir = dataDir.replace(/\\/g, '/');
if (dataDir[dataDir.length - 1] !== '/') dataDir += '/';
var parts = dataDir.split('/');
parts.pop();// remove data or appName-data
parts.pop();
return parts.join('/') + '/backups/';
}
function copyFileSync(source, target) {
var targetFile = target;
// if target is a directory a new file with the same name will be created
if (fs.existsSync(target)) {
if (fs.lstatSync(target).isDirectory()) {
targetFile = pathLib.join(target, pathLib.basename(source));
}
}
fs.writeFileSync(targetFile, fs.readFileSync(source));
}
function copyFolderRecursiveSync(source, target) {
var files = [];
if (!fs.existsSync(target)) fs.mkdirSync(target);
// check if folder needs to be created or integrated
var targetFolder = pathLib.join(target, pathLib.basename(source));
if (!fs.existsSync(targetFolder)) fs.mkdirSync(targetFolder);
// copy
if (fs.lstatSync(source).isDirectory() ) {
files = fs.readdirSync(source);
files.forEach(function (file) {
var curSource = pathLib.join(source, file);
if (fs.lstatSync(curSource).isDirectory()) {
copyFolderRecursiveSync(curSource, targetFolder);
} else {
copyFileSync(curSource, targetFolder);
}
});
}
}
this.createBackup = function (name, callback) {
var count = 0;
if (!name) {
var d = new Date();
name = d.getFullYear() + '_' +
('0' + (d.getMonth() + 1)).slice(-2) + '_' +
('0' + d.getDate()).slice(-2) + '-' +
('0' + d.getHours()).slice(-2) + '_' +
('0' + d.getMinutes()).slice(-2) + '_' +
('0' + d.getSeconds()).slice(-2) + '_backup' + tools.appName;
}
name = name.replace(/\\/g, '/');
if (name.indexOf('/') === -1) {
var path = getBackupDir();
// create directory if not exists
if (!fs.existsSync(path)) {
fs.mkdirSync(path);
}
if (name.indexOf('.tar.gz') === -1) {
name = path + name + '.tar.gz';
} else {
name = path + name;
}
}
objects.getObjectList({include_docs: true}, function (err, res) {
var result = {objects: null, states: {}, config: null};
if (err) {
console.error('host.' + hostname + ' Cannot get objects: ' + err);
} else {
result.objects = res.rows;
}
if (fs.existsSync(tools.getConfigFileName())) result.config = JSON.parse(fs.readFileSync(tools.getConfigFileName(), 'utf8'));
states.getKeys('io.*', function (err, keys) {
/*for (var i = keys.length - 1; i >= 0; i--) {
if (keys[i].match(/^messagebox\./) || keys[i].match(/^log\./)) {
keys.splice(i, 1);
}
}*/
states.getStates(keys, function (err, obj) {
var hostname = tools.getHostName();
var r = new RegExp('^system\\.host\\.' + hostname + '\\.(\\w+)$');
for (var i = 0; i < keys.length; i++) {
if (obj[i].from === 'system.host.' + hostname) {
obj[i].from = 'system.host.$$__hostname__$$';
}
if (r.test(keys[i])) {
keys[i] = keys[i].replace(hostname, '$$__hostname__$$');
}
result.states[keys[i]] = obj[i];
}
if (!fs.existsSync(bkpDir)) fs.mkdirSync(bkpDir);
if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir);
if (!fs.existsSync(tmpDir + '/backup')) fs.mkdirSync(tmpDir + '/backup');
if (!fs.existsSync(tmpDir + '/backup/files')) fs.mkdirSync(tmpDir + '/backup/files');
// try to find user files
for (var j = 0; j < result.objects.length; j++) {
if (!result.objects[j].value || !result.objects[j].value._id) continue;
//if (result.objects[j].doc) delete result.objects[j].doc;
if (result.objects[j].value._id.match(/^system\.adapter\.([\w\d_-]+).(\d+)$/) &&
result.objects[j].value.common.host === hostname) {
result.objects[j].value.common.host = '$$__hostname__$$';
if (result.objects[j].doc) {
result.objects[j].doc.common.host = '$$__hostname__$$';
}
} else
if (r.test(result.objects[j].value._id)) {
result.objects[j].value._id = result.objects[j].value._id.replace(hostname, '$$__hostname__$$');
result.objects[j].id = result.objects[j].value._id;
if (result.objects[j].doc) {
result.objects[j].doc._id = result.objects[j].value._id;
}
} else if (result.objects[j].value._id === 'system.host.' + hostname) {
result.objects[j].value._id = 'system.host.$$__hostname__$$';
result.objects[j].value.common.name = result.objects[j].value._id;
result.objects[j].value.common.hostname = '$$__hostname__$$';
if (result.objects[j].value.native && result.objects[j].value.native.os) {
result.objects[j].value.native.os.hostname = '$$__hostname__$$';
}
result.objects[j].id = result.objects[j].value._id;
if (result.objects[j].doc) {
result.objects[j].doc._id = result.objects[j].value._id;
result.objects[j].doc.common.name = result.objects[j].value._id;
result.objects[j].doc.common.hostname = '$$__hostname__$$';
if (result.objects[j].doc.native && result.objects[j].value.native.os) {
result.objects[j].doc.native.os.hostname = '$$__hostname__$$';
}
}
}
// Read all files
if (result.objects[j].value.type === 'meta' &&
result.objects[j].value.common &&
result.objects[j].value.common.type === 'meta.user') {
count++;
copyDir(result.objects[j].id, '', tmpDir + '/backup/files/' + result.objects[j].id, function () {
count--;
if (!count) {
// todo: store letsencrypt files too => change it as letitbit will be better integrated
var configDir = tools.getConfigFileName().split('/');
configDir.pop();
configDir.push('letsencrypt');
var letsEncrypt = configDir.join('/');
if (fs.existsSync(letsEncrypt)) {
copyFolderRecursiveSync(letsEncrypt, tmpDir + '/backup');
}
var tar = require('tar');
var f = fs.createWriteStream(name);
f.on('finish', function () {
tools.rmdirRecursiveSync(tmpDir + '/backup');
if (callback) callback(pathLib.normalize(name));
});
f.on('error', function (err) {
console.error('host.' + hostname + ' Cannot pack directory ' + pathLib.normalize(tmpDir + '/backup') + ': ' + err);
processExit(9);
});
try {
tar.create({gzip: true, cwd: tmpDir + '/'}, ['backup']).pipe(f);
} catch (err) {
console.error('host.' + hostname + ' Cannot pack directory ' + pathLib.normalize(tmpDir + '/backup') + ': ' + err);
processExit(9);
}
}
});
}
}
fs.writeFileSync(tmpDir + '/backup/backup.json', JSON.stringify(result, null, 2));
if (!count) {
var tar = require('tar');
var f = fs.createWriteStream(name);
f.on('finish', function () {
tools.rmdirRecursiveSync(tmpDir + '/backup');
if (callback) callback(pathLib.normalize(name));
});
f.on('error', function (err) {
console.error('host.' + hostname + ' Cannot pack directory ' + pathLib.normalize(tmpDir + '/backup') + ': ' + err);
processExit(9);
});
try {
tar.create({gzip: true, cwd: tmpDir + '/'}, ['backup']).pipe(f);
} catch (err) {
console.error('host.' + hostname + ' Cannot pack directory ' + pathLib.normalize(tmpDir + '/backup') + ': ' + err);
processExit(9);
}
}
});
});
});
};
//--------------------------------------- RESTORE ---------------------------------------------------
function _setStateHelper(_index, statesList, stateObjects, callback) {
states.setRawState(statesList[_index], stateObjects[statesList[_index]], function () {
if ((_index % 200) === 0) console.log('host.' + hostname + ' Processed ' + _index + '/' + statesList.length + ' states');
_index++;
if (_index < statesList.length) {
setImmediate(_setStateHelper, _index, statesList, stateObjects, callback);
} else {
if (callback) callback();
}
});
}
function _setObjHelper(_index, _objects, callback) {
// Disable all adapters.
if (_objects[_index].id.match(/^system\.adapter\./) && !_objects[_index].id.match(/^system\.adapter\.admin/)) {
if (_objects[_index].doc.common && _objects[_index].doc.common.enabled) {
_objects[_index].doc.common.enabled = false;
}
}
if (_objects[_index].doc && _objects[_index].doc._rev) delete _objects[_index].doc._rev;
objects.setObject(_objects[_index].id, _objects[_index].doc, function (err /* , obj */) {
if (err) {
console.warn('host.' + hostname + ' Cannot restore ' + _objects[_index].id + ': ' + err);
}
if ((_index % 200) === 0) console.log('host.' + hostname + ' Processed ' + _index + '/' + _objects.length + ' objects');
_index++;
if (_index < _objects.length) {
setImmediate(_setObjHelper, _index, _objects, callback);
} else {
if (callback) callback();
}
});
}
function reloadAdapterObject(index, objectList, callback) {
if (objectList && index < objectList.length) {
objects.getObject(objectList[index]._id, function (err, obj) {
if (err || !obj) {
objects.setObject(objectList[index]._id, objectList[index], function () {
console.log('host.' + hostname + ' object ' + objectList[index]._id + ' created');
index++;
setImmediate(reloadAdapterObject, index, objectList, callback);
});
} else {
index++;
setImmediate(reloadAdapterObject, index, objectList, callback);
}
});
} else {
if (callback) callback();
}
}
function reloadAdaptersObjects(callback, dirs, index) {
if (!dirs) {
dirs = [];
var _modules;
if (fs.existsSync(__dirname + '/../../node_modules')) {
_modules = fs.readdirSync(__dirname + '/../../node_modules');
if (_modules) {
var regEx = new RegExp('^' + tools.appName + '\\.', 'i');
for (var i = 0; i < _modules.length; i++) {
if (regEx.test(_modules[i]) &&
dirs.indexOf(_modules[i].substring(tools.appName.length + 1)) === -1) {
dirs.push(_modules[i]);
}
}
}
}
// if installed as npm
if (fs.existsSync(__dirname + '/../../../../node_modules/' + tools.appName + '.js-controller')) {
_modules = fs.readdirSync(__dirname + '/../../..');
var regEx_ = new RegExp('^' + tools.appName + '\\.', 'i');
for (var j = 0; j < _modules.length; j++) {
// if starting from application name + '.'
if (regEx_.test(_modules[j]) &&
// If not js-controller
(_modules[j].substring(tools.appName.length + 1) !== 'js-controller') &&
dirs.indexOf(_modules[j].substring(tools.appName.length + 1)) === -1) dirs.push(_modules[j]);
}
}
if (dirs.length) {
reloadAdaptersObjects(callback, dirs, 0);
} else {
if (callback) callback();
}
} else {
if (index < dirs.length) {
upload.uploadAdapter(dirs[index], false, true, function () {
upload.uploadAdapter(dirs[index], true, true, function () {
var pkg = null;
if (!dirs[index]) {
console.error('Wrong');
}
var adapterDir = tools.getAdapterDir(dirs[index]);
if (fs.existsSync(adapterDir + '/io-package.json')) {
pkg = JSON.parse(fs.readFileSync(adapterDir + '/io-package.json', 'utf8'));
}
if (pkg && pkg.objects && pkg.objects.length) {
console.log('host.' + hostname + ' Setup "' + dirs[index] + '" adapter');
reloadAdapterObject(0, pkg.objects, function () {
index++;
setImmediate(reloadAdaptersObjects, callback, dirs, index);
});
} else {
index++;
reloadAdaptersObjects(callback, dirs, index);
}
});
});
} else {
if (callback) callback();
}
}
}
function uploadUserFiles(root, path, callback) {
if (typeof path === 'function') {
callback = path;
path = '';
}
var called = false;
if (!fs.existsSync(root)) {
callback();
return;
}
var files = fs.readdirSync(root + path);
var count = files.length;
for (var i = 0; i < files.length; i++) {
var stat = fs.statSync(root + path + '/' + files[i]);
if (stat.isDirectory()) {
called = true;
uploadUserFiles(root, path + '/' + files[i], function (err) {
if (err) console.error('Error: ' + err);
if (!--count) setImmediate(callback);
});
} else {
var parts = path.split('/');
var adapter = parts.splice(0, 2);
adapter = adapter[1];
var _path = parts.join('/') + '/' + files[i];
console.log('host.' + hostname + ' Upload user file "' + adapter + "/" + _path);
called = true;
objects.writeFile(adapter, _path, fs.readFileSync(root + path + '/' + files[i]), null, function (err) {
if (err) console.error('Error: ' + err);
if (!--count) setImmediate(callback);
});
}
}
if (!called) callback();
}
function restoreAfterStop(restartOnFinish, callback) {
// Open file
var data = fs.readFileSync(tmpDir + '/backup/backup.json').toString();
var hostname = tools.getHostName();
data = data.replace(/\$\$__hostname__\$\$/g, hostname);
fs.writeFileSync(tmpDir + '/backup/backup_.json', data);
var restore;
try {
restore = JSON.parse(data);
} catch (e) {
console.error('Cannot parse "' + tmpDir + '/backup/backup_.json": ' + e);
if (callback) callback(31);
}
// stop all adapters
console.log('host.' + hostname + ' Clear all objects and states...');
cleanDatabase(false, function () {
console.log('host.' + hostname + ' done.');
// upload all data into DB
// restore ioBorker.json
if (restore.config) fs.writeFileSync(tools.getConfigFileName(), JSON.stringify(restore.config, null, 2));
var sList = [];
for (var state in restore.states) {
if (restore.states.hasOwnProperty(state)) {
sList.push(state);
}
}
_setStateHelper(0, sList, restore.states, function () {
console.log(sList.length + ' states restored.');
_setObjHelper(0, restore.objects, function () {
console.log(restore.objects.length + ' objects restored.');
// Required for upload adapter
mime = require('mime');
// Load user files into DB
uploadUserFiles(tmpDir + '/backup/files', function () {
// reload objects of adapters
reloadAdaptersObjects(function () {
// Reload host objects
var pckgio = JSON.parse(fs.readFileSync(__dirname + '/../../io-package.json', 'utf8'));
reloadAdapterObject(0, pckgio ? pckgio.objects : null, function () {
if (restartOnFinish) {
restartController(callback);
} else {
if (callback) callback();
}
});
});
});
});
});
});
}
this.listBackups = function () {
var dir = getBackupDir();
var result = [];
if (fs.existsSync(dir)) {
var files = fs.readdirSync(dir);
for (var i = 0; i < files.length; i++) {
if (files[i].match(/\.tar\.gz$/i)) {
result.push(files[i]);
}
}
return result;
} else {
return result;
}
};
this.restoreBackup = function (name, callback) {
var backups;
if (!name && name !== 0) {
// List all available backups
console.log('Please specify one of the backup names:');
backups = this.listBackups();
backups.sort(function (a, b) {
return b > a;
});
if (backups.length) {
for (var t = 0; t < backups.length; t++){
console.log(backups[t] + ' or ' + backups[t].replace('_backup' + tools.appName + '.tar.gz', '') + ' or ' + t);
}
} else {
console.warn('No backups found');
}
processExit(10);
}
if (!options.cleanDatabase) throw "Invalid arguments: cleanDatabase is missing";
if (!options.restartController) throw "Invalid arguments: restartController is missing";
if (parseInt(name, 10).toString() === name.toString()) {
backups = this.listBackups();
backups.sort(function (a, b) {
return b > a;
});
name = backups[parseInt(name, 10)];
console.log('host.' + hostname + ' Using backup file ' + name);
}
name = (name || '').toString().replace(/\\/g, '/');
if (name.indexOf('/') === -1) {
name = getBackupDir() + name;
var regEx = new RegExp('_backup' + tools.appName, 'i');
if (!regEx.test(name)) name += '_backup' + tools.appName;
if (!name.match(/\.tar\.gz$/i)) name += '.tar.gz';
}
if (!fs.existsSync(name)) {
console.error('host.' + hostname + ' Cannot find ' + name);
processExit(11);
}
var tar = require('tar');
if (fs.existsSync(tmpDir + '/backup/backup.json')) {
fs.unlinkSync(tmpDir + '/backup/backup.json');
}
tar.extract({
file: name,
cwd: tmpDir
}, function (err) {
if (err) {
console.error('host.' + hostname + ' Cannot extract from file "' + name + '"');
processExit(9);
}
if (!fs.existsSync(tmpDir + '/backup/backup.json')) {
console.error('host.' + hostname + ' Cannot find extracted file from file "' + tmpDir + '/backup/backup.json"');
processExit(9);
}
// Stop controller
var daemon = require('daemonize2').setup({
main: '../../controller.js',
name: tools.appName + ' controller',
pidfile: __dirname + '/../' + tools.appName + '.pid',
cwd: '../../',
stopTimeout: 1000
});
daemon.on('error', function (/* error */) {
restoreAfterStop(false, callback);
});
daemon.on('stopped', function () {
restoreAfterStop(true, callback);
});
daemon.on('notrunning', function () {
console.log('host.' + hostname + ' OK.');
restoreAfterStop(false, callback);
});
daemon.stop();
});
}
}
module.exports = BackupRestore;