commit 541e22701e3189733016e74bb87d613be5795760 Author: zhongjin Date: Sat Sep 15 11:49:47 2018 +0800 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..89f1ea8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules +.idea +admin/i18n/flat.txt +admin/i18n/*/flat.txt +package-lock.* +iob_npm.done \ No newline at end of file diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..d0dfc01 --- /dev/null +++ b/.npmignore @@ -0,0 +1,11 @@ +gulpfile.js +tasks +.idea +node_modules +.git +test +.travis.yml +appveyor.yml +admin/i18n +package-lock.json +iob_npm.done \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2456688 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +os: + - linux + - osx +language: node_js +node_js: + - '4' + - '6' + - '8' + - '10' +before_script: + - export NPMVERSION=$(echo "$($(which npm) -v)"|cut -c1) + - 'if [[ $NPMVERSION == 5 ]]; then npm install -g npm@5; fi' + - npm -v + - npm install winston@2.3.1 + - 'npm install https://github.com/ioBroker/ioBroker.js-controller/tarball/master --production' +env: + - CXX=g++-4.8 +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c6f4b80 --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +The MIT License (MIT) + +Copyright (c) 2014-2018 Bluefox , Apollon77 + +Copyright (c) 2016 Smiling_Jack + +Copyright (c) 2014 hobbyquaker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..17f903c --- /dev/null +++ b/README.md @@ -0,0 +1,399 @@ +![Logo](admin/history.png) +# ioBroker.history +=============================== + +[![NPM version](http://img.shields.io/npm/v/iobroker.history.svg)](https://www.npmjs.com/package/iobroker.history) +[![Downloads](https://img.shields.io/npm/dm/iobroker.history.svg)](https://www.npmjs.com/package/iobroker.history) +[![Tests](http://img.shields.io/travis/ioBroker/ioBroker.history/master.svg)](https://travis-ci.org/ioBroker/ioBroker.history) + +[![NPM](https://nodei.co/npm/iobroker.history.png?downloads=true)](https://nodei.co/npm/iobroker.history/) + +This adapter saves state history in a two-staged process. +At first data points are stored in RAM, as soon as they reach maxLength they will be stored on disk. + +To set up some data points to be stored they must be configured in admin "Objects" Tab (last button). + +## Settings + +- **Storage directory** - Path to the directory, where the files will be stored. It can be done relative to "iobroker-data" or absolute, like "/mnt/history" or "D:/History" +- **Maximal number of stored in RAM values** - After this number of values reached in RAM they will be saved on disk. +- **Store origin of value** - If "from" field will be stored too. Can save place on disk. +- **De-bounce interval** - Protection against too often changes of some value and defined tha time in ms in which after one value change other changes are not logged +- **Storage retention** - How many values in the past will be stored on disk. +- **Log unchanged values any(s)** - When using "log changes only" you can set a time interval in seconds here after which also unchanged values will be re-logged into the DB + +Most of these values will be pre filled on the detail settings for the datapoint and can be changed there. Additionally you can an "alias ID" on the datapoint page. With this you can, e.g. after switching a device and datapoint names changed, still log the data to the former ID by just entering this ID there and all data will be logged as this one. + + +## Access values from Javascript adapter +The sorted values can be accessed from Javascript adapter. E.g. with following code you can read the list of events for last hour: + +``` +// Get 50 last stored events for all IDs +sendTo('history.0', 'getHistory', { + id: '*', + options: { + end: new Date().getTime(), + count: 50, + aggregate: 'onchange' + } +}, function (result) { + for (var i = 0; i < result.result.length; i++) { + console.log(result.result[i].id + ' ' + new Date(result.result[i].ts).toISOString()); + } +}); + +// Get stored values for "system.adapter.admin.0.memRss" in last hour +var end = new Date().getTime(); +sendTo('history.0', 'getHistory', { + id: 'system.adapter.admin.0.memRss', + options: { + start: end - 3600000, + end: end, + aggregate: 'onchange' + } +}, function (result) { + for (var i = 0; i < result.result.length; i++) { + console.log(result.result[i].id + ' ' + new Date(result.result[i].ts).toISOString()); + } +}); +``` + +Possible options: +- **start** - (optional) time in ms - *new Date().getTime()*' +- **end** - (optional) time in ms - *new Date().getTime()*', by default is (now + 5000 seconds) +- **step** - (optional) used in aggregate (m4, max, min, average, total) step in ms of intervals +- **count** - number of values if aggregate is 'onchange' or number of intervals if other aggregate method. Count will be ignored if step is set. +- **from** - if *from* field should be included in answer +- **ack** - if *ack* field should be included in answer +- **q** - if *q* field should be included in answer +- **addId** - if *id* field should be included in answer +- **limit** - do not return more entries than limit +- **ignoreNull** - if null values should be include (false), replaced by last not null value (true) or replaced with 0 (0) +- **aggregate** - aggregate method: + - *minmax* - used special algorithm. Splice the whole time range in small intervals and find for every interval max, min, start and end values. + - *max* - Splice the whole time range in small intervals and find for every interval max value and use it for this interval (nulls will be ignored). + - *min* - Same as max, but take minimal value. + - *average* - Same as max, but take average value. + - *total* - Same as max, but calculate total value. + - *count* - Same as max, but calculate number of values (nulls will be calculated). + - *none* - No aggregation at all. Only raw values in given period. + +The first and last points will be calculated for aggregations, except aggregation "none". +If you manually request some aggregation you should ignore first and last values, because they are calculated from values outside of period. + +## storeState +If you want to write other data into the InfluxDB you can use the build in system function **storeState**. +This function can also be used to convert data from other History adapters like History or SQL. + +The given ids are not checked against the ioBroker database and do not need to be set up there, but can only be accessed directly. + +The Message can have one of the following three formats: +* one ID and one state object +* one ID and array of state objects +* array of multiple IDs with state objects + +## History Logging Management via Javascript +The adapter supports enabling and disabling of history logging via JavaScript and also retrieving the list of enabled datapoints with their settings. + +### enable +The message requires to have the "id" of the datapoint.Additionally optional "options" to define the datapoint specific settings: + +``` +sendTo('history.0', 'enableHistory', { + id: 'system.adapter.history.0.memRss', + options: { + changesOnly: true, + debounce: 0, + retention: 31536000, + maxLength: 3, + changesMinDelta: 0.5, + aliasId: '' + } +}, function (result) { + if (result.error) { + console.log(result.error); + } + if (result.success) { + //successfull enabled + } +}); +``` + +### disable +The message requires to have the "id" of the datapoint. + +``` +sendTo('history.0', 'disableHistory', { + id: 'system.adapter.history.0.memRss', +}, function (result) { + if (result.error) { + console.log(result.error); + } + if (result.success) { + //successfull enabled + } +}); +``` + +### get List +The message has no parameters. + +``` +sendTo('history.0', 'getEnabledDPs', {}, function (result) { + //result is object like: + { + "system.adapter.history.0.memRss": { + "changesOnly":true, + "debounce":0, + "retention":31536000, + "maxLength":3, + "changesMinDelta":0.5, + "enabled":true, + "changesRelogInterval":0, + "aliasId": "" + } + ... + } +}); +``` + +## Data converter +### General idea +When you have more data over time then the history adapter may not be the best choice and a real database is better. For this there are two more History-Adapters for SQL databases (PostgreSQL, MS-SQL, MySQL, SQLite) and InfluxDB. +With this change the quesion comes up how to convert the collected data from the past to these new adapters. + +For this some converter scripts have been prepared that can help and do the job. These scripts are called from the command line. + +### Prepare and analyze existing data in transfer target +When converting data only those data should be transferred that are not already there. Therefor the first set of scripts exists called **analyze.js**. This script should be called once at the beginning to collect some data for existing data and store them in local .json files to be used by the real converter script. +Two kind of data are collected: +- **earliest value for datapoint ID**: The timestamp of the very first entry for each existing datapoint is stored and is used by imported to ignore all newer values by default. The assumption is that the data are filled completely beginning with this first entry and all earlier values would else be duplicated. This assumption can be overwritten on import by parameters. +- **existing values per day per datapoint ID**: The existing data are analyzed on a per day basis and each day is stored where data exist already. This can be used as alternative to the first data to be able to also fill "holes" in the data. + +#### analyzeinflux.js +The analyzeinflux.js can be found in the directory "converter". +This script will collect the above mentioned data for an InfluxDB instance. + +**Usage**: nodejs analyzeinflux.js [] [] [--deepAnalyze] +**Example**: nodejs analyzeinflux.js influxdb.0 info --deepAnalyze + +Parameters: +- ****: which influxdb-Adapter instance should be used? (Default: influxdb.0) If set needs to be first parameter after scriptname. +- ****: Loglevel for output (Default: info). If set needs to be second parameter after scriptname. +- **--deepAnalyze**: collect the existing values per day too, by default only the earliest value is queried. + +The script will then generate one or three .json files with the collected data. These files are then used by the real converter script. + +#### analyzesql.js +The analyzesql.js can be found in the directory "converter". +This script will collect parts of the above mentioned data for an SQL instance. + +**Usage**: nodejs analyzesql.js [] [] +**Example**: nodejs analyzesql.js sql.0 info + +Parameters: +- ****: which SQL-Adapter instance should be used? (Default: sql.0) If set needs to be first parameter after scriptname. +- ****: Loglevel for output (Default: info). If set needs to be second parameter after scriptname. + +The script will then generate two .json files with the collected data. These files are then used by the real converter script. +Currently --processNonExistingValuesOnly for converter script can not be used because the data are not collected. + +### Convert History-Data to a DB +The history2db.js can be found in the directory "converter". + +The script will directly use the generated JSON files from the history adapter on disk to transfer them into the Database. +Additionally it uses the pre-generated data files for already existing values in the target DB to only convert not existing data. + +The script can be run without any analyze step beforehand then you need to set the startdata as parameter and it will simply convert anything from that timepoint backwards in time. +When you have run an analyze before and the earliestDBValues.json file exists then only these datapoints are converted, unless you use parameters to change that. +When an analyze was run before and the datafiles are used, they are also updated with all converted data, so a second run will normally not generate duplicates. +To reset the data delete the File "earliestDBValues.json", "existingDBValues.json" and/or "existingDBTypes.json". + +The Converter then goes backward in time through all the days available as data and will determine which data to transfer to InfluxDB. + +If you want to abort the process you can press "x" or "" and the converter aborts after the current datafile. + +The converter script itself should work with all History adapters that support "storeState" methods. + +Note: Migrating many data will produce a certain load on the system, especially when converter and target database instance are running on the same machine. Monitor your systems load and performance during the action and maybe use the "delayMultiplicator" parameter to increase delays in the converter. + +**Usage:** nodejs history2influx.js DB-Instance [Loglevel] [Date-to-start|0] [path-to-Data] [delayMultiplicator] [--logChangesOnly [relog-Interval(m)]] [--ignoreExistingDBValues] [--processNonExistingValuesOnly] [--processAllDPs] [--simulate] +**Example**: nodejs history2influx.js influxdb.0 info 20161001 /path/to/data 2 --logChangesOnly 30 --processNonExistingValuesOnly + +Possible options and Parameter: +- **DB-Instance**: DB-Instance to send the data to.Required parameter. Needs to be first parameter after scriptname. +- **Loglevel**: Loglevel for output (Default: info). If set needs to be second parameter after scriptname. +- **Date-to-start**: Day to start in format yyyymmdd (e.g. 20161028). Use "0" to use detected earliest values. If set needs to be third parameter after scriptname. +- **path-to-Data**: Path to the datafiles. Defauls to iobroker-install-directory/iobroker-data/history-data . If set needs to be fourth parameter after scriptname. +- ****: Modify the delays between several actions in the script by a multiplicator. "2" would mean that the delays the converted had calculated by itself are doubled. If set needs to be fifth parameter after scriptname. +- **--logChangesOnly [relog-Interval(m)]**: when --logChangesOnly is set the data are parsed and reduced, so that only changed values are stored in InfluxDB. Additionally a "relog-Interval(s)"" can be set in minutes to re-log unchanged values after this interval. +- **--ignoreExistingDBValues**: With this parameter all existing data are ignored and all data are inserted into DB. Please make sure that no duplicates are generated. This option is usefull to fix "holes" in the data where some data are missing. By default it only fills all datapoints with at least one entry in the DB. This can be overwritten by --processAllDPs +- **--processNonExistingValuesOnly**: With this parameter the "existing datapoints by day" file from the analyze script is used and checked for each day and datapoint. In this mode the existing-DB-Values are always ignored, and also not updated, so please do another analyze run after using that mode!!! +- **--processAllDPs**: With this parameter you make sure that all existing datapoints from the history files is transferred into the DB, also if these are not existing in that DB so far. +- **--simulate**: With this parameter you enable the simulation mode, means that no real write happends and also the analyze-datafiles will not be updated on exit. + +## Changelog +## 1.8.5 (2018-07-02) +* (Apollon77) Error fixed in storeState + +## 1.8.4 (2018-06-24) +* (Apollon77) Fixing/allow to disable writing of start and end values + +## 1.8.0 (2018-06-19/24) +* (Apollon77) Add option to write data to a different ID to make device changes easier. Retrieving data works for both IDs + +## 1.7.4 (2018-04-03) +* (AlCalzone) Fix filename handling for states with special characters + +## 1.7.3 (2018-03-28) +* (Apollon77) Respect 'keep forever' setting for retention from datapoint configuration + +## 1.7.2 (2018-02-05) +* (bondrogeen) Admin3 Fixes + +## 1.7.1 (2018-01-31) +* (Bluefox) Admin3 Fixes + +## 1.7.0 (2018-01-17) +* (bluefox) Ready for Admin3 + +## 1.6.6 (2017-12-20) +* (bluefox) translations + +## 1.6.5 (2017-10-05) +* (Apollon77) fix relog value feature + +## 1.6.4 (2017-08-12) +* (bluefox) add "save last value" option + +## 1.6.3 (2017-08-03) +* (Apollon77) fix behaviour of log interval to always log the current value + +## 1.6.2 (2017-04-07) +* fix in datatype conversions + +### 1.6.0 (2017-02-28) +* (Apollon77) Replace some characters in history filenames + +### 1.5.3 (2017-02-22) +* (Apollon77) Small fix for older configurations + +### 1.5.2 +* (Apollon77) Enhance Min-Delta logic for datapoints from type mixed + +### 1.5.1 (2017-01-16) +* (bluefox) Fix handling of float values in Adapter config and Datapoint config. + +### 1.5.0 (2016-12-01) +* (Apollon77) Add messages enableHistory/disableHistory +* (Apollon77) add support to log changes only if value differs a minimum value for numbers +* (Apollon77) Fixing aggregate calculation + +### 1.4.0 (2016-10-29) +* (Apollon77) add option to re-log unchanged values to make it easier for visualization +* (Apollon77) added converter scripts to move history data to db + +### 1.3.1 (2016-09-25) +* (Apollon77) Fixed: ts is assigned as val +* (bluefox) Fix selector for history objects + +### 1.3.0 (2016-08-30) +* (bluefox) сompatible only with new admin + +### 1.2.0 (2016-08-27) +* (bluefox) change name of object from history to custom + +### 1.1.0 (2016-08-27) +* (bluefox) fix aggregation of last point +* (bluefox) aggregation none just deliver the raw data without any aggregation + +### 1.0.5 (2016-07-24) +* (bluefox) fix aggregation on large intervals + +### 1.0.4 (2016-07-05) +* (bluefox) fix aggregation on seconds + +### 1.0.3 (2016-05-31) +* (bluefox) draw line to the end if ignore null + +### 1.0.2 (2016-05-29) +* (bluefox) switch max and min with each other + +### 1.0.1 (2016-05-28) +* (bluefox) calculate end/start values for "on change" too + +### 1.0.0 (2016-05-20) +* (bluefox) change default aggregation name + +### 0.4.1 (2016-05-14) +* (bluefox) support sessionId + +### 0.4.0 (2016-05-05) +* (bluefox) use aggregation file from sql adapter +* (bluefox) fix the values storage on exit +* (bluefox) store all cached data every 5 minutes +* (bluefox) support of ms + +### 0.2.1 (2015-12-14) +* (bluefox) add description of settings +* (bluefox) place aggregate function into separate file to enable sharing with other adapters +* (smiling-Jack) Add generate Demo data +* (smiling-Jack) get history in own fork +* (bluefox) add storeAck flag +* (bluefox) mockup for onchange + +### 0.2.0 (2015-11-15) +* (Smiling_Jack) save and load in adapter and not in js-controller +* (Smiling_Jack) aggregation of data points +* (Smiling_Jack) support of storage path + +### 0.1.3 (2015-02-19) +* (bluefox) fix small error in history (Thanks on Dschaedl) +* (bluefox) update admin page + +### 0.1.2 (2015-01-20) +* (bluefox) enable save&close button by config + +### 0.1.1 (2015-01-10) +* (bluefox) check if state was not deleted + +### 0.1.0 (2015-01-02) +* (bluefox) enable npm install + +### 0.0.8 (2014-12-25) +* (bluefox) support of de-bounce interval + +### 0.0.7 (2014-11-01) +* (bluefox) store every change and not only lc != ts + +### 0.0.6 (2014-10-19) +* (bluefox) add configuration page + +## License + +The MIT License (MIT) + +Copyright (c) 2014-2018 Bluefox , Apollon77 + +Copyright (c) 2016 Smiling_Jack + +Copyright (c) 2014 hobbyquaker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/admin/.gitignore b/admin/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/admin/custom.html b/admin/custom.html new file mode 100644 index 0000000..bcbadd2 --- /dev/null +++ b/admin/custom.html @@ -0,0 +1,164 @@ + + + + diff --git a/admin/custom_m.html b/admin/custom_m.html new file mode 100644 index 0000000..c507afb --- /dev/null +++ b/admin/custom_m.html @@ -0,0 +1,140 @@ + + + + diff --git a/admin/history.png b/admin/history.png new file mode 100644 index 0000000..88e7cb6 Binary files /dev/null and b/admin/history.png differ diff --git a/admin/index.html b/admin/index.html new file mode 100644 index 0000000..0685b32 --- /dev/null +++ b/admin/index.html @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + + + + + + + +
+ + + + + +

History adapter settings

+
+ +
+ + + + + + + + + + + + + + + + + + + +
+

Storage settings:

+
note
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Default settings:

0 = disable
0 = disable delta check
+
+
+
+ + diff --git a/admin/index_m.html b/admin/index_m.html new file mode 100644 index 0000000..d63fda7 --- /dev/null +++ b/admin/index_m.html @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + +
+
+
+ +
+
+
+
+ +
+
+
+
+
note
+
+
+ +
+
+ + +
+
+ + storeFrom +
+
+ + storeAck +
+
+
+
+
+
+ + +
+
+ + +
+
+ + ( + 0 = disable) +
+
+ + ( + 0 = disable delta check) +
+
+ + +
+
+
+ + +
+
+
+
+
+
+ + diff --git a/admin/words.js b/admin/words.js new file mode 100644 index 0000000..b30268d --- /dev/null +++ b/admin/words.js @@ -0,0 +1,402 @@ +// DO NOT EDIT THIS FILE!!! IT WILL BE AUTOMATICALLY GENERATED FROM src/i18n +/*global systemDictionary:true */ +'use strict'; + +systemDictionary = { + "0 = disable": { + "en": "0 = disable", + "de": "0 = Deaktivieren", + "ru": "0 = не активно", + "pt": "0 = desativar", + "nl": "0 = uitschakelen", + "fr": "0 = désactiver", + "it": "0 = disabilita", + "es": "0 = disable", + "pl": "0 = wyłącz" + }, + "0 = disable delta check": { + "en": "0 = disable delta check", + "de": "0 = Abweichungsprüfung deaktivieren", + "ru": "0 = не проверять минимальное изменение", + "pt": "0 = desativar o cheque delta", + "nl": "0 = delta-controle uitschakelen", + "fr": "0 = désactiver le contrôle delta", + "it": "0 = disabilita il controllo delta", + "es": "0 = disable delta check", + "pl": "0 = wyłącz kontrolę delta" + }, + "1 day": { + "en": "1 day", + "de": "1 Tag", + "ru": "1 день", + "pt": "1 dia", + "nl": "1 dag", + "fr": "1 jour", + "it": "1 giorno", + "es": "1 day", + "pl": "1 dzień" + }, + "1 months": { + "en": "1 months", + "de": "1 Monat", + "ru": "1 месяц", + "pt": "1 mês", + "nl": "1 maanden", + "fr": "1 mois", + "it": "1 mese", + "es": "1 months", + "pl": "1 miesiąc" + }, + "1 week": { + "en": "1 week", + "de": "1 Woche", + "ru": "1 неделя", + "pt": "1 semana", + "nl": "1 week", + "fr": "1 semaine", + "it": "1 settimana", + "es": "1 week", + "pl": "1 tydzień" + }, + "1 year": { + "en": "1 year", + "de": "1 Jahr", + "ru": "1 год", + "pt": "1 ano", + "nl": "1 jaar", + "fr": "1 an", + "it": "1 anno", + "es": "1 year", + "pl": "1 rok" + }, + "2 weeks": { + "en": "2 weeks", + "de": "2 Wochen", + "ru": "2 недели", + "pt": "2 semanas", + "nl": "2 weken", + "fr": "2 semaines", + "it": "2 settimane", + "es": "2 weeks", + "pl": "2 tygodnie" + }, + "2 years": { + "en": "2 years", + "de": "2 Jahre", + "ru": "2 года", + "pt": "2 anos", + "nl": "2 jaar", + "fr": "2 ans", + "it": "2 anni", + "es": "2 years", + "pl": "2 lata" + }, + "3 days": { + "en": "3 days", + "de": "3 Tage", + "ru": "3 дня", + "pt": "3 dias", + "nl": "3 dagen", + "fr": "3 jours", + "it": "3 giorni", + "es": "3 days", + "pl": "3 dni" + }, + "3 months": { + "en": "3 months", + "de": "3 Monate", + "ru": "3 месяца", + "pt": "3 meses", + "nl": "3 maanden", + "fr": "3 mois", + "it": "3 mesi", + "es": "3 months", + "pl": "3 miesiące" + }, + "5 days": { + "en": "5 days", + "de": "5 Tage", + "ru": "5 дней", + "pt": "5 dias", + "nl": "5 dagen", + "fr": "5 jours", + "it": "5 giorni", + "es": "5 days", + "pl": "5 dni" + }, + "6 months": { + "en": "6 months", + "de": "6 Monate", + "ru": "6 месяцев", + "pt": "6 meses", + "nl": "6 maanden", + "fr": "6 mois", + "it": "6 mesi", + "es": "6 months", + "pl": "6 miesięcy" + }, + "Curve": { + "en": "Curve", + "de": "Kurve", + "ru": "Тип", + "pt": "Curva", + "nl": "Kromme", + "fr": "Courbe", + "it": "Curva", + "es": "Curve", + "pl": "Krzywa" + }, + "DEC": { + "en": "DEC", + "de": "DEC", + "ru": "DEC", + "pt": "DEC", + "nl": "december", + "fr": "DEC", + "it": "dicembre", + "es": "DEC", + "pl": "DEC" + }, + "DEC/INC": { + "en": "DEC/INC", + "de": "DEC/INC", + "ru": "DEC/INC", + "pt": "DEC / INC", + "nl": "DEC / INC", + "fr": "DEC / INC", + "it": "DEC / INC", + "es": "DEC/INC", + "pl": "DEC / INC" + }, + "Default settings": { + "en": "Default settings", + "de": "Standardeinstellungen für Zustände", + "ru": "Настройки по умолчанию для состояний", + "pt": "Configurações padrão", + "nl": "Standaard instellingen", + "fr": "Paramètres par défaut", + "it": "Impostazioni predefinite", + "es": "Default settings", + "pl": "Ustawienia domyślne" + }, + "End": { + "en": "End", + "de": "Ende", + "ru": "Конец", + "pt": "Fim", + "nl": "Einde", + "fr": "Fin", + "it": "Fine", + "es": "End", + "pl": "Koniec" + }, + "Generate": { + "en": "Generate", + "de": "Generate", + "ru": "Генерировать", + "pt": "Gerar", + "nl": "voortbrengen", + "fr": "Générer", + "it": "creare", + "es": "Generate", + "pl": "Generować" + }, + "History adapter settings": { + "en": "History adapter settings", + "de": "History-Adapter Einstellungen", + "ru": "Настройки профайлера", + "pt": "Configurações do adaptador de histórico", + "nl": "Geschiedenis adapter instellingen", + "fr": "Paramètres de l'adaptateur d'historique", + "it": "Impostazioni della scheda Cronologia", + "es": "History adapter settings", + "pl": "Ustawienia adaptera historii" + }, + "ID": { + "en": "ID", + "de": "ID", + "ru": "ID", + "pt": "identidade", + "nl": "ID kaart", + "fr": "ID", + "it": "ID", + "es": "ID", + "pl": "ID" + }, + "INC": { + "en": "INC", + "de": "INC", + "ru": "INC", + "pt": "INC", + "nl": "INC", + "fr": "INC", + "it": "INC", + "es": "INC", + "pl": "INC" + }, + "Sinus": { + "en": "Sinus", + "de": "Sinus", + "ru": "Синус", + "pt": "Seio", + "nl": "Sinus", + "fr": "Sinus", + "it": "Seno", + "es": "Sinus", + "pl": "Zatoka" + }, + "Start": { + "en": "Start", + "de": "Start", + "ru": "Начало", + "pt": "Começar", + "nl": "Begin", + "fr": "Début", + "it": "Inizio", + "es": "Start", + "pl": "Początek" + }, + "Steps": { + "en": "Steps", + "de": "Schritte", + "ru": "Шаги", + "pt": "Passos", + "nl": "Stappen", + "fr": "Pas", + "it": "passi", + "es": "Steps", + "pl": "Kroki" + }, + "Storage settings": { + "en": "Storage settings", + "de": "Storage-Einstellungen", + "ru": "Настройки сохранения", + "pt": "Configurações de armazenamento", + "nl": "Opslaginstellingen", + "fr": "Paramètres de stockage", + "it": "Impostazioni di archiviazione", + "es": "Storage settings", + "pl": "Ustawienia pamięci" + }, + "debounce": { + "en": "De-bounce interval (ms)", + "de": "Entprellzeit (ms)", + "ru": "Минимальный интервал (ms)", + "pt": "Intervalo de desvio (ms)", + "nl": "De-bounce interval (ms)", + "fr": "Intervalle de rebond (ms)", + "it": "Intervallo di rimbalzo (ms)", + "es": "De-bounce interval (ms)", + "pl": "Interwał odbicia (ms)" + }, + "keep forever": { + "en": "keep forever", + "de": "Keine automatische Löschung", + "ru": "хранить вечно", + "pt": "mantenha para sempre", + "nl": "blijf voor altijd", + "fr": "Garder pour toujours", + "it": "tienilo per sempre", + "es": "keep forever", + "pl": "zachowaj na zawsze" + }, + "log changes interval(s)": { + "en": "Log unchanged values any (s)", + "de": "Gleiche Werte alle x Sekunden aufzeichnen", + "ru": "Запись неизменённых значений каждые (сек)", + "pt": "Registre valores inalterados qualquer (s)", + "nl": "Log onveranderde waarden voor elke", + "fr": "Consigner les valeurs inchangées toutes les x secondes", + "it": "Registra valori invariati nessuno (i)", + "es": "Log unchanged values any (s)", + "pl": "Zaloguj niezmienione wartości dowolne (s)" + }, + "log changes minimal delta": { + "en": "Minimum difference from last value to log", + "de": "Minimale Abweichung vom letzten Wert für Aufzeichnung", + "ru": "Минимальная разница с последним записанным значением", + "pt": "Diferença mínima do último valor ao log", + "nl": "Minimaal verschil van laatste waarde tot log", + "fr": "Différence minimale à la dernière valeur", + "it": "Differenza minima dall'ultimo valore da registrare", + "es": "Minimum difference from last value to log", + "pl": "Minimalna różnica od ostatniej wartości do dziennika" + }, + "maxLength": { + "en": "Maximal number of stored in RAM values", + "de": "Maximale Anzahl von Werten in RAM", + "ru": "Количество событий в RAM", + "pt": "Número máximo de valores armazenados em RAM", + "nl": "Maximaal aantal opgeslagen in RAM-waarden", + "fr": "Nombre maximal de valeurs stockées dans la RAM", + "it": "Numero massimo di valori memorizzati in RAM", + "es": "Maximal number of stored in RAM values", + "pl": "Maksymalna liczba przechowywanych w pamięci RAM" + }, + "note": { + "en": "History settings must be done for every state individually on the tab \"States\" and only if adapter is active", + "de": "History-Einstellungen mussen für jeden State einzeln gemacht werden auf dem Reiter \"States\" und nur wenn der Adapter aktiv ist", + "ru": "Настройки профайлера делаются для каждого состояния отдельно в закладке \"Состояния\" и только, когда драйвер активирован", + "pt": "As configurações de histórico devem ser feitas para cada estado individual na guia \"Estados\" e somente se o adaptador estiver ativo", + "nl": "Geschiedenisinstellingen moeten voor elke afzonderlijke status worden uitgevoerd op het tabblad \"Staten\" en alleen als de adapter actief is", + "fr": "Les paramètres d'historique doivent être définis pour chaque état individuellement dans l'onglet \"Etats\" et uniquement si l'adaptateur est actif", + "it": "Le impostazioni della cronologia devono essere eseguite singolarmente per ogni stato nella scheda \"Stati\" e solo se l'adattatore è attivo", + "es": "History settings must be done for every state individually on the tab \"States\" and only if adapter is active", + "pl": "Ustawienia historii muszą być wykonane dla każdego stanu indywidualnie na karcie \"Stany\" i tylko wtedy, gdy adapter jest aktywny" + }, + "retention": { + "en": "Storage retention", + "de": "Storage Vorhaltezeit", + "ru": "Время хранения в базе", + "pt": "Retenção de armazenamento", + "nl": "Opslag retentie", + "fr": "Conservation du stockage", + "it": "Conservazione di memoria", + "es": "Storage retention", + "pl": "Przechowywanie danych" + }, + "storeAck": { + "en": "Store acknowledge of value", + "de": "Speichere ACK vom Ereignis mit", + "ru": "Сохранять подтверждение события", + "pt": "Recorde de valor", + "nl": "Bewaar erkenning van waarde", + "fr": "Enregistrer la confirmation (ACK) de la valeur", + "it": "Memorizza valore di negozio", + "es": "Store acknowledge of value", + "pl": "Zapisz potwierdzenie wartości" + }, + "storeDir": { + "en": "Storage directory", + "de": "Speicherverzeichnis", + "ru": "Путь для файлов", + "pt": "Diretório de armazenamento", + "nl": "Opslagmap", + "fr": "Répertoire de stockage", + "it": "Directory di archiviazione", + "es": "Storage directory", + "pl": "Katalog pamięci" + }, + "storeFrom": { + "en": "Store origin of value", + "de": "Speichere Quelle vom Ereignis mit", + "ru": "Сохранять источник события", + "pt": "Armazenar origem do valor", + "nl": "Bewaar oorsprong van waarde", + "fr": "Enregistrer l'origine de la valeur", + "it": "Memorizza l'origine del valore", + "es": "Store origin of value", + "pl": "Przechowuj pochodzenie wartości:", + }, + "Write NULL values on start/stop boundaries:": { + "en": "Write NULL values on start/stop boundaries:", + "de": "Schreibe NULL-Werte an Start- / Stop-Grenzen:", + "ru": "Запись значений NULL на границах начала / остановки:", + "pt": "Escreva valores NULL nos limites de início / parada:", + "nl": "Schrijf NULL-waarden op start / stop-grenzen:", + "fr": "Écrire des valeurs NULL sur les limites de démarrage / arrêt:", + "it": "Scrivi valori NULL sui limiti di avvio / arresto:", + "es": "Escribe valores NULL en los límites de inicio / finalización:", + "pl": "Napisz wartości NULL na granicach start / stop:" + } +}; diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..84ca0a9 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,25 @@ +version: 'test-{build}' +environment: + matrix: + - nodejs_version: '4' + - nodejs_version: '6' + - nodejs_version: '8' + - nodejs_version: '10' +platform: + - x86 + - x64 +clone_folder: 'c:\projects\%APPVEYOR_PROJECT_NAME%' +install: + - ps: 'Install-Product node $env:nodejs_version $env:platform' + - ps: '$NpmVersion = (npm -v).Substring(0,1)' + - ps: 'if($NpmVersion -eq 5) { npm install -g npm@5 }' + - ps: npm --version + - npm install + - npm install winston@2.3.1 + - 'npm install https://github.com/ioBroker/ioBroker.js-controller/tarball/master --production' +test_script: + - echo %cd% + - node --version + - npm --version + - npm test +build: 'off' diff --git a/converter/analyzeinflux.js b/converter/analyzeinflux.js new file mode 100644 index 0000000..7db30b4 --- /dev/null +++ b/converter/analyzeinflux.js @@ -0,0 +1,149 @@ +/* jshint -W097 */// jshint strict:false +/*jslint node: true */ +"use strict"; + +//noinspection JSUnresolvedFunction + +//usage: nodejs analyzeinflux.js [] +//usage: nodejs analyzeinflux.js influxdb.0 info +var utils = require(__dirname + '/../lib/utils'); // Get common adapter utils + +var fs = require('fs'); +var path = require('path'); + +var deepAnalyze = false; +var influxInstance = "influxdb.0"; +if (process.argv.indexOf('--deepAnalyze') !== -1) deepAnalyze = true; +if (process.argv[2] && (process.argv[2].indexOf('influxdb') === 0)) { + influxInstance = process.argv[2]; +} +process.argv[2] = "--install"; +console.log('Query Data from ' + influxInstance); +if (deepAnalyze) console.log('Do deep analysis to find holes in data'); + +var earliestDBValue = {}; +var earliesValCachefile = __dirname + '/earliestDBValues.json'; +var existingData = {}; +var existingDataCachefile = __dirname + '/existingDBValues.json'; +var existingTypes = {}; +var existingTypesCachefile = __dirname + '/existingDBTypes.json'; + +var adapter = utils.Adapter('history'); + +var breakIt = false; + +var stdin = process.stdin; +// without this, we would only get streams once enter is pressed +stdin.setRawMode( true ); +// resume stdin in the parent process (node app won't quit all by itself +// unless an error or process.exit() happens) +stdin.resume(); +// i don't want binary, do you? +stdin.setEncoding( 'utf8' ); +// on any data into stdin +stdin.on( 'data', function( key ){ + // ctrl-c ( end of text ) + if ( key === 'x' || key === '\u0003') { + breakIt = true; + } + // write the key to stdout all normal like + console.log('Received Keypress: ' + key); +}); + +adapter.on('ready', function () { + main(); +}); + +function main() { + var inited = false; + var counter = 0; + adapter.sendTo(influxInstance, "query", "SHOW MEASUREMENTS", function(result) { + if (result.error) { + console.error(result.error); + } else { + // show result + console.log('Datapoints found: ' + result.result[0].length); + var dp_list = result.result[0]; + + function analyze() { + if (breakIt) process.exit(); + if (dp_list.length>0) { + counter++; + if (counter%100 === 0) { + setTimeout(analyze,5000); + } + else { + var dp = dp_list.shift(); + var query = "SELECT FIRST(ack) AS val FROM \"" + dp.name + "\""; + if (deepAnalyze) { + query += ";SELECT count(ack) AS val FROM \"" + dp.name + "\" where time ' + new Date(earliestDBValue[dp.name]).toString()); + } + if ((deepAnalyze) && (resultDP.result[1])) { + existingData[dp.name]=[]; + for (var j = 0;j < resultDP.result[1].length; j++) { + if (resultDP.result[1][j].val > 0) { + var ts = new Date(resultDP.result[1][j].ts); + existingData[dp.name].push(parseInt(ts2day(ts), 10)); + } + } + console.log('DayVals ID: '+dp.name+': '+JSON.stringify(existingData[dp.name])); + } + if ((deepAnalyze) && (resultDP.result[2]) && (resultDP.result[2][0])) { + existingTypes[dp.name]=typeof resultDP.result[2][0].val; + console.log('ValType ID: '+dp.name+': '+JSON.stringify(existingTypes[dp.name])); + } + } + setTimeout(analyze,500); + }); + } + } + else { + if (deepAnalyze) { + fs.writeFileSync(existingDataCachefile, JSON.stringify(existingData)); + fs.writeFileSync(existingTypesCachefile, JSON.stringify(existingTypes)); + } + fs.writeFileSync(earliesValCachefile, JSON.stringify(earliestDBValue)); + process.exit(); + } + } + + analyze(); + } + }); +} + + +function ts2day(ts) { + if (ts < 946681200000) ts *= 1000; + var dateObj = new Date(ts); + + var text = dateObj.getFullYear().toString(); + var v = dateObj.getMonth() + 1; + if (v < 10) text += '0'; + text += v.toString(); + + v = dateObj.getDate(); + if (v < 10) text += '0'; + text += v.toString(); + + return text; +} + +process.on('SIGINT', function () { + breakIt = true; +}); + +process.on('uncaughtException', function (err) { + console.log(err); + breakIt = true; +}); diff --git a/converter/analyzesql.js b/converter/analyzesql.js new file mode 100644 index 0000000..0cfa546 --- /dev/null +++ b/converter/analyzesql.js @@ -0,0 +1,69 @@ +/* jshint -W097 */// jshint strict:false +/*jslint node: true */ +"use strict"; + +//noinspection JSUnresolvedFunction + +//usage: nodejs analyzesql.js [] +//usage: nodejs analyzesql.js sql.0 info +var utils = require(__dirname + '/../lib/utils'); // Get common adapter utils + +var fs = require('fs'); +var path = require('path'); + +var deepAnalyze = false; +var dbInstance = "sql.0"; +if (process.argv.indexOf('--deepAnalyze') !== -1) deepAnalyze = true; +if (process.argv[2] && (process.argv[2].indexOf('sql') === 0)) { + dbInstance = process.argv[2]; +} +process.argv[2] = "--install"; +console.log('Query Data from ' + dbInstance); +if (deepAnalyze) { + console.log('Deep Analyze not supported'); + process.exit(); +} + +var earliestDBValue = {}; +var earliesValCachefile = __dirname + '/earliestDBValues.json'; +//var existingData = {}; +//var existingDataCachefile = __dirname + '/existingDBValues.json'; +var existingTypes = {}; +var existingTypesCachefile = __dirname + '/existingDBTypes.json'; + +var adapter = utils.Adapter('history'); + +adapter.on('ready', function () { + main(); +}); + +function main() { + console.log('Send'); + adapter.sendTo(dbInstance, "getDpOverview", "", function(result) { + console.log(JSON.stringify(result)); + if (result.error) { + console.error(result.error); + } else { + // show result + console.log('Datapoints found: ' + result.result.length); + console.log(JSON.stringify(result.result)); + for (var id in result.result) { + earliestDBValue[id] = result.result[id].ts; + if (earliestDBValue[id] < 946681200000) earliestDBValue[id] = new Date().getTime(); // mysterious timestamp, ignore + if (result.result[id].type !== 'undefined') existingTypes[id] = result.result[id].type; + } + fs.writeFileSync(existingTypesCachefile, JSON.stringify(existingTypes)); + fs.writeFileSync(earliesValCachefile, JSON.stringify(earliestDBValue)); + } + process.exit(); + }); +} + +process.on('SIGINT', function () { + process.exit(); +}); + +process.on('uncaughtException', function (err) { + console.log(err); + process.exit(); +}); diff --git a/converter/history2db.js b/converter/history2db.js new file mode 100644 index 0000000..6aa40e5 --- /dev/null +++ b/converter/history2db.js @@ -0,0 +1,430 @@ +/* jshint -W097 */// jshint strict:false +/*jslint node: true */ +"use strict"; + +//noinspection JSUnresolvedFunction + +//usage: nodejs history2db.js [] [] [|0] [] [] [--logChangesOnly []] [--ignoreExistingDBValues] +//usage: nodejs history2db.js influxdb.0 info 20161001 /path/to/data +var utils = require(__dirname + '/../lib/utils'); // Get common adapter utils + +var fs = require('fs'); +var path = require('path'); +var dataDir = path.normalize(utils.controllerDir + '/' + require(utils.controllerDir + '/lib/tools').getDefaultDataDir()); +var historydir = dataDir + 'history-data'; + +var earliestDBValue = {}; +var earliesValCachefile = __dirname + '/earliestDBValues.json'; +var earliesValCachefileExists = false; + +var existingDBValues = {}; +var existingDataCachefile = __dirname + '/existingDBValues.json'; +var processNonExistingValues = false; + +var existingTypes = {}; +var existingTypesCachefile = __dirname + '/existingDBTypes.json'; +var existingTypesCachefileExists = false; + +var processAllDPs = false; +var simulate = false; + +var dbInstance = ""; +var endDay = 0; // 20160917; // or 0 +var ignoreEarliesDBValues = false; +var logChangesOnly = false; +var logChangesOnlyTime = 60*60*1000; +var delayMultiplicator = 1; +var processCounter = 0; + +if (process.argv[2]) { + dbInstance = process.argv[2]; + if ((process.argv[4]) && (parseInt(process.argv[4],10) > 0)) endDay = parseInt(process.argv[4], 10); + if (process.argv[5]) historydir = process.argv[5]; + if ((process.argv[6]) && (!isNaN(parseFloat(process.argv[6])))) delayMultiplicator = parseFloat(process.argv[6]); + if (process.argv.indexOf('--ignoreExistingDBValues') !== -1) ignoreEarliesDBValues = true; + if (process.argv.indexOf('--processNonExistingValuesOnly') !== -1) { + ignoreEarliesDBValues = true; + processNonExistingValues = true; + } + if (process.argv.indexOf('--processAllDPs') !== -1) processAllDPs = true; + if (process.argv.indexOf('--simulate') !== -1) simulate = true; + var logchangesPos = process.argv.indexOf('--logChangesOnly'); + if (logchangesPos !== -1) { + logChangesOnly = true; + if (process.argv[logchangesPos+1]) { + var logTime = parseInt(process.argv[logchangesPos+1], 10); + if ((! isNaN(logTime)) && (logTime > 0)) { + logChangesOnlyTime = logTime * 60000; + } + } + } + process.argv[2] = "--install"; +} +else { + console.log("ERROR: DB-Instance missing"); + process.exit(); +} +console.log('Send Data to ' + dbInstance); +if (endDay !== 0) console.log('Start at ' + endDay); +console.log('Use historyDir ' + historydir); +if (delayMultiplicator != 1) console.log('Use Delay multiplicator ' + delayMultiplicator); +if (logChangesOnly) console.log('Log changes only once per ' + (logChangesOnlyTime/60000) + ' minutes'); + +var adapter = utils.Adapter('history'); + +var breakIt = false; + +var stdin = process.stdin; +// without this, we would only get streams once enter is pressed +stdin.setRawMode( true ); +// resume stdin in the parent process (node app won't quit all by itself +// unless an error or process.exit() happens) +stdin.resume(); +// i don't want binary, do you? +stdin.setEncoding( 'utf8' ); +// on any data into stdin +stdin.on( 'data', function( key ){ + // ctrl-c ( end of text ) + if ( key === 'x' || key === '\u0003') { + breakIt = true; + } + // write the key to stdout all normal like + console.log('Received Keypress: ' + key); +}); + +adapter.on('ready', function () { + main(); +}); + +function main() { + if (processNonExistingValues) { + try { + if (fs.statSync(existingDataCachefile).isFile()) { + var exFileContent = fs.readFileSync(existingDataCachefile); + existingDBValues = JSON.parse(exFileContent); + ignoreEarliesDBValues = true; + console.log('ExistingDBValues initialized from cache ' + Object.keys(existingDBValues).length); + } + } + catch (err) { + console.log("ExistingValues File not existing, but should be used. EXIT"); + process.exit(); + } + } + var fileContent; + try { + if (fs.statSync(earliesValCachefile).isFile()) { + fileContent = fs.readFileSync(earliesValCachefile); + earliestDBValue = JSON.parse(fileContent); + console.log('EarliesDBValues initialized from cache ' + Object.keys(earliestDBValue).length); + earliesValCachefileExists = true; + if (ignoreEarliesDBValues) { + var dateNow = new Date().getTime(); + for (var id in earliestDBValue) { + if (!earliestDBValue.hasOwnProperty(id)) continue; + earliestDBValue[id] = dateNow; + } + console.log('EarliesDBValues overwritten with ' + dateNow); + } + } + else { + earliesValCachefileExists = false; + } + } + catch (err) { + console.log('No stored earliesDBValues found'); + } + try { + if (fs.statSync(existingTypesCachefile).isFile()) { + fileContent = fs.readFileSync(existingTypesCachefile); + existingTypes = JSON.parse(fileContent); + console.log('ExistingDBTypes initialized from cache ' + Object.keys(existingTypes).length); + existingTypesCachefileExists = true; + } + else { + existingTypesCachefileExists = false; + } + } + catch (err) { + console.log('No stored existingDBTypes found'); + } + processFiles(); +} + +var allFiles = {}; + +function processFiles() { + console.log('started processFiles with ' + Object.keys(earliestDBValue).length + ' known db values'); + + if (endDay === 0) { + var endDayTs = 0; + for (var id in earliestDBValue) { + if (!earliestDBValue.hasOwnProperty(id)) continue; + if (earliestDBValue[id] > endDayTs) { + endDayTs = earliestDBValue[id]; + //console.log('new minimum = ' + id + '(' + endDayTs + ')'); + } + } + endDay = parseInt(ts2day(endDayTs), 10); + } + console.log('We start earliest at ' + endDay); + + // get list of directories + var dayList = getDirectories(historydir).sort(function (a, b) { + return b - a; + }); + + for (var i = 0; i < dayList.length; i++) { + var day = parseInt(dayList[i], 10); + if ((!isNaN(day)) && (day<=endDay)) { + var dir = historydir + '/' + dayList[i].toString() + '/'; + allFiles[dayList[i].toString()]={}; + allFiles[dayList[i].toString()].dirname = dir; + allFiles[dayList[i].toString()].files = getFiles(dir); + } + } + processFile(); +} + +function processFile() { + if (breakIt) finish(true); + if (Object.keys(allFiles).length === 0) finish(true); + + var day = parseInt(Object.keys(allFiles)[Object.keys(allFiles).length-1], 10); + + if (allFiles[day].files.length>0) { + var dir = allFiles[day].dirname; + var file = allFiles[day].files.shift(); + var id = file.substring(8,file.length-5); + var weatherunderground_special_handling = ((id.indexOf('weatherunderground') !== -1) && (id.indexOf('current.precip') !== -1)); + console.log('Day ' + day + ' - ' + file ); + + if (earliesValCachefileExists) { + if ((!earliestDBValue[id]) || (earliestDBValue[id] === 0)) { + console.log(' Ignore ID ' + file +': ' + id); + setTimeout(processFile,10); + return; + } + } + if (!earliesValCachefileExists || processAllDPs) { + if (!earliestDBValue[id]) earliestDBValue[id] = new Date().getTime(); + } + if (processNonExistingValues) { + console.log("Check: "+day+" / pos " + existingDBValues[id].indexOf(day) /*+ " :" +JSON.stringify(existingDBValues[id])*/); + if ((existingDBValues[id]) && (existingDBValues[id].indexOf(day) !== -1)) { + console.log(' Ignore existing ID ' + file +': ' + id); + setTimeout(processFile,10); + return; + } + } + + var fileData; + try { + var fileContent = fs.readFileSync(dir + '/' + file); + + fileData = JSON.parse(fileContent, function (key, value) { + if (key === 'ts') { + if (value < 946681200000) value *= 1000; + } + else if (key === 'ack') { + value = !!value; + } + else if (key === 'val') { + if ((weatherunderground_special_handling) && (typeof value === 'string')) value = parseInt(value, 10); + } + return value; + }); + } catch (e) { + console.log('Cannot parse file ' + dir + '/' + file + ': ' + e.message); + } + + //console.log(' File ' + j +': ' + id + ' --> ' + fileData.length); + + if (fileData[fileData.length-1].ts >= earliestDBValue[id]) { + var k; + for (k = 0; k < fileData.length; k++) { + if (fileData[k].ts >= earliestDBValue[id]) break; + } + fileData = fileData.slice(0,k); + console.log('cut filedata to ' + fileData.length); + } + var lastValue = null; + var lastTime = null; + if (fileData.length > 0) { + var sendData = {}; + sendData.id = id; + sendData.state = []; + for (var j = 0; j< fileData.length; j++) { + if (fileData[j].ts < earliestDBValue[id]) earliestDBValue[id] = fileData[j].ts; + if ((lastValue === null) || (fileData[j].val != lastValue) || (!logChangesOnly) || ((logChangesOnly) && (Math.abs(fileData[j].ts-lastTime) > logChangesOnlyTime))) { + sendData.state.push(fileData[j]); + lastValue = fileData[j].val; + lastTime = fileData[j].ts; + //console.log('use value = ' + fileData[j].val) + } + //else console.log('not use value = ' + fileData[j].val) + } + console.log(' datapoints reduced from ' + fileData.length + ' --> ' + sendData.state.length); + if (existingTypesCachefileExists) { + if (!existingTypes[id]) { + existingTypes[id] = typeof sendData.state[sendData.state.length-1].val; + console.log(' used last value to initialize type: ' + existingTypes[id]); + } + var sortedOut = 0; + for (var jj = 0; jj< sendData.state.length; jj++) { + var currType = typeof sendData.state[jj].val; + if (currType != existingTypes[id]) { + switch (existingTypes[id]) { + case 'number': switch (currType) { + case 'boolean': if (sendData.state[jj].val === false) sendData.state[jj].val = 0; + else sendData.state[jj].val = 1; + break; + case 'string': if (sendData.state[jj].val === "true") { + sendData.state[jj].val = 1; + } else if (sendData.state[jj].val === "false") { + sendData.state[jj].val = 0; + } else { + sendData.state[jj].val = parseFloat(sendData.state[jj].val); + if (isNaN(sendData.state[jj].val)) { + sendData.state[jj].val = null; + sortedOut++; + } + } + break; + default: sendData.state[jj].val = null; // value will be sorted out! + sortedOut++; + } + break; + case 'boolean': switch (currType) { + case 'number': if (sendData.state[jj].val === 0) sendData.state[jj].val = false; + else sendData.state[jj].val = true; + break; + case 'string': if (sendData.state[jj].val === "true") { + sendData.state[jj].val = true; + } else if (sendData.state[jj].val === "false") { + sendData.state[jj].val = false; + } else { + sendData.state[jj].val = parseInt(sendData.state[jj].val); + if (sendData.state[jj].val === 0) sendData.state[jj].val = false; + else sendData.state[jj].val = true; + } + break; + default: sendData.state[jj].val = null; // value will be sorted out! + sortedOut++; + } + break; + } + console.log(' type mismatch ' + existingTypes[id] + ' vs. ' + currType + ': fixed=' + (sendData.state[jj].val !== null) + ' --> ' + sendData.state[jj].val); + } + } + console.log(' sorted out ' + sortedOut + ' values'); + if (sortedOut === sendData.state.length) { + sendData.state = []; + } + } + if (sendData.state.length > 0) { + processCounter++; + if (processNonExistingValues) { + if (!existingDBValues[id]) existingDBValues[id] = []; + existingDBValues[id].push(day); + } + if (!simulate) { + adapter.sendTo(dbInstance, "storeState", sendData, function (result) { + if (result.error) { + console.error(result.error); + finish(false); + } + if (result.success && !result.connected) { + console.error('Data stored but db not available anymore, break. ' + JSON.stringify(result)); + finish(true); + } + var delay = 300; + if (result.success && result.seriesBufferFlushPlanned) { + delay = 1500; // 1,5 seconds + if (result.seriesBufferCounter > 1000) delay += 500*(result.seriesBufferCounter/1000); + } + delay = delay * delayMultiplicator; + setTimeout(processFile, delay); + }); + } else { + console.log(' SIMULATE: Not really writing ... ' + sendData.state.length + ' values for ' + id); + setTimeout(processFile, 10); + } + } + else { + setTimeout(processFile,10); + } + } + else { + setTimeout(processFile,10); + } + } + else { + delete allFiles[day]; + if (!ignoreEarliesDBValues && !simulate) fs.writeFileSync(earliesValCachefile, JSON.stringify(earliestDBValue)); + if (processNonExistingValues && !simulate) fs.writeFileSync(existingDataCachefile, JSON.stringify(existingDBValues)); + if (existingTypesCachefileExists && !simulate) fs.writeFileSync(existingTypesCachefile, JSON.stringify(existingTypes)); + console.log("Day end"); + var dayDelay = 30000; + if (processCounter < 10) dayDelay = 1000; + if (processCounter === 0) dayDelay = 10; + processCounter = 0; + setTimeout(processFile, dayDelay); + } +} + + +function finish(updateData) { + console.log('DONE'); + if (updateData && !ignoreEarliesDBValues && !simulate) fs.writeFileSync(earliesValCachefile, JSON.stringify(earliestDBValue)); + if (updateData && processNonExistingValues && !simulate) fs.writeFileSync(existingDataCachefile, JSON.stringify(existingDBValues)); + if (updateData && existingTypesCachefileExists && !simulate) fs.writeFileSync(existingTypesCachefile, JSON.stringify(existingTypes)); + process.exit(); +} + + +function getDirectories(path) { + try { + return fs.readdirSync(path).filter(function (file) { + return fs.statSync(path + '/' + file).isDirectory(); + }); + } + catch (e) { + return []; + } +} + +function getFiles(path) { + try { + return fs.readdirSync(path).filter(function (file) { + return fs.statSync(path + '/' + file).isFile(); + }); + } + catch (e) { + return []; + } +} + +function ts2day(ts) { + if (ts < 946681200000) ts *= 1000; + var dateObj = new Date(ts); + + var text = dateObj.getFullYear().toString(); + var v = dateObj.getMonth() + 1; + if (v < 10) text += '0'; + text += v.toString(); + + v = dateObj.getDate(); + if (v < 10) text += '0'; + text += v.toString(); + + return text; +} + +process.on('SIGINT', function () { + breakIt = true; +}); + +process.on('uncaughtException', function (err) { + breakIt = true; +}); diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000..017ccba --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,401 @@ +'use strict'; + +var gulp = require('gulp'); +var fs = require('fs'); +var pkg = require('./package.json'); +var iopackage = require('./io-package.json'); +var version = (pkg && pkg.version) ? pkg.version : iopackage.common.version; +/*var appName = getAppName(); + +function getAppName() { + var parts = __dirname.replace(/\\/g, '/').split('/'); + return parts[parts.length - 1].split('.')[0].toLowerCase(); +} +*/ +const fileName = 'words.js'; +var languages = { + en: {}, + de: {}, + ru: {}, + pt: {}, + nl: {}, + fr: {}, + it: {}, + es: {}, + pl: {} +}; + +function lang2data(lang, isFlat) { + var str = isFlat ? '' : '{\n'; + var count = 0; + for (var w in lang) { + if (lang.hasOwnProperty(w)) { + count++; + if (isFlat) { + str += (lang[w] === '' ? (isFlat[w] || w) : lang[w]) + '\n'; + } else { + var key = ' "' + w.replace(/"/g, '\\"') + '": '; + str += key + '"' + lang[w].replace(/"/g, '\\"') + '",\n'; + } + } + } + if (!count) return isFlat ? '' : '{\n}'; + if (isFlat) { + return str; + } else { + return str.substring(0, str.length - 2) + '\n}'; + } +} + +function readWordJs(src) { + try { + var words; + if (fs.existsSync(src + 'js/' + fileName)) { + words = fs.readFileSync(src + 'js/' + fileName).toString(); + } else { + words = fs.readFileSync(src + fileName).toString(); + } + + var lines = words.split(/\r\n|\r|\n/g); + var i = 0; + while (!lines[i].match(/^systemDictionary = {/)) { + i++; + } + lines.splice(0, i); + + // remove last empty lines + i = lines.length - 1; + while (!lines[i]) { + i--; + } + if (i < lines.length - 1) { + lines.splice(i + 1); + } + + lines[0] = lines[0].replace('systemDictionary = ', ''); + lines[lines.length - 1] = lines[lines.length - 1].trim().replace(/};$/, '}'); + words = lines.join('\n'); + var resultFunc = new Function('return ' + words + ';'); + + return resultFunc(); + } catch (e) { + return null; + } +} +function padRight(text, totalLength) { + return text + (text.length < totalLength ? new Array(totalLength - text.length).join(' ') : ''); +} +function writeWordJs(data, src) { + var text = '// DO NOT EDIT THIS FILE!!! IT WILL BE AUTOMATICALLY GENERATED FROM src/i18n\n'; + text += '/*global systemDictionary:true */\n'; + text += '\'use strict\';\n\n'; + + text += 'systemDictionary = {\n'; + for (var word in data) { + if (data.hasOwnProperty(word)) { + text += ' ' + padRight('"' + word.replace(/"/g, '\\"') + '": {', 50); + var line = ''; + for (var lang in data[word]) { + if (data[word].hasOwnProperty(lang)) { + line += '"' + lang + '": "' + padRight(data[word][lang].replace(/"/g, '\\"') + '",', 50) + ' '; + } + } + if (line) { + line = line.trim(); + line = line.substring(0, line.length - 1); + } + text += line + '},\n'; + } + } + text = text.replace(/},\n$/, '}\n'); + text += '};'; + + if (fs.existsSync(src + 'js/' + fileName)) { + fs.writeFileSync(src + 'js/' + fileName, text); + } else { + fs.writeFileSync(src + '' + fileName, text); + } +} + +const EMPTY = ''; + +function words2languages(src) { + var langs = Object.assign({}, languages); + var data = readWordJs(src); + if (data) { + for (var word in data) { + if (data.hasOwnProperty(word)) { + for (var lang in data[word]) { + if (data[word].hasOwnProperty(lang)) { + langs[lang][word] = data[word][lang]; + // pre-fill all other languages + for (var j in langs) { + if (langs.hasOwnProperty(j)) { + langs[j][word] = langs[j][word] || EMPTY; + } + } + } + } + } + } + if (!fs.existsSync(src + 'i18n/')) { + fs.mkdirSync(src + 'i18n/'); + } + for (var l in langs) { + if (!langs.hasOwnProperty(l)) continue; + var keys = Object.keys(langs[l]); + //keys.sort(); + var obj = {}; + for (var k = 0; k < keys.length; k++) { + obj[keys[k]] = langs[l][keys[k]]; + } + if (!fs.existsSync(src + 'i18n/' + l)) { + fs.mkdirSync(src + 'i18n/' + l); + } + + fs.writeFileSync(src + 'i18n/' + l + '/translations.json', lang2data(obj)); + } + } else { + console.error('Cannot read or parse ' + fileName); + } +} +function words2languagesFlat(src) { + var langs = Object.assign({}, languages); + var data = readWordJs(src); + if (data) { + for (var word in data) { + if (data.hasOwnProperty(word)) { + for (var lang in data[word]) { + if (data[word].hasOwnProperty(lang)) { + langs[lang][word] = data[word][lang]; + // pre-fill all other languages + for (var j in langs) { + if (langs.hasOwnProperty(j)) { + langs[j][word] = langs[j][word] || EMPTY; + } + } + } + } + } + } + var keys = Object.keys(langs.en); + //keys.sort(); + for (var l in langs) { + if (!langs.hasOwnProperty(l)) continue; + var obj = {}; + for (var k = 0; k < keys.length; k++) { + obj[keys[k]] = langs[l][keys[k]]; + } + langs[l] = obj; + } + if (!fs.existsSync(src + 'i18n/')) { + fs.mkdirSync(src + 'i18n/'); + } + for (var ll in langs) { + if (!langs.hasOwnProperty(ll)) continue; + if (!fs.existsSync(src + 'i18n/' + ll)) { + fs.mkdirSync(src + 'i18n/' + ll); + } + + fs.writeFileSync(src + 'i18n/' + ll + '/flat.txt', lang2data(langs[ll], langs.en)); + } + fs.writeFileSync(src + 'i18n/flat.txt', keys.join('\n')); + } else { + console.error('Cannot read or parse ' + fileName); + } +} +function languagesFlat2words(src) { + var dirs = fs.readdirSync(src + 'i18n/'); + var langs = {}; + var bigOne = {}; + var order = Object.keys(languages); + dirs.sort(function (a, b) { + var posA = order.indexOf(a); + var posB = order.indexOf(b); + if (posA === -1 && posB === -1) { + if (a > b) return 1; + if (a < b) return -1; + return 0; + } else if (posA === -1) { + return -1; + } else if (posB === -1) { + return 1; + } else { + if (posA > posB) return 1; + if (posA < posB) return -1; + return 0; + } + }); + var keys = fs.readFileSync(src + 'i18n/flat.txt').toString().split('\n'); + + for (var l = 0; l < dirs.length; l++) { + if (dirs[l] === 'flat.txt') continue; + var lang = dirs[l]; + var values = fs.readFileSync(src + 'i18n/' + lang + '/flat.txt').toString().split('\n'); + langs[lang] = {}; + keys.forEach(function (word, i) { + langs[lang][word] = values[i].replace(/<\/ i>/g, '').replace(/<\/ b>/g, '').replace(/<\/ span>/g, '').replace(/% s/g, ' %s'); + }); + + var words = langs[lang]; + for (var word in words) { + if (words.hasOwnProperty(word)) { + bigOne[word] = bigOne[word] || {}; + if (words[word] !== EMPTY) { + bigOne[word][lang] = words[word]; + } + } + } + } + // read actual words.js + var aWords = readWordJs(); + + var temporaryIgnore = ['pt', 'fr', 'nl', 'flat.txt']; + if (aWords) { + // Merge words together + for (var w in aWords) { + if (aWords.hasOwnProperty(w)) { + if (!bigOne[w]) { + console.warn('Take from actual words.js: ' + w); + bigOne[w] = aWords[w] + } + dirs.forEach(function (lang) { + if (temporaryIgnore.indexOf(lang) !== -1) return; + if (!bigOne[w][lang]) { + console.warn('Missing "' + lang + '": ' + w); + } + }); + } + } + + } + + writeWordJs(bigOne, src); +} +function languages2words(src) { + var dirs = fs.readdirSync(src + 'i18n/'); + var langs = {}; + var bigOne = {}; + var order = Object.keys(languages); + dirs.sort(function (a, b) { + var posA = order.indexOf(a); + var posB = order.indexOf(b); + if (posA === -1 && posB === -1) { + if (a > b) return 1; + if (a < b) return -1; + return 0; + } else if (posA === -1) { + return -1; + } else if (posB === -1) { + return 1; + } else { + if (posA > posB) return 1; + if (posA < posB) return -1; + return 0; + } + }); + for (var l = 0; l < dirs.length; l++) { + if (dirs[l] === 'flat.txt') continue; + var lang = dirs[l]; + langs[lang] = fs.readFileSync(src + 'i18n/' + lang + '/translations.json').toString(); + langs[lang] = JSON.parse(langs[lang]); + var words = langs[lang]; + for (var word in words) { + if (words.hasOwnProperty(word)) { + bigOne[word] = bigOne[word] || {}; + if (words[word] !== EMPTY) { + bigOne[word][lang] = words[word]; + } + } + } + } + // read actual words.js + var aWords = readWordJs(); + + var temporaryIgnore = ['pt', 'fr', 'nl', 'it']; + if (aWords) { + // Merge words together + for (var w in aWords) { + if (aWords.hasOwnProperty(w)) { + if (!bigOne[w]) { + console.warn('Take from actual words.js: ' + w); + bigOne[w] = aWords[w] + } + dirs.forEach(function (lang) { + if (temporaryIgnore.indexOf(lang) !== -1) return; + if (!bigOne[w][lang]) { + console.warn('Missing "' + lang + '": ' + w); + } + }); + } + } + + } + + writeWordJs(bigOne, src); +} + +gulp.task('adminWords2languages', function (done) { + words2languages('./admin/'); + done(); +}); + +gulp.task('adminWords2languagesFlat', function (done) { + words2languagesFlat('./admin/'); + done(); +}); + +gulp.task('adminLanguagesFlat2words', function (done) { + languagesFlat2words('./admin/'); + done(); +}); + +gulp.task('adminLanguages2words', function (done) { + languages2words('./admin/'); + done(); +}); + + +gulp.task('updatePackages', function (done) { + iopackage.common.version = pkg.version; + iopackage.common.news = iopackage.common.news || {}; + if (!iopackage.common.news[pkg.version]) { + var news = iopackage.common.news; + var newNews = {}; + + newNews[pkg.version] = { + en: 'news', + de: 'neues', + ru: 'новое' + }; + iopackage.common.news = Object.assign(newNews, news); + } + fs.writeFileSync('io-package.json', JSON.stringify(iopackage, null, 4)); + done(); +}); + +gulp.task('updateReadme', function (done) { + var readme = fs.readFileSync('README.md').toString(); + var pos = readme.indexOf('## Changelog\n'); + if (pos !== -1) { + var readmeStart = readme.substring(0, pos + '## Changelog\n'.length); + var readmeEnd = readme.substring(pos + '## Changelog\n'.length); + + if (readme.indexOf(version) === -1) { + var timestamp = new Date(); + var date = timestamp.getFullYear() + '-' + + ('0' + (timestamp.getMonth() + 1).toString(10)).slice(-2) + '-' + + ('0' + (timestamp.getDate()).toString(10)).slice(-2); + + var news = ''; + if (iopackage.common.news && iopackage.common.news[pkg.version]) { + news += '* ' + iopackage.common.news[pkg.version].en; + } + + fs.writeFileSync('README.md', readmeStart + '### ' + version + ' (' + date + ')\n' + (news ? news + '\n\n' : '\n') + readmeEnd); + } + } + done(); +}); + +gulp.task('default', ['updatePackages', 'updateReadme']); \ No newline at end of file diff --git a/history.js b/history.js new file mode 100644 index 0000000..f3e9eb4 --- /dev/null +++ b/history.js @@ -0,0 +1,1078 @@ +/* jshint -W097 */// jshint strict:false +/*jslint node: true */ +'use strict'; +var cp = require('child_process'); +var utils = require(__dirname + '/lib/utils'); // Get common adapter utils +var path = require('path'); +var dataDir = path.normalize(utils.controllerDir + '/' + require(utils.controllerDir + '/lib/tools').getDefaultDataDir()); +var fs = require('fs'); +var GetHistory = require(__dirname + '/lib/getHistory.js'); +var Aggregate = require(__dirname + '/lib/aggregate.js'); + +var history = {}; +var aliasMap = {}; +var subscribeAll = false; +var bufferChecker = null; +var tasksStart = []; +var finished = false; + +var adapter = new utils.Adapter({ + + name: 'history', + + objectChange: function (id, obj) { + var formerAliasId = aliasMap[id] ? aliasMap[id] : id; + if (obj && obj.common && + ( + // todo remove history sometime (2016.08) - Do not forget object selector in io-package.json + (obj.common.history && obj.common.history[adapter.namespace] && obj.common.history[adapter.namespace].enabled) || + (obj.common.custom && obj.common.custom[adapter.namespace] && obj.common.custom[adapter.namespace].enabled) + ) + ) { + var realId = id; + var checkForRemove = true; + if (obj.common.custom && obj.common.custom[adapter.namespace] && obj.common.custom[adapter.namespace].aliasId) { + if (obj.common.custom[adapter.namespace].aliasId !== id) { + aliasMap[id] = obj.common.custom[adapter.namespace].aliasId; + adapter.log.debug('Registered Alias: ' + id + ' --> ' + aliasMap[id]); + id = aliasMap[id]; + checkForRemove = false; + } + else { + adapter.log.warn('Ignoring Alias-ID because identical to ID for ' + id); + obj.common.custom[adapter.namespace].aliasId = ''; + } + } + if (checkForRemove && aliasMap[id]) { + adapter.log.debug('Removed Alias: ' + id + ' !-> ' + aliasMap[id]); + delete aliasMap[id]; + } + + var writeNull = !history[id]; + var state = history[id] ? history[id].state : null; + var list = history[id] ? history[id].list : null; + var timeout = history[id] ? history[id].timeout : null; + + if (!history[formerAliasId] && !subscribeAll) { + // unsubscribe + for (var _id in history) { + if (history.hasOwnProperty(history[_id].realId)) { + adapter.unsubscribeForeignStates(history[_id].realId); + } + } + subscribeAll = true; + adapter.subscribeForeignStates('*'); + } + + if (history[formerAliasId] && history[formerAliasId].relogTimeout) clearTimeout(history[formerAliasId].relogTimeout); + + // todo remove history somewhen (2016.08) + history[id] = obj.common.custom || obj.common.history; + history[id].state = state; + history[id].list = list; + history[id].timeout = timeout; + history[id].realId = realId; + + if (!history[id][adapter.namespace].maxLength && history[id][adapter.namespace].maxLength !== '0' && history[id][adapter.namespace].maxLength !== 0) { + history[id][adapter.namespace].maxLength = parseInt(adapter.config.maxLength, 10) || 960; + } else { + history[id][adapter.namespace].maxLength = parseInt(history[id][adapter.namespace].maxLength, 10); + } + if (!history[id][adapter.namespace].retention && history[id][adapter.namespace].retention !== '0' && history[id][adapter.namespace].retention !== 0) { + history[id][adapter.namespace].retention = parseInt(adapter.config.retention, 10) || 0; + } else { + history[id][adapter.namespace].retention = parseInt(history[id][adapter.namespace].retention, 10) || 0; + } + if (!history[id][adapter.namespace].debounce && history[id][adapter.namespace].debounce !== '0' && history[id][adapter.namespace].debounce !== 0) { + history[id][adapter.namespace].debounce = parseInt(adapter.config.debounce, 10) || 1000; + } else { + history[id][adapter.namespace].debounce = parseInt(history[id][adapter.namespace].debounce, 10); + } + history[id][adapter.namespace].changesOnly = history[id][adapter.namespace].changesOnly === 'true' || history[id][adapter.namespace].changesOnly === true; + if (history[id][adapter.namespace].changesRelogInterval !== undefined && history[id][adapter.namespace].changesRelogInterval !== null && history[id][adapter.namespace].changesRelogInterval !== '') { + history[id][adapter.namespace].changesRelogInterval = parseInt(history[id][adapter.namespace].changesRelogInterval, 10) || 0; + } else { + history[id][adapter.namespace].changesRelogInterval = adapter.config.changesRelogInterval; + } + if (history[id][adapter.namespace].changesRelogInterval > 0) { + history[id].relogTimeout = setTimeout(reLogHelper, (history[id][adapter.namespace].changesRelogInterval * 500 * Math.random()) + history[id][adapter.namespace].changesRelogInterval * 500, id); + } + if (history[id][adapter.namespace].changesMinDelta !== undefined && history[id][adapter.namespace].changesMinDelta !== null && history[id][adapter.namespace].changesMinDelta !== '') { + history[id][adapter.namespace].changesMinDelta = parseFloat(history[id][adapter.namespace].changesMinDelta.toString().replace(/,/g, '.')) || 0; + } else { + history[id][adapter.namespace].changesMinDelta = adapter.config.changesMinDelta; + } + + + // add one day if retention is too small + if (history[id][adapter.namespace].retention && history[id][adapter.namespace].retention <= 604800) { + history[id][adapter.namespace].retention += 86400; + } + if (writeNull && adapter.config.writeNulls) { + writeNulls(id); + } + + adapter.log.info('enabled logging of ' + id + ', Alias=' + (id !== realId)); + } else { + if (aliasMap[id]) { + adapter.log.debug('Removed Alias: ' + id + ' !-> ' + aliasMap[id]); + delete aliasMap[id]; + } + id = formerAliasId; + if (history[id]) { + adapter.log.info('disabled logging of ' + id); + if (history[id].relogTimeout) clearTimeout(history[id].relogTimeout); + if (history[id].timeout) clearTimeout(history[id].timeout); + storeCached(true, id); + delete history[id]; + } + } + }, + + stateChange: function (id, state) { + id = aliasMap[id] ? aliasMap[id] : id; + pushHistory(id, state); + }, + + unload: function (callback) { + finish(callback); + }, + + ready: function () { + main(); + }, + + message: function (obj) { + processMessage(obj); + } +}); + +process.on('SIGINT', function () { + if (adapter && adapter.setState) { + finish(); + } +}); +process.on('SIGTERM', function () { + if (adapter && adapter.setState) { + finish(); + } +}); + +function storeCached(isFinishing, onlyId) { + var now = new Date().getTime(); + + for (var id in history) { + if (onlyId !== undefined && onlyId !== id) continue; + if (isFinishing) { + if (history[id].skipped) { + history[id].list.push(history[id].skipped); + history[id].skipped = null; + } + if (adapter.config.writeNulls) { + var nullValue = {val: null, ts: now, lc: now, q: 0x40, from: 'system.adapter.' + adapter.namespace}; + if (history[id][adapter.namespace].changesOnly && history[id].state && history[id].state !== null) { + var state = Object.assign({}, history[id].state); + state.ts = now; + state.from = 'system.adapter.' + adapter.namespace; + history[id].list.push(state); + nullValue.ts += 1; + nullValue.lc += 1; + } + + // terminate values with null to indicate adapter stop. + history[id].list.push(nullValue); + } + } + + if (history[id].list && history[id].list.length) { + adapter.log.debug('Store the rest for ' + id); + appendFile(id, history[id].list); + } + } +} + +function finish(callback) { + adapter.unsubscribeForeignStates('*'); + if (bufferChecker) { + clearInterval(bufferChecker); + bufferChecker = null; + } + for (var id in history) { + if (history[id].relogTimeout) { + clearTimeout(history[id].relogTimeout); + history[id].relogTimeout = null; + } + if (history[id].timeout) { + clearTimeout(history[id].timeout); + history[id].timeout = null; + } + } + + if (!finished) { + finished = true; + storeCached(true); + } + + if (callback) { + callback(); + } +} + +function processMessage(msg) { + if (msg.command === 'getHistory') { + getHistory(msg); + } else if (msg.command === 'storeState') { + storeState(msg); + } else if (msg.command === 'enableHistory') { + enableHistory(msg); + } else if (msg.command === 'disableHistory') { + disableHistory(msg); + } else if (msg.command === 'getEnabledDPs') { + getEnabledDPs(msg); + } else if (msg.command === 'stopInstance') { + finish(function () { + if (msg.callback) { + adapter.sendTo(msg.from, msg.command, 'stopped', msg.callback); + setTimeout(function () { + process.exit(0); + }, 200); + } + }); + } +} + +function fixSelector(callback) { + // fix _design/custom object + adapter.getForeignObject('_design/custom', function (err, obj) { + if (!obj || obj.views.state.map.indexOf('common.history') === -1 || obj.views.state.map.indexOf('common.custom') === -1) { + obj = { + _id: '_design/custom', + language: 'javascript', + views: { + state: { + map: 'function(doc) { if (doc.type===\'state\' && (doc.common.custom || doc.common.history)) emit(doc._id, doc.common.custom || doc.common.history) }' + } + } + }; + adapter.setForeignObject('_design/custom', obj, function (err) { + if (callback) callback(err); + }); + } else { + if (callback) callback(err); + } + }); +} + +function processStartValues() { + if (tasksStart && tasksStart.length) { + var task = tasksStart.shift(); + if (history[task.id][adapter.namespace].changesOnly) { + adapter.getForeignState(history[task.id].realId, function (err, state) { + var now = task.now || new Date().getTime(); + pushHistory(task.id, { + val: null, + ts: state ? now - 4 : now, // 4ms because of MS-SQL + ack: true, + q: 0x40, + from: 'system.adapter.' + adapter.namespace}); + + if (state) { + state.ts = now; + state.from = 'system.adapter.' + adapter.namespace; + pushHistory(task.id, state); + } + setTimeout(processStartValues, 0); + }); + } else { + pushHistory(task.id, { + val: null, + ts: task.now || new Date().getTime(), + ack: true, + q: 0x40, + from: 'system.adapter.' + adapter.namespace}); + setTimeout(processStartValues, 0); + } + } +} + +function writeNulls(id, now) { + if (!id) { + now = new Date().getTime(); + for (var _id in history) { + if (history.hasOwnProperty(_id)) { + writeNulls(_id, now); + } + } + } else { + now = now || new Date().getTime(); + tasksStart.push({id: id, now: now}); + if (tasksStart.length === 1) { + processStartValues(); + } + if (history[id][adapter.namespace].changesRelogInterval > 0) { + if (history[id].relogTimeout) clearTimeout(history[id].relogTimeout); + history[id].relogTimeout = setTimeout(reLogHelper, (history[id][adapter.namespace].changesRelogInterval * 500 * Math.random()) + history[id][adapter.namespace].changesRelogInterval * 500, id); + } + } +} + +function main() { + adapter.config.storeDir = adapter.config.storeDir || 'history'; + adapter.config.storeDir = adapter.config.storeDir.replace(/\\/g, '/'); + if (adapter.config.writeNulls === undefined) adapter.config.writeNulls = true; + + // remove last "/" + if (adapter.config.storeDir[adapter.config.storeDir.length - 1] === '/') { + adapter.config.storeDir = adapter.config.storeDir.substring(0, adapter.config.storeDir.length - 1); + } + + if (adapter.config.storeDir[0] !== '/' && !adapter.config.storeDir.match(/^\w:\//)) { + adapter.config.storeDir = dataDir + adapter.config.storeDir; + } + adapter.config.storeDir += '/'; + + if (adapter.config.changesRelogInterval !== null && adapter.config.changesRelogInterval !== undefined) { + adapter.config.changesRelogInterval = parseInt(adapter.config.changesRelogInterval, 10); + } else { + adapter.config.changesRelogInterval = 0; + } + + if (adapter.config.changesMinDelta !== null && adapter.config.changesMinDelta !== undefined) { + adapter.config.changesMinDelta = parseFloat(adapter.config.changesMinDelta.toString().replace(/,/g, '.')); + } else { + adapter.config.changesMinDelta = 0; + } + + // create directory + if (!fs.existsSync(adapter.config.storeDir)) { + fs.mkdirSync(adapter.config.storeDir); + } + + fixSelector(function () { + adapter.objects.getObjectView('custom', 'state', {}, function (err, doc) { + var count = 0; + if (doc && doc.rows) { + for (var i = 0, l = doc.rows.length; i < l; i++) { + if (doc.rows[i].value) { + var id = doc.rows[i].id; + var realId = id; + if (doc.rows[i].value[adapter.namespace] && doc.rows[i].value[adapter.namespace].aliasId) { + aliasMap[id] = doc.rows[i].value[adapter.namespace].aliasId; + adapter.log.debug('Found Alias: ' + id + ' --> ' + aliasMap[id]); + id = aliasMap[id]; + } + history[id] = doc.rows[i].value; + + // todo remove it somewhen (2016.08) + // convert old value + if (history[id].enabled !== undefined) { + history[id] = history[id].enabled ? {'history.0': history[id]} : null; + if (!history[id]) { + delete history[id]; + continue; + } + } + if (!history[id][adapter.namespace] || history[id][adapter.namespace].enabled === false) { + delete history[id]; + } else { + count++; + adapter.log.info('enabled logging of ' + id + ' (Count=' + count + '), Alias=' + (id !== realId)); + if (!history[id][adapter.namespace].maxLength && history[id][adapter.namespace].maxLength !== '0' && history[id][adapter.namespace].maxLength !== 0) { + history[id][adapter.namespace].maxLength = parseInt(adapter.config.maxLength, 10) || 960; + } else { + history[id][adapter.namespace].maxLength = parseInt(history[id][adapter.namespace].maxLength, 10); + } + if (!history[id][adapter.namespace].retention && history[id][adapter.namespace].retention !== '0' && history[id][adapter.namespace].retention !== 0) { + history[id][adapter.namespace].retention = parseInt(adapter.config.retention, 10) || 0; + } else { + history[id][adapter.namespace].retention = parseInt(history[id][adapter.namespace].retention, 10) || 0; + } + if (!history[id][adapter.namespace].debounce && history[id][adapter.namespace].debounce !== '0' && history[id][adapter.namespace].debounce !== 0) { + history[id][adapter.namespace].debounce = parseInt(adapter.config.debounce, 10) || 1000; + } else { + history[id][adapter.namespace].debounce = parseInt(history[id][adapter.namespace].debounce, 10); + } + history[id][adapter.namespace].changesOnly = history[id][adapter.namespace].changesOnly === 'true' || history[id][adapter.namespace].changesOnly === true; + if (history[id][adapter.namespace].changesRelogInterval !== undefined && history[id][adapter.namespace].changesRelogInterval !== null && history[id][adapter.namespace].changesRelogInterval !== '') { + history[id][adapter.namespace].changesRelogInterval = parseInt(history[id][adapter.namespace].changesRelogInterval, 10) || 0; + } else { + history[id][adapter.namespace].changesRelogInterval = adapter.config.changesRelogInterval; + } + if (history[id][adapter.namespace].changesRelogInterval > 0) { + history[id].relogTimeout = setTimeout(reLogHelper, (history[id][adapter.namespace].changesRelogInterval * 500 * Math.random()) + history[id][adapter.namespace].changesRelogInterval * 500, id); + } + if (history[id][adapter.namespace].changesMinDelta !== undefined && history[id][adapter.namespace].changesMinDelta !== null && history[id][adapter.namespace].changesMinDelta !== '') { + history[id][adapter.namespace].changesMinDelta = parseFloat(history[id][adapter.namespace].changesMinDelta) || 0; + } else { + history[id][adapter.namespace].changesMinDelta = adapter.config.changesMinDelta; + } + + // add one day if retention is too small + if (history[id][adapter.namespace].retention && history[id][adapter.namespace].retention <= 604800) { + history[id][adapter.namespace].retention += 86400; + } + + history[id].realId = realId; + } + } + } + } + if (count < 20) { + for (var _id in history) { + if (history.hasOwnProperty(_id)) { + adapter.subscribeForeignStates(history[_id].realId); + } + } + } else { + subscribeAll = true; + adapter.subscribeForeignStates('*'); + } + + if (adapter.config.writeNulls) writeNulls(); + + // store all buffered data every 10 minutes to not lost the data + bufferChecker = setInterval(function () { + storeCached(); + }, 10 * 60000); + }); + }); + + adapter.subscribeForeignObjects('*'); +} + +function pushHistory(id, state, timerRelog) { + if (timerRelog === undefined) timerRelog = false; + // Push into history + if (history[id]) { + var settings = history[id][adapter.namespace]; + + if (!settings || !state) return; + + if (state.ts < 946681200000) state.ts *= 1000; + if (state.lc < 946681200000) state.lc *= 1000; + + if (typeof state.val === 'string') { + var f = parseFloat(state.val); + if (f == state.val) { + state.val = f; + } + } + if (history[id].state && settings.changesOnly && !timerRelog) { + if (settings.changesRelogInterval === 0) { + if ((history[id].state.val !== null || state.val === null) && state.ts !== state.lc) { + history[id].skipped = state; // remember new timestamp + adapter.log.debug('value not changed ' + id + ', last-value=' + history[id].state.val + ', new-value=' + state.val + ', ts=' + state.ts); + return; + } + } else if (history[id].lastLogTime) { + if ((history[id].state.val !== null || state.val === null) && (state.ts !== state.lc) && (Math.abs(history[id].lastLogTime - state.ts) < settings.changesRelogInterval * 1000)) { + history[id].skipped = state; // remember new timestamp + adapter.log.debug('value not changed ' + id + ', last-value=' + history[id].state.val + ', new-value=' + state.val + ', ts=' + state.ts); + return; + } + if (state.ts !== state.lc) { + adapter.log.debug('value-changed-relog ' + id + ', value=' + state.val + ', lastLogTime=' + history[id].lastLogTime + ', ts=' + state.ts); + } + } + if (history[id].state.val !== null && (settings.changesMinDelta !== 0) && (typeof state.val === 'number') && (Math.abs(history[id].state.val - state.val) < settings.changesMinDelta)) { + history[id].skipped = state; // remember new timestamp + adapter.log.debug('Min-Delta not reached ' + id + ', last-value=' + history[id].state.val + ', new-value=' + state.val + ', ts=' + state.ts); + return; + } + else if (typeof state.val === 'number') { + adapter.log.debug('Min-Delta reached ' + id + ', last-value=' + history[id].state.val + ', new-value=' + state.val + ', ts=' + state.ts); + } + else { + adapter.log.debug('Min-Delta ignored because no number ' + id + ', last-value=' + history[id].state.val + ', new-value=' + state.val + ', ts=' + state.ts); + } + } + + if (history[id].relogTimeout) { + clearTimeout(history[id].relogTimeout); + history[id].relogTimeout = null; + } + if (settings.changesRelogInterval > 0) { + history[id].relogTimeout = setTimeout(reLogHelper, settings.changesRelogInterval * 1000, id); + } + + var ignoreDebonce = false; + if (timerRelog) { + state.ts = new Date().getTime(); + adapter.log.debug('timed-relog ' + id + ', value=' + state.val + ', lastLogTime=' + history[id].lastLogTime + ', ts=' + state.ts); + ignoreDebonce = true; + } else { + if (settings.changesOnly && history[id].skipped) { + history[id].state = history[id].skipped; + pushHelper(id); + } + if (history[id].state && ((history[id].state.val === null && state.val !== null) || (history[id].state.val !== null && state.val === null))) { + ignoreDebonce = true; + } else if (!history[id].state && state.val === null) { + ignoreDebonce = true; + } + // only store state if really changed + history[id].state = state; + } + history[id].lastLogTime = state.ts; + history[id].skipped = null; + if (settings.debounce && !ignoreDebonce) { + // Discard changes in de-bounce time to store last stable value + if (history[id].timeout) clearTimeout(history[id].timeout); + history[id].timeout = setTimeout(pushHelper, settings.debounce, id); + } else { + pushHelper(id); + } + } +} + +function reLogHelper(_id) { + if (!history[_id]) { + adapter.log.info('non-existing id ' + _id); + return; + } + history[_id].relogTimeout = null; + if (history[_id].skipped) { + history[_id].state = history[_id].skipped; + history[_id].state.from = 'system.adapter.' + adapter.namespace; + history[_id].skipped = null; + pushHistory(_id, history[_id].state, true); + } + else { + adapter.getForeignState(history[_id].realId, function (err, state) { + if (err) { + adapter.log.info('init timed Relog: can not get State for ' + _id + ' : ' + err); + } + else if (!state) { + adapter.log.info('init timed Relog: disable relog because state not set so far ' + _id + ': ' + JSON.stringify(state)); + } + else { + adapter.log.debug('init timed Relog: getState ' + _id + ': Value=' + state.val + ', ack=' + state.ack + ', ts=' + state.ts + ', lc=' + state.lc); + history[_id].state = state; + pushHistory(_id, history[_id].state, true); + } + }); + } +} + +function pushHelper(_id) { + if (!history[_id] || !history[_id].state) return; + var _settings = history[_id][adapter.namespace]; + // if it was not deleted in this time + if (_settings) { + history[_id].timeout = null; + history[_id].list = history[_id].list || []; + + if (typeof history[_id].state.val === 'string') { + var f = parseFloat(history[_id].state.val); + if (f == history[_id].state.val) { + history[_id].state.val = f; + } else if (history[_id].state.val === 'true') { + history[_id].state.val = true; + } else if (history[_id].state.val === 'false') { + history[_id].state.val = false; + } + } + if (history[_id].state.lc !== undefined) delete history[_id].state.lc; + if (!adapter.config.storeAck && history[_id].state.ack !== undefined) { + delete history[_id].state.ack; + } else { + history[_id].state.ack = history[_id].state.ack ? 1 : 0; + } + if (!adapter.config.storeFrom && history[_id].state.from !== undefined) delete history[_id].state.from; + + history[_id].list.push(history[_id].state); + + if (history[_id].list.length > _settings.maxLength) { + adapter.log.debug('moving ' + history[_id].list.length + ' entries from '+ _id +' to file'); + appendFile(_id, history[_id].list); + checkRetention(_id); + } + } +} + +function checkRetention(id) { + if (history[id][adapter.namespace].retention) { + var d = new Date(); + var dt = d.getTime(); + // check every 6 hours + if (!history[id].lastCheck || dt - history[id].lastCheck >= 21600000/* 6 hours */) { + history[id].lastCheck = dt; + // get list of directories + var dayList = getDirectories(adapter.config.storeDir).sort(function (a, b) { + return a - b; + }); + // calculate date + d.setSeconds(-(history[id][adapter.namespace].retention)); + var day = GetHistory.ts2day(d.getTime()); + for (var i = 0; i < dayList.length; i++) { + if (dayList[i] < day) { + const file = GetHistory.getFilenameForID(adapter.config.storeDir, dayList[i], id); + if (fs.existsSync(file)) { + adapter.log.info('Delete old history "' + file + '"'); + try { + fs.unlinkSync(file); + } catch (ex) { + adapter.log.error('Cannot delete file "' + file + '": ' + ex); + } + var files = fs.readdirSync(adapter.config.storeDir + dayList[i]); + if (!files.length) { + adapter.log.info('Delete old history dir "' + adapter.config.storeDir + dayList[i] + '"'); + try { + fs.unlinkSync(adapter.config.storeDir + dayList[i]); + } catch (ex) { + adapter.log.error('Cannot delete directory "' + adapter.config.storeDir + dayList[i] + '": ' + ex); + } + } + } + } else { + break; + } + } + } + } +} + +function appendFile(id, states) { + var day = GetHistory.ts2day(states[states.length - 1].ts); + + const file = GetHistory.getFilenameForID(adapter.config.storeDir, day, id); + var data; + + var i; + for (i = states.length - 1; i >= 0; i--) { + if (!states[i]) continue; + if (GetHistory.ts2day(states[i].ts) !== day) { + break; + } + } + data = states.splice(i - states.length + 1); + + if (fs.existsSync(file)) { + try { + data = JSON.parse(fs.readFileSync(file)).concat(data); + } catch (err) { + adapter.log.error('Cannot read file ' + file + ': ' + err); + } + } + + try { + // create directory + if (!fs.existsSync(adapter.config.storeDir + day)) { + fs.mkdirSync(adapter.config.storeDir + day); + } + fs.writeFileSync(file, JSON.stringify(data, null, 2)); + } catch (ex) { + adapter.log.error('Cannot store file ' + file + ': ' + ex); + } + + if (states.length) { + appendFile(id, states); + } +} + +function getOneCachedData(id, options, cache, addId) { + addId = addId || options.addId; + + if (history[id]) { + var res = history[id].list; + // todo can be optimized + if (res) { + var iProblemCount = 0; + var vLast = null; + for (var i = res.length - 1; i >= 0 ; i--) { + if (!res[i]) { + iProblemCount++; + continue; + } + if (options.start && res[i].ts < options.start) { + if (options.ack) res[i].ack = !!res[i].ack; + if (addId) res[i].id = id; + // add one before start + cache.unshift(res[i]); + break; + } else if (res[i].ts > options.end) { + // add one after end + vLast = res[i]; + continue; + } + if (options.ack) res[i].ack = !!res[i].ack; + + if (vLast) { + if (options.ack) vLast.ack = !!vLast.ack; + if (addId) res[i].id = id; + cache.unshift(vLast); + vLast = null; + } + + if (addId) res[i].id = id; + cache.unshift(res[i]); + + if (!options.start && cache.length >= options.count) break; + } + if (iProblemCount) adapter.log.warn('got null states ' + iProblemCount + ' times for ' + options.id); + + adapter.log.debug('got ' + res.length + ' datapoints for ' + options.id); + } else { + adapter.log.debug('datapoints for ' + options.id + ' do not yet exist'); + } + } +} + +function getCachedData(options, callback) { + var cache = []; + + if (options.id && options.id !== '*') { + getOneCachedData(options.id, options, cache); + } else { + for (var id in history) { + getOneCachedData(id, options, cache, true); + } + } + options.length = cache.length; + callback(cache, !options.start && cache.length >= options.count); +} + +function tsSort(a, b) { + return b.ts - a.ts; +} + +function getOneFileData(dayList, dayStart, dayEnd, id, options, data, addId) { + addId = addId || options.addId; + + // get all files in directory + for (var i = 0; i < dayList.length; i++) { + var day = parseInt(dayList[i], 10); + + if (!isNaN(day) && day > 20100101 && day >= dayStart && day <= dayEnd) { + const file = GetHistory.getFilenameForID(options.path, dayList[i], id); + + if (fs.existsSync(file)) { + try { + var _data = JSON.parse(fs.readFileSync(file)).sort(tsSort); + var last = false; + + for (var ii in _data) { + if (!_data.hasOwnProperty(ii)) continue; + if (options.ack) { + _data[ii].ack = !!_data[ii].ack; + } + if (addId) { + _data[ii].id = id; + } + data.push(_data[ii]); + if (!options.start && data.length >= options.count) break; + if (last) break; + if (options.start && _data[ii].ts < options.start) last = true; + } + } catch (e) { + console.log('Cannot parse file ' + file + ': ' + e.message); + } + } + } + if (!options.start && data.length >= options.count) break; + if (day > dayEnd) break; + } +} + +function getFileData(options, callback) { + var dayStart = options.start ? parseInt(GetHistory.ts2day(options.start), 10) : 0; + var dayEnd = parseInt(GetHistory.ts2day(options.end), 10); + var fileData = []; + + // get list of directories + var dayList = getDirectories(options.path).sort(function (a, b) { + return b - a; + }); + + if (options.id && options.id !== '*') { + getOneFileData(dayList, dayStart, dayEnd, options.id, options, fileData); + } else { + for (var id in history) { + getOneFileData(dayList, dayStart, dayEnd, id, options, fileData, true); + } + } + + callback(fileData); +} + +function sortByTs(a, b) { + var aTs = a.ts; + var bTs = b.ts; + return ((aTs < bTs) ? -1 : ((aTs > bTs) ? 1 : 0)); +} + +function getHistory(msg) { + var startTime = new Date().getTime(); + var options = { + id: msg.message.id ? msg.message.id : null, + path: adapter.config.storeDir, + start: msg.message.options.start, + end: msg.message.options.end || ((new Date()).getTime() + 5000000), + step: parseInt(msg.message.options.step, 10) || null, + count: parseInt(msg.message.options.count, 10) || 500, + from: msg.message.options.from || false, + ack: msg.message.options.ack || false, + q: msg.message.options.q || false, + ignoreNull: msg.message.options.ignoreNull, + aggregate: msg.message.options.aggregate || 'average', // One of: max, min, average, total + limit: parseInt(msg.message.options.limit || adapter.config.limit || 2000, 10), + addId: msg.message.options.addId || false, + sessionId: msg.message.options.sessionId + }; + if (options.id && aliasMap[options.id]) { + options.id = aliasMap[options.id]; + } + if (options.start > options.end) { + var _end = options.end; + options.end = options.start; + options.start = _end; + } + + // if less 2000.01.01 00:00:00 + if (options.start < 946681200000) { + options.start *= 1000; + if (options.step !== null && options.step !== undefined) options.step *= 1000; + } + + // if less 2000.01.01 00:00:00 + if (options.end < 946681200000) options.end *= 1000; + + if ((!options.start && options.count) || options.aggregate === 'onchange' || options.aggregate === '' || options.aggregate === 'none') { + getCachedData(options, function (cacheData, isFull) { + adapter.log.debug('after getCachedData: length = ' + cacheData.length + ', isFull=' + isFull); + // if all data read + if (isFull && cacheData.length) { + cacheData = cacheData.sort(sortByTs); + if ((options.count) && (cacheData.length > options.count) && (options.aggregate === 'none')) { + cacheData = cacheData.slice(0, options.count); + adapter.log.debug('cut cacheData to ' + options.count + ' values'); + } + adapter.log.debug('Send: ' + cacheData.length + ' values in: ' + (new Date().getTime() - startTime) + 'ms'); + adapter.sendTo(msg.from, msg.command, { + result: cacheData, + step: null, + error: null + }, msg.callback); + } else { + var origCount = options.count; + options.count -= cacheData.length; + getFileData(options, function (fileData) { + adapter.log.debug('after getFileData: cacheData.length = ' + cacheData.length + ', fileData.length = ' + fileData.length); + cacheData = cacheData.concat(fileData); + cacheData = cacheData.sort(sortByTs); + options.result = cacheData; + options.count = origCount; + Aggregate.beautify(options); + + adapter.log.debug('Send: ' + options.result.length + ' values in: ' + (new Date().getTime() - startTime) + 'ms'); + adapter.sendTo(msg.from, msg.command, { + result: options.result, + step: null, + error: null + }, msg.callback); + }); + } + }); + } else { + // to use parallel requests activate this. + if (1 || typeof GetHistory === 'undefined') { + adapter.log.debug('use parallel requests'); + var gh = cp.fork(__dirname + '/lib/getHistory.js', [JSON.stringify(options)], {silent: false}); + + var ghTimeout = setTimeout(function () { + try { + gh.kill('SIGINT'); + } + catch (err) { + adapter.log.error(err); + } + }, 120000); + + gh.on('message', function (data) { + var cmd = data[0]; + if (cmd === 'getCache') { + var settings = data[1]; + getCachedData(settings, function (cacheData) { + gh.send(['cacheData', cacheData]); + }); + } else if (cmd === 'response') { + clearTimeout(ghTimeout); + ghTimeout = null; + + var result = data[1]; + var overallLength = data[2]; + var step = data[3]; + if (result) { + adapter.log.debug('Send: ' + result.length + ' of: ' + overallLength + ' in: ' + (new Date().getTime() - startTime) + 'ms'); + adapter.sendTo(msg.from, msg.command, { + result: result, + step: step, + error: null + }, msg.callback); + } else { + adapter.log.info('No Data'); + adapter.sendTo(msg.from, msg.command, { + result: [], + step: null, + error: null + }, msg.callback); + } + } + }); + } else { + GetHistory.initAggregate(options); + GetHistory.getFileData(options); + getCachedData(options, function (cachedData) { + GetHistory.aggregation(options, cachedData); + var data = GetHistory.response(options); + + if (data[0] === 'response') { + if (data[1]) { + adapter.log.debug('Send: ' + data[1].length + ' of: ' + data[2] + ' in: ' + (new Date().getTime() - startTime) + 'ms'); + adapter.sendTo(msg.from, msg.command, { + result: data[1], + step: data[3], + error: null + }, msg.callback); + } else { + adapter.log.info('No Data'); + adapter.sendTo(msg.from, msg.command, { + result: [], + step: null, + error: null + }, msg.callback); + } + } else { + adapter.log.error('Unknown response type: ' + data[0]); + } + }); + } + } +} + +function getDirectories(path) { + return fs.readdirSync(path).filter(function (file) { + return fs.statSync(path + '/' + file).isDirectory(); + }); +} + +function storeState(msg) { + if (!msg.message || !msg.message.id || !msg.message.state) { + adapter.log.error('storeState called with invalid data'); + adapter.sendTo(msg.from, msg.command, { + error: 'Invalid call' + }, msg.callback); + return; + } + var id; + if (Array.isArray(msg.message)) { + adapter.log.debug('storeState: store ' + msg.message.length + ' states for multiple ids'); + for (var i = 0; i < msg.message.length; i++) { + id = aliasMap[msg.message[i].id] ? aliasMap[msg.message[i].id] : msg.message[i].id; + if (history[id]) { + history[id].state = msg.message[i].state; + pushHelper(id); + } + else { + adapter.log.warn('storeState: history not enabled for ' + msg.message[i].id + '. Ignoring'); + } + } + } else if (Array.isArray(msg.message.state)) { + adapter.log.debug('storeState: store ' + msg.message.state.length + ' states for ' + msg.message.id); + for (var j = 0; j < msg.message.state.length; j++) { + id = aliasMap[msg.message.id] ? aliasMap[msg.message.id] : msg.message.id; + if (history[id]) { + history[id].state = msg.message.state[j]; + pushHelper(id); + } + else { + adapter.log.warn('storeState: history not enabled for ' + msg.message.id + '. Ignoring'); + } + } + } else { + adapter.log.debug('storeState: store 1 state for ' + msg.message.id); + id = aliasMap[msg.message.id] ? aliasMap[msg.message.id] : msg.message.id; + if (history[id]) { + history[id].state = msg.message.state; + pushHelper(id); + } + else { + adapter.log.warn('storeState: history not enabled for ' + msg.message.id + '. Ignoring'); + } + } + + adapter.sendTo(msg.from, msg.command, { + success: true + }, msg.callback); +} + +function enableHistory(msg) { + if (!msg.message || !msg.message.id) { + adapter.log.error('enableHistory called with invalid data'); + adapter.sendTo(msg.from, msg.command, { + error: 'Invalid call' + }, msg.callback); + return; + } + var obj = {}; + obj.common = {}; + obj.common.custom = {}; + if (msg.message.options) { + obj.common.custom[adapter.namespace] = msg.message.options; + } + else { + obj.common.custom[adapter.namespace] = {}; + } + obj.common.custom[adapter.namespace].enabled = true; + adapter.extendForeignObject(msg.message.id, obj, function (err) { + if (err) { + adapter.log.error('enableHistory: ' + err); + adapter.sendTo(msg.from, msg.command, { + error: err + }, msg.callback); + } else { + adapter.log.info(JSON.stringify(obj)); + adapter.sendTo(msg.from, msg.command, { + success: true + }, msg.callback); + } + }); +} + +function disableHistory(msg) { + if (!msg.message || !msg.message.id) { + adapter.log.error('disableHistory called with invalid data'); + adapter.sendTo(msg.from, msg.command, { + error: 'Invalid call' + }, msg.callback); + return; + } + var obj = {}; + obj.common = {}; + obj.common.custom = {}; + obj.common.custom[adapter.namespace] = {}; + obj.common.custom[adapter.namespace].enabled = false; + adapter.extendForeignObject(msg.message.id, obj, function (err) { + if (err) { + adapter.log.error('disableHistory: ' + err); + adapter.sendTo(msg.from, msg.command, { + error: err + }, msg.callback); + } else { + adapter.log.info(JSON.stringify(obj)); + adapter.sendTo(msg.from, msg.command, { + success: true + }, msg.callback); + } + }); +} + +function getEnabledDPs(msg) { + var data = {}; + for (var id in history) { + if (!history.hasOwnProperty(id)) continue; + data[history[id].realId] = history[id][adapter.namespace]; + } + + adapter.sendTo(msg.from, msg.command, data, msg.callback); +} diff --git a/io-package.json b/io-package.json new file mode 100644 index 0000000..b76eb19 --- /dev/null +++ b/io-package.json @@ -0,0 +1,219 @@ +{ + "common": { + "name": "history", + "title": "History", + "titleLang": { + "en": "History", + "de": "History", + "ru": "History", + "pt": "History", + "nl": "History", + "fr": "History", + "it": "History", + "es": "History", + "pl": "History" + }, + "desc": { + "en": "Logging of the state history in files", + "de": "Protokollierung der Zustandsgeschichte in Dateien", + "ru": "Ведение истории событий в файлах", + "pt": "Registro do histórico de estado em arquivos", + "nl": "Logboekregistratie van de statusgeschiedenis in bestanden", + "fr": "Consignation de l'historique des états dans les fichiers", + "it": "Registrazione della cronologia di stato nei file" + }, + "version": "1.8.5", + "news": { + "1.8.5": { + "en": "error fixed in storeState", + "de": "Fehler in storeState behoben", + "ru": "исправлена ​​ошибка в storeState", + "pt": "erro corrigido no storeState", + "nl": "fout opgelost in storeState", + "fr": "erreur corrigée dans storeState", + "it": "errore riparato in storeState", + "es": "error corregido en storeState", + "pl": "błąd został naprawiony w storeState" + }, + "1.8.4": { + "en": "error fixed for allow to disable writing of start and end values", + "de": "Fehler behoben für das Schreiben von Start- und Endwerten zu deaktivieren", + "ru": "исправлена ​​ошибка, позволяющая отключить запись начальных и конечных значений", + "pt": "erro corrigido para permitir desativar a escrita dos valores inicial e final", + "nl": "error fixed for allow om schrijven van begin- en eindwaarden uit te schakelen", + "fr": "erreur corrigée pour permettre de désactiver l'écriture des valeurs de début et de fin", + "it": "errore risolto per consentire di disabilitare la scrittura dei valori iniziali e finali", + "es": "error corregido para permitir deshabilitar la escritura de los valores de inicio y fin", + "pl": "błąd został usunięty, aby umożliwić wyłączenie zapisywania wartości początkowej i końcowej" + }, + "1.8.3": { + "en": "Add option to write data to a different ID to make device changes easier. Retrieving data works for both IDs, also allow to disable writing of start and end values", + "de": "Option zum Schreiben von Daten in eine andere ID hinzufügen, um Geräteänderungen zu vereinfachen. Das Abrufen von Daten funktioniert für beide IDs und ermöglicht auch das Schreiben von Start- und Endwerten zu deaktivieren", + "ru": "Добавьте параметр для записи данных на другой идентификатор, чтобы упростить внесение изменений. Получение данных для обоих идентификаторов также позволяет отключить запись начальных и конечных значений", + "pt": "Adicione a opção de gravar dados em um ID diferente para facilitar as alterações no dispositivo. Recuperar dados funciona para ambos os IDs, também permite desativar a gravação de valores iniciais e finais", + "nl": "Optie toevoegen om gegevens naar een andere ID te schrijven om apparaatwijzigingen gemakkelijker te maken. Het ophalen van gegevens werkt voor beide ID's en staat ook toe om het schrijven van begin- en eindwaarden uit te schakelen", + "fr": "Ajoutez une option pour écrire des données sur un ID différent afin de faciliter les changements de périphérique. La récupération des données pour les deux ID permet également de désactiver l'écriture des valeurs de début et de fin", + "it": "Aggiungi l'opzione per scrivere i dati su un ID diverso per facilitare le modifiche ai dispositivi. Il recupero dei dati funziona per entrambi gli ID, inoltre consente di disabilitare la scrittura dei valori iniziali e finali", + "es": "Agregue la opción para escribir datos en una ID diferente para facilitar los cambios en el dispositivo. La recuperación de datos funciona para ambos ID, también permite deshabilitar la escritura de los valores de inicio y fin", + "pl": "Dodaj opcję zapisywania danych do innego identyfikatora, aby ułatwić zmianę urządzenia. Pobieranie danych działa dla obu identyfikatorów, pozwala również wyłączyć zapisywanie wartości początkowej i końcowej" + }, + "1.7.4": { + "en": "Fix filename handling for states with special characters", + "de": "Behebt die Behandlung von Dateinamen für Zustände mit Sonderzeichen", + "ru": "Исправить обработку имен файлов для состояний со специальными символами", + "pt": "Corrigir manipulação de nome de arquivo para estados com caracteres especiais", + "nl": "Bestandsnaamafhandeling voor staten met speciale tekens oplossen", + "fr": "Correction de la gestion des noms de fichiers pour les états avec des caractères spéciaux", + "it": "Correggere la gestione dei nomi dei file per gli stati con caratteri speciali", + "es": "Corregir el manejo del nombre de archivo para estados con caracteres especiales", + "pl": "Popraw obsługę plików dla stanów ze znakami specjalnymi" + }, + "1.7.3": { + "en": "Respect 'keep forever' setting for retention from datapoint configuration", + "de": "Beachten der 'Keep forever' Einstellung für die Aufbewahrung von Datenpunktkonfiguration", + "ru": "Упорядочить «сохранить навсегда» для сохранения из конфигурации данных", + "pt": "Respeito 'manter sempre' configurado para retenção da configuração do ponto de dados", + "nl": "Respecteer 'blijf voor altijd' instelling voor retentie van datapuntconfiguratie", + "fr": "Respectez le paramètre 'keep forever' pour la rétention à partir de la configuration du point de données", + "it": "Rispettare l'impostazione 'keep forever' per la conservazione dalla configurazione del datapoint", + "es": "Respetar la configuración 'mantener para siempre' para la retención desde la configuración del punto de datos", + "pl": "Szanuj ustawienie \"zachowaj na zawsze\", aby zachować dane z konfiguracji punktu danych" + }, + "1.7.2": { + "en": "Admin3 Fixes", + "de": "Admin3-Korrekturen", + "ru": "Исправления Admin3", + "pt": "Admin3 Fixes", + "nl": "Admin3 Fixes", + "fr": "Corrections Admin3", + "it": "Correzioni di Admin3", + "es": "Correcciones de Admin3", + "pl": "Poprawki Admin3" + }, + "1.7.1": { + "en": "Admin3 Fixes", + "de": "Admin3-Korrekturen", + "ru": "Исправления Admin3", + "pt": "Admin3 Fixes", + "nl": "Admin3 Fixes", + "fr": "Corrections Admin3", + "it": "Correzioni di Admin3", + "es": "Correcciones de Admin3", + "pl": "Poprawki Admin3" + }, + "1.7.0": { + "en": "Ready for Admin3", + "de": "Bereit für Admin3", + "ru": "Готово для Admin3", + "pt": "Pronto para Admin3", + "nl": "Klaar voor Admin3", + "fr": "Prêt pour Admin3", + "it": "Pronto per Admin3", + "es": "Listo para Admin3", + "pl": "Gotowy na Admin3" + }, + "1.6.6": { + "en": "Translations", + "de": "Übersetzungen", + "ru": "Переводы", + "pt": "Traduções", + "nl": "vertaalwerk", + "fr": "Traductions", + "it": "Traduzioni" + }, + "1.6.5": { + "en": "fix relog value feature", + "de": "Erneutes Logging korrigiert", + "ru": "fix relog value feature" + }, + "1.6.4": { + "en": "add \"save last value\" option", + "de": "Die Option um vorheriges Wert zu speichern ist hinzugefügt", + "ru": "Возможность сохранять предыдущее значение при сохранении только изменений" + }, + "1.6.3": { + "en": "fix behaviour of log interval to always log the current value", + "de": "Verhalten vom Logging in bestimmten Zeitabständen geändert, sodass immer der aktuellste Wert geloggt wird", + "ru": "fix behaviour of log interval to always log the current value" + }, + "1.6.2": { + "en": "fix in datatype conversions", + "de": "Fehlerbehebung in Datentypkonvertierungen", + "ru": "fix in datatype conversions" + }, + "1.6.1": { + "en": "Changed Debounce-Feature to log the last stable value that did not changed for given time!! Replace some characters in history filenames", + "de": "Debounce-Feature geändert, sodass der letzte Wert der in der angegeben Zeit stabil geblieben ist geloggt wird!! Ersetzung einiger Zeichen in Dateinamen", + "ru": "Changed Debounce-Feature to log the last stable value that did not changed for given time!! Replace some characters in history filenames" + }, + "1.5.3": { + "en": "Small fix for older configurations", + "de": "Kleine Fehlerbehebung für ältere Konfigurationen", + "ru": "Small fix for older configurations" + }, + "1.5.2": { + "en": "Enhance Min-Delta logic for datapoints from type mixed", + "de": "Min-Delta-Logik verbessert für Mixed-Datenpunkte", + "ru": "Enhance Min-Delta logic for datapoints from type mixed" + }, + "1.5.1": { + "en": "Fix handling of float values in Adapter config and Datapoint config.", + "de": "Fehlerbehebung bei der Eingabe von Zahlen mit Kommastellen in der Adapter- und Datenpunk-Konfiguration.", + "ru": "Fix handling of float values in Adapter config and Datapoint config." + }, + "1.5.0": { + "en": "Add messages enableHistory/disableHistory/getEnabledDPs, add support to log changes only if value differs a minimum value for numbers. Fixing aggregate calculation", + "de": "Messages enableHistory/disableHistory/getEnabledDPs hinzugefügt, Unterstützung zum Loggen wenn der Zahlenwert um einen Mindestwert abweicht. Fehler in Aggregationsberechnung behoben", + "ru": "Add messages enableHistory/disableHistory/getEnabledDPs, add support to log changes only if value differs a minimum value for numbers. Fixing aggregate calculation" + } + }, + "mode": "daemon", + "platform": "Javascript/Node.js", + "loglevel": "info", + "messagebox": true, + "subscribe": "messagebox", + "keywords": ["charts", "history", "logging", "graphs", "Graphen", "data", "archive"], + "main": "history.js", + "enabled": true, + "preserveSettings": "custom", + "authors": [ + "bluefox ", + "hobbyquaker ", + "Apollon77 " + ], + "license": "MIT", + "readme": "https://github.com/ioBroker/ioBroker.history/blob/master/README.md", + "icon": "history.png", + "extIcon": "https://raw.githubusercontent.com/ioBroker/ioBroker.history/master/admin/history.png", + "type": "storage", + "supportCustoms": true, + "materialize": true, + "getHistory": true, + "dependencies": [{"js-controller": ">=0.12.0","admin": ">=1.6.10"}], + "supportStopInstance": 2000, + "stopTimeout": 10000 + }, + "native": { + "maxLength": 960, + "limit": 2000, + "storeDir": "", + "debounce": 1000, + "retention": 31536000, + "storeFrom": false, + "storeAck": true, + "changesRelogInterval": 0, + "changesMinDelta": 0, + "writeNulls": true + }, + "objects": [ + { + "_id": "_design/custom", + "language": "javascript", + "views": { + "state": { + "map": "function(doc) { if (doc.type==='state' && (doc.common.custom || doc.common.history)) emit(doc._id, doc.common.custom || doc.common.history) }" + } + } + } + ] +} diff --git a/lib/aggregate.js b/lib/aggregate.js new file mode 100644 index 0000000..e86531b --- /dev/null +++ b/lib/aggregate.js @@ -0,0 +1,476 @@ +/* jshint -W097 */// jshint strict:false +/*jslint node: true */ +"use strict"; +// THIS file should be identical with sql and history adapter's one + +function initAggregate(options) { + + //step; // 1 Step is 1 second + if (options.step === null) { + options.step = (options.end - options.start) / options.count; + } + + // Limit 2000 + if ((options.end - options.start) / options.step > options.limit) { + options.step = (options.end - options.start) / options.limit; + } + + options.maxIndex = Math.ceil(((options.end - options.start) / options.step) - 1); + options.result = []; + options.averageCount = []; + options.aggregate = options.aggregate || 'minmax'; + options.overallLength = 0; + + // pre-fill the result with timestamps (add one before start and one after end) + for (var i = 0; i <= options.maxIndex + 2; i++){ + + options.result[i] = { + val: {ts: null, val: null}, + max: {ts: null, val: null}, + min: {ts: null, val: null}, + start: {ts: null, val: null}, + end: {ts: null, val: null} + }; + + if (options.aggregate === 'average') options.averageCount[i] = 0; + } + return options; +} + +function aggregation(options, data) { + var index; + var preIndex; + for (var i = 0; i < data.length; i++) { + if (!data[i]) continue; + if (typeof data[i].ts !== 'number') data[i].ts = parseInt(data[i].ts, 10); + if (data[i].ts < 946681200000) data[i].ts *= 1000; + + preIndex = Math.floor((data[i].ts - options.start) / options.step); + + // store all border values + if (preIndex < 0) { + index = 0; + } else if (preIndex > options.maxIndex) { + index = options.maxIndex + 2; + } else { + index = preIndex + 1; + } + options.overallLength++; + + if (!options.result[index]) { + console.error('Cannot find index ' + index); + continue; + } + + if (options.aggregate === 'max') { + if (!options.result[index].val.ts) options.result[index].val.ts = Math.round(options.start + (((index - 1) + 0.5) * options.step)); + if (options.result[index].val.val === null || options.result[index].val.val < data[i].val) options.result[index].val.val = data[i].val; + } else if (options.aggregate === 'min') { + if (!options.result[index].val.ts) options.result[index].val.ts = Math.round(options.start + (((index - 1) + 0.5) * options.step)); + if (options.result[index].val.val === null || options.result[index].val.val > data[i].val) options.result[index].val.val = data[i].val; + } else if (options.aggregate === 'average') { + if (!options.result[index].val.ts) options.result[index].val.ts = Math.round(options.start + (((index - 1) + 0.5) * options.step)); + options.result[index].val.val += parseFloat(data[i].val); + options.averageCount[index]++; + } else if (options.aggregate === 'total') { + if (!options.result[index].val.ts) options.result[index].val.ts = Math.round(options.start + (((index - 1) + 0.5) * options.step)); + options.result[index].val.val += parseFloat(data[i].val); + } else if (options.aggregate === 'minmax') { + if (options.result[index].min.ts === null) { + options.result[index].min.ts = data[i].ts; + options.result[index].min.val = data[i].val; + + options.result[index].max.ts = data[i].ts; + options.result[index].max.val = data[i].val; + + options.result[index].start.ts = data[i].ts; + options.result[index].start.val = data[i].val; + + options.result[index].end.ts = data[i].ts; + options.result[index].end.val = data[i].val; + } else { + if (data[i].val !== null) { + if (data[i].val > options.result[index].max.val) { + options.result[index].max.ts = data[i].ts; + options.result[index].max.val = data[i].val; + } else if (data[i].val < options.result[index].min.val) { + options.result[index].min.ts = data[i].ts; + options.result[index].min.val = data[i].val; + } + if (data[i].ts > options.result[index].end.ts) { + options.result[index].end.ts = data[i].ts; + options.result[index].end.val = data[i].val; + } + } else { + if (data[i].ts > options.result[index].end.ts) { + options.result[index].end.ts = data[i].ts; + options.result[index].end.val = null; + } + } + } + } + } + + return {result: options.result, step: options.step, sourceLength: data.length} ; +} + +function finishAggregation(options) { + if (options.aggregate === 'minmax') { + for (var ii = options.result.length - 1; ii >= 0; ii--) { + // no one value in this period + if (options.result[ii].start.ts === null) { + options.result.splice(ii, 1); + } else { + // just one value in this period: max == min == start == end + if (options.result[ii].start.ts === options.result[ii].end.ts) { + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } else + if (options.result[ii].min.ts === options.result[ii].max.ts) { + // if just 2 values: start == min == max, end + if (options.result[ii].start.ts === options.result[ii].min.ts || + options.result[ii].end.ts === options.result[ii].min.ts) { + options.result.splice(ii + 1, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } // if just 3 values: start, min == max, end + else { + options.result.splice(ii + 1, 0, { + ts: options.result[ii].max.ts, + val: options.result[ii].max.val + }); + options.result.splice(ii + 2, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } + } else + if (options.result[ii].start.ts === options.result[ii].max.ts) { + // just one value in this period: start == max, min == end + if (options.result[ii].min.ts === options.result[ii].end.ts) { + options.result.splice(ii + 1, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } // start == max, min, end + else { + options.result.splice(ii + 1, 0, { + ts: options.result[ii].min.ts, + val: options.result[ii].min.val + }); + options.result.splice(ii + 2, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } + } else + if (options.result[ii].end.ts === options.result[ii].max.ts) { + // just one value in this period: start == min, max == end + if (options.result[ii].min.ts === options.result[ii].start.ts) { + options.result.splice(ii + 1, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } // start, min, max == end + else { + + options.result.splice(ii + 1, 0, { + ts: options.result[ii].min.ts, + val: options.result[ii].min.val + }); + options.result.splice(ii + 2, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } + } else + if (options.result[ii].start.ts === options.result[ii].min.ts || + options.result[ii].end.ts === options.result[ii].min.ts) { + // just one value in this period: start == min, max, end + options.result.splice(ii + 1, 0, { + ts: options.result[ii].max.ts, + val: options.result[ii].max.val + }); + options.result.splice(ii + 2, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } else { + // just one value in this period: start == min, max, end + if (options.result[ii].max.ts > options.result[ii].min.ts) { + options.result.splice(ii + 1, 0, { + ts: options.result[ii].min.ts, + val: options.result[ii].min.val + }); + options.result.splice(ii + 2, 0, { + ts: options.result[ii].max.ts, + val: options.result[ii].max.val + }); + } else { + options.result.splice(ii + 1, 0, { + ts: options.result[ii].max.ts, + val: options.result[ii].max.val + }); + options.result.splice(ii + 2, 0, { + ts: options.result[ii].min.ts, + val: options.result[ii].min.val + }); + } + options.result.splice(ii + 3, 0, { + ts: options.result[ii].end.ts, + val: options.result[ii].end.val + }); + options.result[ii] = { + ts: options.result[ii].start.ts, + val: options.result[ii].start.val + }; + } + } + } + } else if (options.aggregate === 'average') { + for (var k = options.result.length - 1; k >= 0; k--) { + if (options.result[k].val.ts) { + options.result[k] = { + ts: options.result[k].val.ts, + val: (options.result[k].val.val !== null) ? Math.round(options.result[k].val.val / options.averageCount[k] * 100) / 100 : null + }; + } else { + // no one value in this interval + options.result.splice(k, 1); + } + } + } else { + for (var j = options.result.length - 1; j >= 0; j--) { + if (options.result[j].val.ts) { + options.result[j] = { + ts: options.result[j].val.ts, + val: options.result[j].val.val + }; + } else { + // no one value in this interval + options.result.splice(j, 1); + } + } + } + beautify(options); +} + +function beautify(options) { + var preFirstValue = null; + var postLastValue = null; + if (options.ignoreNull === 'true') options.ignoreNull = true; // include nulls and replace them with last value + if (options.ignoreNull === 'false') options.ignoreNull = false; // include nulls + if (options.ignoreNull === '0') options.ignoreNull = 0; // include nulls and replace them with 0 + if (options.ignoreNull !== true && options.ignoreNull !== false && options.ignoreNull !== 0) options.ignoreNull = false; + + // process null values, remove points outside the span and find first points after end and before start + for (var i = 0; i < options.result.length; i++) { + if (options.ignoreNull !== false) { + // if value is null + if (options.result[i].val === null) { + // null value must be replaced with last not null value + if (options.ignoreNull === true) { + // remove value + options.result.splice(i, 1); + i--; + continue; + } else { + // null value must be replaced with 0 + options.result[i].val = options.ignoreNull; + } + } + } + + // remove all not requested points + if (options.result[i].ts < options.start) { + preFirstValue = options.result[i].val !== null ? options.result[i] : null; + options.result.splice(i, 1); + i--; + continue; + } + + postLastValue = options.result[i].val !== null ? options.result[i] : null; + + if (options.result[i].ts > options.end) { + options.result.splice(i, options.result.length - i); + break; + } + } + + // check start and stop + if (options.result.length && options.aggregate !== 'none') { + var firstTS = options.result[0].ts; + + if (firstTS > options.start) { + if (preFirstValue) { + var firstY = options.result[0].val; + // if steps + if (options.aggregate === 'onchange' || !options.aggregate) { + if (preFirstValue.ts !== firstTS) { + options.result.unshift({ts: options.start, val: preFirstValue.val}); + } else { + if (options.ignoreNull) { + options.result.unshift({ts: options.start, val: firstY}); + } + } + } else { + if (preFirstValue.ts !== firstTS) { + if (firstY !== null) { + // interpolate + var y = preFirstValue.val + (firstY - preFirstValue.val) * (options.start - preFirstValue.ts) / (firstTS - preFirstValue.ts); + options.result.unshift({ts: options.start, val: y}); + } else { + options.result.unshift({ts: options.start, val: null}); + } + } else { + if (options.ignoreNull) { + options.result.unshift({ts: options.start, val: firstY}); + } + } + } + } else { + if (options.ignoreNull) { + options.result.unshift({ts: options.start, val: options.result[0].val}); + } else { + options.result.unshift({ts: options.start, val: null}); + } + } + } + + var lastTS = options.result[options.result.length - 1].ts; + if (lastTS < options.end) { + if (postLastValue) { + // if steps + if (options.aggregate === 'onchange' || !options.aggregate) { + // if more data following, draw line to the end of chart + if (postLastValue.ts !== lastTS) { + options.result.push({ts: options.end, val: postLastValue.val}); + } else { + if (options.ignoreNull) { + options.result.push({ts: options.end, val: postLastValue.val}); + } + } + } else { + if (postLastValue.ts !== lastTS) { + var lastY = options.result[options.result.length - 1].val; + if (lastY !== null) { + // make interpolation + var _y = lastY + (postLastValue.val - lastY) * (options.end - lastTS) / (postLastValue.ts - lastTS); + options.result.push({ts: options.end, val: _y}); + } else { + options.result.push({ts: options.end, val: null}); + } + } else { + if (options.ignoreNull) { + options.result.push({ts: options.end, val: postLastValue.val}); + } + } + } + } else { + if (options.ignoreNull) { + var lastY = options.result[options.result.length - 1].val; + // if no more data, that means do not draw line + options.result.push({ts: options.end, val: lastY}); + } else { + // if no more data, that means do not draw line + options.result.push({ts: options.end, val: null}); + } + } + } + } + else if (options.aggregate === 'none') { + if ((options.count) && (options.result.length > options.count)) { + options.result = options.result.slice(0, options.count); + } + } +} + +function sendResponse(adapter, msg, options, data, startTime) { + var aggregateData; + if (typeof data === 'string') { + adapter.log.error(data); + return adapter.sendTo(msg.from, msg.command, { + result: [], + step: 0, + error: data, + sessionId: options.sessionId + }, msg.callback); + } + + if (options.count && !options.start && data.length > options.count) { + data.splice(0, data.length - options.count); + } + if (data[0]) { + options.start = options.start || data[0].ts; + + if (!options.aggregate || options.aggregate === 'onchange' || options.aggregate === 'none') { + aggregateData = {result: data, step: 0, sourceLength: data.length}; + + // convert ack from 0/1 to false/true + if (options.ack && aggregateData.result) { + for (var i = 0; i < aggregateData.result.length; i++) { + aggregateData.result[i].ack = !!aggregateData.result[i].ack; + } + } + options.result = aggregateData.result; + beautify(options); + if ((options.aggregate === 'none') && (options.count) && (options.result.length > options.count)) { + options.result = options.result.slice(0, options.count); + } + aggregateData.result = options.result; + } else { + initAggregate(options); + aggregateData = aggregation(options, data); + finishAggregation(options); + aggregateData.result = options.result; + } + + adapter.log.debug('Send: ' + aggregateData.result.length + ' of: ' + aggregateData.sourceLength + ' in: ' + (new Date().getTime() - startTime) + 'ms'); + adapter.sendTo(msg.from, msg.command, { + result: aggregateData.result, + step: aggregateData.step, + error: null, + sessionId: options.sessionId + }, msg.callback); + } else { + adapter.log.info('No Data'); + adapter.sendTo(msg.from, msg.command, {result: [], step: null, error: null, sessionId: options.sessionId}, msg.callback); + } +} + +module.exports.sendResponse = sendResponse; +module.exports.initAggregate = initAggregate; +module.exports.aggregation = aggregation; +module.exports.beautify = beautify; +module.exports.finishAggregation = finishAggregation; diff --git a/lib/getHistory.js b/lib/getHistory.js new file mode 100644 index 0000000..100b383 --- /dev/null +++ b/lib/getHistory.js @@ -0,0 +1,134 @@ +'use strict'; +// todo add cache data +// todo error tests +// todo clean up + +var fs = require('fs'); +var Aggregate = require(__dirname + '/aggregate.js'); + +var gOptions; + +if (typeof module === 'undefined' || !module || !module.parent) { + gOptions = JSON.parse(process.argv[2]); +} + +var finished = false; + +if (typeof module === 'undefined' || !module || !module.parent) { + process.on('message', function (msg){ + if (msg[0] == 'cacheData'){ + if (msg[1]) Aggregate.aggregation(gOptions, msg[1]); + + if (finished) { + response(gOptions); + } else { + finished = true; + } + } + }); +} + +function getDirectories(path) { + return fs.readdirSync(path).filter(function (file) { + return fs.statSync(path + '/' + file).isDirectory(); + }); +} + +function tsSort(a, b) { + return b.ts - a.ts; +} + +function getFilenameForID(path, date, id) { + const safeId = id.replace(/[\u0000:\/\\]/g, "~"); + return `${path}${date.toString()}/history.${safeId}.json`; +} + +function getFileData(options) { + var dayStart = parseInt(ts2day(options.start), 10); + var dayEnd = parseInt(ts2day(options.end), 10); + var fileData = []; + + // get list of directories + var dayList = getDirectories(options.path).sort(function (a, b) { + return a - b; + }); + + // get all files in directory + for (var i = 0; i < dayList.length; i++) { + var day = parseInt(dayList[i], 10); + + if (!isNaN(day) && day > 20100101 && day >= dayStart && day <= dayEnd) { + const file = getFilenameForID(options.path, dayList[i], options.id); + + if (fs.existsSync(file)) { + try { + fileData = JSON.parse(fs.readFileSync(file)).sort(tsSort); + } catch (e) { + fileData = null; + } + if (fileData) Aggregate.aggregation(options, fileData); + } + } + if (day > dayEnd) break; + } +} + +function ts2day(ts) { + if (ts < 946681200000) ts *= 1000; + var dateObj = new Date(ts); + + var text = dateObj.getFullYear().toString(); + var v = dateObj.getMonth() + 1; + if (v < 10) text += '0'; + text += v.toString(); + + v = dateObj.getDate(); + if (v < 10) text += '0'; + text += v.toString(); + + return text; +} + +function response(options){ + Aggregate.finishAggregation(options); + if (typeof module === 'undefined' || !module || !module.parent) { + if (process.send) process.send(['response', options.result, options.overallLength, options.step]); + setTimeout(function () { + process.exit(); + }, 500); + } else { + return ['response', options.result, options.overallLength, options.step]; + } +} + +if (typeof module === 'undefined' || !module || !module.parent) { + Aggregate.initAggregate(gOptions); + if (process.send) { + process.send(['getCache', gOptions]); + } else { + finished = true; + } + getFileData(gOptions); + + if (finished) { + response(gOptions); + } else { + finished = true; + } +} + + +if (typeof module !== 'undefined' && module.parent) { + module.exports.initAggregate = Aggregate.initAggregate; + module.exports.aggregation = Aggregate.aggregation; + module.exports.finishAggregation = Aggregate.finishAggregation; + module.exports.getFileData = getFileData; + module.exports.ts2day = ts2day; + module.exports.response = response; + module.exports.getFilenameForID = getFilenameForID; + + // how to use: + // var options = initAggregate(options); + // getFileData(options); + // var result = response(options); +} diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..c8a0eb7 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,83 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +let controllerDir; +let appName; + +/** + * returns application name + * + * The name of the application can be different and this function finds it out. + * + * @returns {string} + */ + function getAppName() { + const parts = __dirname.replace(/\\/g, '/').split('/'); + return parts[parts.length - 2].split('.')[0]; +} + +/** + * looks for js-controller home folder + * + * @param {boolean} isInstall + * @returns {string} + */ +function getControllerDir(isInstall) { + // Find the js-controller location + const possibilities = [ + 'iobroker.js-controller', + 'ioBroker.js-controller', + ]; + /** @type {string} */ + let controllerPath; + for (const pkg of possibilities) { + try { + const possiblePath = require.resolve(pkg); + if (fs.existsSync(possiblePath)) { + controllerPath = possiblePath; + break; + } + } catch (e) { /* not found */ } + } + if (controllerPath == null) { + if (!isInstall) { + console.log('Cannot find js-controller'); + process.exit(10); + } else { + process.exit(); + } + } + // we found the controller + return path.dirname(controllerPath); +} + +/** + * reads controller base settings + * + * @alias getConfig + * @returns {object} + */ + function getConfig() { + let configPath; + if (fs.existsSync( + configPath = path.join(controllerDir, 'conf', appName + '.json') + )) { + return JSON.parse(fs.readFileSync(configPath, 'utf8')); + } else if (fs.existsSync( + configPath = path.join(controllerDir, 'conf', + appName.toLowerCase() + '.json') + )) { + return JSON.parse(fs.readFileSync(configPath, 'utf8')); + } else { + throw new Error('Cannot find ' + controllerDir + '/conf/' + appName + '.json'); + } +} +appName = getAppName(); +controllerDir = getControllerDir(typeof process !== 'undefined' && process.argv && process.argv.indexOf('--install') !== -1); +const adapter = require(path.join(controllerDir, 'lib/adapter.js')); + +exports.controllerDir = controllerDir; +exports.getConfig = getConfig; +exports.Adapter = adapter; +exports.appName = appName; diff --git a/log/.gitignore b/log/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json new file mode 100644 index 0000000..3625779 --- /dev/null +++ b/package.json @@ -0,0 +1,42 @@ +{ + "name": "iobroker.history", + "description": "Log state history in a two-stages process (first to RAM, then to disk)", + "version": "1.8.5", + "author": "hobbyquaker ", + "contributors": [ + "bluefox ", + "smiling-Jack", + "hobbyquaker ", + "Apollon77" + ], + "homepage": "https://github.com/ioBroker/ioBroker.history", + "repository": { + "type": "git", + "url": "https://github.com/ioBroker/ioBroker.history" + }, + "licenses": [ + { + "type": "MIT", + "url": "https://github.com/ioBroker/ioBroker.history/blob/master/LICENSE" + } + ], + "keywords": [ + "ioBroker", + "log data", + "home automation" + ], + "dependencies": {}, + "devDependencies": { + "gulp": "^3.9.1", + "mocha": "^4.1.0", + "chai": "^4.1.2" + }, + "bugs": { + "url": "https://github.com/ioBroker/ioBroker.history/issues" + }, + "main": "history.js", + "scripts": { + "test": "node node_modules/mocha/bin/mocha --exit" + }, + "license": "MIT" +} diff --git a/test/lib/setup.js b/test/lib/setup.js new file mode 100644 index 0000000..16857ed --- /dev/null +++ b/test/lib/setup.js @@ -0,0 +1,728 @@ +/* jshint -W097 */// jshint strict:false +/*jslint node: true */ +// check if tmp directory exists +var fs = require('fs'); +var path = require('path'); +var child_process = require('child_process'); +var rootDir = path.normalize(__dirname + '/../../'); +var pkg = require(rootDir + 'package.json'); +var debug = typeof v8debug === 'object'; +pkg.main = pkg.main || 'main.js'; + +var adapterName = path.normalize(rootDir).replace(/\\/g, '/').split('/'); +adapterName = adapterName[adapterName.length - 2]; +var adapterStarted = false; + +function getAppName() { + var parts = __dirname.replace(/\\/g, '/').split('/'); + return parts[parts.length - 3].split('.')[0]; +} + +var appName = getAppName().toLowerCase(); + +var objects; +var states; + +var pid = null; + +function copyFileSync(source, target) { + + var targetFile = target; + + //if target is a directory a new file with the same name will be created + if (fs.existsSync(target)) { + if ( fs.lstatSync( target ).isDirectory() ) { + targetFile = path.join(target, path.basename(source)); + } + } + + try { + fs.writeFileSync(targetFile, fs.readFileSync(source)); + } + catch (err) { + console.log("file copy error: " +source +" -> " + targetFile + " (error ignored)"); + } +} + +function copyFolderRecursiveSync(source, target, ignore) { + var files = []; + + var base = path.basename(source); + if (base === adapterName) { + base = pkg.name; + } + //check if folder needs to be created or integrated + var targetFolder = path.join(target, base); + if (!fs.existsSync(targetFolder)) { + fs.mkdirSync(targetFolder); + } + + //copy + if (fs.lstatSync(source).isDirectory()) { + files = fs.readdirSync(source); + files.forEach(function (file) { + if (ignore && ignore.indexOf(file) !== -1) { + return; + } + + var curSource = path.join(source, file); + var curTarget = path.join(targetFolder, file); + if (fs.lstatSync(curSource).isDirectory()) { + // ignore grunt files + if (file.indexOf('grunt') !== -1) return; + if (file === 'chai') return; + if (file === 'mocha') return; + copyFolderRecursiveSync(curSource, targetFolder, ignore); + } else { + copyFileSync(curSource, curTarget); + } + }); + } +} + +if (!fs.existsSync(rootDir + 'tmp')) { + fs.mkdirSync(rootDir + 'tmp'); +} + +function storeOriginalFiles() { + console.log('Store original files...'); + var dataDir = rootDir + 'tmp/' + appName + '-data/'; + + var f = fs.readFileSync(dataDir + 'objects.json'); + var objects = JSON.parse(f.toString()); + if (objects['system.adapter.admin.0'] && objects['system.adapter.admin.0'].common) { + objects['system.adapter.admin.0'].common.enabled = false; + } + if (objects['system.adapter.admin.1'] && objects['system.adapter.admin.1'].common) { + objects['system.adapter.admin.1'].common.enabled = false; + } + + fs.writeFileSync(dataDir + 'objects.json.original', JSON.stringify(objects)); + try { + f = fs.readFileSync(dataDir + 'states.json'); + fs.writeFileSync(dataDir + 'states.json.original', f); + } + catch (err) { + console.log('no states.json found - ignore'); + } +} + +function restoreOriginalFiles() { + console.log('restoreOriginalFiles...'); + var dataDir = rootDir + 'tmp/' + appName + '-data/'; + + var f = fs.readFileSync(dataDir + 'objects.json.original'); + fs.writeFileSync(dataDir + 'objects.json', f); + try { + f = fs.readFileSync(dataDir + 'states.json.original'); + fs.writeFileSync(dataDir + 'states.json', f); + } + catch (err) { + console.log('no states.json.original found - ignore'); + } + +} + +function checkIsAdapterInstalled(cb, counter, customName) { + customName = customName || pkg.name.split('.').pop(); + counter = counter || 0; + var dataDir = rootDir + 'tmp/' + appName + '-data/'; + console.log('checkIsAdapterInstalled...'); + + try { + var f = fs.readFileSync(dataDir + 'objects.json'); + var objects = JSON.parse(f.toString()); + if (objects['system.adapter.' + customName + '.0']) { + console.log('checkIsAdapterInstalled: ready!'); + setTimeout(function () { + if (cb) cb(); + }, 100); + return; + } else { + console.warn('checkIsAdapterInstalled: still not ready'); + } + } catch (err) { + + } + + if (counter > 20) { + console.error('checkIsAdapterInstalled: Cannot install!'); + if (cb) cb('Cannot install'); + } else { + console.log('checkIsAdapterInstalled: wait...'); + setTimeout(function() { + checkIsAdapterInstalled(cb, counter + 1); + }, 1000); + } +} + +function checkIsControllerInstalled(cb, counter) { + counter = counter || 0; + var dataDir = rootDir + 'tmp/' + appName + '-data/'; + + console.log('checkIsControllerInstalled...'); + try { + var f = fs.readFileSync(dataDir + 'objects.json'); + var objects = JSON.parse(f.toString()); + if (objects['system.adapter.admin.0']) { + console.log('checkIsControllerInstalled: installed!'); + setTimeout(function () { + if (cb) cb(); + }, 100); + return; + } + } catch (err) { + + } + + if (counter > 20) { + console.log('checkIsControllerInstalled: Cannot install!'); + if (cb) cb('Cannot install'); + } else { + console.log('checkIsControllerInstalled: wait...'); + setTimeout(function() { + checkIsControllerInstalled(cb, counter + 1); + }, 1000); + } +} + +function installAdapter(customName, cb) { + if (typeof customName === 'function') { + cb = customName; + customName = null; + } + customName = customName || pkg.name.split('.').pop(); + console.log('Install adapter...'); + var startFile = 'node_modules/' + appName + '.js-controller/' + appName + '.js'; + // make first install + if (debug) { + child_process.execSync('node ' + startFile + ' add ' + customName + ' --enabled false', { + cwd: rootDir + 'tmp', + stdio: [0, 1, 2] + }); + checkIsAdapterInstalled(function (error) { + if (error) console.error(error); + console.log('Adapter installed.'); + if (cb) cb(); + }); + } else { + // add controller + var _pid = child_process.fork(startFile, ['add', customName, '--enabled', 'false'], { + cwd: rootDir + 'tmp', + stdio: [0, 1, 2, 'ipc'] + }); + + waitForEnd(_pid, function () { + checkIsAdapterInstalled(function (error) { + if (error) console.error(error); + console.log('Adapter installed.'); + if (cb) cb(); + }); + }); + } +} + +function waitForEnd(_pid, cb) { + if (!_pid) { + cb(-1, -1); + return; + } + _pid.on('exit', function (code, signal) { + if (_pid) { + _pid = null; + cb(code, signal); + } + }); + _pid.on('close', function (code, signal) { + if (_pid) { + _pid = null; + cb(code, signal); + } + }); +} + +function installJsController(cb) { + console.log('installJsController...'); + if (!fs.existsSync(rootDir + 'tmp/node_modules/' + appName + '.js-controller') || + !fs.existsSync(rootDir + 'tmp/' + appName + '-data')) { + // try to detect appName.js-controller in node_modules/appName.js-controller + // travis CI installs js-controller into node_modules + if (fs.existsSync(rootDir + 'node_modules/' + appName + '.js-controller')) { + console.log('installJsController: no js-controller => copy it from "' + rootDir + 'node_modules/' + appName + '.js-controller"'); + // copy all + // stop controller + console.log('Stop controller if running...'); + var _pid; + if (debug) { + // start controller + _pid = child_process.exec('node ' + appName + '.js stop', { + cwd: rootDir + 'node_modules/' + appName + '.js-controller', + stdio: [0, 1, 2] + }); + } else { + _pid = child_process.fork(appName + '.js', ['stop'], { + cwd: rootDir + 'node_modules/' + appName + '.js-controller', + stdio: [0, 1, 2, 'ipc'] + }); + } + + waitForEnd(_pid, function () { + // copy all files into + if (!fs.existsSync(rootDir + 'tmp')) fs.mkdirSync(rootDir + 'tmp'); + if (!fs.existsSync(rootDir + 'tmp/node_modules')) fs.mkdirSync(rootDir + 'tmp/node_modules'); + + if (!fs.existsSync(rootDir + 'tmp/node_modules/' + appName + '.js-controller')){ + console.log('Copy js-controller...'); + copyFolderRecursiveSync(rootDir + 'node_modules/' + appName + '.js-controller', rootDir + 'tmp/node_modules/'); + } + + console.log('Setup js-controller...'); + var __pid; + if (debug) { + // start controller + _pid = child_process.exec('node ' + appName + '.js setup first --console', { + cwd: rootDir + 'tmp/node_modules/' + appName + '.js-controller', + stdio: [0, 1, 2] + }); + } else { + __pid = child_process.fork(appName + '.js', ['setup', 'first', '--console'], { + cwd: rootDir + 'tmp/node_modules/' + appName + '.js-controller', + stdio: [0, 1, 2, 'ipc'] + }); + } + waitForEnd(__pid, function () { + checkIsControllerInstalled(function () { + // change ports for object and state DBs + var config = require(rootDir + 'tmp/' + appName + '-data/' + appName + '.json'); + config.objects.port = 19001; + config.states.port = 19000; + fs.writeFileSync(rootDir + 'tmp/' + appName + '-data/' + appName + '.json', JSON.stringify(config, null, 2)); + console.log('Setup finished.'); + + copyAdapterToController(); + + installAdapter(function () { + storeOriginalFiles(); + if (cb) cb(true); + }); + }); + }); + }); + } else { + // check if port 9000 is free, else admin adapter will be added to running instance + var client = new require('net').Socket(); + client.connect(9000, '127.0.0.1', function() { + console.error('Cannot initiate fisrt run of test, because one instance of application is running on this PC. Stop it and repeat.'); + process.exit(0); + }); + + setTimeout(function () { + client.destroy(); + if (!fs.existsSync(rootDir + 'tmp/node_modules/' + appName + '.js-controller')) { + console.log('installJsController: no js-controller => install from git'); + + child_process.execSync('npm install https://github.com/' + appName + '/' + appName + '.js-controller/tarball/master --prefix ./ --production', { + cwd: rootDir + 'tmp/', + stdio: [0, 1, 2] + }); + } else { + console.log('Setup js-controller...'); + var __pid; + if (debug) { + // start controller + child_process.exec('node ' + appName + '.js setup first', { + cwd: rootDir + 'tmp/node_modules/' + appName + '.js-controller', + stdio: [0, 1, 2] + }); + } else { + child_process.fork(appName + '.js', ['setup', 'first'], { + cwd: rootDir + 'tmp/node_modules/' + appName + '.js-controller', + stdio: [0, 1, 2, 'ipc'] + }); + } + } + + // let npm install admin and run setup + checkIsControllerInstalled(function () { + var _pid; + + if (fs.existsSync(rootDir + 'node_modules/' + appName + '.js-controller/' + appName + '.js')) { + _pid = child_process.fork(appName + '.js', ['stop'], { + cwd: rootDir + 'node_modules/' + appName + '.js-controller', + stdio: [0, 1, 2, 'ipc'] + }); + } + + waitForEnd(_pid, function () { + // change ports for object and state DBs + var config = require(rootDir + 'tmp/' + appName + '-data/' + appName + '.json'); + config.objects.port = 19001; + config.states.port = 19000; + fs.writeFileSync(rootDir + 'tmp/' + appName + '-data/' + appName + '.json', JSON.stringify(config, null, 2)); + + copyAdapterToController(); + + installAdapter(function () { + storeOriginalFiles(); + if (cb) cb(true); + }); + }); + }); + }, 1000); + } + } else { + setTimeout(function () { + console.log('installJsController: js-controller installed'); + if (cb) cb(false); + }, 0); + } +} + +function copyAdapterToController() { + console.log('Copy adapter...'); + // Copy adapter to tmp/node_modules/appName.adapter + copyFolderRecursiveSync(rootDir, rootDir + 'tmp/node_modules/', ['.idea', 'test', 'tmp', '.git', appName + '.js-controller']); + console.log('Adapter copied.'); +} + +function clearControllerLog() { + var dirPath = rootDir + 'tmp/log'; + var files; + try { + if (fs.existsSync(dirPath)) { + console.log('Clear controller log...'); + files = fs.readdirSync(dirPath); + } else { + console.log('Create controller log directory...'); + files = []; + fs.mkdirSync(dirPath); + } + } catch(e) { + console.error('Cannot read "' + dirPath + '"'); + return; + } + if (files.length > 0) { + try { + for (var i = 0; i < files.length; i++) { + var filePath = dirPath + '/' + files[i]; + fs.unlinkSync(filePath); + } + console.log('Controller log cleared'); + } catch (err) { + console.error('cannot clear log: ' + err); + } + } +} + +function clearDB() { + var dirPath = rootDir + 'tmp/iobroker-data/sqlite'; + var files; + try { + if (fs.existsSync(dirPath)) { + console.log('Clear sqlite DB...'); + files = fs.readdirSync(dirPath); + } else { + console.log('Create controller log directory...'); + files = []; + fs.mkdirSync(dirPath); + } + } catch(e) { + console.error('Cannot read "' + dirPath + '"'); + return; + } + if (files.length > 0) { + try { + for (var i = 0; i < files.length; i++) { + var filePath = dirPath + '/' + files[i]; + fs.unlinkSync(filePath); + } + console.log('Clear sqlite DB'); + } catch (err) { + console.error('cannot clear DB: ' + err); + } + } +} + +function setupController(cb) { + installJsController(function (isInited) { + clearControllerLog(); + clearDB(); + + if (!isInited) { + restoreOriginalFiles(); + copyAdapterToController(); + } + // read system.config object + var dataDir = rootDir + 'tmp/' + appName + '-data/'; + + var objs; + try { + objs = fs.readFileSync(dataDir + 'objects.json'); + objs = JSON.parse(objs); + } + catch (e) { + console.log('ERROR reading/parsing system configuration. Ignore'); + objs = {'system.config': {}}; + } + if (!objs || !objs['system.config']) { + objs = {'system.config': {}}; + } + + if (cb) cb(objs['system.config']); + }); +} + +function startAdapter(objects, states, callback) { + if (adapterStarted) { + console.log('Adapter already started ...'); + if (callback) callback(objects, states); + return; + } + adapterStarted = true; + console.log('startAdapter...'); + if (fs.existsSync(rootDir + 'tmp/node_modules/' + pkg.name + '/' + pkg.main)) { + try { + if (debug) { + // start controller + pid = child_process.exec('node node_modules/' + pkg.name + '/' + pkg.main + ' --console silly', { + cwd: rootDir + 'tmp', + stdio: [0, 1, 2] + }); + } else { + // start controller + pid = child_process.fork('node_modules/' + pkg.name + '/' + pkg.main, ['--console', 'silly'], { + cwd: rootDir + 'tmp', + stdio: [0, 1, 2, 'ipc'] + }); + } + } catch (error) { + console.error(JSON.stringify(error)); + } + } else { + console.error('Cannot find: ' + rootDir + 'tmp/node_modules/' + pkg.name + '/' + pkg.main); + } + if (callback) callback(objects, states); +} + +function startController(isStartAdapter, onObjectChange, onStateChange, callback) { + if (typeof isStartAdapter === 'function') { + callback = onStateChange; + onStateChange = onObjectChange; + onObjectChange = isStartAdapter; + isStartAdapter = true; + } + + if (onStateChange === undefined) { + callback = onObjectChange; + onObjectChange = undefined; + } + + if (pid) { + console.error('Controller is already started!'); + } else { + console.log('startController...'); + adapterStarted = false; + var isObjectConnected; + var isStatesConnected; + + var Objects = require(rootDir + 'tmp/node_modules/' + appName + '.js-controller/lib/objects/objectsInMemServer'); + objects = new Objects({ + connection: { + "type" : "file", + "host" : "127.0.0.1", + "port" : 19001, + "user" : "", + "pass" : "", + "noFileCache": false, + "connectTimeout": 2000 + }, + logger: { + silly: function (msg) { + console.log(msg); + }, + debug: function (msg) { + console.log(msg); + }, + info: function (msg) { + console.log(msg); + }, + warn: function (msg) { + console.warn(msg); + }, + error: function (msg) { + console.error(msg); + } + }, + connected: function () { + isObjectConnected = true; + if (isStatesConnected) { + console.log('startController: started!'); + if (isStartAdapter) { + startAdapter(objects, states, callback); + } else { + if (callback) { + callback(objects, states); + callback = null; + } + } + } + }, + change: onObjectChange + }); + + // Just open in memory DB itself + var States = require(rootDir + 'tmp/node_modules/' + appName + '.js-controller/lib/states/statesInMemServer'); + states = new States({ + connection: { + type: 'file', + host: '127.0.0.1', + port: 19000, + options: { + auth_pass: null, + retry_max_delay: 15000 + } + }, + logger: { + silly: function (msg) { + console.log(msg); + }, + debug: function (msg) { + console.log(msg); + }, + info: function (msg) { + console.log(msg); + }, + warn: function (msg) { + console.log(msg); + }, + error: function (msg) { + console.log(msg); + } + }, + connected: function () { + isStatesConnected = true; + if (isObjectConnected) { + console.log('startController: started!!'); + if (isStartAdapter) { + startAdapter(objects, states, callback); + } else { + if (callback) { + callback(objects, states); + callback = null; + } + } + } + }, + change: onStateChange + }); + } +} + +function stopAdapter(cb) { + if (!pid) { + console.error('Controller is not running!'); + if (cb) { + setTimeout(function () { + cb(false); + }, 0); + } + } else { + adapterStarted = false; + pid.on('exit', function (code, signal) { + if (pid) { + console.log('child process terminated due to receipt of signal ' + signal); + if (cb) cb(); + pid = null; + } + }); + + pid.on('close', function (code, signal) { + if (pid) { + if (cb) cb(); + pid = null; + } + }); + + pid.kill('SIGTERM'); + } +} + +function _stopController() { + if (objects) { + objects.destroy(); + objects = null; + } + if (states) { + states.destroy(); + states = null; + } +} + +function stopController(cb) { + var timeout; + if (objects) { + console.log('Set system.adapter.' + pkg.name + '.0'); + objects.setObject('system.adapter.' + pkg.name + '.0', { + common:{ + enabled: false + } + }); + } + + stopAdapter(function () { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + + _stopController(); + + if (cb) { + cb(true); + cb = null; + } + }); + + timeout = setTimeout(function () { + timeout = null; + console.log('child process NOT terminated'); + + _stopController(); + + if (cb) { + cb(false); + cb = null; + } + pid = null; + }, 5000); +} + +// Setup the adapter +function setAdapterConfig(common, native, instance) { + var objects = JSON.parse(fs.readFileSync(rootDir + 'tmp/' + appName + '-data/objects.json').toString()); + var id = 'system.adapter.' + adapterName.split('.').pop() + '.' + (instance || 0); + if (common) objects[id].common = common; + if (native) objects[id].native = native; + fs.writeFileSync(rootDir + 'tmp/' + appName + '-data/objects.json', JSON.stringify(objects)); +} + +// Read config of the adapter +function getAdapterConfig(instance) { + var objects = JSON.parse(fs.readFileSync(rootDir + 'tmp/' + appName + '-data/objects.json').toString()); + var id = 'system.adapter.' + adapterName.split('.').pop() + '.' + (instance || 0); + return objects[id]; +} + +if (typeof module !== undefined && module.parent) { + module.exports.getAdapterConfig = getAdapterConfig; + module.exports.setAdapterConfig = setAdapterConfig; + module.exports.startController = startController; + module.exports.stopController = stopController; + module.exports.setupController = setupController; + module.exports.stopAdapter = stopAdapter; + module.exports.startAdapter = startAdapter; + module.exports.installAdapter = installAdapter; + module.exports.appName = appName; + module.exports.adapterName = adapterName; + module.exports.adapterStarted = adapterStarted; +} diff --git a/test/testAdapter.js b/test/testAdapter.js new file mode 100644 index 0000000..cc41541 --- /dev/null +++ b/test/testAdapter.js @@ -0,0 +1,419 @@ +/* jshint -W097 */// jshint strict:false +/*jslint node: true */ +/*jshint expr: true*/ +var expect = require('chai').expect; +var setup = require(__dirname + '/lib/setup'); + +var objects = null; +var states = null; +var onStateChanged = null; +//var onObjectChanged = null; +var sendToID = 1; + +var adapterShortName = setup.adapterName.substring(setup.adapterName.indexOf('.')+1); + +var now; + +function checkConnectionOfAdapter(cb, counter) { + counter = counter || 0; + console.log('Try check #' + counter); + if (counter > 30) { + if (cb) cb('Cannot check connection'); + return; + } + + console.log('Checking alive key for key : ' + adapterShortName); + states.getState('system.adapter.' + adapterShortName + '.0.alive', function (err, state) { + if (err) console.error(err); + if (state && state.val) { + if (cb) cb(); + } else { + setTimeout(function () { + checkConnectionOfAdapter(cb, counter + 1); + }, 1000); + } + }); +} +/* +function checkValueOfState(id, value, cb, counter) { + counter = counter || 0; + if (counter > 20) { + if (cb) cb('Cannot check value Of State ' + id); + return; + } + + states.getState(id, function (err, state) { + if (err) console.error(err); + if (value === null && !state) { + if (cb) cb(); + } else + if (state && (value === undefined || state.val === value)) { + if (cb) cb(); + } else { + setTimeout(function () { + checkValueOfState(id, value, cb, counter + 1); + }, 500); + } + }); +} +*/ +function sendTo(target, command, message, callback) { + onStateChanged = function (id, state) { + if (id === 'messagebox.system.adapter.test.0') { + callback(state.message); + } + }; + + states.pushMessage('system.adapter.' + target, { + command: command, + message: message, + from: 'system.adapter.test.0', + callback: { + message: message, + id: sendToID++, + ack: false, + time: (new Date()).getTime() + } + }); +} + +describe('Test ' + adapterShortName + ' adapter', function() { + before('Test ' + adapterShortName + ' adapter: Start js-controller', function (_done) { + this.timeout(600000); // because of first install from npm + + setup.setupController(function () { + var config = setup.getAdapterConfig(); + // enable adapter + config.common.enabled = true; + config.common.loglevel = 'debug'; + + //config.native.dbtype = 'sqlite'; + + setup.setAdapterConfig(config.common, config.native); + + setup.startController(true, function(id, obj) {}, function (id, state) { + if (onStateChanged) onStateChanged(id, state); + }, + function (_objects, _states) { + objects = _objects; + states = _states; + objects.setObject('history.0.testValue', { + common: { + type: 'number', + role: 'state', + custom: { + "history.0": { + enabled: true, + changesOnly: true, + debounce: 0, + retention: 31536000, + maxLength: 3, + changesMinDelta: 0.5 + } + } + }, + type: 'state' + }, _done); + }); + }); + }); + + it('Test ' + adapterShortName + ' adapter: Check if adapter started', function (done) { + this.timeout(60000); + checkConnectionOfAdapter(function (res) { + if (res) console.log(res); + expect(res).not.to.be.equal('Cannot check connection'); + objects.setObject('system.adapter.test.0', { + common: { + + }, + type: 'instance' + }, + function () { + states.subscribeMessage('system.adapter.test.0'); + objects.setObject('history.0.testValue2', { + common: { + type: 'number', + role: 'state' + }, + type: 'state' + }, + function () { + sendTo('history.0', 'enableHistory', { + id: 'history.0.testValue2', + options: { + changesOnly: true, + debounce: 0, + retention: 31536000, + maxLength: 3, + changesMinDelta: 0.5, + aliasId: 'history.0.testValue2-alias' + } + }, function (result) { + expect(result.error).to.be.undefined; + expect(result.success).to.be.true; + // wait till adapter receives the new settings + setTimeout(function () { + done(); + }, 2000); + }); + }); + }); + }); + }); + it('Test ' + adapterShortName + ': Check Enabled Points after Enable', function (done) { + this.timeout(5000); + + sendTo('history.0', 'getEnabledDPs', {}, function (result) { + console.log(JSON.stringify(result)); + expect(Object.keys(result).length).to.be.equal(2); + expect(result['history.0.testValue'].enabled).to.be.true; + done(); + }); + }); + it('Test ' + adapterShortName + ': Write values into DB', function (done) { + this.timeout(25000); + now = new Date().getTime(); + + states.setState('history.0.testValue', {val: 1, ts: now + 1000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue', {val: 2, ts: now + 10000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue', {val: 2, ts: now + 13000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue', {val: 2, ts: now + 15000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue', {val: 2.2, ts: now + 16000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue', {val: 2.5, ts: now + 17000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue', {val: 3, ts: now + 19000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue2', {val: 1, ts: now + 12000}, function (err) { + if (err) { + console.log(err); + } + setTimeout(function () { + states.setState('history.0.testValue2', {val: 3, ts: now + 19000}, function (err) { + if (err) { + console.log(err); + } + done(); + }); + }, 100); + }); + }, 100); + }); + }, 100); + }); + }, 100); + }); + }, 100); + }); + }, 100); + }); + }, 100); + }); + }, 100); + }); + }); + it('Test ' + adapterShortName + ': Read values from DB using GetHistory', function (done) { + this.timeout(25000); + + sendTo('history.0', 'getHistory', { + id: 'history.0.testValue', + options: { + start: now, + end: now + 30000, + count: 50, + aggregate: 'none' + } + }, function (result) { + console.log(JSON.stringify(result.result, null, 2)); + expect(result.result.length).to.be.at.least(4); + var found = 0; + for (var i = 0; i < result.result.length; i++) { + if (result.result[i].val >= 1 && result.result[i].val <= 3) found ++; + } + expect(found).to.be.equal(5); // additionally null value by start of adapter. + + sendTo('history.0', 'getHistory', { + id: 'history.0.testValue', + options: { + start: now + 15000, + end: now + 30000, + count: 2, + aggregate: 'none' + } + }, function (result) { + console.log(JSON.stringify(result.result, null, 2)); + expect(result.result.length).to.be.equal(2); + var found = 0; + for (var i = 0; i < result.result.length; i++) { + if (result.result[i].val >= 2 && result.result[i].val < 3) found ++; + } + expect(found).to.be.equal(2); + done(); + }); + }); + }); + + it('Test ' + adapterShortName + ': Read average from DB using GetHistory', function (done) { + this.timeout(25000); + + sendTo('history.0', 'getHistory', { + id: 'history.0.testValue', + options: { + start: now + 100, + end: now + 30001, + count: 1, + aggregate: 'average', + ignoreNull: true + } + }, function (result) { + console.log(JSON.stringify(result.result, null, 2)); + expect(result.result.length).to.be.equal(3); + expect(result.result[1].val).to.be.equal(2.14); + expect(result.result[2].val).to.be.equal(2.14); + done(); + }); + }); + + it('Test ' + adapterShortName + ': Read values from DB using GetHistory for aliased testValue2', function (done) { + this.timeout(25000); + + sendTo('history.0', 'getHistory', { + id: 'history.0.testValue2', + options: { + start: now, + end: now + 30000, + count: 50, + aggregate: 'none' + } + }, function (result) { + console.log(JSON.stringify(result.result, null, 2)); + expect(result.result.length).to.be.equal(2); + + sendTo('history.0', 'getHistory', { + id: 'history.0.testValue2-alias', + options: { + start: now, + end: now + 30000, + count: 50, + aggregate: 'none' + } + }, function (result2) { + console.log(JSON.stringify(result2.result, null, 2)); + expect(result2.result.length).to.be.equal(2); + for (var i = 0; i < result2.result.length; i++) { + expect(result2.result[i].val).to.be.equal(result.result[i].val); + } + + done(); + }); + }); + }); + + it('Test ' + adapterShortName + ': Remove Alias-ID', function (done) { + this.timeout(5000); + + sendTo('history.0', 'enableHistory', { + id: 'history.0.testValue2', + options: { + aliasId: '' + } + }, function (result) { + expect(result.error).to.be.undefined; + expect(result.success).to.be.true; + // wait till adapter receives the new settings + setTimeout(function () { + done(); + }, 2000); + }); + }); + it('Test ' + adapterShortName + ': Add Alias-ID again', function (done) { + this.timeout(5000); + + sendTo('history.0', 'enableHistory', { + id: 'history.0.testValue2', + options: { + aliasId: 'this.is.a.test-value' + } + }, function (result) { + expect(result.error).to.be.undefined; + expect(result.success).to.be.true; + // wait till adapter receives the new settings + setTimeout(function () { + done(); + }, 2000); + }); + }); + it('Test ' + adapterShortName + ': Change Alias-ID', function (done) { + this.timeout(5000); + + sendTo('history.0', 'enableHistory', { + id: 'history.0.testValue2', + options: { + aliasId: 'this.is.another.test-value' + } + }, function (result) { + expect(result.error).to.be.undefined; + expect(result.success).to.be.true; + // wait till adapter receives the new settings + setTimeout(function () { + done(); + }, 2000); + }); + }); + + it('Test ' + adapterShortName + ': Disable Datapoint again', function (done) { + this.timeout(5000); + + sendTo('history.0', 'disableHistory', { + id: 'history.0.testValue' + }, function (result) { + expect(result.error).to.be.undefined; + expect(result.success).to.be.true; + done(); + }); + }); + it('Test ' + adapterShortName + ': Check Enabled Points after Disable', function (done) { + this.timeout(5000); + + sendTo('history.0', 'getEnabledDPs', {}, function (result) { + console.log(JSON.stringify(result)); + expect(Object.keys(result).length).to.be.equal(1); + done(); + }); + }); + + after('Test ' + adapterShortName + ' adapter: Stop js-controller', function (done) { + this.timeout(10000); + + setup.stopController(function (normalTerminated) { + console.log('Adapter normal terminated: ' + normalTerminated); + done(); + }); + }); +}); diff --git a/test/testPackageFiles.js b/test/testPackageFiles.js new file mode 100644 index 0000000..c600a60 --- /dev/null +++ b/test/testPackageFiles.js @@ -0,0 +1,91 @@ +/* jshint -W097 */ +/* jshint strict:false */ +/* jslint node: true */ +/* jshint expr: true */ +var expect = require('chai').expect; +var fs = require('fs'); + +describe('Test package.json and io-package.json', function() { + it('Test package files', function (done) { + console.log(); + + var fileContentIOPackage = fs.readFileSync(__dirname + '/../io-package.json', 'utf8'); + var ioPackage = JSON.parse(fileContentIOPackage); + + var fileContentNPMPackage = fs.readFileSync(__dirname + '/../package.json', 'utf8'); + var npmPackage = JSON.parse(fileContentNPMPackage); + + expect(ioPackage).to.be.an('object'); + expect(npmPackage).to.be.an('object'); + + expect(ioPackage.common.version, 'ERROR: Version number in io-package.json needs to exist').to.exist; + expect(npmPackage.version, 'ERROR: Version number in package.json needs to exist').to.exist; + + expect(ioPackage.common.version, 'ERROR: Version numbers in package.json and io-package.json needs to match').to.be.equal(npmPackage.version); + + if (!ioPackage.common.news || !ioPackage.common.news[ioPackage.common.version]) { + console.log('WARNING: No news entry for current version exists in io-package.json, no rollback in Admin possible!'); + console.log(); + } + + expect(npmPackage.author, 'ERROR: Author in package.json needs to exist').to.exist; + expect(ioPackage.common.authors, 'ERROR: Authors in io-package.json needs to exist').to.exist; + + if (ioPackage.common.name.indexOf('template') !== 0) { + if (Array.isArray(ioPackage.common.authors)) { + expect(ioPackage.common.authors.length, 'ERROR: Author in io-package.json needs to be set').to.not.be.equal(0); + if (ioPackage.common.authors.length === 1) { + expect(ioPackage.common.authors[0], 'ERROR: Author in io-package.json needs to be a real name').to.not.be.equal('my Name '); + } + } + else { + expect(ioPackage.common.authors, 'ERROR: Author in io-package.json needs to be a real name').to.not.be.equal('my Name '); + } + } + else { + console.log('WARNING: Testing for set authors field in io-package skipped because template adapter'); + console.log(); + } + expect(fs.existsSync(__dirname + '/../README.md'), 'ERROR: README.md needs to exist! Please create one with description, detail information and changelog. English is mandatory.').to.be.true; + if (!ioPackage.common.titleLang || typeof ioPackage.common.titleLang !== 'object') { + console.log('WARNING: titleLang is not existing in io-package.json. Please add'); + console.log(); + } + if ( + ioPackage.common.title.indexOf('iobroker') !== -1 || + ioPackage.common.title.indexOf('ioBroker') !== -1 || + ioPackage.common.title.indexOf('adapter') !== -1 || + ioPackage.common.title.indexOf('Adapter') !== -1 + ) { + console.log('WARNING: title contains Adapter or ioBroker. It is clear anyway, that it is adapter for ioBroker.'); + console.log(); + } + + if (ioPackage.common.name.indexOf('vis-') !== 0) { + if (!ioPackage.common.materialize || !fs.existsSync(__dirname + '/../admin/index_m.html') || !fs.existsSync(__dirname + '/../gulpfile.js')) { + console.log('WARNING: Admin3 support is missing! Please add it'); + console.log(); + } + if (ioPackage.common.materialize) { + expect(fs.existsSync(__dirname + '/../admin/index_m.html'), 'Admin3 support is enabled in io-package.json, but index_m.html is missing!').to.be.true; + } + } + + var licenseFileExists = fs.existsSync(__dirname + '/../LICENSE'); + var fileContentReadme = fs.readFileSync(__dirname + '/../README.md', 'utf8'); + if (fileContentReadme.indexOf('## Changelog') === -1) { + console.log('Warning: The README.md should have a section ## Changelog'); + console.log(); + } + expect((licenseFileExists || fileContentReadme.indexOf('## License') !== -1), 'A LICENSE must exist as LICENSE file or as part of the README.md').to.be.true; + if (!licenseFileExists) { + console.log('Warning: The License should also exist as LICENSE file'); + console.log(); + } + if (fileContentReadme.indexOf('## License') === -1) { + console.log('Warning: The README.md should also have a section ## License to be shown in Admin3'); + console.log(); + } + done(); + }); +});