- Messages used to get ignored if they were over 1 second old. This caused issues with messages being ignored on servers that took longer than a second to respond (*cough* matrix.org *cough*). Now it accepts all new messages. Moving forward any messages that are sent before Node-RED starts or gets deployed will be ignored (so that old messages don't pop up since the cache clears every deploy/node-red restart so old messages look like new ones).

- When the matrix-server node would shut down (from a deployment) it wasn't properly shutting the old instance of the client down. This would cause messages to start duplicating X times (X being however many times you deployed since you last started Node-RED). This is now fixed.
- Upgraded to matrix-js-sdk 15.3.0
- Message processing is now done on the matrix-server node. Before if you had multiple matrix-receive nodes it would output one line per matrix-receive node into the log. Now it only outputs from the matrix-server node. Note that all messages get logged that are sent to rooms the bot is in whereas before it would only log message in rooms the matrix-receive node was in.
- Fixed Shared secret registration failing if user_type was defined.

README.md

@@ -3,6 +3,8 @@ Matrix chat server client for [Node-RED](https://nodered.org/)
 
 ***Currently we are in beta. We ask that you open any issues you have on our repository to help us reach a stable well tested version. Things may change & break before our first release so check changelog before updating.***
 
+If you need help with this feel free to join our public matrix room at [#node-red-contrib-matrix-chat:skylar.tech](https://app.element.io/#/room/#node-red-contrib-matrix-chat:skylar.tech)
+
 ### Features
 
 The following is supported from this package:
@@ -66,4 +68,4 @@ You will need a user to use this module. Luckily this module comes with a node t
 ### Contributing
 All contributions are welcome! If you do add a feature please do a pull request so that everyone benefits :)
 
-Sharing is caring!
+Sharing is caring!

package.json

@@ -1,14 +1,15 @@
 {
   "name": "node-red-contrib-matrix-chat",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Matrix chat server client for Node-RED",
   "dependencies": {
     "fs-extra": "^9.1.0",
     "got": "^11.8.2",
     "isomorphic-webcrypto": "^2.3.8",
-    "matrix-js-sdk": "^12.2.0",
+    "matrix-js-sdk": "^15.3.0",
     "node-localstorage": "^2.2.1",
-    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
+    "process": "^0.11.10",
     "utf8": "^3.0.0"
   },
   "node-red": {

src/matrix-create-room.html

@@ -39,7 +39,7 @@
         <dt class="optional">msg.payload
             <span class="property-type">string | object | null</span>
         </dt>
-        <dd> a list of options to pass to the /createRoom API. <a href="https://matrix.org/docs/spec/client_server/r0.4.0#id266">Click here</a> for information about what can be passed. If this is a string it just sets the room name and ignores other options.</dd>
+        <dd> a list of options to pass to the /createRoom API. <a href="https://matrix.org/docs/spec/client_server/latest#post-matrix-client-r0-createroom">Click here</a> for information about what can be passed. If this is a string it just sets the room name and ignores other options.</dd>
     </dl>
 
     <h3>Outputs</h3>

src/matrix-invite-room.html

@@ -8,7 +8,8 @@
         outputs: 2,
         defaults: {
             name: { value: null },
-            server: { value: "", type: "matrix-server-config" }
+            server: { value: "", type: "matrix-server-config" },
+            roomId: { value: null },
         },
         label: function() {
             return this.name || "Room Invite";
@@ -27,6 +28,11 @@
         <label for="node-input-server"><i class="fa fa-user"></i> Matrix Server Config</label>
         <input type="text" id="node-input-server">
     </div>
+
+    <div class="form-row">
+        <label for="node-input-roomId"><i class="fa fa-user"></i> Room ID</label>
+        <input type="text" id="node-input-roomId" placeholder="msg.topic">
+    </div>
 </script>
 
 <script type="text/html" data-help-name="matrix-invite-room">

src/matrix-invite-room.js

@@ -6,6 +6,7 @@ module.exports = function(RED) {
 
         this.name = n.name;
         this.server = RED.nodes.getNode(n.server);
+        this.roomId = n.roomId;
 
         if(!this.server) {
             node.error('Server must be configured on the node.');
@@ -47,7 +48,7 @@ module.exports = function(RED) {
 
             msg.topic = node.roomId || msg.topic;
             if(!msg.topic) {
-                node.error("msg.topic must be defined or configured on the node.");
+                node.error("room must be defined in either msg.topic or in node config");
                 return;
             }
 

src/matrix-receive.js

@@ -22,7 +22,7 @@ module.exports = function(RED) {
             return;
         }
 
-        node.server.on("disconnected", function() {
+        node.server.on("disconnected", function(){
             node.status({ fill: "red", shape: "ring", text: "disconnected" });
         });
 
@@ -30,43 +30,12 @@ module.exports = function(RED) {
             node.status({ fill: "green", shape: "ring", text: "connected" });
         });
 
-        node.server.on("Room.timeline", async function(event, room, toStartOfTimeline, data) {
-            if (toStartOfTimeline) {
-                return; // ignore paginated results
-            }
-            if (!event.getSender() || event.getSender() === node.server.userId) {
-                return; // ignore our own messages
-            }
-            if (!event.getUnsigned() || event.getUnsigned().age > 1000) {
-                return; // ignore old messages
-            }
-
+        node.server.on("Room.timeline", async function(event, room, toStartOfTimeline, removed, data, msg) {
             // if node has a room ID set we only listen on that room
             if(node.roomIds.length && node.roomIds.indexOf(room.roomId) === -1) {
                 return;
             }
 
-            try {
-                await node.server.matrixClient.decryptEventIfNeeded(event);
-            } catch (error) {
-                node.error(error);
-                return;
-            }
-
-            let msg = {
-                encrypted : event.isEncrypted(),
-                redacted  : event.isRedacted(),
-                content   : event.getContent(),
-                type      : (event.getContent()['msgtype'] || event.getType()) || null,
-                payload   : (event.getContent()['body'] || event.getContent()) || null,
-                userId    : event.getSender(),
-                topic     : event.getRoomId(),
-                eventId   : event.getId(),
-                event     : event,
-            };
-
-            node.log("Received" + (msg.encrypted ? ' encrypted' : '') +" timeline event [" + msg.type + "]: (" + room.name + ") " + event.getSender() + " :: " + msg.content.body);
-
             switch(msg.type) {
                 case 'm.emote':
                     if(!node.acceptEmotes) return;

src/matrix-send-file.js

@@ -80,7 +80,7 @@ module.exports = function(RED) {
                         .sendMessage(msg.topic, content)
                             .then(function(e) {
                                 node.log("File message sent: " + e);
-                                msg.eventId = e.eventId;
+                                msg.eventId = e.event_id;
                                 node.send([msg, null]);
                             })
                             .catch(function(e){

src/matrix-send-image.html

@@ -34,7 +34,7 @@
     </div>
     <div class="form-row">
         <label for="node-input-contentType"><i class="fa fa-user"></i> Content-Type</label>
-        <input type="text" id="node-input-contentType" placeholder="application/png">
+        <input type="text" id="node-input-contentType" placeholder="msg.contentType">
     </div>
     <div class="form-tips">
         Must be a valid <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types" target="_blank">MIME Type</a> (ex: image/png) or left empty

src/matrix-send-message.js

@@ -89,6 +89,7 @@ module.exports = function(RED) {
             if(!node.server.isConnected()) {
                 node.error("Matrix server connection is currently closed");
                 node.send([null, msg]);
+                return;
             }
 
             msg.topic = node.roomId || msg.topic;
@@ -109,13 +110,16 @@ module.exports = function(RED) {
 
             if(msgFormat === 'html') {
                 content.format = "org.matrix.custom.html";
-                content.formatted_body = msg.formatted_payload || msg.payload;
+                content.formatted_body =
+                    (typeof msg.formatted_payload !== 'undefined' && msg.formatted_payload)
+                        ? msg.formatted_payload.toString()
+                        : msg.payload.toString();
             }
 
             node.server.matrixClient.sendMessage(msg.topic, content)
                 .then(function(e) {
                     node.log("Message sent: " + msg.payload);
-                    msg.eventId = e.eventId;
+                    msg.eventId = e.event_id;
                     node.send([msg, null]);
                 })
                 .catch(function(e){

src/matrix-server-config.html

@@ -54,7 +54,7 @@
             id="node-config-input-autoAcceptRoomInvites"
             style="width: auto; margin-left: 125px; vertical-align: top"
         />
-        <label for="node-config-input-autoAcceptRoomInvites" style="width: auto">
+        <label for="node-config-input-autoAcceptRoomInvites" style="width: auto;max-width:50%;">
             Auto join invited rooms
         </label>
     </div>
@@ -64,13 +64,10 @@
                 id="node-config-input-enableE2ee"
                 style="width: auto; margin-left: 125px; vertical-align: top"
         />
-        <label for="node-config-input-enableE2ee" style="width: auto">
-            Enable end-to-end encryption (requires DeviceID)
+        <label for="node-config-input-enableE2ee" style="width: auto;max-width:50%;">
+            Enable end-to-end encryption (requires a Device ID to be set.)
         </label>
     </div>
-    <div class="form-tips" style="margin-bottom: 12px;">
-        E2ee requires a Device ID to be set.
-    </div>
 
     <div class="form-row">
         <input
@@ -82,13 +79,23 @@
             Global access to Matrix Client
         </label>
         <div class="form-tips" style="margin-bottom: 12px;">
-            If enabled this allows you to access the matrix client directly with a Function node. This way you can do whatever you want with the client.<br />
-            Example: <code>let client = global.get("matrixClient['@bot:example.com']")</code>
+            If enabled this allows you to access the matrix client directly with a Function node. This way you can do <a href="https://github.com/Skylar-Tech/node-red-contrib-matrix-chat/tree/master/examples#use-function-node-to-run-any-command" target="_blank">whatever you want</a> with the client. Example:<br>
+            <code style="white-space: normal;">let client = global.get("matrixClient['@bot:example.com']");</code>
         </div>
     </div>
 </script>
 
 <script type="text/html" data-help-name="matrix-server-config">
     <h3>Details</h3>
-    <p>Matrix client configuration node</p>
+    <p>Matrix client connection configuration</p>
+
+    <h3>Setting up an account</h3>
+    <div>
+        <p>
+            You need an account for your client to use. If you are going to be using End-to-End Encryption you should generate the bot and only use it within Node-RED otherwise if you have other clients connected on the same user it could cause problems with e2ee (key sharing is currently not supported).
+        </p>
+        <p>If you have access to the server directly you can use Shared Secret Registration as described <a href="https://github.com/Skylar-Tech/node-red-contrib-matrix-chat/tree/master/examples#create-user-with-shared-secret-registration" target="_blank" style="text-decoration: underline;">here</a>.</p>
+        <p>If this is a server you do not administrate/have access to follow these instructions:</p>
+        <ol><li>In a private/incognito browser window, open Element.</li><li>Log in to the account you want to get the access token for, such as the bot's account. <strong>Do not setup key storage</strong>.</li><li>Click on the bot's name in the top left corner then "Settings".</li><li>(Optional) Set your bot's display name and avatar.</li><li>Click the "Help &amp; About" tab (left side of the dialog).</li><li>Scroll to the bottom and click the <code>&lt;click to reveal&gt;</code> part of <code>Access Token: &lt;click to reveal&gt;</code>.</li><li>Copy your access token to a safe place, like the bot's configuration file.</li><li><strong>Do not log out.</strong> Instead, just close the window. If you used a private browsing session, you should be able to still use Element for your own account. Logging out deletes the access token from the server, making the bot unable to use it.</li></ol>
+    </div>
 </script>

src/matrix-server-config.js

@@ -35,6 +35,7 @@ module.exports = function(RED) {
         this.enableE2ee = n.enableE2ee || false;
         this.e2ee = (this.enableE2ee && this.deviceId);
         this.globalAccess = n.global;
+        this.initializedAt = new Date();
 
         if(!this.credentials.accessToken) {
             node.log("Matrix connection failed: missing access token.");
@@ -43,9 +44,13 @@ module.exports = function(RED) {
         } else if(!this.userId) {
             node.log("Matrix connection failed: missing user ID.");
         } else {
-            node.setConnected = function(connected) {
+            node.setConnected = function(connected, cb) {
                 if (node.connected !== connected) {
                     node.connected = connected;
+                    if(typeof cb === 'function') {
+                        cb(connected);
+                    }
+
                     if (connected) {
                         node.log("Matrix server connection ready.");
                         node.emit("connected");
@@ -81,7 +86,6 @@ module.exports = function(RED) {
 
             node.on('close', function(done) {
                 if(node.matrixClient) {
-                    node.matrixClient.close();
                     node.matrixClient.stopClient();
                     node.setConnected(false);
                 }
@@ -93,8 +97,41 @@ module.exports = function(RED) {
                 return node.connected;
             };
 
-            node.matrixClient.on("Room.timeline", async function(event, room, toStartOfTimeline, data) {
-                node.emit("Room.timeline", event, room, toStartOfTimeline, data);
+            node.matrixClient.on("Room.timeline", async function(event, room, toStartOfTimeline, removed, data) {
+                if (toStartOfTimeline) {
+                    return; // ignore paginated results
+                }
+                if (!event.getSender() || event.getSender() === node.userId) {
+                    return; // ignore our own messages
+                }
+                if (!data || !data.liveEvent) {
+                    return; // ignore old message (we only want live events)
+                }
+                if(node.initializedAt > event.getDate()) {
+                    return; // skip events that occurred before our client initialized
+                }
+
+                try {
+                    await node.matrixClient.decryptEventIfNeeded(event);
+                } catch (error) {
+                    node.error(error);
+                    return;
+                }
+
+                let msg = {
+                    encrypted : event.isEncrypted(),
+                    redacted  : event.isRedacted(),
+                    content   : event.getContent(),
+                    type      : (event.getContent()['msgtype'] || event.getType()) || null,
+                    payload   : (event.getContent()['body'] || event.getContent()) || null,
+                    userId    : event.getSender(),
+                    topic     : event.getRoomId(),
+                    eventId   : event.getId(),
+                    event     : event,
+                };
+
+                node.log("Received" + (msg.encrypted ? ' encrypted' : '') +" timeline event [" + msg.type + "]: (" + room.name + ") " + event.getSender() + " :: " + msg.content.body + (data.liveEvent ? ' [LIVE]' : '') + (toStartOfTimeline ? ' [PAGINATED]' : ''));
+                node.emit("Room.timeline", event, room, toStartOfTimeline, removed, data, msg);
             });
 
             // node.matrixClient.on("RoomMember.typing", async function(event, member) {
@@ -129,29 +166,70 @@ module.exports = function(RED) {
             });
 
             node.matrixClient.on("sync", async function(state, prevState, data) {
-                switch (state) {
-                    case "ERROR":
+                node.debug("SYNC [STATE=" + state + "] [PREVSTATE=" + prevState + "]");
+                if(prevState === null && state === "PREPARED" ) {
+                    // Occurs when the initial sync is completed first time.
+                    // This involves setting up filters and obtaining push rules.
+                    node.setConnected(true, function(){
+                        node.log("Matrix client connected");
+                    });
+                } else if(prevState === null && state === "ERROR") {
+                    // Occurs when the initial sync failed first time.
+                    node.setConnected(false, function(){
+                        node.error("Failed to connect to Matrix server");
+                    });
+                } else if(prevState === "ERROR" && state === "PREPARED") {
+                    // Occurs when the initial sync succeeds
+                    // after previously failing.
+                    node.setConnected(true, function(){
+                        node.log("Matrix client connected");
+                    });
+                } else if(prevState === "PREPARED" && state === "SYNCING") {
+                    // Occurs immediately after transitioning to PREPARED.
+                    // Starts listening for live updates rather than catching up.
+                    node.setConnected(true, function(){
+                        node.log("Matrix client connected");
+                    });
+                } else if(prevState === "SYNCING" && state === "RECONNECTING") {
+                    // Occurs when the live update fails.
+                    node.setConnected(false, function(){
                         node.error("Connection to Matrix server lost");
-                        node.setConnected(false);
-                        break;
-
-                    case "RECONNECTING":
-                    case "STOPPED":
-                        node.setConnected(false);
-                        break;
-
-                    case "SYNCING":
-                        break;
-
-                    case "PREPARED":
-                        node.setConnected(true);
-                        break;
-
-                    // case "PREPARED":
-                    //     // the client instance is ready to be queried.
-                    //     node.log("Matrix server connection ready.");
-                    //     node.setConnected(true);
-                    //     break;
+                    });
+                } else if(prevState === "RECONNECTING" && state === "RECONNECTING") {
+                    // Can occur if the update calls continue to fail,
+                    // but the keepalive calls (to /versions) succeed.
+                    node.setConnected(false, function(){
+                        node.error("Connection to Matrix server lost");
+                    });
+                } else if(prevState === "RECONNECTING" && state === "ERROR") {
+                    // Occurs when the keepalive call also fails
+                    node.setConnected(false, function(){
+                        node.error("Connection to Matrix server lost");
+                    });
+                } else if(prevState === "ERROR" && state === "SYNCING") {
+                    // Occurs when the client has performed a
+                    // live update after having previously failed.
+                    node.setConnected(true, function(){
+                        node.log("Matrix client connected");
+                    });
+                } else if(prevState === "ERROR" && state === "ERROR") {
+                    // Occurs when the client has failed to
+                    // keepalive for a second time or more.
+                    node.setConnected(false, function(){
+                        node.error("Connection to Matrix server lost");
+                    });
+                } else if(prevState === "SYNCING" && state === "SYNCING") {
+                    // Occurs when the client has performed a live update.
+                    // This is called <i>after</i> processing.
+                    node.setConnected(true, function(){
+                        node.log("Matrix client connected");
+                    });
+                } else if(state === "STOPPED") {
+                    // Occurs once the client has stopped syncing or
+                    // trying to sync after stopClient has been called.
+                    node.setConnected(false, function(){
+                        node.error("Connection to Matrix server lost");
+                    });
                 }
             });
 
@@ -178,7 +256,9 @@ module.exports = function(RED) {
                         node.matrixClient.setGlobalErrorOnUnknownDevices(false);
                     }
                     node.log("Connecting to Matrix server...");
-                    await node.matrixClient.startClient({ initialSyncLimit: 8 });
+                    await node.matrixClient.startClient({
+                        initialSyncLimit: 8
+                    });
                 } catch(error){
                     node.error(error);
                 }

src/matrix-synapse-join-room.js

@@ -64,7 +64,7 @@ module.exports = function(RED) {
                     'POST',
                     node.encodeUri(
                         "/_synapse/admin/v1/join/$room_id_or_alias",
-                        { $room_id_or_alias: roomId },
+                        { $room_id_or_alias: msg.topic },
                     ),
                     undefined,
                     { "user_id": msg.userId },

src/matrix-synapse-register.html

@@ -78,7 +78,10 @@
         <dt class="optional">msg.payload.user_type
             <span class="property-type">string | null</span>
         </dt>
-        <dd> Set the user type. Leave this to null if you don't know what it is for. Check <a href="https://github.com/matrix-org/synapse/blob/master/synapse/api/constants.py">here</a> and look for <code>class UserTypes</code> to figure out what is valid.</dd>
+        <dd>
+            Set the user type. Leave this to null if you don't know what it is for. Check <a href="https://github.com/matrix-org/synapse/blob/master/synapse/api/constants.py">here</a> and look for <code>class UserTypes</code> to figure out what is valid.<br>
+            NOTE: Testing shows that "bot" and "support" do not work for this. Generate a normal user to use as a bot.
+        </dd>
     </dl>
 
     <h3>Outputs</h3>

src/matrix-synapse-register.js

@@ -62,7 +62,13 @@ module.exports = function(RED) {
                     .update(utf8.encode(msg.payload.password))
                     .update("\x00")
                     .update(msg.payload.admin ? "admin" : "notadmin")
-                    .digest('hex');
+
+                if(msg.payload.user_type || null) {
+                    hmac.update("\x00")
+                        .update(msg.payload.user_type);
+                }
+
+                hmac = hmac.digest('hex');
 
                 try {
                     response = await got.post(this.server + '/_synapse/admin/v1/register', {

src/matrix-synapse-users.js

@@ -39,7 +39,7 @@ module.exports = function(RED) {
             };
 
             if(msg.guests) {
-                queryParams['guests'] = msg.guests;
+                queryParams['guests'] = msg.guests ? 'true' : 'false';
             }
 
             if(msg.order_by) {