websocket reconnect support, docs

README.md

@@ -3,8 +3,10 @@
 A WebSocket-compatible kill & fleet tracker for EVE Online, built [on Sails.js](https://sailsjs.com/). Part of the magic behind the [Gloss](https://github.com/dougestey/gloss) project.
 
 ## Features ##
-- [ESI](https://esi.tech.ccp.is/) support
-- Live reporting from zKillboard's [RedisQ](https://github.com/zKillboard/RedisQ) service
+- Live reporting from zKillboard's [WebSocket service](https://github.com/zKillboard/zKillboard/wiki/Websocket)
+- Identifies fleet patterns based on recorded kills
+- Resolves characters, corporations and alliances via [ESI](https://esi.tech.ccp.is/)
+- Resolves shiptypes and systems via the [EVE Online SDE](https://developers.eveonline.com/resource/resources)
 - [Database-agnostic](https://sailsjs.com/documentation/reference/configuration/sails-config-datastores#?supported-databases) (Sentinel doesn't care if you use PostgreSQL, Mongo, or even your [RAM](https://github.com/balderdashy/sails-disk))
 - Sophisticated job queue scheduler via [Kue](https://github.com/Automattic/kue)
 
@@ -23,11 +25,11 @@ The app will refuse to run without a valid root-level `.env` - see the [example
     $ yarn
     $ node app.js
 
-And now you're listening for kills and saving them to the DB in real time. RedisQ will remember who you are for up to 3 hours if you go offline, otherwise you're considered a new listener.
+And now you're listening for kills and saving them to the DB in real time.
 
 Provided it's not firewalled, a frontend to the job queue will be available at `http://<BASE_URL>:6574`.
 
-If you're going to leave this thing running permanently, you should run it `NODE_ENV=production` (i.e. `npm start`).
+If you're going to leave this thing running permanently, you should run it with `NODE_ENV=production` (i.e. `npm start`).
 
 ## Support ##
 

api/models/Kill.js

@@ -1,7 +1,7 @@
 /**
  * Kill.js
  *
- * @description :: Kill report as retrieved from the zKill/Push service.
+ * @description :: Resolved kill report from zKillboard.
  * @docs        :: http://sailsjs.org/documentation/concepts/models-and-orm/models
  */
 

api/services/Fuzzworks.js

@@ -20,11 +20,11 @@ module.exports = {
         json: true
       }, (error, response, body) => {
         if (error || !body) {
-          sails.log.error(`[Fuzzworks.nearestCelestial] ${response.statusCode} ${error}`);
+          sails.log.error(`[${new Date().toLocaleTimeString()}] [Fuzzworks.nearestCelestial] ${response.statusCode} ${error}`);
           return reject();
         }
 
-        sails.log.debug(`[Fuzzworks.nearestCelestial] Success.`);
+        sails.log.silly(`[Fuzzworks.nearestCelestial] Success.`);
         sails.log.silly(`[Fuzzworks.nearestCelestial] ${body}`);
 
         return resolve(body);

api/services/Identifier.js

@@ -13,22 +13,22 @@
 let _resolveCharactersToIds = async(ids) => {
   let resolved = [];
 
-  sails.log.debug(`[Identifier._resolveCharactersToIds] Resolving ${ids.length} characters...`);
+  sails.log.silly(`[Identifier._resolveCharactersToIds] Resolving ${ids.length} characters...`);
 
   for (let characterId of ids) {
     let character;
 
     try {
       character = await Swagger.character(characterId);
     } catch(e) {
-      sails.log.error(`[Identifier._resolveCharactersToIds] ${JSON.stringify(e)}`);
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [Identifier._resolveCharactersToIds] ${JSON.stringify(e)}`);
     }
 
     if (character && character.id)
       resolved.push(character.id);
   }
 
-  sails.log.debug('[Identifier._resolveCharactersToIds] End');
+  sails.log.silly('[Identifier._resolveCharactersToIds] End');
 
   return _.compact(resolved);
 };
@@ -77,7 +77,7 @@ let _createFleet = async(killmail, kill, system) => {
     isActive,
     system
   })
-  .intercept((e) => sails.log.error('Fleet create error:', e))
+  .intercept((e) => sails.log.error(`[${new Date().toLocaleTimeString()}] [Identifier._createFleet] Fleet create error:`, e))
   .fetch();
 
   await Fleet.addToCollection(fleet.id, 'characters').members(characters);
@@ -89,7 +89,7 @@ let _createFleet = async(killmail, kill, system) => {
 
 let _updateFleet = async(fleet, kill, system) => {
   if (!fleet) {
-    return sails.log.error('[Identifier._updateFleet] No fleet to update');
+    return sails.log.error(`[${new Date().toLocaleTimeString()}] [Identifier._updateFleet] No fleet to update`);
   }
 
   // First we add the kill to the fleet's collection, as it will impact other data.
@@ -148,7 +148,7 @@ let Identifier = {
     if (!attackersWithIds.length)
       return;
 
-    sails.log.debug(`[Identifier.fleet] Fetching active fleet records related to km...`);
+    sails.log.silly(`[Identifier.fleet] Fetching active fleet records related to km...`);
 
     let fleetIds = [];
 
@@ -182,7 +182,7 @@ let Identifier = {
     }
 
     // We have more than one fleet candidate, so let's score them.
-    sails.log.debug(`[Identifier.fleet] Scoring ${fleets.length} fleets...`);
+    sails.log.silly(`[Identifier.fleet] Scoring ${fleets.length} fleets...`);
 
     let candidates = fleets.map((candidate) => {
       let { id, characters } = candidate;
@@ -205,7 +205,7 @@ let Identifier = {
       return { id, similarity, size: characters.length };
     });
 
-    sails.log.debug(`[Identifier.fleet] Sorting potential matches...`);
+    sails.log.silly(`[Identifier.fleet] Sorting potential matches...`);
 
     candidates = _.sortByOrder(candidates, ['similarity', 'size'], ['desc', 'desc']);
 

api/services/Resolver.js

@@ -9,7 +9,7 @@ let Resolver = {
 
   async position(position, systemId) {
     if (!position || !systemId) {
-      sails.log.error('[Resolver] ESI failure: Incomplete position data');
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [Resolver] ESI failure: Incomplete position data`);
 
       return 'Unknown';
     }
@@ -19,7 +19,7 @@ let Resolver = {
     try {
       response = await Fuzzworks.nearestCelestial(position, systemId);
     } catch(e) {
-      sails.log.error(`[Resolver] Fuzzworks failure: ${e}`);
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [Resolver] Fuzzworks failure: ${e}`);
       return 'Unknown';
     }
 

api/services/ZkillResolve.js

@@ -27,12 +27,12 @@ module.exports = {
     let existingRecord = await Kill.findOne({ killId });
 
     if (existingRecord) {
-      return sails.log.debug(`[ZkillResolve.kill] Ignoring ${killId} - already in database.`);
+      return sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillResolve.kill] Ignoring ${killId} - already in database.`);
     }
 
     if (!characterId || !shipTypeId || !systemId) {
-      sails.log.debug(`[ZkillResolve.kill] Issue with record: characterId ${characterId} || shipTypeId ${shipTypeId} || systemId ${systemId}`);
-      sails.log.debug('[ZkillResolve.kill] Cancelling resolve.');
+      sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillResolve.kill] Issue with record: characterId ${characterId} || shipTypeId ${shipTypeId} || systemId ${systemId}`);
+      sails.log.debug('[${new Date().toLocaleTimeString()}] [ZkillResolve.kill] Cancelling resolve.');
       return;
     }
 
@@ -44,31 +44,31 @@ module.exports = {
     try {
       shipRes = await Type.findOne(shipTypeId);
     } catch(e) {
-      sails.log.error('[ZkillResolve] ESI failure.');
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [ZkillResolve] ESI failure.`);
       sails.log.error(e);
       return;
     }
 
     try {
       victimRes = await Swagger.character(characterId);
     } catch(e) {
-      sails.log.error('[ZkillResolve] ESI failure.');
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [ZkillResolve] ESI failure.`);
       sails.log.error(e);
       return;
     }
 
     try {
       systemRes = await System.findOne(systemId);
     } catch(e) {
-      sails.log.error('[ZkillResolve] ESI failure.');
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [ZkillResolve] ESI failure.`);
       sails.log.error(e);
       return;
     }
 
     try {
       positionRes = await Resolver.position(position, systemId);
     } catch(e) {
-      sails.log.error('[ZkillResolve] ESI failure.');
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [ZkillResolve] ESI failure.`);
       sails.log.error(e);
       return;
     }
@@ -95,7 +95,7 @@ module.exports = {
       victim,
       system
     })
-    .intercept('E_UNIQUE', (e) => { return sails.log.error(`[ZkillResolve.kill] Race condition: Tried to create a kill that already exists. ${e}`) })
+    .intercept('E_UNIQUE', (e) => { return sails.log.error(`[${new Date().toLocaleTimeString()}] [ZkillResolve.kill] Race condition: Tried to create a kill that already exists. ${e}`) })
     .fetch();
 
     let now = moment(),

api/services/ZkillSocket.js

@@ -6,22 +6,60 @@
  */
 
 const WebSocket = require('ws');
-const socket = new WebSocket('wss://zkillboard.com:2096');
 
 let ZkillSocket = {
 
+  connect() {
+    this.socket = new WebSocket('wss://zkillboard.com:2096');
+
+    sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillSocket] Initializing new socket instance.`);
+
+    this.initialize();
+  },
+
+  ping() {
+    this.socket.ping();
+  },
+
   initialize() {
+    // Set up heartbeat.
+    if (this.heartbeatCheck) {
+      clearInterval(this.heartbeatCheck);
+    }
+
+    this.isAlive = true;
+
+    this.heartbeatCheck = setInterval(() => {
+      if (!this.isAlive) {
+        sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillSocket] No heartbeat detected in over 30 seconds.`);
+
+        sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillSocket] Will attempt to reconnect in 5 seconds.`);
+
+        return setTimeout(this.connect, 5000);
+      }
+
+      this.isAlive = false;
+
+      this.ping();
+    }, 30000);
+
+    this.socket.on('pong', () => {
+      this.isAlive = true;
+
+      sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillSocket] Heartbeat response received from Zkill.`);
+    });
+
     // Subscribe to the full killstream.
-    socket.on('open', () => {
-      sails.log.debug('[ZkillSocket] Connected.');
+    this.socket.on('open', () => {
+      sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillSocket] Connected.`);
 
-      socket.send(JSON.stringify({
+      this.socket.send(JSON.stringify({
         action: 'sub',
         channel: 'killstream'
       }));
     });
 
-    socket.on('message', async(data) => {
+    this.socket.on('message', async(data) => {
       let package = await Package.create({ body: data }).fetch();
 
       let job = sails.config.jobs.create('process_zkill_package', { id: package.id });
@@ -30,12 +68,26 @@ let ZkillSocket = {
       job.attempts(3).backoff({ type:'exponential' });
 
       job.on('failed', function(err) {
-        sails.log.error('[Zkill.processZkillPackage] Job failed');
+        sails.log.error(`[${new Date().toLocaleTimeString()}] [Zkill.processZkillPackage] Job failed`);
         sails.log.error(err);
       });
 
       job.save();
     });
+
+    this.socket.on('close', (code, reason) => {
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [ZkillSocket] Connection was closed: ${code} ${reason}`);
+
+      sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillSocket] Attempting to re-establish...`);
+
+      setTimeout(this.connect, 2000);
+    });
+
+    this.socket.on('error', (error) => {
+      sails.log.error(`[${new Date().toLocaleTimeString()}] [ZkillSocket] Error from Zkill:`, error);
+    });
+
+    sails.log.debug(`[${new Date().toLocaleTimeString()}] [ZkillSocket] All message handlers initialized.`);
   },
 
 };

config/bootstrap.js

@@ -16,7 +16,7 @@ module.exports.bootstrap = async function(done) {
   // Job queue kickoff (config/jobs.js)
   sails.config.jobs.init();
 
-  ZkillSocket.initialize();
+  ZkillSocket.connect();
 
   // Needed for Sails to complete lift.
   return done();

config/jobs.js

@@ -115,18 +115,22 @@ function init() {
   // Zkill
 
   jobs.process('process_zkill_package', (job, done) => {
+    sails.log.debug(`[${new Date().toLocaleTimeString()}] [Kue] Processing job for ${job.data.id}`);
+
     Package.findOne(job.data.id)
       .then((package) => {
         let resolvedPackage = JSON.parse(package.body);
 
         if (!_shouldTrack(resolvedPackage)) {
-          sails.log.debug(`[Zkill.processZkillPackage] Not tracking ${resolvedPackage.killmail_id}`);
+          sails.log.debug(`[${new Date().toLocaleTimeString()}] [Zkill.processZkillPackage] Not tracking ${resolvedPackage.killmail_id}.`);
 
           return done();
         }
 
         ZkillResolve.kill(resolvedPackage)
           .then(() => {
+            sails.log.debug(`[${new Date().toLocaleTimeString()}] [Kue] Job for ${job.data.id} finished.`);
+
             done();
           })
           .catch((err) => {
@@ -149,7 +153,7 @@ function init() {
       .limit(10)
       .then((characters) => {
         if (characters && characters instanceof Error) {
-          sails.log.error(`[Job.update_danger_ratios] ${characters}`);
+          sails.log.error(`[${new Date().toLocaleTimeString()}] [Job.update_danger_ratios] ${characters}`);
           done(characters);
         }
 
@@ -162,7 +166,7 @@ function init() {
               await Character.update(character.id, { dangerRatio, lastZkillUpdate });
             })
             .catch((error) => {
-              sails.log.error(`[Job.update_danger_ratios] ${error}`);
+              sails.log.error(`[${new Date().toLocaleTimeString()}] [Job.update_danger_ratios] ${error}`);
             });
         }
 
@@ -184,7 +188,7 @@ function init() {
   jobs.on('job complete', function(id) {
     kue.Job.get(id, function(err, job) {
       if (err) {
-        console.log(`Job ${id} failed: ${err}`);
+        console.log(`[${new Date().toLocaleTimeString()}] [Kue] Job ${id} failed: ${err}`);
       }
 
       if (err) { return; }

package.json

@@ -1,9 +1,13 @@
 {
   "name": "sentinel",
   "private": false,
-  "version": "2.1.0",
+  "version": "2.1.1",
   "description": "A WebSocket-compatible fleet tracker for EVE Online.",
-  "keywords": [],
+  "keywords": [
+    "eve online",
+    "eve",
+    "mmorpg"
+  ],
   "dependencies": {
     "async": "2.0.1",
     "dotenv-safe": "^5.0.1",