Merge pull request #204 from RADAR-base/release-2.1.0

Release 2.1.0

.travis.yml

@@ -24,3 +24,9 @@ script:
   - ./setup-env.sh
   - cd ../
   - bin/radar-docker install && sleep 120 && bin/radar-docker ps && [ -z "$(bin/radar-docker ps | tail -n +3 | grep " Exit " | grep -v "kafka-init_1")" ]
+
+  # Firebase messaging stack
+  - cd  ../firebase-app-server
+  - cp etc/env.template .env
+  - sudo $DOCKER_COMPOSE up -d --build && sleep 15 && [ -z "$(sudo $DOCKER_COMPOSE ps | tail -n +3 | grep " Exit ")" ]
+  - sudo $DOCKER_COMPOSE down

README.md

@@ -1,4 +1,4 @@
-# RADAR-Docker 2.0.1
+# RADAR-Docker 2.1.0
 
 The dockerized RADAR stack for deploying the RADAR-base platform. Component repositories can be found at [RADAR-base DockerHub org](https://hub.docker.com/u/radarbase/dashboard/)
 

dcompose-stack/.gitignore

@@ -0,0 +1 @@
+.DS_Store

dcompose-stack/firebase-app-server/.gitignore

@@ -0,0 +1 @@
+.env

dcompose-stack/firebase-app-server/README.md

@@ -0,0 +1,50 @@
+# Firebase Notification server
+
+This directory provides services and accompanying files required to run the FCM notifications server. Currently, [XMPP App server](https://github.com/RADAR-base/fcmxmppserverv2) is provided but this may later change to the [RADAR-AppServer](https://github.com/RADAR-base/RADAR-Appserver).
+
+## Configuration
+
+Copy `etc/env.template` to `.env`. Set the values of the new file to the desired values.
+
+```sh
+FCM_XMPP_APP_SERVER_DB_PATH= # The path on the host where to store database files
+FCM_XMPP_APP_SERVER_LOGS_PATH= # The path on the host where to store the log files generated by the server
+FCM_SENDER_KEY= # The Firebase Cloud Messaging Sender ID
+FCM_SERVER_KEY= # # The Firebase Cloud Messaging Server Key
+```
+
+For more information, take a look at the instruction in the [README](https://github.com/RADAR-base/fcmxmppserverv2).
+
+If the server is not running in a secure environment, it may be essential to explicitly add username and password to the database.
+This can be done by changing the `server.database.n` properties in the [server.properties](/etc/server.properties) file and appending with your username and password as stated [here](http://hsqldb.org/doc/guide/guide.html#N15798). Then add the same to the environment of the `xmppserver` in [docker-compose.yml](/docker-compose.yml) file with keys `RADAR_XMPP_DB_USER` and `RADAR_XMPP_DB_PASS`.
+
+## Usage
+
+Since the App Server does not depend on any other services, it can be run separately by running -
+
+```sh
+bin/start-xmpp -d
+```
+
+It can be further controlled with `docker-compose` and `docker` commands.
+
+## Extras
+
+There are some extra scripts provided for convenience when administering the server in `bin/` folder.
+
+- `get-subject-data.sh` -  this can be used for getting the notifications, and other data for a particular subject using their subject Id. Note that this will require the [sqltool.jar](http://hsqldb.org/doc/2.0/util-guide/sqltool-chapt.html) from hsqldb. You will also need to update any settings in (sqltool.rc)[/etc/sqltool.rc] according to your DB setup. Once configured, The script can be used as follows -
+    ```sh
+    ./get-subject-data.sh <subject_id> <path_to_sqltool.jar_file>
+    ```
+
+- `log-parser.py` - This is used for parsing information out of the logs files generated by the server. This is supposed to be run as a cron job for creating CSV files for Execution, delivery and error messages for each notification request over a long period of time. This can be run in cron per day as follows -
+    ```sh
+    0 16 * * * python3 /home/ubuntu/xmpp-server-extras/logs-parser/log-parser.py /usr/local/var/lib/radar/xmpp/hsql/logs/ /home/ubuntu/xmpp-server-extras/logs-parser/files >> /home/ubuntu/log-parser-run.log 2>&1
+    ```
+    This will output the CSV files in the directory `/home/ubuntu/xmpp-server-extras/logs-parser/files` which contain three files for each day (One for Executions, One for Delivery and One for Errors).
+
+- `install-systemd` Run the xmpp server in a systemd environment as part of system startup.
+
+## Other information
+
+If you are in a development environment, It may be worth trying to use the new and improved [RADAR-AppServer](https://github.com/RADAR-base/RADAR-Appserver/tree/dev). It exposes REST endpoints along with supporting the legacy XMPP and has secure integration with the Management Portal.

dcompose-stack/firebase-app-server/bin/get-subject-data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+raw_subject=${1}
+#echo $HOME
+if [[ $raw_subject = 'help' ]];then
+	sudo docker-compose -f ../xmpp-server/docker-compose.yml exec hsqldb sh -c "java -jar /opt/hsqldb/lib/sqltool.jar --help"
+else
+	match="-"
+	repl="\\002d"
+	subject=${raw_subject//$match/$repl}
+#	echo $subject
+	java -jar ${2} --rcFile=../etc/sqltool.rc --debug --sql="
+	select status_info.subject_id, notification_info.title, notification_info.ttl_seconds, notification_info.message, notification_info.execution_time from notification_info inner join status_info on notification_info.notification_task_uuid = status_info.notification_task_uuid where status_info.subject_id=U&'${subject}';" db
+fi

dcompose-stack/firebase-app-server/bin/install-systemd

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+echo $(pwd)
+
+. ../radar-cp-hadoop-stack/lib/util.sh
+. .env
+
+if [ "$(id -un)" == "root" ] || id -Gn | grep -qe '\<sudo\>'; then
+  BASE=/etc/systemd/system
+  SYSTEMCTL_OPTS=()
+else
+  BASE=$HOME/.config/systemd/user
+  mkdir -p $BASE
+  SYSTEMCTL_OPTS=(--user)
+  export XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-/run/user/$UID}
+fi
+
+echo "==> Copying templates"
+copy_template $BASE/radar-xmpp-server.service lib/systemd/radar-xmpp-server.service.template
+
+echo "==> Inlining variables"
+inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-xmpp-server.service
+inline_variable 'ExecStart=' "$PWD/bin/start-xmpp" $BASE/radar-xmpp-server.service
+
+echo "==> Reloading systemd"
+systemctl "${SYSTEMCTL_OPTS[@]}" daemon-reload
+systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-xmpp-server
+systemctl "${SYSTEMCTL_OPTS[@]}" start radar-xmpp-server
+

dcompose-stack/firebase-app-server/bin/log-parser.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import sys, csv, json
+import os
+import os.path
+
+def parse_logs(input_directory, output_directory):
+    subject_token = {}
+
+    for log_file in os.listdir(input_directory):
+        # Finding out the executed ones
+        if not log_file.endswith('.log'):
+            continue
+
+        print("Parsing {}...".format(log_file))
+
+        output_file_name = log_file.split('.')[0]
+        log_path = os.path.join(input_directory, log_file)
+        exec_path = os.path.join(output_directory, "{}_executions.csv".format(output_file_name))
+        delivery_path = os.path.join(output_directory, "{}_delivery.csv".format(output_file_name))
+        error_path = os.path.join(output_directory, "{}_error.csv".format(output_file_name))
+
+        with open(log_path) as f,\
+                 open(exec_path, 'w') as exec_output,\
+                 open(delivery_path, 'w') as delivery_output,\
+                 open(error_path, 'w') as error_output:
+            exec_writer = csv.writer(exec_output, delimiter=',',
+                    quotechar='|', quoting=csv.QUOTE_MINIMAL)
+            delivery_writer = csv.writer(delivery_output, delimiter=',',
+                    quotechar='|', quoting=csv.QUOTE_MINIMAL)
+            error_writer = csv.writer(error_output, delimiter=',',
+                    quotechar='|', quoting=csv.QUOTE_MINIMAL)
+
+            exec_writer.writerow(['date', 'time', 'subject_id', 'fcm_token', 'scheduled_time', 'executed?'])
+            delivery_writer.writerow(['date', 'time', 'subject_id', 'fcm_token', 'message_sent_timestamp', 'message_status'])
+            for line in f:
+                data = line.split()
+                if len(data) < 2:
+                    continue
+                date = data[0]
+                time = data[1]
+                if "Executing Execution" in line:
+                    #print(line)
+                    #print(str(data))
+                    ids = data[7].split('=')[1].split('+')
+                    fcm_token = ids[0]
+                    subject_id = ids[1]
+                    scheduled_time = data[8].split('=')[1].split(',')[0]
+                    subject_token[fcm_token] = subject_id
+                    exec_writer.writerow([date, time, subject_id, fcm_token, scheduled_time, 'true'])
+                    #print(str(ids))
+                    #break
+                elif "message_type\":\"receipt" in line:
+                    #print(line)
+                    parsed_data = data[9].split('>')[1].split('<')[0]
+                    json_data = json.loads(parsed_data)
+                    message_status = json_data['data']['message_status']
+                    message_sent_timestamp = json_data['data']['message_sent_timestamp']
+                    fcm_token = json_data['data']['device_registration_id']
+                    try:
+                        subject_id = subject_token[fcm_token]
+                    except KeyError:
+                        subject_id = 'unknown'
+                        pass
+                    #print(message_status, message_sent_timestamp, fcm_token, subject_id)
+                    delivery_writer.writerow([date, time, subject_id, fcm_token, message_sent_timestamp, message_status])
+                elif 'message_type":"nack' in line:
+                    error_writer.writerow([line])
+                    print(line)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print("Usage: {} <input_directory> <output_directory>".format(sys.argv[0]))
+        sys.exit(1)
+
+    input_directory = sys.argv[1]
+    output_directory = sys.argv[2]
+    parse_logs(input_directory, output_directory)

dcompose-stack/firebase-app-server/bin/start-xmpp

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+. ../radar-cp-hadoop-stack/lib/util.sh
+
+copy_template_if_absent .env etc/env.template
+
+. .env
+
+ensure_env_password FCM_SENDER_KEY "Firebase Cloud Messaging Sender ID"
+ensure_env_password FCM_SERVER_KEY "Firebase Cloud Messaging Server Key"
+
+mkdir -p "$FCM_XMPP_APP_SERVER_DB_PATH"
+chown -R 9999:999 "$FCM_XMPP_APP_SERVER_DB_PATH"
+
+exec docker-compose up "$@"

dcompose-stack/firebase-app-server/docker-compose.yml

@@ -0,0 +1,38 @@
+version: '2.1'
+
+networks:
+  db:
+    driver: bridge
+    internal: true
+
+services:
+  hsqldb:
+    build: ../../images/hsqldb/
+    image: radarbase/hsqldb:2.5.0
+    restart: always
+    networks:
+      - db
+      - default
+    ports:
+      - 9001:9001
+    volumes:
+      - ${FCM_XMPP_APP_SERVER_DB_PATH}:/var/opt/hsqldb/data
+      - ./etc/server.properties:/etc/opt/hsqldb/conf/server.properties
+
+  xmppserver:
+    image: radarbase/fcmxmppserverv2:0.1.5
+    restart: always
+    networks:
+      - default
+      - db
+    depends_on:
+      - hsqldb
+    volumes:
+      - ${FCM_XMPP_APP_SERVER_LOGS_PATH}:/usr/local/radar/xmpp-server/logs/
+    environment:
+      RADAR_XMPP_FCM_SENDER_KEY: ${FCM_SENDER_KEY}
+      RADAR_XMPP_FCM_SERVER_KEY: ${FCM_SERVER_KEY}
+      RADAR_XMPP_SCHEDULER_TYPE: "server"
+      RADAR_XMPP_DB_PATH: "//hsqldb:9001/notification"
+      RADAR_XMPP_CACHE_EXPIRY: 300
+      JAVA_OPTS: "-Xms256m -Xmx1G"

dcompose-stack/firebase-app-server/etc/env.template

@@ -0,0 +1,4 @@
+FCM_XMPP_APP_SERVER_DB_PATH=/usr/local/var/lib/radar/xmpp/hsql
+FCM_XMPP_APP_SERVER_LOGS_PATH=/usr/local/var/lib/radar/xmpp/hsql/logs/
+FCM_SENDER_KEY=
+FCM_SERVER_KEY=

dcompose-stack/firebase-app-server/etc/server.properties

@@ -0,0 +1,5 @@
+server.database.0=file:/var/opt/hsqldb/data/notification
+server.dbname.0=notification
+
+server.database.1=file:/var/opt/hsqldb/data/status
+server.dbname.1=status

dcompose-stack/firebase-app-server/etc/sqltool.rc

@@ -0,0 +1,4 @@
+urlid db
+url jdbc:hsqldb:hsql://hsqldb/status
+username SA
+password

dcompose-stack/firebase-app-server/lib/systemd/radar-xmpp-server.service.template

@@ -0,0 +1,25 @@
+[Unit]
+Description=RADAR FCM XMPP service
+
+[Service]
+TimeoutStartSec=0
+TimeoutStopSec=90
+Restart=always
+RestartSec=10
+
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=radar-xmpp-server
+
+WorkingDirectory=
+
+ExecStart=./bin/start-xmpp.sh
+
+ExecReload=/usr/local/bin/docker-compose restart
+
+ExecStop=/usr/local/bin/docker-compose down
+
+NotifyAccess=all
+
+[Install]
+WantedBy=default.target