Issue #32: Enhanced debug logging

dagda/api/dagda_server.py

@@ -20,6 +20,7 @@
 import os
 import json
 import datetime
+import traceback
 from flask import Flask
 from flask_cors import CORS, cross_origin
 from api.internal.internal_server import InternalServer
@@ -55,10 +56,11 @@ class DagdaServer:
     # DagdaServer Constructor
     def __init__(self, dagda_server_host='127.0.0.1', dagda_server_port=5000, mongodb_host='127.0.0.1',
                  mongodb_port=27017, mongodb_ssl=False, mongodb_user=None, mongodb_pass=None,
-                 falco_rules_filename=None, external_falco_output_filename=None):
+                 falco_rules_filename=None, external_falco_output_filename=None, debug_logging=False):
         super(DagdaServer, self).__init__()
         self.dagda_server_host = dagda_server_host
         self.dagda_server_port = dagda_server_port
+        InternalServer.set_debug_logging_enabled(debug_logging)
         InternalServer.set_mongodb_driver(mongodb_host, mongodb_port, mongodb_ssl, mongodb_user, mongodb_pass)
         self.sysdig_falco_monitor = SysdigFalcoMonitor(InternalServer.get_docker_driver(),
                                                        InternalServer.get_mongodb_driver(),
@@ -150,6 +152,8 @@ def _init_or_update_db():
         except Exception as ex:
             message = "Unexpected exception of type {0} occured: {1!r}".format(type(ex).__name__,  ex.args)
             DagdaLogger.get_logger().error(message)
+            if InternalServer.is_debug_logging_enabled():
+                traceback.print_exc()
             InternalServer.get_mongodb_driver().insert_init_db_process_status(
                     {'status': message, 'timestamp': datetime.datetime.now().timestamp()})
 

dagda/api/internal/internal_server.py

@@ -32,6 +32,7 @@ class InternalServer:
     _mongodb_driver = MongoDbDriver()
     _docker_driver = DockerDriver()
     _external_falco = False
+    _debug_logging = False
 
     # -- Static methods
 
@@ -82,3 +83,12 @@ def is_runtime_analysis_enabled():
         return InternalServer._external_falco or \
                len(InternalServer._docker_driver.get_docker_container_ids_by_image_name('sysdig/falco')) > 0
 
+    # Sets if debug logging is enabled
+    @staticmethod
+    def set_debug_logging_enabled(debug_logging):
+        InternalServer._debug_logging = debug_logging
+
+    # Is debug logging enabled
+    @staticmethod
+    def is_debug_logging_enabled():
+        return InternalServer._debug_logging

dagda/cli/command/start_cli_parser.py

@@ -31,6 +31,7 @@ class StartCLIParser:
     def __init__(self):
         super(StartCLIParser, self).__init__()
         self.parser = DagdaStartParser(prog='dagda.py start', usage=start_parser_text)
+        self.parser.add_argument('-d','--debug', action='store_true')
         self.parser.add_argument('-s', '--server_host', type=str)
         self.parser.add_argument('-p', '--server_port', type=int)
         self.parser.add_argument('-m', '--mongodb_host', type=str)
@@ -48,6 +49,10 @@ def __init__(self):
 
     # -- Getters
 
+    # Gets if debug logging is required
+    def is_debug_logging_required(self):
+        return self.args.debug
+
     # Gets server host
     def get_server_host(self):
         return self.args.server_host
@@ -140,7 +145,7 @@ def format_help(self):
 
 # Custom text
 
-start_parser_text = '''usage: dagda.py start [-h] [--server_host SERVER_HOST] [--server_port SERVER_PORT]
+start_parser_text = '''usage: dagda.py start [-h] [-d] [--server_host SERVER_HOST] [--server_port SERVER_PORT]
                   [--mongodb_host MONGODB_HOST] [--mongodb_port MONGODB_PORT]
                   [--mongodb_ssl] [--mongodb_user MONGODB_USER] [--mongodb_pass MONGODB_PASS]
                   [--falco_rules_file RULES_FILE] [--external_falco OUTPUT_FILE]
@@ -149,6 +154,7 @@ def format_help(self):
 
 Optional Arguments:
   -h, --help            show this help message and exit
+  -d, --debug           enable debug logging
 
   -s SERVER_HOST, --server_host SERVER_HOST
                         address/interface where the server binds itself. By

dagda/cli/dagda_cli.py

@@ -38,7 +38,8 @@ def execute_dagda_cmd(cmd, args):
                          mongodb_user=args.get_mongodb_user(),
                          mongodb_pass=args.get_mongodb_pass(),
                          falco_rules_filename=args.get_falco_rules_filename(),
-                         external_falco_output_filename=args.get_external_falco_output_filename())
+                         external_falco_output_filename=args.get_external_falco_output_filename(),
+                         debug_logging=args.is_debug_logging_required())
         ds.run()
 
     # Executes agent sub-command

dagda/log/dagda_logger.py

@@ -25,8 +25,10 @@
 class DagdaLogger(logging.Logger):
 
     # -- Init
-    logging.basicConfig(format='<%(asctime)s> <%(levelname)s> <DagdaServer> <%(module)s> <%(message)s>')
+    logging.basicConfig(format='<%(asctime)s> <%(levelname)s> <DagdaServer> <%(module)s> <%(funcName)s:%(lineno)d> ' +
+                               '<%(message)s>')
     _logger = logging.getLogger('DagdaLogger')
+    _logger.setLevel('DEBUG')
 
     # -- Static methods
 

dagda/vulnDB/db_composer.py

@@ -19,6 +19,7 @@
 
 import io
 from datetime import date
+from log.dagda_logger import DagdaLogger
 from api.internal.internal_server import InternalServer
 from vulnDB.ext_source_util import get_bug_traqs_lists_from_file
 from vulnDB.ext_source_util import get_bug_traqs_lists_from_online_mode
@@ -47,10 +48,19 @@ def __init__(self):
 
     # Compose vuln DB
     def compose_vuln_db(self):
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('ENTRY to the method for composing VulnDB')
+
         # -- CVE
         # Adding or updating CVEs
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('Updating CVE collection ...')
+
         first_year = self.mongoDbDriver.remove_only_cve_for_update()
         for i in range(first_year, next_year):
+            if InternalServer.is_debug_logging_enabled():
+                DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i))
+
             compressed_content = get_http_resource_content(
                 "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz")
             cve_list = get_cve_list_from_file(compressed_content, i)
@@ -68,8 +78,14 @@ def compose_vuln_db(self):
             if len(cve_ext_info_list) > 0:
                 self.mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
 
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('CVE collection updated')
+
         # -- Exploit DB
         # Adding or updating Exploit_db and Exploit_db info
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('Updating Exploit DB collection ...')
+
         self.mongoDbDriver.delete_exploit_db_collection()
         self.mongoDbDriver.delete_exploit_db_info_collection()
         csv_content = get_http_resource_content(
@@ -78,8 +94,14 @@ def compose_vuln_db(self):
         self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list)
         self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list)
 
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('Exploit DB collection updated')
+
         # -- BID
         # Adding BugTraqs from 20180328_sf_db.json.gz, where 103525 is the max bid in the gz file
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('Updating BugTraqs Id collection ...')
+
         max_bid = self.mongoDbDriver.get_max_bid_inserted()
         if max_bid < 103525:
             # Clean
@@ -113,8 +135,14 @@ def compose_vuln_db(self):
             self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array)
             bid_detail_array.clear()
 
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('BugTraqs Id collection updated')
+
         # -- RHSA (Red Hat Security Advisory) and RHBA (Red Hat Bug Advisory)
         # Adding or updating rhsa and rhba collections
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('Updating RHSA & RHBA collections ...')
+
         self.mongoDbDriver.delete_rhba_collection()
         self.mongoDbDriver.delete_rhba_info_collection()
         self.mongoDbDriver.delete_rhsa_collection()
@@ -125,3 +153,9 @@ def compose_vuln_db(self):
         self.mongoDbDriver.bulk_insert_rhba(rhba_list)
         self.mongoDbDriver.bulk_insert_rhsa_info(rhsa_info_list)
         self.mongoDbDriver.bulk_insert_rhba_info(rhba_info_list)
+
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('RHSA & RHBA collections updated')
+
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('EXIT from the method for composing VulnDB')