Enhanced debug logging

dagda/analysis/analyzer.py

@@ -20,6 +20,7 @@
 import datetime
 import requests
 import json
+import traceback
 from analysis.static.os import os_info_extractor
 from analysis.static.dependencies import dep_info_extractor
 from analysis.static.av import malware_extractor
@@ -48,6 +49,9 @@ def __init__(self, dagda_server_url=None):
 
     # Evaluate image from image name or container id
     def evaluate_image(self, image_name, container_id):
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('ENTRY to the method for analyzing a docker image')
+
         # Init
         data = {}
 
@@ -60,6 +64,9 @@ def evaluate_image(self, image_name, container_id):
         temp_dir = None
         try:
             # Get OS packages
+            if InternalServer.is_debug_logging_enabled():
+                DagdaLogger.get_logger().debug('Retrieving OS packages from the docker image ...')
+
             if container_id is None:  # Scans the docker image
                 os_packages = os_info_extractor.get_soft_from_docker_image(docker_driver=self.dockerDriver,
                                                                            image_name=image_name)
@@ -71,25 +78,46 @@ def evaluate_image(self, image_name, container_id):
                 temp_dir = extract_filesystem_bundle(docker_driver=self.dockerDriver,
                                                      container_id=container_id)
 
+            if InternalServer.is_debug_logging_enabled():
+                DagdaLogger.get_logger().debug('OS packages from the docker image retrieved')
+
             # Get malware binaries
+            if InternalServer.is_debug_logging_enabled():
+                DagdaLogger.get_logger().debug('Retrieving malware files from the docker image ...')
+
             malware_binaries = malware_extractor.get_malware_included_in_docker_image(docker_driver=self.dockerDriver,
                                                                                       temp_dir=temp_dir)
 
+            if InternalServer.is_debug_logging_enabled():
+                DagdaLogger.get_logger().debug('Malware files from the docker image retrieved')
+
             # Get programming language dependencies
+            if InternalServer.is_debug_logging_enabled():
+                DagdaLogger.get_logger().debug('Retrieving dependencies from the docker image ...')
+
             dependencies = dep_info_extractor.get_dependencies_from_docker_image(docker_driver=self.dockerDriver,
                                                                                  image_name=image_name,
                                                                                  temp_dir=temp_dir)
+
+            if InternalServer.is_debug_logging_enabled():
+                DagdaLogger.get_logger().debug('Dependencies from the docker image retrieved')
+
         except Exception as ex:
             message = "Unexpected exception of type {0} occured: {1!r}"\
                 .format(type(ex).__name__,  ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args)
             DagdaLogger.get_logger().error(message)
+            if InternalServer.is_debug_logging_enabled():
+                traceback.print_exc()
             data['status'] = message
 
         # -- Cleanup
         if temp_dir is not None:
             clean_up(temporary_dir=temp_dir)
 
         # -- Prepare output
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('Preparing analysis output ...')
+
         if dependencies is not None:
             data['status'] = 'Completed'
         else:
@@ -99,7 +127,13 @@ def evaluate_image(self, image_name, container_id):
         data['timestamp'] = datetime.datetime.now().timestamp()
         data['static_analysis'] = self.generate_static_analysis(image_name, os_packages, dependencies, malware_binaries)
 
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('Analysis output completed')
+
         # -- Return
+        if InternalServer.is_debug_logging_enabled():
+            DagdaLogger.get_logger().debug('EXIT from the method for analyzing a docker image')
+
         return data
 
     # Generates the result of the static analysis

dagda/analysis/runtime/docker_events_monitor.py

@@ -19,6 +19,10 @@
 
 import json
 import requests
+import bson
+import traceback
+from log.dagda_logger import DagdaLogger
+from api.internal.internal_server import InternalServer
 
 
 # Docker daemon events monitor class
@@ -40,8 +44,19 @@ def run(self):
             try:
                 for event in self.docker_driver.docker_events():
                     e = json.loads(event.decode('UTF-8').replace("\n", ""))
+                    if 'Actor' in e and 'Attributes' in e['Actor']:
+                        for key in e['Actor']['Attributes']:
+                            if '.' in key:
+                                del e['Actor']['Attributes'][key]
+                                break
                     # Bulk insert
                     self.mongodb_driver.bulk_insert_docker_daemon_events([e])
             except requests.packages.urllib3.exceptions.ReadTimeoutError:
                 # Nothing to do
                 pass
+            except bson.errors.InvalidDocument as ex:
+                message = "Unexpected exception of type {0} occured: {1!r}" \
+                    .format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args)
+                DagdaLogger.get_logger().error(message)
+                if InternalServer.is_debug_logging_enabled():
+                    traceback.print_exc()

dagda/dagda.py

@@ -18,6 +18,7 @@
 #
 
 import json
+import traceback
 from cli.dagda_cli import execute_dagda_cmd
 from cli.dagda_cli_parser import DagdaCLIParser
 from log.dagda_logger import DagdaLogger
@@ -43,6 +44,7 @@ def main(parsed_args):
                 DagdaLogger.get_logger().error(str(err))
     except BaseException as err:
         DagdaLogger.get_logger().error(str(err))
+        traceback.print_exc()
 
 
 if __name__ == "__main__":