Merge pull request #1235 from splunk/TR-407_orphan_tests

TR-407_orphan_tests

bin/validate.py

@@ -13,6 +13,7 @@
 import datetime
 import string
 import re
+from pathlib import Path
 from os import path, walk
 
 
@@ -84,6 +85,10 @@ def validate_objects(REPO_PATH, objects, verbose):
     for object in objects['baselines']:
         errors = errors + validate_baseline_search(object, objects['macros'])
 
+
+    for object in objects['tests']:
+        errors = errors + validate_tests(REPO_PATH, object)
+
     errors = lookup_errors + errors
 
     return errors
@@ -238,10 +243,25 @@ def validate_lookups_content(REPO_PATH, lookup_path, lookup):
 
     return errors
 
+
+def validate_tests(REPO_PATH, object):
+    errors = []
+
+    # check detection file exists
+    for test in object['tests']:
+        if 'file' in test:
+            detection_file_path = Path(REPO_PATH + '/detections/' + test['file'])
+            if not detection_file_path.is_file():
+                errors.append('ERROR: orphaned test: {0}, detection file: {1} no longer exists or incorrect detection path under `file`'.format(object['name'], detection_file_path))
+        else:
+            errors.append('ERROR: test: {0} does not have a detection `file` associated with detection: {1}'.format(object['name'], test['name']))
+        test['file']
+    return errors
+
 def main(REPO_PATH, verbose):
 
-    validation_objects = ['macros','lookups','stories','detections','baselines','response_tasks','responses','deployments']
-    
+    validation_objects = ['macros','lookups','stories','detections','baselines','response_tasks','responses','deployments', 'tests']
+
     objects = {}
     schema_error = False
     schema_errors = []
@@ -277,4 +297,4 @@ def main(REPO_PATH, verbose):
     REPO_PATH = args.path
     verbose = args.verbose
 
-    main(REPO_PATH, verbose)
+    main(REPO_PATH, verbose)

spec/tests.spec.json

@@ -0,0 +1,57 @@
+{
+  "$id": "https://raw.githubusercontent.com/splunk/security_content/develop/docs/spec/tests.spec.json",
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "additionalProperties": true,
+  "default": {},
+  "description": "schema test file",
+  "properties": {
+    "name": {
+      "$id": "#/properties/name",
+      "default": "",
+      "description": "Name of the Analytics Story",
+      "examples": [
+        "Credential Dumping"
+      ],
+      "type": "string"
+    },
+    "tests": {
+      "$id": "#/properties/tests",
+      "additionalProperties": true,
+      "default": {},
+      "description": "a test description.",
+      "examples": [
+        [
+          {
+            "name": "Detect Activity Related to Pass the Hash Attacks",
+            "file": "endpoint/detect_activity_related_to_pass_the_hash_attacks.yml",
+            "pass_condition": "| stats count | where count > 0",
+            "earliest_time": "-24h",
+            "latest_time": "now",
+            "attack_data": [
+              {
+                "file_name": "windows-security.log",
+                "data": "https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1550.002/atomic_red_team/windows-security.log",
+                "source": "WinEventLog:Security",
+                "sourcetype": "WinEventLog",
+                "update_timestamp": true
+              }
+            ]
+          }
+        ]
+      ],
+      "minItems": 1,
+      "type": "array",
+      "items": [
+   {
+     "type": "object"
+   }
+ ]
+    }
+  },
+  "required": [
+    "name",
+    "tests"
+  ],
+  "title": "Test file schema",
+  "type": "object"
+}

tests/cloud/abnormally_high_cloud_instances_destroyed.test.yml

@@ -1,7 +1,7 @@
 name: Abnormally High Number Of Cloud Instances Destroyed Unit Test
 tests:
 - name: Abnormally High Number Of Cloud Instances Destroyed
-  file: cloud/abnormally_high_cloud_instances_destroyed.yml
+  file: experimental/cloud/abnormally_high_cloud_instances_destroyed.yml
   pass_condition: '| stats count | where count > 0'
   earliest_time: '-24h'
   latest_time: 'now'

tests/cloud/abnormally_high_cloud_instances_launched.test.yml

@@ -1,7 +1,7 @@
 name: Abnormally High Number of Cloud Instances Launched Unit Test
 tests:
 - name: Abnormally High Number of Cloud Instances Launched
-  file: cloud/abnormally_high_cloud_instances_launched.yml
+  file: experimental/cloud/abnormally_high_cloud_instances_launched.yml
   pass_condition: '| stats count | where count > 0'
   earliest_time: '-24h'
   latest_time: 'now'

tests/cloud/high_number_of_login_failures_from_a_single_source.test.yml

@@ -1,7 +1,7 @@
 name: High Number of Login Failures from a single source Unit Test
 tests:
 - name: High Number of Login Failures from a single source
-  file: cloud/high_number_of_login_failures_from_a_single_source.yml
+  file: experimental/cloud/high_number_of_login_failures_from_a_single_source.yml
   pass_condition: '| stats count | where count > 0'
   earliest_time: '-24h'
   latest_time: 'now'

tests/endpoint/windows_adfind_exe.test.yml

@@ -1,7 +1,7 @@
 name: Windows AdFind Exe Unit Test
 tests:
 - name: Windows AdFind Exe
-  file: endpoint/windows_adfind_exe.yml
+  file: experimental/endpoint/windows_adfind_exe.yml
   pass_condition: '| stats count | where count > 0'
   earliest_time: '-24h'
   latest_time: 'now'

tests/network/detect_ipv6_network_infrastructure_threats.test.yml

@@ -1,7 +1,7 @@
 name: IPv6 Network Infrastructure Threats
 tests:
   - name: Detect IPv6 Network Infrastructure Threats
-    file: network/detect_ipv6_network_infrastructure_threats.yml
+    file: experimental/network/detect_ipv6_network_infrastructure_threats.yml
     pass_condition: '| stats count | where count > 0'
     earliest_time: '-24h'
     latest_time: 'now'

tests/network/detect_snicat_sni_exfiltration.test.yml

@@ -1,7 +1,7 @@
 name: Detect SNICat SNI Exfiltration Unit Test
 tests:
 - name: Detect SNICat SNI Exfiltration
-  file: network/detect_snicat_sni_exfiltration.yml
+  file: experimental/network/detect_snicat_sni_exfiltration.yml
   pass_condition: '| stats count | where count > 0'
   earliest_time: '-24h'
   latest_time: 'now'