Merge pull request #1242 from splunk/TR-381_add_required_fields

Tr 381 add required fields

bin/content_changer.py

@@ -0,0 +1,75 @@
+import glob
+import yaml
+import sys
+import re
+import argparse
+
+from os import path
+
+
+def load_objects(file_path, REPO_PATH):
+    files = []
+    manifest_files = path.join(path.expanduser(REPO_PATH), file_path)
+    for file in sorted(glob.glob(manifest_files)):
+        files.append(load_file(file))
+    return files
+
+
+def load_file(file_path):
+    with open(file_path, 'r', encoding="utf-8") as stream:
+        try:
+            file = list(yaml.safe_load_all(stream))[0]
+        except yaml.YAMLError as exc:
+            print(exc)
+            sys.exit("ERROR: reading {0}".format(file_path))
+    return file
+
+
+def load_content(old_project):
+    stories = load_objects("stories/*.yml", old_project)
+    macros = load_objects("macros/*.yml", old_project)
+    lookups = load_objects("lookups/*.yml", old_project)
+    baselines = load_objects("baselines/*.yml", old_project)
+    responses = load_objects("responses/*.yml", old_project)
+    response_tasks = load_objects("response_tasks/*.yml", old_project)
+    deployments = load_objects("deployments/*.yml", old_project)
+
+    # process all detections
+    detections = []
+    detections = load_objects("detections/*/*.yml", old_project)
+    detections.extend(load_objects("detections/*/*/*.yml", old_project))
+
+    #print(len(detections))
+
+    return detections, stories, macros, lookups, baselines, responses, response_tasks, deployments
+
+
+def add_required_field(detections, new_project):
+    #for detection in detections:
+    matches = re.findall(r'(?<key>[^\s]*)=', detections[0])
+    for match in matches:
+        print(match)
+
+
+def main(new_project, old_project, change):
+
+    detections, stories, macros, lookups, baselines, responses, response_tasks, deployments = load_content(old_project)
+
+    if change == "add_required_field":
+        add_required_field(detections, new_project)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description="applies security content changes to the whole project")
+    parser.add_argument("-np", "--new_project", required=True, help="the security content project to write the new configs in to")
+    parser.add_argument("-op", "--old_project", required=True, help="the security content project to read the files from")
+    parser.add_argument("-c", "--change", required=True, help="the name of your change")
+
+   # parse them
+    args = parser.parse_args()
+    new_project = args.new_project
+    old_project = args.old_project
+    change = args.change
+
+    main(new_project, old_project, change)

bin/jinja2_templates/detection.j2

@@ -31,3 +31,5 @@ tags:
   {% for product in products -%}
   - {{product}}
   {% endfor -%}
+  required_fields:
+  - _time

bin/validate.py

@@ -79,6 +79,7 @@ def validate_objects(REPO_PATH, objects, verbose):
     for object in objects['detections']:
         if object['type'] == 'batch':
             errors = errors + validate_detection_search(object, objects['macros'])
+            errors = errors + validate_fields(object)
 
     for object in objects['baselines']:
         errors = errors + validate_baseline_search(object, objects['macros'])
@@ -88,6 +89,21 @@ def validate_objects(REPO_PATH, objects, verbose):
     return errors
 
 
+def validate_fields(object):
+    errors = []
+
+    if 'tags' in object:
+
+        # check if required_fields is present
+        if 'required_fields' not in object['tags']:
+            errors.append("ERROR: a `required_fields` tag is required for object: %s" % object['name'])
+
+        if 'security_domain' not in object['tags']:
+            errors.append("ERROR: a `security_domain` tag is required for object: %s" % object['name'])
+
+    return errors
+
+
 def validate_standard_fields(object, uuids):
 
     errors = []

detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml

@@ -46,6 +46,11 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.command
+  - All_Changes.user
+  - All_Changes.status
   risk_object: user
   risk_object_type: user
   risk_score: 25

detections/cloud/abnormally_high_number_of_cloud_security_group_api_calls.yml

@@ -47,6 +47,12 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.command
+  - All_Changes.object_category
+  - All_Changes.status
+  - All_Changes.user
   risk_object: user
   risk_object_type: user
   risk_score: 25

detections/cloud/aws_cross_account_activity_from_previously_unseen_account.yml

@@ -50,6 +50,13 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - Authentication.signature
+  - Authentication.vendor_account
+  - Authentication.user
+  - Authentication.user_role
+  - Authentication.src
   risk_object: user
   risk_object_type: user
   risk_score: 15

detections/cloud/aws_detect_users_creating_keys_with_encrypt_policy_without_mfa.yml

@@ -38,4 +38,12 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - eventName
+  - eventSource
+  - eventID
+  - awsRegion
+  - requestParameters.policy
+  - userIdentity.principalId
   security_domain: threat

detections/cloud/aws_detect_users_with_kms_keys_performing_encryption_s3.yml

@@ -33,4 +33,13 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - eventName
+  - requestParameters.x-amz-server-side-encryption
+  - requestParameters.bucketName
+  - requestParameters.x-amz-copy-source
+  - requestParameters.key
+  - userAgent
+  - region
   security_domain: threat

detections/cloud/aws_network_access_control_list_created_with_all_open_ports.yml

@@ -44,6 +44,18 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - eventName
+  - requestParameters.ruleAction
+  - requestParameters.egress
+  - requestParameters.aclProtocol
+  - requestParameters.portRange.to
+  - requestParameters.portRange.from
+  - requestParameters.cidrBlock
+  - userName
+  - userIdentity.principalId
+  - userAgent
   risk_object: userName
   risk_object_type: user
   risk_score: 10

detections/cloud/aws_network_access_control_list_deleted.yml

@@ -40,6 +40,14 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - eventName
+  - requestParameters.egress
+  - userName
+  - userIdentity.principalId
+  - src
+  - userAgent
   risk_object: userName
   risk_object_type: user
   risk_score: 5

detections/cloud/aws_saml_access_by_provider_user_and_principal.yml

@@ -40,4 +40,14 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - eventName
+  - requestParameters.principalArn
+  - requestParameters.roleArn
+  - requestParameters.roleSessionName
+  - recipientAccountId
+  - responseElements.issuer
+  - sourceIPAddress
+  - userAgent
   security_domain: threat

detections/cloud/aws_saml_update_identity_provider.yml

@@ -36,4 +36,13 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - eventName
+  - eventType
+  - requestParameters.sAMLProviderArn
+  - userIdentity.sessionContext.sessionIssuer.arn
+  - sourceIPAddress
+  - userIdentity.accessKeyId
+  - userIdentity.principalId
   security_domain: threat

detections/cloud/cloud_api_calls_from_previously_unseen_user_roles.yml

@@ -44,6 +44,13 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.user
+  - All_Changes.user_type
+  - All_Changes.status
+  - All_Changes.command
+  - All_Changes.object
   risk_object: user
   risk_object_type: user
   risk_score: 25

detections/cloud/cloud_compute_instance_created_by_previously_unseen_user.yml

@@ -42,6 +42,12 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.object
+  - All_Changes.action
+  - All_Changes.user
+  - All_Changes.vendor_region
   risk_object: user
   risk_object_type: user
   risk_score: 20

detections/cloud/cloud_compute_instance_created_in_previously_unused_region.yml

@@ -48,6 +48,12 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.object_id
+  - All_Changes.action
+  - All_Changes.vendor_region
+  - All_Changes.user
   risk_object: user
   risk_object_type: user
   risk_score: 20

detections/cloud/cloud_compute_instance_created_with_previously_unseen_image.yml

@@ -43,6 +43,12 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.object_id
+  - All_Changes.action
+  - All_Changes.Instance_Changes.image_id
+  - All_Changes.user
   risk_object: user
   risk_object_type: user
   risk_score: 20

detections/cloud/cloud_compute_instance_created_with_previously_unseen_instance_type.yml

@@ -43,6 +43,12 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.object_id
+  - All_Changes.action
+  - All_Changes.Instance_Changes.instance_type
+  - All_Changes.user
   risk_object: user
   risk_object_type: user
   risk_score: 20

detections/cloud/cloud_instance_modified_with_previously_unseen_user.yml

@@ -43,6 +43,14 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.object_id
+  - All_Changes.command
+  - All_Changes.action
+  - All_Changes.change_type
+  - All_Changes.status
+  - All_Changes.user
   risk_object: user
   risk_object_type: user
   risk_score: 10

detections/cloud/cloud_provisioning_from_previously_unseen_city.yml

@@ -58,6 +58,14 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.action
+  - All_Changes.status
+  - All_Changes.src
+  - All_Changes.user
+  - All_Changes.object
+  - All_Changes.command
   risk_object: user
   risk_object_type: user
   risk_score: 10

detections/cloud/cloud_provisioning_from_previously_unseen_country.yml

@@ -58,6 +58,14 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.action
+  - All_Changes.status
+  - All_Changes.src
+  - All_Changes.user
+  - All_Changes.object
+  - All_Changes.command
   risk_object: user
   risk_object_type: user
   risk_score: 5

detections/cloud/cloud_provisioning_from_previously_unseen_ip_address.yml

@@ -57,6 +57,14 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.object_id
+  - All_Changes.action
+  - All_Changes.status
+  - All_Changes.src
+  - All_Changes.user
+  - All_Changes.command
   risk_object: user
   risk_object_type: user
   risk_score: 5

detections/cloud/cloud_provisioning_from_previously_unseen_region.yml

@@ -58,6 +58,14 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - All_Changes.action
+  - All_Changes.status
+  - All_Changes.src
+  - All_Changes.user
+  - All_Changes.object
+  - All_Changes.command
   risk_object: user
   risk_object_type: user
   risk_score: 5

detections/cloud/detect_aws_console_login_by_new_user.yml

@@ -48,6 +48,10 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
+  required_fields:
+  - _time
+  - Authentication.signature
+  - Authentication.user
   risk_object: user
   risk_object_type: user
   risk_score: 30