Merge pull request #809 from splunk/develop

update master for v3.0.8

.circleci/config.yml

@@ -2,11 +2,16 @@
 #
 # Check https://circleci.com/docs/2.0/language-python/ for more details
 #
+#############
+# Automatically generated by ci-generator.py in splunk/security-content
+# On Date: 2020-07-30T03:07:52 UTC
+# Author: Splunk Security Research
+# Contact: research@splunk.com
+#############
 
 version: 2.1
 orbs:
   aws-cli: circleci/aws-cli@0.1.19
-  slack: circleci/slack@3.4.2
 
 dependencies:
   cache_directories:
@@ -82,10 +87,7 @@ jobs:
             cd security-content
             source venv/bin/activate
             python bin/doc-gen.py --path . --output docs -v
-      - slack/status:
-          webhook: '${SLACK_WEBHOOK}'
-          fail_only: true
-
+
   build-sources:
     executor: content-executor
     steps:
@@ -151,10 +153,7 @@ jobs:
           root: security-content/
           paths:
               - content-pack-build.tar.gz
-      - slack/status:
-          fail_only: true
-          webhook: '${SLACK_WEBHOOK}'
-
+
   build-package:
     executor: content-executor
     steps:
@@ -192,10 +191,7 @@ jobs:
           root: ~/dist
           paths:
              - DA-ESS-ContentUpdate-latest.tar.gz
-      - slack/status:
-          fail_only: true
-          webhook: '${SLACK_WEBHOOK}'
-
+
   run-appinspect:
     executor: content-executor
     steps:
@@ -205,7 +201,7 @@ jobs:
       - run:
           name: grab appinspect
           command: |
-            curl -Ls https://download.splunk.com/misc/appinspect/splunk-appinspect-2.0.0.tar.gz -o appinspect-lastest.tar.gz
+            curl -Ls https://download.splunk.com/misc/appinspect/splunk-appinspect-latest.tar.gz -o appinspect-lastest.tar.gz
             mkdir appinspect-latest
             tar -zxf appinspect-lastest.tar.gz -C appinspect-latest --strip-components=1
       - run:
@@ -228,10 +224,7 @@ jobs:
           root: ~/
           paths:
              - DA-ESS-ContentUpdate-latest.tar.gz
-      - slack/status:
-          fail_only: true
-          webhook: '${SLACK_WEBHOOK}'
-
+
   community-api-update:
     executor: aws-cli/default
     steps:
@@ -265,6 +258,7 @@ jobs:
           root: ~/
           paths:
              - DA-ESS-ContentUpdate-latest.tar.gz
+
   update-sources-github:
     executor: content-executor
     steps:
@@ -295,6 +289,23 @@ jobs:
             cd security-content
             source venv/bin/activate
             python bin/doc-gen.py --path . --output docs -v
+      - run:
+          name: get cti repo for mitre-maps
+          command: |
+            cd security-content
+            git clone https://github.com/mitre/cti.git
+      - run:
+          name: run generate-actors-map
+          command: |
+            cd security-content
+            source venv/bin/activate
+            python bin/generate-actors-map.py --projects_path . --output docs/mitre-map/
+      - run:
+          name: run generate-coverage-map
+          command: |
+            cd security-content
+            source venv/bin/activate
+            python bin/generate-coverage-map.py --projects_path . --output docs/mitre-map
       - run:
           name: update github with new docs and package bits
           command: |
@@ -315,10 +326,7 @@ jobs:
           root: ~/
           paths:
               - DA-ESS-ContentUpdate-latest.tar.gz
-      - slack/status:
-          fail_only: true
-          webhook: '${SLACK_WEBHOOK}'
-
+
   publish-github-release:
     docker:
       - image: cibuilds/github:0.10
@@ -334,10 +342,7 @@ jobs:
           root: ~/
           paths:
               - DA-ESS-ContentUpdate-latest.tar.gz
-      - slack/status:
-          fail_only: true
-          webhook: '${SLACK_WEBHOOK}'
-
+
   attack-range-update:
     executor: aws-cli/default
     steps:
@@ -351,10 +356,7 @@ jobs:
             aws s3 cp ~/DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries/
             # make the file public since it is not by default
             aws s3api put-object-acl --bucket attack-range-appbinaries --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read
-      - slack/status:
-          fail_only: true
-          webhook: '${SLACK_WEBHOOK}'
-
+
   master-api-update:
     executor: aws-cli/default
     steps:
@@ -382,10 +384,7 @@ jobs:
             aws s3 cp lookups s3://security-content/lookups --recursive --exclude "*" --include "*.csv"
             aws s3 cp macros s3://security-content/macros --recursive --exclude "*" --include "*.yml"
             aws s3 cp deployments s3://security-content/deployments --recursive --exclude "*" --include "*.yml"
-      - slack/status:
-          fail_only: true
-          webhook: '${SLACK_WEBHOOK}'
-
+
 workflows:
   version: 2.1
   validate-and-build:

.gitignore

@@ -82,6 +82,7 @@ celerybeat-schedule
 .venv
 env/
 venv/
+automated_detection_testing/venv/
 ENV/
 env.bak/
 venv.bak/

.gitlab-ci.yml

@@ -0,0 +1,15 @@
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+
+stages:
+  - ssa-validate
+
+validate:
+  stage: ssa-validate
+  image: python:3.7
+  before_script:
+    - pip3 install -r requirements.txt
+  script:
+    - python3 bin/validate.py -p . -v
+
+

README.md

@@ -1,6 +1,3 @@
-
-
-
 # Splunk Security Content
 ![security-content](docs/static/logo.png)
 =====
@@ -12,13 +9,13 @@
 
 Welcome to the Splunk Security Content
 
-This project gives you access to our repository of Analytic Stories that are security guides which provide background on TTPs, mapped to the MITRE framework, the Lockheed Martin Kill Chain, and CIS controls. They include Splunk searches, machine-learning algorithms, and Splunk Phantom playbooks (where available)—all designed to work together to detect, investigate, and respond to threats.
+This project gives you access to our repository of Analytic Stories that are security guides the provide background on TTPs, mapped to the MITRE framework, the Lockheed Martin Kill Chain, and CIS controls. They include Splunk searches, machine-learning algorithms, and Splunk Phantom playbooks (where available)—all designed to work together to detect, investigate, and respond to threats.
 
-# Usage
+# Usage🛡
 The Splunk Security Content can be used via:
 
 #### [Splunk App](https://github.com/splunk/security-content/releases)
-Grab the latest release of DA-ESS-ContentUpdate and install it on a Splunk Enterprise instance.
+Grab the latest release of DA-ESS-ContentUpdate and install it on a Splunk Enterprise instance. Alternatively, you can download it from [splunkbase](https://splunkbase.splunk.com/app/3449/), it is currently a Splunk Supported App.
 
 #### [API](https://docs.splunkresearch.com/?version=latest)
 ```
@@ -29,56 +26,75 @@ curl -s https://content.splunkresearch.com | jq
 ```
 
 #### [GitHub Workflow](https://github.com/splunk/security-content/wiki/Installation-and-Usage)
-Create your customized version of Security Content by forking this project and following this guide.
+Create your customized version of Security Content by forking this project and following this [guide](https://github.com/splunk/security-content/wiki/Installation-and-Usage#github-workflow).
 
-# What's in an Analytic Story?
-[Analytic Stories](https://github.com/splunk/security-content/blob/develop/docs/stories_categories.md) and their corresponding searches are composed of **.yml** files (manifests) and associated .conf files. The stories reside in [/stories](https://github.com/splunk/security-content/tree/develop/stories) and the searches live in [/detections](https://github.com/splunk/security-content/tree/develop/detections).
+# MITRE ATT&CK
+### Detection Coverage
+To view an up-to-date detection coverage map for all the content tagged with MITRE techniques visit: [https://mitremap.splunkresearch.com/](https://mitremap.splunkresearch.com/) under the **Detection Coverage** layer. Below is a snapshot in time of what technique we currently have some detection coverage for. The darker the shade of blue the more detections we have for this particular technique. This map is automatically updated on every release and generated from the [generate-coverage-map.py](https://github.com/splunk/security-content/blob/mitre_maps/bin/generate-coverage-map.py).
 
-Manifests contain a number of mandatory and optional fields. You can see the full field list for each piece of content [here](https://github.com/splunk/security-content/tree/develop/docs#spec-documentation).
+![](docs/mitre-map/coverage.png)
 
-# Customize to your Environment
+### Detection Priority by Threat Actors
+If curious about how the Threat Research team prioritizes what content to build refer to our **Detection Priority by Threat Actors** layer. Using the actor data from [MITRE CTI](https://github.com/mitre/cti) we add a point for every threat actor that uses a particular technique, and then subtract a point of every detection we have mapped to that technique. The resulting map below is how we prioritize what techniques and detections to focus on next. This map is automatically updated on every release and is generated by the [generate-actors-map.py](https://github.com/splunk/security-content/blob/mitre_maps/bin/generate-actors-map.py) script.
 
-After release [1.0.46](https://github.com/splunk/security-content/releases) we introduced a concept of **input(pre-filter)** and **output(post-filter)** macros for each of our detection search. The intention behind introducing these macros is primarily to help our users to update the macro definition “once” and those changes will be applicable across all detections that leverage that macro and  local to your Splunk Environment.
+![](docs/mitre-map/priority.png)
 
-**input(pre-filter):** This macro is to  specify your environment-specific configurations (index, source, sourcetype, etc.) to get the specific data sources that you would like to bring in. Replace the macro definition with configurations for your Splunk environment. For example the [sysmon](macros/sysmon.yml) **input macro** can be modified to the local splunk deployments index or sourcetype.
+# Customize to your Environment 🏗
+Customize your content to change how [often detections run](https://github.com/splunk/security-content/wiki/Customize-to-Your-Environment#customizing-scheduling-and-alert-actions-with-deployments), or what the right source type for [sysmon](https://github.com/splunk/security-content/wiki/Customize-to-Your-Environment#customizing-source-types-with-macros) in your environment is please follow this [guide](https://github.com/splunk/security-content/wiki/Customize-to-Your-Environment).  
 
-**output(post-filter):** This macro is to  specify your environment-specific values (eg: dest, user), to filter out known false positives.. Replace the macro definition with values that you’d like to exclude from detection results. Think of this as a whitelisting/blacklisting using macros. A good example
+# Writing Content 📓
+Please see the Developing Content [guide](https://github.com/splunk/security-content/wiki/Developing-Content) for instructions.
 
+# What's in an Analytic Story?
+A complete use case, specifically built to detect, investigate, and respond to a specific threat like [Credential Dumping](https://github.com/splunk/security-content/blob/develop/stories/credential_dumping.yml) or [Ransomware](). A group of detections and a response make up an analytic story, they are associated with the tag `analytics_story: <name>`.  
 
-# Execute an Analytic Story
-
-Download and install the latest version of [Splunk Analytic Story Execution]
-(https://github.com/splunk/analytic_story_execution/releases). This Splunk application will help the user do the following:
+# Execute an Analytic Story 🏃‍♀️
+Download and install the latest version of [Splunk Analytic Story Execution](https://github.com/splunk/analytic_story_execution/releases). This Splunk application will help the user do the following:
 
-1. Execute an analytic story in an adhoc mode and view the results.
+1. Execute an analytic story in an ad-hoc mode and view the results.
 2. Schedule all the detection searches in an analytic story.
 3. Update security-content via an API
 
+# Content Parts 🧩
 
-# Writing Content
-Before you begin, follow the steps to install **dependencies and pre-commit hooks** under [Developing Content](https://github.com/splunk/security-content/wiki/Developing-Content).
-
-# Security Content
-
-#### Content Parts
-* [stories/](stories/): All Analytic Stories
-* [detections/](detections/): Splunk Enterprise, Splunk UBA, and Splunk Phantom detections that power Analytic Stories
-* [response_tasks/](response_tasks/): Splunk Enterprise and Splunk Phantom investigative searches and playbooks employed by Analytic Stories
-* [responses/](responses/): Automated Splunk Enterprise and Splunk Phantom responses triggered by Analytic Stories
-* [baselines/](baselines/): Splunk Phantom and Splunk Enterprise baseline searches needed to support detection searches in Analytic Stories
+* [detections/](detections/): Contains all 209 detection searches to-date and growing.
+* [stories/](stories/): All Analytic Stories that are group detections or also known as Use Cases
+* [deployments/](deployments/): Configuration for the schedule and alert action for all content
+* [responses/](responses/): Incident Response Playbooks/Workflow for responding to a specific Use Case or Threat.
+* [response_tasks/](response_tasks/): Individual steps in responses that help the user investigate via a Splunk search, automate via a phantom playbook, and visualize via dashboards threats.
+* [baselines/](baselines/): Searches that must be executed before a detection runs. It is specifically useful for collecting data on a system before running your detection on the collected data.
+* [dashboards/](dashboards/): JSON definitions of Mission Control dashboards, to be used as a response task. Currently not used.
+* [macros/](macros/): Implements Splunk’s search macros, shortcuts to commonly used search patterns like sysmon source type. More on how macros are used to customize content below.
+* [lookups/](lookups/): Implements Splunk’s lookup, usually to provide a list of static values like commonly used ransomware extensions.
 
 #### Supporting Parts
 * [package/](package/): Splunk content app-source files, including lookups, binaries, and default config files
 * [bin/](bin/): All binaries required to produce and test content
 
-# Contribution
+# Contribution 🥰
 We welcome feedback and contributions from the community! Please see our [contributing to the project](https://github.com/splunk/security-content/wiki/Contributing-to-the-Project) for more information on how to get involved.
 
-## Support
+## Support 💪
 Please use the [GitHub Issue Tracker](https://github.com/splunk/security-content/issues) to submit bugs or request features.
 
 If you have questions or need support, you can:
 
 * Post a question to [Splunk Answers](http://answers.splunk.com)
 * Join the [#security-research](https://splunk-usergroups.slack.com/messages/C1RH09ERM/) room in the [Splunk Slack channel](http://splunk-usergroups.slack.com)
 * If you are a Splunk Enterprise customer with a valid support entitlement contract and have a Splunk-related question, you can also open a support case on the https://www.splunk.com/ support portal
+
+
+## License
+Copyright 2020 Splunk Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

automated_detection_testing/.dockerignore

@@ -0,0 +1,6 @@
+
+venv
+security-content
+attack_range
+attack_range.log
+Dockerfile

automated_detection_testing/Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:18.04
+MAINTAINER Patrick Bareiss
+
+RUN apt-get update
+RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
+RUN apt-get install -y python3-dev git python-dev unzip python3-pip awscli
+RUN apt-get install -y python-gitdb
+RUN apt-get install -y wget unzip
+
+RUN wget --quiet https://releases.hashicorp.com/terraform/0.13.1/terraform_0.13.1_linux_amd64.zip \
+  && unzip terraform_0.13.1_linux_amd64.zip \
+  && mv terraform /usr/bin \
+  && rm terraform_0.13.1_linux_amd64.zip
+
+ADD config /root/.aws/config
+ADD . /app
+
+WORKDIR /app
+RUN pip3 install -r requirements.txt
+
+ENTRYPOINT ["python3", "detection_service.py"]
+CMD ["-tfn", "T1003_002"]