diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/build.yml similarity index 67% rename from .github/workflows/validate-and-build.yml rename to .github/workflows/build.yml index 1e4ddc884f..9d15eb041c 100644 --- a/.github/workflows/validate-and-build.yml +++ b/.github/workflows/build.yml @@ -1,51 +1,36 @@ -name: validate-and-build +name: build on: - push: pull_request: - types: [opened, reopened] + types: [opened, reopened, synchronize] jobs: - - validate-and-build: + build: #Note that the CircleCI job used a Container. The way to do this with Github Actions #is to first start up a Virtual Machine, then we can by following: # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainer runs-on: ubuntu-latest steps: - name: Check out the repository code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | - python3.11 -m venv .venv - source .venv/bin/activate pip install contentctl git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git - - name: content_ctl validate - run: | - source .venv/bin/activate - contentctl validate - - - name: contentctl generate + - name: Running build with enrichments run: | - source .venv/bin/activate contentctl build --enrichments mkdir artifacts mv dist/DA-ESS-ContentUpdate-latest.tar.gz artifacts/ - name: store_artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: content-latest path: | diff --git a/.github/workflows/detection-testing.yml b/.github/workflows/detection-testing.yml deleted file mode 100644 index df6254ce12..0000000000 --- a/.github/workflows/detection-testing.yml +++ /dev/null @@ -1,347 +0,0 @@ -# name: detection-testing -# on: -# push: -# pull_request: -# types: [opened, reopened] -# schedule: -# - cron: "44 4 * * *" -# jobs: - -# validate-tag-if-present: -# runs-on: ubuntu-latest - -# steps: -# - name: TAGGED, Validate that the tag is in the correct format - -# run: | -# echo "The GITHUB_REF: $GITHUB_REF" -# #First check to see if the release is a tag -# if [[ $GITHUB_REF =~ refs/tags/* ]]; then -# #Yes, this is a tag, so we need to test to make sure that the tag -# #is in the correct format (like v1.10.20) -# if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then -# echo "PASS: Tagged release with good format" -# exit 0 -# else -# echo "FAIL: Tagged release with bad format" -# exit 1 -# fi -# else -# echo "PASS: Not a tagged release" -# exit 0 -# fi - -# quit-for-dependabot: -# runs-on: ubuntu-latest -# if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' -# steps: -# - name: "Placeholder" -# run: | -# echo "yes it ran" - -# docker-detection-testing-setup: -# runs-on: ubuntu-latest -# if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job -# needs: [validate-tag-if-present, quit-for-dependabot] -# steps: -# - name: Get branch and PR required for detection testing main.py -# id: vars -# run: | -# echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - -# - name: Checkout Repo -# uses: actions/checkout@v2 -# #with: -# # ref: develop - - - -# - uses: actions/setup-python@v2 -# with: -# python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy -# architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified -# cache: 'pip' - -# - name: Install Python Dependencies -# run: | -# python -m venv .venv -# source .venv/bin/activate -# python -m pip install wheel -# python -m pip install -r requirements.txt - -# - name: Run the CI -# run: | -# source .venv/bin/activate -# cd bin/docker_detection_tester -# echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]" -# echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]" -# echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]" -# echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]" -# echo "github.event_name : [${{ github.event_name }}]" - - -# if [[ ${{ github.event_name }} == schedule ]]; then -# # Note that scheduled actions ONLY run on the default branch, so it won't run on all other branches! -# echo "Running a nightly test on all detections OR a commit was made directly to develop" -# python detection_testing_execution.py run --branch develop --mode all --mock --config_file test_config_github_actions.json -# elif [[ ! -z "${{ github.event.pull_request.head.ref }}" && ! -z "${{ github.event.pull_request.number }}" ]]; then -# echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]" -# python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }} --mode changes --mock --config_file test_config_github_actions.json -# else -# echo "Push from branch [${{ steps.vars.outputs.branch }}]" -# python detection_testing_execution.py run --branch ${{ steps.vars.outputs.branch }} --mode changes --mock --config_file test_config_github_actions.json -# fi - -# mv *-test-run.json replicate_test.json -# - name: Upload Test Results Files -# uses: actions/upload-artifact@v2 -# with: -# name: testing-results-config -# path: | -# bin/docker_detection_tester/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz -# bin/docker_detection_tester/prior_config/config_tests_0.json -# bin/docker_detection_tester/prior_config/config_tests_1.json -# bin/docker_detection_tester/prior_config/config_tests_2.json -# bin/docker_detection_tester/prior_config/config_tests_3.json -# bin/docker_detection_tester/prior_config/config_tests_4.json -# bin/docker_detection_tester/prior_config/config_tests_5.json -# bin/docker_detection_tester/prior_config/config_tests_6.json -# bin/docker_detection_tester/prior_config/config_tests_7.json -# bin/docker_detection_tester/prior_config/config_tests_8.json -# bin/docker_detection_tester/prior_config/config_tests_9.json - -# - name: Upload File to Enable Replication of the Test at a Different Time or Place -# uses: actions/upload-artifact@v2 -# with: -# name: replicate_test -# path: | -# bin/docker_detection_tester/replicate_test.json - -# docker-detection-testing-execution: -# runs-on: ubuntu-latest -# if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job -# needs: [docker-detection-testing-setup] -# strategy: -# matrix: -# manifest_filename: ["config_tests_0.json", -# "config_tests_1.json", -# "config_tests_2.json", -# "config_tests_3.json", -# "config_tests_4.json", -# "config_tests_5.json", -# "config_tests_6.json", -# "config_tests_7.json", -# "config_tests_8.json", -# "config_tests_9.json"] -# steps: -# - name: Get branch and PR required for detection testing main.py -# id: vars -# run: | -# echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - -# - name: Checkout Repo -# uses: actions/checkout@v2 -# #with: -# # ref: develop - -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: testing-results-config -# path: bin/docker_detection_tester/prior_config - - -# - uses: actions/setup-python@v2 -# with: -# python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy -# architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified -# cache: 'pip' - -# - name: Install Python Dependencies -# run: | -# python -m venv .venv -# source .venv/bin/activate -# python -m pip install wheel -# python -m pip install -r requirements.txt - -# - name: Run the CI -# run: | -# source .venv/bin/activate -# cd bin/docker_detection_tester - - -# python detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} - - -# - name: Upload Test Results Files -# uses: actions/upload-artifact@v2 -# with: -# name: ${{ matrix.manifest_filename}}.results -# path: | -# bin/docker_detection_tester/test_results/success.csv -# bin/docker_detection_tester/test_results/error.csv -# bin/docker_detection_tester/test_results/failure.csv -# bin/docker_detection_tester/test_results/combined.csv -# bin/docker_detection_tester/test_results/success.json -# bin/docker_detection_tester/test_results/error.json -# bin/docker_detection_tester/test_results/failure.json -# bin/docker_detection_tester/test_results/combined.json - -# bin/docker_detection_tester/test_results/summary.json - -# docker-detection-testing-execution-merge-results: -# runs-on: ubuntu-latest -# if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job -# needs: [docker-detection-testing-setup, docker-detection-testing-execution] - -# steps: -# - name: Get branch and PR required for detection testing main.py -# id: vars -# run: | -# echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - -# - name: Checkout Repo -# uses: actions/checkout@v2 -# #with: -# # ref: develop - -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_0.json.results -# path: bin/docker_detection_tester/results_0 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_1.json.results -# path: bin/docker_detection_tester/results_1 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_2.json.results -# path: bin/docker_detection_tester/results_2 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_3.json.results -# path: bin/docker_detection_tester/results_3 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_4.json.results -# path: bin/docker_detection_tester/results_4 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_5.json.results -# path: bin/docker_detection_tester/results_5 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_6.json.results -# path: bin/docker_detection_tester/results_6 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_7.json.results -# path: bin/docker_detection_tester/results_7 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_8.json.results -# path: bin/docker_detection_tester/results_8 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_9.json.results -# path: bin/docker_detection_tester/results_9 - -# - uses: actions/setup-python@v2 -# with: -# python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy -# architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified -# cache: 'pip' - -# - name: Install Python Dependencies -# run: | -# python -m venv .venv -# source .venv/bin/activate -# python -m pip install wheel -# python -m pip install -r requirements.txt - -# - name: Merge Detections into single File -# run: | -# source .venv/bin/activate -# cd bin/docker_detection_tester -# python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json - - -# - name: Upload Summary Test Results JSON -# uses: actions/upload-artifact@v2 -# if: always() -# with: -# name: SummaryTestResults -# path: | -# bin/docker_detection_tester/summary_test_results.json - -# - name: Upload Failures Manifest on Failure -# uses: actions/upload-artifact@v2 -# if: failure() -# with: -# name: DetectionFailureManifest -# path: | -# bin/docker_detection_tester/detection_failure_manifest.json - - -# #Always clean these up, they make the output messy -# - name: Clean up intermediate Files -# uses: geekyeggo/delete-artifact@v1 -# if: always() -# with: -# name: | -# config_tests_0.json.results -# config_tests_1.json.results -# config_tests_2.json.results -# config_tests_3.json.results -# config_tests_4.json.results -# config_tests_5.json.results -# config_tests_6.json.results -# config_tests_7.json.results -# config_tests_8.json.results -# config_tests_9.json.results - -# - name: Log in to S3 for Artifact Uploads -# if: ${{ github.event_name == 'schedule' }} -# uses: aws-actions/configure-aws-credentials@v1 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - -# aws-region: us-west-2 - -# - name: Upload S3 Badge and Summary Artifacts for Nightly Scheduled Run -# if: ${{ github.event_name == 'schedule' }} -# run: | -# cd bin/docker_detection_tester -# python generate_detection_coverage_badge.py --input_summary_file summary_test_results.json --output_badge_file detection_coverage.svg --badge_string "Pass Rate" - - -# #Upload artifact (summary test results) -# aws s3 cp summary_test_results.json s3://security-content/reporting/summary_test_results.json - -# #Since these reside in a public bucket, no need to explicitly mark as public -# # make the file public since it is not by default -# #aws s3api put-object-acl --bucket security-content --key reporting/summary_test_results.json --acl public-read - - -# #Upload artifact (test results coverage badge) -# aws s3 cp detection_coverage.svg s3://security-content/reporting/detection_coverage.svg - -# #Since these reside in a public bucket, no need to explicitly mark as public -# # make the file public since it is not by default -# #aws s3api put-object-acl --bucket security-content --key reporting/detection_coverage.svg --acl public-read - - - - - diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py new file mode 100644 index 0000000000..d142af3321 --- /dev/null +++ b/.github/workflows/format_test_results.py @@ -0,0 +1,67 @@ +""" +A simple script formatting test_results/summary.yml to display on github actions +""" + +import yaml +import re +import os +import sys + +def main(): + + # Define the path to the YAML file + # yaml_file_path = 'summary.yml' + yaml_file_path = '/home/runner/work/security_content/security_content/test_results/summary.yml' + + # Check if the YAML file exists + if not os.path.exists(yaml_file_path): + print(f"Error: The file {yaml_file_path} does not exist.") + exit(1) # Exit with an error code + + # Load the YAML file + with open(yaml_file_path, 'r') as file: + data = yaml.safe_load(file) + + # Extract total_fail value and debug print it + total_fail = data['summary']['total_fail'] + total_detections = data['summary']['total_detections'] + print("**Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.**\n") + print(" 📝 **Experimental or manual_test detections are not tested** 📝 **\n") + print(f"Extracted total_fail: [{total_fail}]\n") + + # Print all unit test details first + print(" 🏗️⚒️ **Unit Test Details:**\n") + print(f"{'Name':<80} | {'Status':<6} | {'Test Type':<10} | {'Exception':<50}") + print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") + for detection in data['tested_detections']: + for test in detection['tests']: + if test['test_type'].strip() == "unit": # Check if the test type is "unit" + name = detection['name'].strip() + status = 'PASS' if test['success'] else 'FAIL' + test_type = test['test_type'].strip() + exception = test.get('exception', 'N/A') # Get exception if exists, else 'N/A' + if status == 'FAIL': + print(f"{name:<80} | 🔴 {status:<6} | {test_type:<10} | {exception:<50}") + else: + print(f"{name:<80} | 🟢 {status:<6} | {test_type:<10} | {'-':<50}") + # Print table footer + # print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") + + # Check if total_fail is a valid integer and greater than or equal to one + print("\n") # Print a newline for separation + print("**Overall Status**") + print("-------------------------------") + # Continue with additional prints or other logic + if int(total_fail) >=1: + # Print the message in bold + print("🔴 - **CI Failure: There are failed tests.**\n\n") + sys.exit(1) + if int(total_fail) < 1: + print("🟢 - **CI Success: No failed tests.**\n\n") + sys.exit(0) + if int(total_detections) < 1: + print("🔵 - **CI Success: No detections to test**\n\n") + sys.exit(0) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml new file mode 100644 index 0000000000..970339f562 --- /dev/null +++ b/.github/workflows/unit-testing.yml @@ -0,0 +1,59 @@ +name: unit-testing +on: + pull_request: + types: [opened, reopened, synchronize] +jobs: + unit-testing: + runs-on: ubuntu-latest + if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job + # needs: [validate-tag-if-present, quit-for-dependabot] + steps: + - name: Check out the repository code + uses: actions/checkout@v4 + with: + ref: develop + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + + - name: Install Python Dependencies and ContentCTL + run: | + python -m pip install --upgrade pip + pip install contentctl + + # Running contentctl test with a few arguments, before running the command make sure you checkout into the current branch of the pull request. This step only performs unit testing on all the changes against the target-branch. In most cases this target branch will be develop + - name: Run ContentCTL test for changes against target branch + run: | + echo "Current Branch (Head Ref): ${{ github.head_ref }}" + echo "Target Branch (Base Ref): ${{ github.base_ref }}" + git pull > /dev/null 2>&1 + git checkout ${{ github.head_ref }} + echo "The target branch for this PR is ${{ github.base_ref }}" + contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + echo "contentctl test - COMPLETED" + continue-on-error: true + + # Store test_results/summary.yml and dist/DA-ESS-ContentUpdate-latest.tar.gz to job artifact-test_summary_results.zip + - name: store_artifacts + uses: actions/upload-artifact@v4 + with: + name: test_summary_results + path: | + test_results/summary.yml + dist/DA-ESS-ContentUpdate-latest.tar.gz + continue-on-error: true + + # Print entire result summary so that the users can view it in the Github Actions logs + - name: Print entire test_results/summary.yml + run: cat test_results/summary.yml + continue-on-error: true + + # Run a simple custom script created to pretty print results in a markdown friendly format in Github Actions Summary + - name: Check the test_results/summary.yml for pass/fail. + run: | + echo "This job will fail if there are failures in unit-testing" + python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY + echo "The Unit testing is completed. See details in the unit-testing job summary UI " + \ No newline at end of file