conformance-run.yml

name: Conformance Testing

on:
  workflow_dispatch:
  schedule:
    # Every day at midnight
    - cron: '0 0 * * *'

jobs:

  # conformance-run works through a matrix of actors and runs the conformance
  # test suite for each of them. Results are stored in a `reports` artifact for
  # later compilation and publishing.
  conformance-run:
    name: Run Conformance Suite
    runs-on: ubuntu-latest
    strategy:
      # Job must not stop iterating through matrix on failure
      fail-fast: false
      matrix:
        include:
          - name: "Mavennet"
            actor: "MAVENNET_STAGING"
          - name: "mesur.io"
            actor: "MESUR_IO_PRODUCTION"
          - name: "GS1US"
            actor: "GS1US"
          - name: 'BCGov'
            actor: 'BCGOV'
    steps:
      # Check out repo, set up node, and install dependencies.
      # @see https://github.com/actions/setup-node#usage
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: 16.15.1
          cache: 'npm'
      - run: npm ci
      - name: Run Tests
        env:
          organization_did_web: ${{secrets[format('{0}_ORGANIZATION_DID_WEB', matrix.actor)]}}
          client_id: ${{secrets[format('{0}_CLIENT_ID', matrix.actor)]}}
          client_secret: ${{secrets[format('{0}_CLIENT_SECRET', matrix.actor)]}}
          token_audience: ${{secrets[format('{0}_TOKEN_AUDIENCE', matrix.actor)]}}
          token_endpoint: ${{secrets[format('{0}_TOKEN_ENDPOINT', matrix.actor)]}}
          api_base_url: ${{secrets[format('{0}_API_BASE_URL', matrix.actor)]}}
        run: |
          npx newman run ./tests/conformance_suite.postman_collection.json \
          --env-var ORGANIZATION_DID_WEB=$organization_did_web \
          --env-var CLIENT_ID=$client_id \
          --env-var CLIENT_SECRET=$client_secret \
          --env-var TOKEN_AUDIENCE=$token_audience \
          --env-var TOKEN_ENDPOINT=$token_endpoint \
          --env-var API_BASE_URL=$api_base_url \
          --reporters cli,htmlextra,json \
          --reporter-htmlextra-skipSensitiveData \
          --reporter-htmlextra-export "newman/${{format('{0}-{1}-{2}.html', github.run_id, github.job, matrix.name)}}" \
          --reporter-json-export "newman/${{format('{0}-{1}-{2}.json', github.run_id, github.job, matrix.name)}}"

      # Write sanitized Newman output to `./docs/reports`.
      - name: Sanitize Report Output
        if: always() # Run even when postman tests fail
        run: npm run report:sanitize

      # Sanitized reports are needed by subsequent jobs
      - uses: actions/upload-artifact@v3
        if: always() # Run even when postman tests fail
        with:
          name: reports
          path: docs/reports/

  # conformance-compile compiles the sanitized raw report output into an
  # interactive HTML conformance report suitable for publishing.
  conformance-compile:
    name: Compile Reports
    runs-on: ubuntu-latest

    # Run even when postman tests fail
    if: always()

    # Wait for jobs which generate reporting input.
    needs:
      - conformance-run

    steps:
      # Check out repo, setup node, and install dependencies.
      # @see https://github.com/actions/setup-node#usage
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: 16.15.1
          cache: 'npm'
      - run: npm ci

      # Download 'reports' artifact
      - uses: actions/download-artifact@v3
        with:
          name: reports
          path: docs/reports/

      # Build report index
      - name: Build JSON Index
        run: npm run report:index -- --folder conformance

      # Set up Python 3.10
      - name: Set up Python 3.10
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: pip
      - run: pip install -r ./reporting/requirements.txt

      # Compile interactive report
      - name: Build Conformance Report
        working-directory: ./reporting
        run: ./reporter.py --mode ci --conformance

      # Compiled report must be added to artifact
      - uses: actions/upload-artifact@v3
        with:
          name: reports
          path: docs/reports/

  # conformance-publish publishes sanitized raw and interactive HTML reports to
  # the `reports/conformance` directory in GitHub pages. This job is restricted
  # to commits on the `main` branch to prevent feature branches from overwriting
  # production reports.
  conformance-publish:
    name: Publish Report
    runs-on: ubuntu-latest

    # Publishing must only happen when triggered on the 'main' branch to prevent
    # tests on other branches from polluting the published reports. Note that
    # `always()`` is a special flag that is REQUIRED in order to get this job to
    # run if jobs listed in the `needs:` section fail.
    if: github.ref == 'refs/heads/main' && always()

    # Report must be compiled before publishing
    needs:
      - conformance-compile

    steps:
      # Check out repo
      - uses: actions/checkout@v3

      # Download 'reports' artifact
      - uses: actions/download-artifact@v3
        with:
          name: reports
          path: docs/reports/

      # Publish report subfolder to GitHub Pages
      - name: Publish Conformance Report
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./docs/reports
          destination_dir: reports/conformance

      # Prepare an archive folder name and update the index.json appropriately
      - name: Prepare Conformance Archive
        id: archive
        run: |
          FOLDER=conformance-$(date +'%s')
          sed -i "s#reports/conformance/#reports/$FOLDER/#" ./docs/reports/index.json
          echo "::set-output name=folder::$FOLDER"

      # Publish report archive subfolder to GitHub Pages
      - name: Publish Conformance Archive
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./docs/reports
          destination_dir: reports/${{ steps.archive.outputs.folder }}