qa-saas.yml

# qa-saas workflow is for scale testing and/or validation against the Nexodus qa or prod SaaS deployments
name: qa-saas

concurrency:
  group: qa-scale

on:
  workflow_dispatch:
    inputs:
      deployment_size:
        description: 'deployment size: small | medium | large | xlarge'
        required: true
        default: 'small'
        type: string

      pr_or_branch:
        description: 'pull request number or branch name'
        required: true
        default: 'main'

      debug_pause:
        description: 'time in minutes to pause before tearing down the infra for debugging'
        required: false
        default: '0'

      controller_address:
        description: 'options: qa.nexodus.io | try.nexodus.io'
        required: true
        default: 'qa.nexodus.io'
        type: string

  schedule:
    - cron: '0 17 * * *'

jobs:
  deploy-qa:
    name: deploy-qa-ec2
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      AWS_REGION: "us-east-1"
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      ANSIBLE_PRIVATE_KEY_FILE: "nexodus.pem"
      ANSIBLE_HOST_KEY_CHECKING: "false"
      # Set default values for scheduled runs
      DEPLOYMENT_SIZE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deployment_size || 'small' }}
      PR_OR_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.pr_or_branch || 'main' }}
      DEBUG_PAUSE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_pause || '0' }}
      CONTROLLER_ADDRESS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.controller_address || 'qa.nexodus.io' }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Determine if pr_or_branch is a PR number
        id: check_pr
        run: |
          if [[ "${{ github.event.inputs.pr_or_branch }}" =~ ^[0-9]+$ ]]; then
            echo "is_pr=true" >> "$GITHUB_OUTPUT"
          else
            echo "is_pr=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Fetch and checkout PR
        if: steps.check_pr.outputs.is_pr == 'true'
        run: |
          git fetch origin pull/${{ github.event.inputs.pr_or_branch }}/head:pr-${{ github.event.inputs.pr_or_branch }}
          git checkout pr-${{ github.event.inputs.pr_or_branch }}

      - name: Checkout branch
        if: steps.check_pr.outputs.is_pr == 'false'
        run: git checkout ${{ github.event.inputs.pr_or_branch }}

      - name: Setup Go
        uses: ./.github/actions/setup-go-env

      - name: Build
        run: |
          make dist/nexd-linux-amd64
          make dist/nexctl-linux-amd64

      - name: Configure aws credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE }}
          role-session-name: nexodus-ci-deploy
          aws-region: us-east-1

      - name: Copy binaries to s3
        run: |
          aws s3 cp ./dist/nexd-linux-amd64 s3://nexodus-io/ec2-e2e/qa/
          aws s3 cp ./dist/nexctl-linux-amd64 s3://nexodus-io/ec2-e2e/qa/

      - name: Set Controller and Keycloak Credentials
        run: |
          # Set controller address to default if not provided in event inputs
          controller_address=${{ github.event.inputs.controller_address || 'qa.nexodus.io' }}
          echo "controller_address=${controller_address}" >> "$GITHUB_ENV"
          controller_auth_address="auth.$controller_address"
          echo "CONTROLLER_AUTH_ADDRESS=$controller_auth_address" >> "$GITHUB_ENV"
          # Set Keycloak credentials based on controller address
          if [ "$controller_address" = "qa.nexodus.io" ]; then
          echo "KC_USERNAME=${{ secrets.KC_QA_USERNAME }}" >> "$GITHUB_ENV"
          echo "KC_PASSWORD=${{ secrets.KC_QA_PASSWORD }}" >> "$GITHUB_ENV"
          elif [ "$controller_address" = "try.nexodus.io" ]; then
          echo "KC_USERNAME=${{ secrets.KC_PROD_USERNAME }}" >> "$GITHUB_ENV"
          echo "KC_PASSWORD=${{ secrets.KC_PROD_PASSWORD }}" >> "$GITHUB_ENV"
          else
          # Default to qa.nexodus.io credentials if controller_address is not specified
          echo "KC_USERNAME=${{ secrets.KC_QA_USERNAME }}" >> "$GITHUB_ENV"
          echo "KC_PASSWORD=${{ secrets.KC_QA_PASSWORD }}" >> "$GITHUB_ENV"
          fi

      - name:  Build Keycloak Tool
        run: |
          go build -o ./ ./hack/e2e-scripts/kctool/

      - name: Create a Keycloak User
        id: kc-user
        run: |
          output=$(./kctool --create-user \
            -ku "${{ env.KC_USERNAME }}" \
            -kp "${{ env.KC_PASSWORD }}" \
            -u qa \
            -p "${{ secrets.QA_USER_PASSWORD }}" \
            "https://${{ env.CONTROLLER_AUTH_ADDRESS }}")
          echo "USER=$output" >> "$GITHUB_OUTPUT"

      - name: User results from Keycloak
        run: echo "User is ${{ steps.kc-user.outputs.USER }}"

      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install Ansible and Dependencies
        run: pip3.10 install boto boto3 ansible-vault ansible-core

      - name: Install amazon.aws Ansible library
        run: ansible-galaxy collection install amazon.aws

      - name: Set Default Deployment Size if Not Provided
        if: github.event_name == 'schedule' || github.event.inputs.deployment_size == ''
        run: |
          echo "${{ secrets.ANSIBLE_VARS_SMALL_QA }}" > ./ops/ansible/aws/vars.yml

      - name: Set Deployment Size to Small
        if: github.event.inputs.deployment_size == 'small'
        run: |
          echo "${{ secrets.ANSIBLE_VARS_SMALL_QA }}" > ./ops/ansible/aws/vars.yml

      - name: Set Deployment Size to Medium
        if: github.event.inputs.deployment_size == 'medium'
        run: |
          echo "${{ secrets.ANSIBLE_VARS_MEDIUM_QA }}" > ./ops/ansible/aws/vars.yml

      - name: Set Deployment Size to Large
        if: github.event.inputs.deployment_size == 'large'
        run: |
          echo "${{ secrets.ANSIBLE_VARS_LARGE_QA }}" > ./ops/ansible/aws/vars.yml

      - name: Set Deployment Size to XLarge
        if: github.event.inputs.deployment_size == 'xlarge'
        run: |
          echo "${{ secrets.ANSIBLE_VARS_XLARGE_QA }}" > ./ops/ansible/aws/vars.yml

      - name: Create Ansible Secrets
        run: |
          echo "${{ secrets.ANSIBLE_SSH_KEY }}" > nexodus.pem
          chmod 0400 nexodus.pem
          echo "${{ secrets.ANSIBLE_VAULT_PASSWORD }}" > vault-secret.txt
          chmod 0400 vault-secret.txt

      - name: Deploy EC2 Agent Nodes
        run: |
          ansible-playbook -vv ./ops/ansible/aws/deploy-ec2-qa.yml \
          -i ./ops/ansible/aws/inventory.txt \
          --private-key nexodus.pem \
          --vault-password-file vault-secret.txt \
          --extra-vars "nexodus_auth_uid=${{ steps.kc-user.outputs.USER }}" \
          --extra-vars "nexodus_auth_password=${{ secrets.QA_USER_PASSWORD }}" \
          --extra-vars "debug_pause=0" \
          --extra-vars "controller_address=$controller_address"

      - name: Mesh Connectivity Results
        run: |
          set -e
          /bin/sh -c 'cat ./ops/ansible/aws/*-connectivity-results.txt > ./ops/ansible/aws/mesh-connectivity-results.txt'
          cat ./ops/ansible/aws/mesh-connectivity-results.txt
          if grep -iq 'Unreachable' ./ops/ansible/aws/mesh-connectivity-results.txt || grep -iq 'Failed' ./ops/ansible/aws/mesh-connectivity-results.txt; then
            echo "Connectivity results contain 'Unreachable or Failed' nodes, check the connectivity results and artifacts for details. Failing the job"
            exit 1
          else
            echo "Connectivity results do not contain any 'Unreachable' nodes"
          fi

      - name: Terminate EC2 Instances
        if: always()
        run: |
          ansible-playbook -vv ./ops/ansible/aws/terminate-instances.yml \
          --private-key nexodus.pem \
          --vault-password-file vault-secret.txt

      - name: Upload nexd and api-server Logs to Artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: dev-ec2-artifacts
          path: ./ops/ansible/aws/nexd-logs/
          retention-days: 10