Merge branch 'develop' into main

clemlesne · May 10, 2023 · 06fcfb5 · 06fcfb5
2 parents a8e46ab + 7b5b9fb
commit 06fcfb5
Show file tree

Hide file tree

Showing 12 changed files with 449 additions and 77 deletions.
diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml
@@ -37,22 +37,46 @@ env:
   AZURE_CLI_VERSION: 2.48.1
   # https://github.com/stedolan/jq/releases
   JQ_VERSION: 1.6
+  # https://github.com/aws/aws-cli/releases
+  AWS_CLI_VERSION: 2.11.18
   # https://github.com/git-for-windows/git/releases
   GIT_VERSION_WIN: 2.40.1
   # https://github.com/facebook/zstd/releases
   ZSTD_VERSION_WIN: 1.5.5
+  # https://www.python.org/downloads/windows
+  PYTHON_VERSION_WIN: 3.11.3
 
 jobs:
+  sast-creds:
+    name: SAST - Credentials
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.5.2
+        with:
+          # We need all Git history for testing credentials
+          fetch-depth: 0
+          # Ensure all submodules up-to-date
+          submodules: recursive
+      - name: SAST - Credentials
+        uses: trufflesecurity/trufflehog@v3.33.0
+        with:
+          base: ${{ github.event.repository.default_branch }}
+          head: HEAD
+          path: .
   build-helm:
     name: Build Helm chart
+    needs:
+      - sast-creds
+      - sast-semgrep
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3.5.2
         with:
           # We need all Git history for "version.sh"
           fetch-depth: 0
-          # Ensure "version.sh" submodule us up-to-date
+          # Ensure "version.sh" submodule are up-to-date
           submodules: recursive
 
       - name: Version
@@ -117,7 +141,6 @@ jobs:
       - build-publish-linux
       - build-publish-win
       - build-helm
-      - semgrep
     # Only deploy on non-scheduled main branch, as there is only one Helm repo and we cannot override an existing version
     if: (github.event_name != 'schedule') && (github.ref == 'refs/heads/main')
     runs-on: ubuntu-22.04
@@ -150,7 +173,8 @@ jobs:
   build-publish-linux:
     name: Build & deploy Linux image "${{ matrix.os }}"
     needs:
-      - semgrep
+      - sast-semgrep
+      - sast-creds
     runs-on: ubuntu-22.04
     strategy:
       matrix:
@@ -169,7 +193,7 @@ jobs:
         with:
           # We need all Git history for "version.sh"
           fetch-depth: 0
-          # Ensure "version.sh" submodule us up-to-date
+          # Ensure "version.sh" submodule are up-to-date
           submodules: recursive
 
       - name: Configure Git
@@ -237,6 +261,7 @@ jobs:
         uses: docker/build-push-action@v4.0.0
         with:
           build-args: |
+            "AWS_CLI_VERSION=${{ env.AWS_CLI_VERSION }}"
             "AZP_AGENT_VERSION=${{ env.AZP_AGENT_VERSION }}"
             "AZURE_CLI_VERSION=${{ env.AZURE_CLI_VERSION }}"
             "BUILDKIT_VERSION=${{ env.BUILDKIT_VERSION }}"
@@ -275,7 +300,8 @@ jobs:
   build-publish-win:
     name: Build & deploy Windows image "${{ matrix.os }}"
     needs:
-      - semgrep
+      - sast-semgrep
+      - sast-creds
     runs-on: ${{ matrix.runs-on }}
     strategy:
       matrix:
@@ -290,7 +316,7 @@ jobs:
         with:
           # We need all Git history for "version.sh"
           fetch-depth: 0
-          # Ensure "version.sh" submodule us up-to-date
+          # Ensure "version.sh" submodule are up-to-date
           submodules: recursive
 
       - name: Configure Git
@@ -349,11 +375,13 @@ jobs:
         run: |
           $params = @(
             # Required build arguments
+            "--build-arg", "AWS_CLI_VERSION=${{ env.AWS_CLI_VERSION }}",
             "--build-arg", "AZP_AGENT_VERSION=${{ env.AZP_AGENT_VERSION }}",
             "--build-arg", "AZURE_CLI_VERSION=${{ env.AZURE_CLI_VERSION }}",
             "--build-arg", "GIT_VERSION=${{ env.GIT_VERSION_WIN }}",
             "--build-arg", "JQ_VERSION=${{ env.JQ_VERSION }}",
             "--build-arg", "POWERSHELL_VERSION=${{ env.POWERSHELL_VERSION }}",
+            "--build-arg", "PYTHON_VERSION=${{ env.PYTHON_VERSION_WIN }}",
             "--build-arg", "YQ_VERSION=${{ env.YQ_VERSION }}",
             "--build-arg", "ZSTD_VERSION=${{ env.ZSTD_VERSION_WIN }}",
 
@@ -363,9 +391,9 @@ jobs:
           $tags = ('${{ steps.meta.outputs.tags }}').Split([Environment]::NewLine)
           foreach ($tag in $tags) {
             $params += "--tag", $tag
+          }
 
-            # Pull image locally to use as cache
-            docker pull --quiet $tag || true
+          foreach ($tag in $tags) {
             $params += "--cache-from", $tag
           }
 
@@ -374,16 +402,31 @@ jobs:
             $params += "--label", $label
           }
 
-          # Build
-          docker build @params src\docker
+          Write-Host "Build arguments:"
+          $params | ForEach-Object -Begin { $i = 0 } -Process {
+            if ($i % 2 -eq 0) {
+              Write-Host -NoNewline "`n  $_"
+            } else {
+              Write-Host -NoNewline " $_"
+            }
+            $i++
+          }
+          Write-Host
 
-          # Push
+          Write-Host "Pulling images for cache:"
           foreach ($tag in $tags) {
-            docker push --quiet $tag
+            Write-Host "  $tag"
+            docker pull --quiet $tag || true
           }
 
-  semgrep:
-    name: SAST Semgrep
+          Write-Host "Building..."
+          docker build @params src\docker
+
+          Write-Host "Pushing..."
+          docker push --quiet --all-tags ${{ env.CONTAINER_REGISTRY }}/${{ env.CONTAINER_NAME }}
+
+  sast-semgrep:
+    name: SAST - Semgrep
     runs-on: ubuntu-22.04
     container:
       image: returntocorp/semgrep
@@ -394,7 +437,7 @@ jobs:
       - name: Run tests
         run: semgrep ci --sarif --output=semgrep.sarif
         env:
-          SEMGREP_RULES: p/cwe-top-25 p/owasp-top-ten p/secrets p/kubernetes p/dockerfile
+          SEMGREP_RULES: p/cwe-top-25 p/owasp-top-ten p/kubernetes p/dockerfile
 
       - name: Upload results to GitHub CodeQL
         uses: github/codeql-action/upload-sarif@v2.2.12

diff --git a/README.md b/README.md
@@ -65,25 +65,6 @@ helm upgrade --install agent clemlesne-azure-pipelines-agent/azure-pipelines-age
 
 ## Advanced topics
 
-### Provided software
-
-#### Linux
-
-- [Azure Pipelines agent](https://github.com/microsoft/azure-pipelines-agent) (see env var `AZP_AGENT_VERSION` on the container images) + [requirements](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops#linux)
-- [ASP.NET Core Runtime](https://github.com/dotnet/aspnetcore) (required by the Azure Pipelines agent)
-- [Azure CLI](https://github.com/Azure/azure-cli) (required by the Azure Pipelines agent) + requirements ([Python 3.8](https://www.python.org/downloads/release/python-380), [Python 3.9](https://www.python.org/downloads/release/python-390), [Python 3.10](https://www.python.org/downloads/release/python-3100), depending of the system, plus C/Rust build tools for libs non pre-built on the platforms)
-- [PowerShell Core](https://github.com/PowerShell/PowerShell), [bash](https://www.gnu.org/software/bash) and [zsh](https://www.zsh.org) (for inter-operability)
-- [BuildKit](https://github.com/moby/buildkit) + requirements ([dbus-user-session](https://dbus.freedesktop.org), [fuse-overlayfs](https://github.com/containers/fuse-overlayfs), [iptables](https://www.netfilter.org/projects/iptables/index.html), [shadow-utils](https://github.com/shadow-maint/shadow), [uidmap](https://github.com/shadow-maint/shadow))
-- [gzip](https://www.gnu.org/software/gzip), [jq](https://github.com/stedolan/jq), [make](https://www.gnu.org/software/make), [tar](https://www.gnu.org/software/tar), [unzip](https://infozip.sourceforge.net/UnZip.html), [wget](https://www.gnu.org/software/wget), [yq](https://github.com/mikefarah/yq), [zip](https://infozip.sourceforge.net/Zip.html), [zstd](https://github.com/facebook/zstd) (for developer ease-of-life)
-
-#### Windows
-
-- [Azure Pipelines agent](https://github.com/microsoft/azure-pipelines-agent) (see env var `AZP_AGENT_VERSION` on the container images) + [requirements](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops#linux)
-- [.NET Framework Runtime](https://dotnet.microsoft.com) (required by the Azure Pipelines agent)
-- [Azure CLI](https://github.com/Azure/azure-cli) (required by the Azure Pipelines agent)
-- [PowerShell Core](https://github.com/PowerShell/PowerShell) (for inter-operability)
-- [git](https://github.com/git-for-windows/git), [jq](https://github.com/stedolan/jq), [yq](https://github.com/mikefarah/yq), [zstd](https://github.com/facebook/zstd) (for developer ease-of-life)
-
 ### Capabilities
 
 Capabilities are declarative variables you can add to the agents, to allow developers to select the right agent for their pipeline ([official documentation](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml)).
@@ -218,8 +199,8 @@ Linux systems are supported, but not Windows:
 | `ghcr.io/clemlesne/azure-pipelines-agent:focal-main` | ✅ |
 | `ghcr.io/clemlesne/azure-pipelines-agent:jammy-main` | ✅ |
 | `ghcr.io/clemlesne/azure-pipelines-agent:ubi8-main` | ✅ |
-| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019` | ❌ |
-| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022` | ❌ |
+| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2019-main` | ❌ |
+| `ghcr.io/clemlesne/azure-pipelines-agent:win-ltsc2022-main` | ❌ |
 
 #### How to use the bundled BuildKit
 
@@ -283,6 +264,85 @@ steps:
 
 Same way, if you want to use multiple versions of the framework, re-execute the task with the new version. Installations are cached locally.
 
+### Run the agent with a custom root certificate
+
+If you need to run the agent with a custom root certificate, you can use the following Helm values. Format is [PEM certificate](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) and with [UTF-8](https://en.wikipedia.org/wiki/UTF-8) encoding.
+
+Paths are `/app-root/azp-custom-certs` for Linux-based agents and `C:\app-root\azp-custom-certs` for Windows-based agents.
+
+```yaml
+# config-root-ca.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: custom-certs
+data:
+  root-1.crt: |
+    -----BEGIN CERTIFICATE-----
+    ...
+    -----END CERTIFICATE-----
+  root-2.crt: |
+    -----BEGIN CERTIFICATE-----
+    ...
+    -----END CERTIFICATE-----
+```
+
+```yaml
+# values.yaml
+extraVolumes:
+  - name: custom-certs
+    configMap:
+      name: custom-certs
+extraVolumeMounts:
+  - name: custom-certs
+    mountPath: /app-root/azp-custom-certs
+    readOnly: true
+```
+
+### Provided software
+
+#### Linux
+
+- [Azure Pipelines agent](https://github.com/microsoft/azure-pipelines-agent) + [requirements](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops#linux)
+- [BuildKit](https://github.com/moby/buildkit) + requirements ([dbus-user-session](https://dbus.freedesktop.org), [fuse-overlayfs](https://github.com/containers/fuse-overlayfs), [iptables](https://www.netfilter.org/projects/iptables/index.html), [shadow-utils](https://github.com/shadow-maint/shadow), [uidmap](https://github.com/shadow-maint/shadow))
+- Cloud providers CLIs
+  - [AWS CLI](https://github.com/aws/aws-cli)
+  - [Azure CLI](https://github.com/Azure/azure-cli)
+- Shells
+  - [bash](https://www.gnu.org/software/bash)
+  - [PowerShell Core](https://github.com/PowerShell/PowerShell)
+  - [zsh](https://www.zsh.org)
+- Programming languages
+  - [ASP.NET Core Runtime](https://github.com/dotnet/aspnetcore)
+  - Python ([Python 3.8](https://www.python.org/downloads/release/python-380), [Python 3.9](https://www.python.org/downloads/release/python-390), [Python 3.10](https://www.python.org/downloads/release/python-3100), depending of the system, plus C/Rust build tools for libs non pre-built on the platforms)
+- Tools
+  - [gzip](https://www.gnu.org/software/gzip)
+  - [jq](https://github.com/stedolan/jq)
+  - [make](https://www.gnu.org/software/make)
+  - [tar](https://www.gnu.org/software/tar)
+  - [unzip](https://infozip.sourceforge.net/UnZip.html)
+  - [wget](https://www.gnu.org/software/wget)
+  - [yq](https://github.com/mikefarah/yq)
+  - [zip](https://infozip.sourceforge.net/Zip.html)
+  - [zstd](https://github.com/facebook/zstd)
+
+#### Windows
+
+- [Azure Pipelines agent](https://github.com/microsoft/azure-pipelines-agent) + [requirements](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/docker?view=azure-devops#linux)
+- Cloud providers CLIs
+  - [Azure CLI](https://github.com/Azure/azure-cli)
+  - [AWS CLI](https://github.com/aws/aws-cli)
+- Shells
+  - [PowerShell Core](https://github.com/PowerShell/PowerShell)
+- Programming languages
+  - [.NET Framework Runtime](https://dotnet.microsoft.com)
+  - [Python 3.11](https://www.python.org/downloads/release/python-3110)
+- Tools
+  - [git](https://github.com/git-for-windows/git)
+  - [jq](https://github.com/stedolan/jq)
+  - [yq](https://github.com/mikefarah/yq)
+  - [zstd](https://github.com/facebook/zstd)
+
 ### Helm values
 
 | Parameter | Description | Default |

diff --git a/src/docker/Dockerfile-bullseye b/src/docker/Dockerfile-bullseye
@@ -56,6 +56,16 @@ COPY arch.sh .
 RUN chmod +x arch.sh \
     && bash arch.sh
 
+# Upgrade Python build tools
+RUN python3 -m pip \
+    --disable-pip-version-check \
+    --no-cache-dir \
+    --quiet \
+    install  \
+        --compile \
+        --upgrade \
+        pip setuptools wheel
+
 FROM base as rootlesskit
 
 # Install Go, then verify installation
@@ -78,13 +88,28 @@ RUN git clone --depth 1 --branch v${ROOTLESSKIT_VERSION} https://github.com/root
 
 FROM base
 
-# Install Azure CLI, then upgrade the default installation
+# Install Azure CLI, then verify and upgrade the default installation
 ARG AZURE_CLI_VERSION
 ENV AZURE_CLI_VERSION ${AZURE_CLI_VERSION}
-RUN python3 -m pip install --upgrade pip setuptools wheel \
-    && python3 -m pip install "azure-cli==${AZURE_CLI_VERSION}" \
+RUN python3 -m pip \
+        --disable-pip-version-check \
+        --no-cache-dir \
+        --quiet \
+        install \
+            --compile \
+            azure-cli==${AZURE_CLI_VERSION} \
+    && az version \
     && az upgrade --yes
 
+# Install AWS CLI, then verify installation
+ARG AWS_CLI_VERSION
+ENV AWS_CLI_VERSION ${AWS_CLI_VERSION}
+RUN curl -LsSf https://awscli.amazonaws.com/awscli-exe-linux-$(ARCH_X64=x86_64 ARCH_ARM64=aarch64 bash arch.sh).zip -o awscli.zip \
+    && unzip awscli.zip \
+    && ./aws/install \
+    && rm -rf awscli.zip aws \
+    && aws --version
+
 # Install Powershell, then verify installation
 ARG POWERSHELL_VERSION
 ENV POWERSHELL_VERSION ${POWERSHELL_VERSION}
@@ -136,6 +161,9 @@ RUN mkdir -p ${AZP_HOME} \
 ENV AZP_WORK ${HOME}/azp-work
 VOLUME ${AZP_WORK}
 
+# Define path for the custom SSL certificate
+ENV AZP_CUSTOM_CERT_PEM ${HOME}/azp-custom-certs
+
 # Cleanup helper script
 RUN rm arch.sh