NSS corpus update service

services/nss-corpus-update/Dockerfile

@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+FROM ubuntu:20.04
+
+LABEL maintainer Maurice Dauer <mdauer@mozilla.com>
+
+ENV LOGNAME         worker
+ENV HOSTNAME        taskcluster-worker
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+
+COPY recipes/linux/ /src/recipes/
+COPY services/nss-corpus/setup.sh /src/recipes/setup-nss-corpus.sh
+COPY services/nss-corpus/launch.sh /home/worker/
+RUN /src/recipes/setup-nss-corpus.sh
+
+ENV LANG   en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
+USER worker
+WORKDIR /home/worker
+CMD ["/home/worker/launch.sh"]

services/nss-corpus-update/launch.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -e
+set -x
+set -o pipefail
+
+# shellcheck source=recipes/linux/common.sh
+source .local/bin/common.sh
+
+# Clone nss/nspr
+retry hg clone https://hg.mozilla.org/projects/nspr
+retry hg clone https://hg.mozilla.org/projects/nss
+
+# Clone github nss fuzzing corpus repo
+git-clone git@github.com:mozilla/nss-fuzzing-corpus.git
+
+# TODO: Setup github credentials
+
+# Update github nss fuzzing corpus repo
+for file in nss/fuzz/options/*; do
+    name=${file%.options}
+
+    mkdir -p "nss-fuzzing-corpus/$name"
+    pushd "nss-fuzzing-corpus/$name"
+
+    code=$(retry-curl --no-fail -w "%{http_code}" -o /tmp/public.zip "https://storage.googleapis.com/nss-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nss_$name/public.zip")
+    if [[ $code -eq 200 ]]; then
+      rm -rf ./*
+      unzip /tmp/public.zip
+    fi
+    rm -f /tmp/public.zip
+
+    popd
+done
+
+# Commit and push any changes
+pushd nss-fuzzing-corpus
+if [[ -z "$(git status --porcelain)" ]]; then
+    git add -A
+    git commit -m "nss-corpus-update: merge public oss-fuzz corpus"
+    git push
+fi
+popd
+
+# Build nss
+pushd nss
+time ./build.sh -c -v
+popd
+
+# Get list of hosts to collect handshakes
+# TODO
+
+# Run collection scripts
+mkdir -p nss-collected-corpus
+# TODO: Rename script to `collect_handshakes.py` and `--nss` to `--nss-build`.
+#       Also add documentation to nss/fuzz about orion and that for example the
+#       corpus generation scripts should output in directory $name-corpus.
+python nss/fuzz/config/gen_corpus.py --nss ./dist/Debug --hosts TODO --threads 5 --output ./nss-collected-corpus
+
+# Minimize and upload to gcloud bucket
+for directory in nss-collected-corpus/*; do
+    name=${directory%-corpus}
+
+    mkdir -p "nss-collected-corpus/$directory-minimized"
+    dist/Debug/bin/nssfuzz-"$name" -merge=1 "./nss-collected-corpus/$directory-minimized" "./nss-collected-corpus/$directory"
+
+    # TODO: Upload to gcloud bucket (nfm and fm)
+
+    # Free up some disk space
+    rf -rf "./nss-collected-corpus/$directory"
+    rm -rf "./nss-collected-corpus/$directory-minimized"
+done

services/nss-corpus-update/service.yaml

@@ -0,0 +1 @@
+name: nss-corpus-update

services/nss-corpus-update/setup.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -e
+set -x
+set -o pipefail
+
+# shellcheck source=recipes/linux/common.sh
+source "${0%/*}/common.sh"
+
+#### Bootstrap Packages
+
+sys-update
+
+#### Install recipes
+
+cd "${0%/*}"
+./taskcluster.sh
+
+packages=(
+    clang
+    git
+    gyp
+    locales
+    make
+    mercurial
+    ninja-build
+    python-is-python3
+    python3
+)
+
+sys-embed "${packages[@]}"
+
+#### Base System Configuration
+
+# Generate locales
+locale-gen en_US.utf8
+
+#### Base Environment Configuration
+
+mkdir -p /home/worker/.local/bin
+
+# Add `cleanup.sh` to let images perform standard cleanup operations.
+cp "${0%/*}/cleanup.sh" /home/worker/.local/bin/cleanup.sh
+
+# Add shared `common.sh` to Bash
+cp "${0%/*}/common.sh" /home/worker/.local/bin/common.sh
+printf "source ~/.local/bin/common.sh\n" >> /home/worker/.bashrc
+
+/home/worker/.local/bin/cleanup.sh
+
+mkdir -p /home/worker/.ssh /root/.ssh
+chmod 0700 /home/worker/.ssh /root/.ssh
+cat << EOF | tee -a /root/.ssh/config /home/worker/.ssh/config > /dev/null
+Host *
+UseRoaming no
+EOF
+retry ssh-keyscan github.com | tee -a /root/.ssh/known_hosts /home/worker/.ssh/known_hosts > /dev/null
+
+chown -R worker:worker /home/worker
+chmod 0777 /src