From 07adb242e2386a2826f3634672b7e221b096d35f Mon Sep 17 00:00:00 2001 From: Greg Hewett Date: Fri, 8 Sep 2023 20:10:32 -0500 Subject: [PATCH] adding vcpkg as a submodule to speed up CI --- .github/workflows/main_ci.yml | 119 ++++++++++++++++++---------------- .gitmodules | 3 + Makefile | 12 +++- vcpkg | 1 + 4 files changed, 77 insertions(+), 58 deletions(-) create mode 100644 .gitmodules create mode 160000 vcpkg diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml index 8f6c3168..164c2558 100644 --- a/.github/workflows/main_ci.yml +++ b/.github/workflows/main_ci.yml @@ -29,7 +29,7 @@ jobs: - 'cmd' - 'lib' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run clang-format style check for C/C++ programs uses: jidicula/clang-format-action@v4.11.0 @@ -43,22 +43,42 @@ jobs: name: Quick Linux Check and Interop runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + + # check out the repository with recursively pulling the submodules + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + # write the commit hash of vcpkg to a text file so we can use it in the + # hashFiles for cache + - run: | + git -C vcpkg rev-parse HEAD > vcpkg-commit.txt - name: Dependencies (Ubuntu) if: ${{ matrix.os == 'ubuntu-latest' }} run: | sudo apt-get install -y linux-headers-$(uname -r) - - name: Restore cache + # First, attempt to pull key key, if that is not present, pull one of the + # restore-keys so we do not need to build from scratch. + # VCPKG-BinaryCache - description of cache + # v1 - provide a way to reset cache + # matrix.os - cache per OS and version + # hashFiles - Recache if the vcpkg files change + - name: Restore Cache uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: VCPKG-BinaryCache-${{ runner.os }} + key: VCPKG-BinaryCache-ubuntu-latest-v1-${{ hashFiles('vcpkg-commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} + restore-keys: | + VCPKG-BinaryCache-ubuntu-latest-v1 + VCPKG-BinaryCache-ubuntu-latest - name: Build (OpenSSL 1.1) run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" + cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DTESTING=ON -DCMAKE_TOOLCHAIN_FILE="vcpkg/scripts/buildsystems/vcpkg.cmake" cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target all --parallel 2 - name: Unit Test (OpenSSL 1.1) @@ -68,7 +88,7 @@ jobs: - name: Build (Interop Harness) run: | cd cmd/interop - cmake -B build -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" + cmake -B build -DCMAKE_TOOLCHAIN_FILE="../../vcpkg/scripts/buildsystems/vcpkg.cmake" cmake --build build - name: Test self-interop @@ -86,13 +106,17 @@ jobs: - name: Build (OpenSSL 3) run: | - cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" -DCMAKE_TOOLCHAIN_FILE="$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" + cmake -B "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" -DTESTING=ON -DVCPKG_MANIFEST_DIR="alternatives/openssl_3" -DCMAKE_TOOLCHAIN_FILE="vcpkg/scripts/buildsystems/vcpkg.cmake" cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" - name: Unit Test (OpenSSL 3) run: | cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target test + # recently, the oldest supported macos image was added to this matrix. This is + # to ensure backward compatibility on existing integrations. The goal is to + # have the newest and oldest OS's represented for macOs. Only using latest + # windows and linux images because they have not been an challenge. platform-sanitizer-tests: if: github.event.pull_request.draft == false needs: quick-linux-interop-check @@ -101,23 +125,36 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest, macos-11] include: - - os: windows-latest - vcpkg-cmake-file: "$env:VCPKG_INSTALLATION_ROOT\\scripts\\buildsystems\\vcpkg.cmake" - ossl3-vcpkg-dir: "alternatives\\openssl_3" - ctest-target: RUN_TESTS - - os: ubuntu-latest - vcpkg-cmake-file: "$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" - ossl3-vcpkg-dir: "alternatives/openssl_3" - ctest-target: test - - os: macos-latest - vcpkg-cmake-file: "$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" - ossl3-vcpkg-dir: "alternatives/openssl_3" - ctest-target: test + - os: windows-latest + vcpkg-cmake-file: "vcpkg\\scripts\\buildsystems\\vcpkg.cmake" + ossl3-vcpkg-dir: "alternatives\\openssl_3" + ctest-target: RUN_TESTS + - os: ubuntu-latest + vcpkg-cmake-file: "vcpkg/scripts/buildsystems/vcpkg.cmake" + ossl3-vcpkg-dir: "alternatives/openssl_3" + ctest-target: test + - os: macos-latest + vcpkg-cmake-file: "vcpkg/scripts/buildsystems/vcpkg.cmake" + ossl3-vcpkg-dir: "alternatives/openssl_3" + ctest-target: test + - os: macos-11 + vcpkg-cmake-file: "vcpkg/scripts/buildsystems/vcpkg.cmake" + ossl3-vcpkg-dir: "alternatives/openssl_3" + ctest-target: test steps: - - uses: actions/checkout@v3 + - name: Checkout repository and submodules + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + # write the commit hash of vcpkg to a text file so we can use it in the + # hashFiles for cache + - run: | + git -C vcpkg rev-parse HEAD > vcpkg-commit.txt - name: Dependencies (macOs) if: ${{ matrix.os == 'macos-latest' }} @@ -131,11 +168,14 @@ jobs: run: | sudo apt-get install -y linux-headers-$(uname -r) - - name: Restore cache + - name: Restore Cache uses: actions/cache@v3 with: path: ${{ github.workspace }}/build/cache - key: VCPKG-BinaryCache-${{ runner.os }} + key: VCPKG-BinaryCache-${{ matrix.os }}-v1-${{ hashFiles('vcpkg-commit.txt', 'vcpkg.json', 'alternatives/openssl_3/vcpkg.json') }} + restore-keys: | + VCPKG-BinaryCache-${{ matrix.os }}-v1 + VCPKG-BinaryCache-${{ matrix.os }} - name: Build (OpenSSL1.1) run: | @@ -154,36 +194,3 @@ jobs: - name: Unit Test (OpenSSL3) run: | cmake --build "${{ env.CMAKE_BUILD_OPENSSL3_DIR }}" --target "${{ matrix.ctest-target}}" - - old-macos-compatibility: - if: github.event.pull_request.draft == false - needs: quick-linux-interop-check - name: Build for older MacOS - runs-on: macos-latest - - env: - CMAKE_BUILD_DIR: ${{ github.workspace }}/build - TOOLCHAIN_FILE: $VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake - VCPKG_BINARY_SOURCES: files,${{ github.workspace }}/build/cache,readwrite - MACOSX_DEPLOYMENT_TARGET: 10.11 - - steps: - - uses: actions/checkout@v3 - - - name: dependencies - run: | - brew install llvm pkg-config - ln -s "/usr/local/opt/llvm/bin/clang-format" "/usr/local/bin/clang-format" - ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" - - - name: Restore cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/build/cache - key: VCPKG-BinaryCache-${{ runner.os }} - - - name: Build - run: | - cmake -B "${{ env.CMAKE_BUILD_DIR }}" -DCMAKE_TOOLCHAIN_FILE="${{ env.TOOLCHAIN_FILE }}" - cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target mlspp --parallel 2 - diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..6b3519de --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vcpkg"] + path = vcpkg + url = https://github.com/microsoft/vcpkg.git diff --git a/Makefile b/Makefile index 206d326e..f9a35191 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ TEST_DIR=build/test CLANG_FORMAT=clang-format -i CLANG_TIDY=OFF OPENSSL3_MANIFEST=alternatives/openssl_3 +TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake .PHONY: all dev dev3 test ctest dtest dbtest libs test-libs test-all everything ci ci3 clean cclean format @@ -18,14 +19,21 @@ all: ${BUILD_DIR} ${BUILD_DIR}: CMakeLists.txt test/CMakeLists.txt cmake -B${BUILD_DIR} . +vcpkg: + git submodule update --init --recursive + dev: # Only enable testing, not clang-tidy/sanitizers; the latter make the build # too slow, and we can run them in CI cmake -B${BUILD_DIR} -DTESTING=ON -DCMAKE_BUILD_TYPE=Debug . -dev3: +vcpkg-dev: vcpkg + # Like `dev`, but retrieve dependencies using vcpkg + cmake -B${BUILD_DIR} -DTESTING=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} + +dev3: vcpkg # Like `dev`, but using OpenSSL 3 - cmake -B${BUILD_DIR} -DTESTING=ON -DCMAKE_BUILD_TYPE=Debug -DVCPKG_MANIFEST_DIR=${OPENSSL3_MANIFEST} . + cmake -B${BUILD_DIR} -DTESTING=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DVCPKG_MANIFEST_DIR=${OPENSSL3_MANIFEST} . test: ${BUILD_DIR} test/* cmake --build ${BUILD_DIR} --target mlspp_test diff --git a/vcpkg b/vcpkg new file mode 160000 index 00000000..962e5e39 --- /dev/null +++ b/vcpkg @@ -0,0 +1 @@ +Subproject commit 962e5e39f8a25f42522f51fffc574e05a3efd26b