-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Manylinux wheel generation workflow
- Loading branch information
Yngve S. Kristiansen
committed
Oct 31, 2023
1 parent
cc7b32a
commit 338aa5b
Showing
8 changed files
with
374 additions
and
77 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,162 @@ | ||
name: 🏎️ Make & Test Wheels 🏎️ | ||
|
||
on: [pull_request] | ||
|
||
env: | ||
ERT_SHOW_BACKTRACE: 1 | ||
NO_PROJECT_RES: 1 | ||
BOOST_VERSION: 1.83.0 | ||
BOOST_VERSION_UNDERSCORES: 1_83_0 | ||
DAKOTA_VERSION: 6.18.0 | ||
SEBA_TAG: 6.12.0 | ||
INSTALL_DIR: local | ||
FORCE_REBUILD: false | ||
|
||
jobs: | ||
build_wheels: | ||
name: 🛞 Build Wheels 🛞 | ||
timeout-minutes: 120 | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
python-version: ['3.8','3.9','3.10'] # Versions > 3.10 excluded. Reason: not supported by Everest. | ||
os: [ubuntu-latest] | ||
ert-version: ['main'] | ||
|
||
runs-on: ${{ matrix.os }} | ||
|
||
steps: | ||
- uses: actions/cache@v3 | ||
id: cache-package-check | ||
with: | ||
key: carolina_dist_${{ matrix.os }}_python-${{ matrix.python-version }}_boost-${{ env.BOOST_VERSION }}_dakota-${{ env.DAKOTA_VERSION }} | ||
path: ./carolina_dist | ||
|
||
- uses: actions/checkout@v3 | ||
if: env.FORCE_REBUILD == 'true' || steps.cache-package-check.outputs.cache-hit != 'true' | ||
with: | ||
fetch-depth: 0 | ||
|
||
- name: Build Linux Wheel | ||
if: env.FORCE_REBUILD == 'true' || steps.cache-package-check.outputs.cache-hit != 'true' | ||
uses: docker://quay.io/pypa/manylinux2014_x86_64 | ||
env: | ||
PYTHON_VERSION: ${{ matrix.python-version }} | ||
BOOST_VERSION: ${{ env.BOOST_VERSION }} | ||
BOOST_VERSION_UNDERSCORES: ${{ env.BOOST_VERSION_UNDERSCORES }} | ||
DAKOTA_VERSION: ${{ env.DAKOTA_VERSION }} | ||
with: | ||
entrypoint: /bin/bash | ||
args: '-c "sh dakota_manylinux_install_files/build_wheels_gha.sh ${{ matrix.python-version }}"' | ||
|
||
- uses: actions/cache/save@v3 | ||
if: env.FORCE_REBUILD == 'true' || steps.cache-package-check.outputs.cache-hit != 'true' | ||
id: cache-package-store | ||
with: | ||
key: carolina_dist_${{ matrix.os }}_python-${{ matrix.python-version }}_boost-${{ env.BOOST_VERSION }}_dakota-${{ env.DAKOTA_VERSION }} | ||
path: ./carolina_dist | ||
|
||
- name: Get Carolina wheel from cache | ||
uses: actions/cache/restore@v3 | ||
id: restore-cached-package | ||
with: | ||
key: carolina_dist_${{ matrix.os }}_python-${{ matrix.python-version }}_boost-${{ env.BOOST_VERSION }}_dakota-${{ env.DAKOTA_VERSION }} | ||
path: /local/carolina_dist | ||
|
||
- name: Set up Python ${{ matrix.python-version }} | ||
uses: actions/setup-python@v4 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
|
||
- name: Set up python venv | ||
run: | | ||
python -m venv myvenv | ||
- name: Install Carolina from wheel | ||
run: | | ||
source myvenv/bin/activate | ||
pyv=$(echo ${{matrix.python-version}} | sed 's/\.//g') | ||
pip install carolina_dist/carolina-1.0-cp$pyv-cp$pyv-manylinux_2_17_x86_64.manylinux2014_x86_64.whl | ||
- name: Install other Everest dependencies | ||
run: | | ||
source myvenv/bin/activate | ||
pip install git+https://${{ secrets.DEPENDENCY_ACCESS_TOKEN }}@github.com/equinor/spinningjenny.git | ||
pip install git+https://${{ secrets.DEPENDENCY_ACCESS_TOKEN }}@github.com/TNO-Everest/seba.git@${{ env.SEBA_TAG }} | ||
- name: Install Everest | ||
run: | | ||
source myvenv/bin/activate | ||
git clone https://${{ secrets.DEPENDENCY_ACCESS_TOKEN }}@github.com/equinor/everest.git | ||
cd everest | ||
pip install .[test,docs] | ||
- name: Install latest ert main | ||
if: matrix.ert-version == 'main' | ||
run: | | ||
source myvenv/bin/activate | ||
sudo apt-get install build-essential | ||
pip install git+https://github.com/equinor/ert.git | ||
- name: Run Tests | ||
run: | | ||
source myvenv/bin/activate | ||
cd everest | ||
pytest tests -n 4 -m "not ui_test and not integration_test" --dist loadgroup -sv | ||
- name: Run Integration Tests | ||
run: | | ||
source myvenv/bin/activate | ||
cd everest | ||
pytest tests -n 4 -m "integration_test" --dist loadgroup | ||
- name: Build Documentation | ||
run: | | ||
source myvenv/bin/activate | ||
cd everest | ||
mkdir tmp | ||
sphinx-build -W -b html -d tmp/doctrees docs/source {envtmpdir}/html | ||
- name: Run UI Tests | ||
env: | ||
QT_QPA_PLATFORM: 'minimal' | ||
run: | | ||
source myvenv/bin/activate | ||
cd everest | ||
python -m pytest -m "ui_test" | ||
- name: Upload wheel as artifact | ||
if: always() | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: ${{ matrix.os }} Python ${{ matrix.python-version }} wheel | ||
path: | | ||
carolina_dist/* | ||
trace/* | ||
publish: | ||
name: 👾 Publish 👾 | ||
runs-on: ubuntu-latest | ||
needs: [ build_wheels ] | ||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') | ||
steps: | ||
- name: Get wheels | ||
uses: actions/download-artifact@v3 | ||
with: | ||
path: artifacts | ||
|
||
- name: Create dist/ folder with all python wheels | ||
run: | | ||
mkdir dist | ||
find artifacts -name "carolina*.whl" -exec mv '{}' dist/ \; | ||
- name: Upload dist/ folder as artifact (debug) | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
path: dist/* | ||
|
||
- name: Publish to pypi | ||
uses: pypa/gh-action-pypi-publish@v1.8.10 | ||
with: | ||
user: __token__ | ||
password: ${{ secrets.PYPI_TOKEN }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
--- a/dakota-6.18.0-public-src-cli/CMakeLists.txt 2023-05-11 03:08:25 | ||
+++ b/dakota-6.18.0-public-src-cli/CMakeLists.txt 2023-10-26 08:43:44 | ||
@@ -150,10 +150,20 @@ | ||
# Perl is required for examples, docs, and system tests | ||
find_package(Perl REQUIRED) | ||
|
||
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wall -lpthread -lutil") | ||
# Python is optionally required by Dakota, Teuchos, and Acro; probe | ||
# for components here at top-level: | ||
+include_directories(${PYTHON_INCLUDE_DIRS}) | ||
+include_directories(/usr/include) | ||
+include_directories(/usr/lib64) | ||
+include_directories(/tmp/INSTALL_DIR/dakota-6.18.0-public-src-cli/packages/external/eigen3/include/) | ||
+find_package(Threads REQUIRED) | ||
+ | ||
+set (PYBIND11_PYTHON_VERSION "3.10") | ||
+ | ||
include(DakotaFindPython) | ||
dakota_find_python() | ||
+add_link_options() | ||
|
||
# Conditionally find Java JDK needed for input spec, docs, Java API | ||
include(DakotaFindJava) | ||
@@ -186,6 +196,7 @@ | ||
"${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") | ||
endif() | ||
|
||
+link_directories(/usr/lib64) | ||
|
||
include(DakotaFindSystemTPLs) | ||
# Unconditionally find BLAS/LAPACK or equivalent |
11 changes: 11 additions & 0 deletions
11
dakota_manylinux_install_files/DakotaFindPython.cmake.patch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
--- a/dakota-6.18.0-public-src-cli/cmake/DakotaFindPython.cmake 2023-05-11 03:08:25 | ||
+++ b/dakota-6.18.0-public-src-cli/cmake/DakotaFindPython.cmake 2023-10-26 08:43:44 | ||
@@ -17,7 +17,7 @@ | ||
|
||
endif() | ||
|
||
- find_package(Python REQUIRED ${dakota_python_components}) | ||
+ find_package(Python COMPONENTS ${dakota_python_components} REQUIRED) | ||
|
||
# pybind11, C3, Acro, etc., use older CMake FindPythonInterp, so we | ||
# coerce it to use same as Dakota; more complex situations may |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,159 @@ | ||
#!/bin/bash | ||
set -e | ||
|
||
if [ -z "$1" ]; then | ||
echo "Please provide a Python version as an argument (e.g., 3.10)" | ||
exit 1 | ||
fi | ||
|
||
cd /tmp | ||
INSTALL_DIR=/tmp/INSTALL_DIR | ||
|
||
mkdir -p $INSTALL_DIR | ||
mkdir /github/workspace/trace | ||
touch /github/workspace/trace/boost_bootstrap.log | ||
touch /github/workspace/trace/boost_install.log | ||
touch /github/workspace/trace/dakota_bootstrap.log | ||
touch /github/workspace/trace/dakota_install.log | ||
touch /github/workspace/trace/env | ||
|
||
# VERY IMPORTANT: extract python dev headers, | ||
# more info: https://github.com/pypa/manylinux/pull/1250 | ||
pushd /opt/_internal && tar -xJf static-libs-for-embedding-only.tar.xz && popd | ||
|
||
echo "pushd /opt/_internal && tar -xJf static-libs-for-embedding-only.tar.xz && popd" >> /github/workspace/trace/env | ||
echo "INSTALL_DIR=$INSTALL_DIR" >> /github/workspace/trace/env | ||
|
||
yum install lapack-devel -y | ||
yum install python3-devel.x86_64 -y | ||
yum install -y wget | ||
cd /tmp | ||
|
||
wget https://boostorg.jfrog.io/artifactory/main/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORES.tar.bz2 --no-check-certificate > /dev/null | ||
python_exec=$(which python$1) | ||
$python_exec -m venv myvenv | ||
source ./myvenv/bin/activate | ||
pip install numpy | ||
pip install pybind11[global] | ||
|
||
PYTHON_DEV_HEADERS_DIR=$(rpm -ql python3-devel.x86_64 | grep '\.h$' | head -n 1 | xargs dirname) | ||
NUMPY_INCLUDE_PATH=$(find /tmp -type d -path "*site-packages/numpy/core/include") | ||
PYTHON_INCLUDE_PATH=$(python -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") | ||
python_root=$(python -c "import sys; print(sys.prefix)") | ||
python_version=$(python --version | sed -E 's/.*([0-9]+\.[0-9]+)\.([0-9]+).*/\1/') | ||
python_version_no_dots="$(echo "${python_version//\./}")" | ||
python_bin_include_lib=" using python : $python_version : $(python -c "from sysconfig import get_paths as gp; g=gp(); print(f\"$python_exec : {g['include']} : {g['stdlib']} ;\")")" | ||
|
||
echo "Found dev headers $PYTHON_DEV_HEADERS_DIR" | ||
echo "Found numpy include path $NUMPY_INCLUDE_PATH" | ||
echo "Found python include path $PYTHON_INCLUDE_PATH" | ||
echo "Found python root $python_root" | ||
|
||
tar xf boost_$BOOST_VERSION_UNDERSCORES.tar.bz2 | ||
cd boost_$BOOST_VERSION_UNDERSCORES | ||
|
||
echo "python_exec=$python_exec" >> /github/workspace/trace/env | ||
echo "PYTHON_DEV_HEADERS_DIR=$PYTHON_DEV_HEADERS_DIR" >> /github/workspace/trace/env | ||
echo "NUMPY_INCLUDE_PATH=$NUMPY_INCLUDE_PATH" >> /github/workspace/trace/env | ||
echo "PYTHON_INCLUDE_PATH=$PYTHON_INCLUDE_PATH" >> /github/workspace/trace/env | ||
echo "python_root=$python_root" >> /github/workspace/trace/env | ||
echo "python_version=$python_version" >> /github/workspace/trace/env | ||
echo "python_version_no_dots=$python_version_no_dots" >> /github/workspace/trace/env | ||
echo "python_bin_include_lib=$python_bin_include_lib" >> /github/workspace/trace/env | ||
echo "bootstrap_cmd=./bootstrap.sh --with-libraries=python,filesystem,program_options,regex,serialization,system --with-python=$(which python) --with-python-root=$python_root &> "$INSTALL_DIR/boost_bootstrap.log"" >> /github/workspace/trace/env | ||
|
||
./bootstrap.sh --with-libraries=python,filesystem,program_options,regex,serialization,system --with-python=$(which python) --with-python-root="$python_root" &> "$INSTALL_DIR/boost_bootstrap.log" | ||
sed -i -e "s|.*using python.*|$python_bin_include_lib|" project-config.jam | ||
echo "# sed -i -e \"s|.*using python.*|$python_bin_include_lib|\" project-config.jam" >> /github/workspace/trace/env | ||
|
||
./b2 install -j8 -a cxxflags="-std=c++17" --prefix="$INSTALL_DIR" &> /github/workspace/trace/boost_install.log | ||
echo "# ./b2 install -j8 -a cxxflags="-std=c++17" --prefix="$INSTALL_DIR" &> /github/workspace/trace/boost_install.log" >> /github/workspace/trace/env | ||
|
||
cd $INSTALL_DIR | ||
DAKOTA_INSTALL_DIR=/tmp/INSTALL_DIR/dakota | ||
mkdir -p $DAKOTA_INSTALL_DIR | ||
echo "DAKOTA_INSTALL_DIR=$DAKOTA_INSTALL_DIR" >> /github/workspace/trace/env | ||
|
||
wget https://github.com/snl-dakota/dakota/releases/download/v$DAKOTA_VERSION/dakota-$DAKOTA_VERSION-public-src-cli.tar.gz > /dev/null | ||
tar xf dakota-$DAKOTA_VERSION-public-src-cli.tar.gz | ||
|
||
CAROLINA_DIR=/github/workspace | ||
|
||
cd dakota-$DAKOTA_VERSION-public-src-cli | ||
patch -s -p2 < $CAROLINA_DIR/dakota_manylinux_install_files/CMakeLists.txt.patch | ||
patch -s -p2 < $CAROLINA_DIR/dakota_manylinux_install_files/DakotaFindPython.cmake.patch | ||
|
||
mkdir build | ||
cd build | ||
|
||
export PATH=/tmp/INSTALL_DIR/bin:$PATH | ||
export PYTHON_INCLUDE_DIRS="$PYTHON_INCLUDE_PATH $PYTHON_DEV_HEADERS_DIR /tmp/INSTALL_DIR/lib" | ||
export PYTHON_EXECUTABLE=$(which python) | ||
export LD_LIBRARY_PATH="$INSTALL_DIR/lib:/usr/local/lib:$PYTHON_INCLUDE_PATH:$NUMPY_INCLUDE_PATH:$NUMPY_INCLUDE_PATH/numpy:$PYTHON_DEV_HEADERS_DIR:/tmp/INSTALL_DIR/lib:$LD_LIBRARY_PATH" | ||
|
||
echo "export PATH=$PATH" >> /github/workspace/trace/env | ||
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /github/workspace/trace/env | ||
echo "export PYTHON_INCLUDE_DIRS=$PYTHON_INCLUDE_DIRS" >> /github/workspace/trace/env | ||
echo "export PYTHON_EXECUTABLE=$PYTHON_EXECUTABLE" >> /github/workspace/trace/env | ||
|
||
export BOOST_PYTHON="boost_python$python_version_no_dots" | ||
export BOOST_ROOT=$INSTALL_DIR | ||
export PATH="$PATH:$INSTALL_DIR/bin" | ||
|
||
# More stable approach: Go via python | ||
numpy_lib_dir=$(find /tmp/myvenv/ -name numpy.libs) | ||
export LD_LIBRARY_PATH="/usr/lib:/usr/lib64:$INSTALL_DIR/lib:$INSTALL_DIR/bin:$numpy_lib_dir:$NUMPY_INCLUDE_PATH" | ||
export CMAKE_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | sed 's/::/:/g' | sed 's/:/;/g') | ||
export PYTHON_LIBRARIES="/usr/lib64/" | ||
export PYTHON_INCLUDE_DIR="/opt/_internal/cpython-3.7.17/include/python3.7m" | ||
export CMAKE_LINK_OPTS="-Wl,--copy-dt-needed-entries,-l pthread" | ||
|
||
echo "export BOOST_PYTHON=$BOOST_PYTHON" >> /github/workspace/trace/env | ||
echo "export BOOST_ROOT=$BOOST_ROOT" >> /github/workspace/trace/env | ||
echo "export PATH=$PATH" >> /github/workspace/trace/env | ||
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /github/workspace/trace/env | ||
echo "export CMAKE_LIBRARY_PATH=\"$CMAKE_LIBRARY_PATH\"" >> /github/workspace/trace/env | ||
echo "export PYTHON_LIBRARIES=\"$PYTHON_LIBRARIES\"" >> /github/workspace/trace/env | ||
echo "export PYTHON_INCLUDE_DIR=\"$PYTHON_INCLUDE_DIR\"" >> /github/workspace/trace/env | ||
|
||
cmake_command=""" | ||
cmake \ | ||
-DCMAKE_CXX_STANDARD=14 \ | ||
-DBUILD_SHARED_LIBS=ON \ | ||
-DDAKOTA_PYTHON=ON \ | ||
-DDAKOTA_PYTHON_DIRECT_INTERFACE=ON \ | ||
-DDAKOTA_PYTHON_DIRECT_INTERFACE_NUMPY=ON \ | ||
-DDAKOTA_DLL_API=OFF \ | ||
-DHAVE_X_GRAPHICS=OFF \ | ||
-DDAKOTA_ENABLE_TESTS=OFF \ | ||
-DDAKOTA_ENABLE_TPL_TESTS=OFF \ | ||
-DCMAKE_BUILD_TYPE="Release" \ | ||
-DDAKOTA_NO_FIND_TRILINOS:BOOL=TRUE \ | ||
-DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ | ||
-DPYTHON_LIBRARIES=$PYTHON_LIBRARIES \ | ||
-DCMAKE_LINK_OPTIONS=\"$CMAKE_LINK_OPTS\" \ | ||
.. &> "$INSTALL_DIR/dakota_bootstrap.log" | ||
""" | ||
echo "# $cmake_command" >> /github/workspace/trace/env | ||
|
||
echo "Boostrapping Dakota ..." | ||
$($cmake_command &> /github/workspace/trace/dakota_bootstrap.log) | ||
|
||
echo "# make --debug=b -j8 install" >> /github/workspace/trace/env | ||
echo "Building Dakota ..." | ||
make --debug=b -j8 install &> /github/workspace/trace/dakota_install.log | ||
|
||
cd $INSTALL_DIR/.. | ||
|
||
git clone https://github.com/equinor/Carolina.git | ||
cd Carolina | ||
pip install . &> $INSTALL_DIR/carolina_install.log | ||
|
||
pip install pytest | ||
pytest tests | ||
|
||
pip wheel . -w wheelhouse &> $INSTALL_DIR/carolina_pipwheel.log | ||
auditwheel repair wheelhouse/* -w /github/workspace/carolina_dist &> $INSTALL_DIR/carolina_auditwheel_repair.log | ||
|
||
echo "Copied distributables and installation trace" |
Oops, something went wrong.