Create convert-weights.md #93
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Release | ||
| on: | ||
| push: | ||
| tags: | ||
| - v* | ||
| env: | ||
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
| AWS_DEFAULT_REGION: us-east-1 | ||
| PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }} | ||
| DIST_DIR: ${{ github.sha }} | ||
| jobs: | ||
| lock_exists: | ||
| runs-on: ubuntu-latest | ||
| name: Cargo.lock | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Cargo.lock lock exists | ||
| run: cat Cargo.lock | ||
| working-directory: ./bindings/python | ||
| create_wheels_manylinux: | ||
| runs-on: ubuntu-latest | ||
| needs: [lock_exists] | ||
| name: Create wheels for manylinux2014 | ||
| container: quay.io/pypa/manylinux2014_x86_64 | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Install dependencies | ||
| run: yum install -y openssl-devel | ||
| - name: Build and audit wheels | ||
| working-directory: ./bindings/python | ||
| run: sh build-wheels.sh | ||
| create_wheels_windows: | ||
| name: Windows | ||
| runs-on: windows-latest | ||
| needs: [lock_exists] | ||
| strategy: | ||
| matrix: | ||
| python: ["3.7", "3.8", "3.9", "3.10", "3.11"] | ||
| bits: ["32", "64"] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v2 | ||
| - name: Install Rust | ||
| uses: actions-rs/toolchain@v1 | ||
| with: | ||
| toolchain: ${{ matrix.bits == '32' && 'stable-i686-pc-windows-msvc' || 'stable-x86_64-pc-windows-msvc' }} | ||
| override: true | ||
| - name: Override toolchain | ||
| shell: bash | ||
| working-directory: ./bindings/python | ||
| run: echo ${{ matrix.bits == '32' && 'stable-i686-pc-windows-msvc' || 'stable-x86_64-pc-windows-msvc' }} > rust-toolchain | ||
| - name: Install Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ matrix.python }} | ||
| architecture: ${{ matrix.bits == '32' && 'x86' || 'x64' }} | ||
| - name: Install dependencies | ||
| run: | | ||
| # On old versions of python there is an old version of setuptools already installed | ||
| pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall | ||
| - name: Build wheel | ||
| working-directory: ./bindings/python | ||
| run: python setup.py bdist_wheel | ||
| - name: Upload wheels | ||
| shell: bash | ||
| run: | | ||
| pip install awscli | ||
| aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" | ||
| create_wheels_windows: | ||
| name: Windows | ||
| runs-on: windows-latest | ||
| needs: [lock_exists] | ||
| strategy: | ||
| matrix: | ||
| python: ["3.7", "3.8", "3.9", "3.10", "3.11"] | ||
| bits: ["32", "64"] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v2 | ||
| - name: Install Rust 32bits | ||
| if: ${{ matrix.os == '32' }} | ||
| uses: actions-rs/toolchain@v1 | ||
| with: | ||
| toolchain: stable-i686-pc-windows-msvc | ||
| override: true | ||
| - name: Override toolchain | ||
| if: ${{ matrix.os == '32' }} | ||
| shell: bash | ||
| working-directory: ./bindings/python | ||
| run: echo "stable-i686-pc-windows-msvc" > rust-toolchain | ||
| - name: Install Rust 64bits | ||
| if: ${{ matrix.os == '64' }} | ||
| uses: actions-rs/toolchain@v1 | ||
| - name: Install Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ matrix.python }} | ||
| architecture: x86 | ||
| - name: Install dependencies | ||
| run: | | ||
| # On old versions of python there is an old version of setuptools already installed | ||
| pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall | ||
| - name: Build wheel | ||
| working-directory: ./bindings/python | ||
| run: python setup.py bdist_wheel | ||
| - name: Upload wheels | ||
| shell: bash | ||
| run: | | ||
| pip install awscli | ||
| aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" | ||
| create_wheels_macos_conda: | ||
| name: MacOS - Conda | ||
| runs-on: ${{ matrix.os }} | ||
| needs: [lock_exists] | ||
| strategy: | ||
| matrix: | ||
| os: [macos-latest] | ||
| # 3.11 not available on Conda yet. | ||
| python: ["3.7", "3.8", "3.9", "3.10"] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Install miniconda | ||
| uses: conda-incubator/setup-miniconda@v2 | ||
| with: | ||
| auto-update-conda: true | ||
| python-version: ${{ matrix.python }} | ||
| - name: Conda info | ||
| shell: bash -l {0} | ||
| run: conda info | ||
| - name: Install Rust | ||
| uses: actions-rs/toolchain@v1 | ||
| with: | ||
| toolchain: stable | ||
| - name: Setup conda env | ||
| shell: bash -l {0} | ||
| run: | | ||
| conda install setuptools-rust | ||
| conda install -c defaults anaconda-client conda-build | ||
| - name: Extract version | ||
| shell: bash -l {0} | ||
| working-directory: ./bindings/python | ||
| run: echo "SAFETENSORS_VERSION=`python setup.py --version`" >> $GITHUB_ENV | ||
| - name: Build conda packages | ||
| shell: bash -l {0} | ||
| working-directory: ./bindings/python | ||
| run: | | ||
| MACOSX_DEPLOYMENT_TARGET=10.11 python setup.py bdist_wheel | ||
| - name: Upload wheels | ||
| shell: bash | ||
| run: | | ||
| pip install awscli | ||
| aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" | ||
| create_wheels_macos: | ||
| name: MacOS | ||
| runs-on: ${{ matrix.os.os }} | ||
| needs: [lock_exists] | ||
| strategy: | ||
| matrix: | ||
| python: ["3.7", "3.8", "3.9", "3.10", "3.11"] | ||
| # os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}, {os: "macos-13", target: "14.0"}] | ||
| os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v2 | ||
| - name: Install Rust | ||
| uses: actions-rs/toolchain@v1 | ||
| with: | ||
| toolchain: stable | ||
| override: true | ||
| - name: Install Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ matrix.python }} | ||
| - name: Install dependencies | ||
| run: | | ||
| # On old versions of python there is an old version of setuptools already installed | ||
| pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall | ||
| - name: Override target | ||
| if: ${{ matrix.os.target }} | ||
| working-directory: ./bindings/python | ||
| run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.target }}" >> $GITHUB_ENV | ||
| - name: Build wheel | ||
| working-directory: ./bindings/python | ||
| run: echo $MACOX_DEPLOYMENT_TARGET && python setup.py bdist_wheel | ||
| - name: Upload wheels | ||
| shell: bash | ||
| run: | | ||
| pip install awscli | ||
| aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" | ||
| create_wheels_macos_arm64: | ||
| name: MacOS M1 | ||
| runs-on: macos-arm64 | ||
| needs: [lock_exists] | ||
| strategy: | ||
| matrix: | ||
| python: ["3.8.16", "3.9.13", "3.10.6", "3.11.0"] | ||
| # target: ["12.0", "13.0", "14.0"] | ||
| target: ["12.0", "13.0"] | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Install Rust | ||
| uses: actions-rs/toolchain@v1 | ||
| with: | ||
| toolchain: stable | ||
| override: true | ||
| - name: Install Python | ||
| shell: bash | ||
| run: | | ||
| echo $HOME | ||
| export PYENV_ROOT="$HOME/.pyenv" | ||
| command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" | ||
| eval "$(pyenv init -)" | ||
| pyenv shell ${{ matrix.python }} | ||
| which pyenv | ||
| which python | ||
| pip install -U setuptools wheel setuptools-rust awscli | ||
| cd ./bindings/python | ||
| MACOSX_DEPLOYMENT_TARGET=${{ matrix.target }} python setup.py bdist_wheel | ||
| cd ../../ | ||
| aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" | ||
| Upload_package: | ||
| name: Upload package to PyPi | ||
| runs-on: ubuntu-latest | ||
| needs: [create_wheels_manylinux, create_wheels_windows, create_wheels_macos, create_wheels_macos_arm64, create_wheels_macos_conda] | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Install Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: "3.10" | ||
| architecture: x64 | ||
| - name: Retrieve all wheels | ||
| shell: bash | ||
| run: | | ||
| pip install awscli | ||
| aws s3 sync "s3://safetensors-releases/python/$DIST_DIR" ./bindings/python/dist | ||
| - name: Install dependencies | ||
| run: | | ||
| pip install setuptools wheel setuptools-rust | ||
| - name: Create source distribution | ||
| working-directory: ./bindings/python | ||
| run: sh build-sdist.sh | ||
| - name: Upload to PyPi | ||
| working-directory: ./bindings/python | ||
| run: | | ||
| pip install twine | ||
| twine upload dist/* -u __token__ -p "$PYPI_TOKEN" | ||
