diff --git a/.github/workflows/python-release-conda.yml b/.github/workflows/python-release-conda.yml index ddb68aab..11f9b183 100644 --- a/.github/workflows/python-release-conda.yml +++ b/.github/workflows/python-release-conda.yml @@ -2,8 +2,8 @@ name: Python Release - Conda on: push: - tags: - - v* + # tags: + # - v* env: ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} @@ -125,8 +125,8 @@ jobs: conda list conda-build .github/conda --python=${{ matrix.python }} - - name: Upload to Anaconda - shell: bash -l {0} - run: | - source $HOME/miniconda/bin/activate - anaconda upload `conda-build .github/conda --output` --force + # - name: Upload to Anaconda + # shell: bash -l {0} + # run: | + # source $HOME/miniconda/bin/activate + # anaconda upload `conda-build .github/conda --output` --force diff --git a/.github/workflows/python-release-extra.yml b/.github/workflows/python-release-extra.yml index 06dc253a..43691dfd 100644 --- a/.github/workflows/python-release-extra.yml +++ b/.github/workflows/python-release-extra.yml @@ -1,120 +1,120 @@ -name: Python Release extra - -on: - push: - tags: - - v* - -env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }} - DIST_DIR: '${{ github.sha }}_extra' - -jobs: - - create_wheels_manylinux_2014_ppc64le: - runs-on: ubuntu-latest - name: Create wheels for manylinux2014 - PowerPC - steps: - - uses: actions/checkout@v3 - - - name: Upgrade libssl - run: sudo apt-get install -y libssl-dev - - - name: Pull images - run: | - docker pull multiarch/qemu-user-static - docker pull quay.io/pypa/manylinux2014_ppc64le:latest - - - name: Install QEMU - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build and audit wheels - working-directory: ./bindings/python - run: | - docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \ - --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_ppc64le \ - /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh" - - create_wheels_manylinux_2014_aarch64: - runs-on: ubuntu-latest - name: Create wheels for manylinux2014 - Aarch64 - steps: - - uses: actions/checkout@v2 - - - name: Upgrade libssl - run: sudo apt-get install -y libssl-dev - - - name: Pull images - run: | - docker pull multiarch/qemu-user-static - docker pull quay.io/pypa/manylinux2014_aarch64:latest - - - name: Install QEMU - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build and audit wheels - working-directory: ./bindings/python - run: | - docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \ - --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_aarch64 \ - /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh" - - create_wheels_manylinux_2014_x390x: - runs-on: ubuntu-latest - name: Create wheels for manylinux2014 - S390X - steps: - - uses: actions/checkout@v2 - - - name: Upgrade libssl - run: sudo apt-get install -y libssl-dev - - - name: Pull images - run: | - docker pull multiarch/qemu-user-static - docker pull quay.io/pypa/manylinux2014_s390x:latest - - - name: Install QEMU - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build and audit wheels - working-directory: ./bindings/python - run: | - docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \ - --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_s390x \ - /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh" - - upload_package: - name: Upload package to PyPi - runs-on: ubuntu-latest - needs: - - create_wheels_manylinux_2014_ppc64le - - create_wheels_manylinux_2014_aarch64 - - create_wheels_manylinux_2014_x390x - - steps: - - uses: actions/checkout@v2 - - - name: Install Python - uses: actions/setup-python@v4 - - - name: Retrieve all wheels - shell: bash - run: | - pip install awscli - aws s3 sync "s3://safetensors-releases/python/$DIST_DIR" ./bindings/python/dist - - - name: Install dependencies - run: | - pip install setuptools wheel setuptools-rust - - - name: Upload to PyPi - working-directory: ./bindings/python - run: | - pip install twine - twine upload dist/* -u __token__ -p "$PYPI_TOKEN" +# name: Python Release extra +# +# on: +# push: +# tags: +# - v* +# +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: us-east-1 +# PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }} +# DIST_DIR: '${{ github.sha }}_extra' +# +# jobs: +# +# create_wheels_manylinux_2014_ppc64le: +# runs-on: ubuntu-latest +# name: Create wheels for manylinux2014 - PowerPC +# steps: +# - uses: actions/checkout@v3 +# +# - name: Upgrade libssl +# run: sudo apt-get install -y libssl-dev +# +# - name: Pull images +# run: | +# docker pull multiarch/qemu-user-static +# docker pull quay.io/pypa/manylinux2014_ppc64le:latest +# +# - name: Install QEMU +# run: | +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build and audit wheels +# working-directory: ./bindings/python +# run: | +# docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \ +# --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_ppc64le \ +# /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh" +# +# create_wheels_manylinux_2014_aarch64: +# runs-on: ubuntu-latest +# name: Create wheels for manylinux2014 - Aarch64 +# steps: +# - uses: actions/checkout@v2 +# +# - name: Upgrade libssl +# run: sudo apt-get install -y libssl-dev +# +# - name: Pull images +# run: | +# docker pull multiarch/qemu-user-static +# docker pull quay.io/pypa/manylinux2014_aarch64:latest +# +# - name: Install QEMU +# run: | +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build and audit wheels +# working-directory: ./bindings/python +# run: | +# docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \ +# --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_aarch64 \ +# /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh" +# +# create_wheels_manylinux_2014_x390x: +# runs-on: ubuntu-latest +# name: Create wheels for manylinux2014 - S390X +# steps: +# - uses: actions/checkout@v2 +# +# - name: Upgrade libssl +# run: sudo apt-get install -y libssl-dev +# +# - name: Pull images +# run: | +# docker pull multiarch/qemu-user-static +# docker pull quay.io/pypa/manylinux2014_s390x:latest +# +# - name: Install QEMU +# run: | +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build and audit wheels +# working-directory: ./bindings/python +# run: | +# docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \ +# --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_s390x \ +# /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh" +# +# upload_package: +# name: Upload package to PyPi +# runs-on: ubuntu-latest +# needs: +# - create_wheels_manylinux_2014_ppc64le +# - create_wheels_manylinux_2014_aarch64 +# - create_wheels_manylinux_2014_x390x +# +# steps: +# - uses: actions/checkout@v2 +# +# - name: Install Python +# uses: actions/setup-python@v4 +# +# - name: Retrieve all wheels +# shell: bash +# run: | +# pip install awscli +# aws s3 sync "s3://safetensors-releases/python/$DIST_DIR" ./bindings/python/dist +# +# - name: Install dependencies +# run: | +# pip install setuptools wheel setuptools-rust +# +# - name: Upload to PyPi +# working-directory: ./bindings/python +# run: | +# pip install twine +# twine upload dist/* -u __token__ -p "$PYPI_TOKEN" diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 31fd8014..1bef79a5 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,8 +1,8 @@ name: Python Release on: push: - tags: - - v* + # tags: + # - v* env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -21,278 +21,137 @@ jobs: run: cat Cargo.lock working-directory: ./bindings/python - create_wheels_manylinux: - runs-on: ubuntu-latest - needs: [lock_exists] - name: Create wheels for manylinux2014 - container: quay.io/pypa/manylinux2014_x86_64 - steps: - - uses: actions/checkout@v3 - - - name: Install dependencies - run: yum install -y openssl-devel - - - name: Build and audit wheels - working-directory: ./bindings/python - run: sh build-wheels.sh - - create_wheels_windows: - name: Windows - runs-on: windows-latest - needs: [lock_exists] + build: + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) + # only run on push to main and on release + if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') strategy: + fail-fast: false matrix: - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - bits: ["32", "64"] + os: [ubuntu, macos, windows] + target: [x86_64, aarch64] + manylinux: [auto] + include: + - os: ubuntu + platform: linux + - os: windows + ls: dir + interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10 + - os: windows + ls: dir + target: i686 + python-architecture: x86 + interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 + - os: windows + ls: dir + target: aarch64 + interpreter: 3.11 3.12 + - os: macos + target: aarch64 + interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9 pypy3.10 + - os: ubuntu + platform: linux + target: i686 + - os: ubuntu + platform: linux + target: aarch64 + + - os: ubuntu + platform: linux + target: armv7 + interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 + # musllinux + - os: ubuntu + platform: linux + target: x86_64 + manylinux: musllinux_1_1 + - os: ubuntu + platform: linux + target: aarch64 + manylinux: musllinux_1_1 + - os: ubuntu + platform: linux + target: ppc64le + interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 + - os: ubuntu + platform: linux + target: s390x + interpreter: 3.7 3.8 3.9 3.10 3.11 3.12 + # exclude: + # # Optimized PGO builds for x86_64 manylinux and windows follow a different matrix, + # # maybe in future maturin-action can support this automatically + # - os: ubuntu + # target: x86_64 + # manylinux: auto + # - os: windows + # target: x86_64 + # # Windows on arm64 only supports Python 3.11+ + # - os: windows + # target: aarch64 + + runs-on: ${{ matrix.os }}-latest steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.bits == '32' && 'stable-i686-pc-windows-msvc' || 'stable-x86_64-pc-windows-msvc' }} - override: true - - - name: Override toolchain - shell: bash - working-directory: ./bindings/python - run: echo ${{ matrix.bits == '32' && 'stable-i686-pc-windows-msvc' || 'stable-x86_64-pc-windows-msvc' }} > rust-toolchain - - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - architecture: ${{ matrix.bits == '32' && 'x86' || 'x64' }} - - - name: Install dependencies - run: | - # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall - - - name: Build wheel - working-directory: ./bindings/python - run: python setup.py bdist_wheel - - - name: Upload wheels - shell: bash - run: | - pip install awscli - aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" - - create_wheels_windows: - name: Windows - runs-on: windows-latest - needs: [lock_exists] - strategy: - matrix: - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - bits: ["32", "64"] - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install Rust 32bits - if: ${{ matrix.os == '32' }} - uses: actions-rs/toolchain@v1 - with: - toolchain: stable-i686-pc-windows-msvc - override: true - - - name: Override toolchain - if: ${{ matrix.os == '32' }} - shell: bash - working-directory: ./bindings/python - run: echo "stable-i686-pc-windows-msvc" > rust-toolchain - - - name: Install Rust 64bits - if: ${{ matrix.os == '64' }} - uses: actions-rs/toolchain@v1 - + - uses: actions/checkout@v3 - - name: Install Python + - name: set up python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} - architecture: x86 - - - name: Install dependencies - run: | - # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall - - - name: Build wheel - working-directory: ./bindings/python - run: python setup.py bdist_wheel - - - name: Upload wheels - shell: bash - run: | - pip install awscli - aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" - - create_wheels_macos_conda: - name: MacOS - Conda - runs-on: ${{ matrix.os }} - needs: [lock_exists] - strategy: - matrix: - os: [macos-latest] - # 3.11 not available on Conda yet. - python: ["3.7", "3.8", "3.9", "3.10"] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python }} + python-version: '3.11' + architecture: ${{ matrix.python-architecture || 'x64' }} - - name: Conda info - shell: bash -l {0} - run: conda info + - run: pip install -U twine - - name: Install Rust - uses: actions-rs/toolchain@v1 + - name: build wheels + uses: PyO3/maturin-action@v1 with: - toolchain: stable - - - name: Setup conda env - shell: bash -l {0} - run: | - conda install setuptools-rust - conda install -c defaults anaconda-client conda-build - - - name: Extract version - shell: bash -l {0} - working-directory: ./bindings/python - run: echo "SAFETENSORS_VERSION=`python setup.py --version`" >> $GITHUB_ENV - - - name: Build conda packages - shell: bash -l {0} - working-directory: ./bindings/python - run: | - MACOSX_DEPLOYMENT_TARGET=10.11 python setup.py bdist_wheel + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'auto' }} + container: ${{ matrix.container }} + args: --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }} ${{ matrix.extra-build-args }} + rust-toolchain: stable + docker-options: -e CI - - name: Upload wheels - shell: bash - run: | - pip install awscli - aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" + - run: ${{ matrix.ls || 'ls -lh' }} dist/ - create_wheels_macos: - name: MacOS - runs-on: ${{ matrix.os.os }} - needs: [lock_exists] - strategy: - matrix: - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - # os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}, {os: "macos-13", target: "14.0"}] - os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}] - steps: - - name: Checkout repository - uses: actions/checkout@v2 + - run: twine check --strict dist/* - - name: Install Rust - uses: actions-rs/toolchain@v1 + - uses: actions/upload-artifact@v3 with: - toolchain: stable - override: true + name: pypi_files + path: dist - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: Install dependencies - run: | - # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall - - name: Override target - if: ${{ matrix.os.target }} - working-directory: ./bindings/python - run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.target }}" >> $GITHUB_ENV + # upload_package: + # name: Upload package to PyPi + # runs-on: ubuntu-latest + # needs: [build] - - name: Build wheel - working-directory: ./bindings/python - run: echo $MACOX_DEPLOYMENT_TARGET && python setup.py bdist_wheel + # steps: + # - uses: actions/checkout@v3 - - name: Upload wheels - shell: bash - run: | - pip install awscli - aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" + # - name: Install Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + # architecture: x64 + # - name: Retrieve all wheels + # shell: bash + # run: | + # pip install awscli + # aws s3 sync "s3://safetensors-releases/python/$DIST_DIR" ./bindings/python/dist - create_wheels_macos_arm64: - name: MacOS M1 - runs-on: macos-arm64 - needs: [lock_exists] - strategy: - matrix: - python: ["3.8.16", "3.9.13", "3.10.6", "3.11.0"] - # target: ["12.0", "13.0", "14.0"] - target: ["12.0", "13.0"] - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - - name: Install Python - shell: bash - run: | - echo $HOME - export PYENV_ROOT="$HOME/.pyenv" - command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell ${{ matrix.python }} - which pyenv - which python - pip install -U setuptools wheel setuptools-rust awscli - cd ./bindings/python - MACOSX_DEPLOYMENT_TARGET=${{ matrix.target }} python setup.py bdist_wheel - cd ../../ - aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" + # - name: Install dependencies + # run: | + # pip install setuptools wheel setuptools-rust - Upload_package: - name: Upload package to PyPi - runs-on: ubuntu-latest - needs: [create_wheels_manylinux, create_wheels_windows, create_wheels_macos, create_wheels_macos_arm64, create_wheels_macos_conda] - - steps: - - uses: actions/checkout@v3 + # - name: Create source distribution + # working-directory: ./bindings/python + # run: sh build-sdist.sh - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - architecture: x64 - - - name: Retrieve all wheels - shell: bash - run: | - pip install awscli - aws s3 sync "s3://safetensors-releases/python/$DIST_DIR" ./bindings/python/dist - - - name: Install dependencies - run: | - pip install setuptools wheel setuptools-rust - - - name: Create source distribution - working-directory: ./bindings/python - run: sh build-sdist.sh - - - name: Upload to PyPi - working-directory: ./bindings/python - run: | - pip install twine - twine upload dist/* -u __token__ -p "$PYPI_TOKEN" + # - name: Upload to PyPi + # working-directory: ./bindings/python + # run: | + # pip install twine + # twine upload dist/* -u __token__ -p "$PYPI_TOKEN" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 554dd832..aaa8e689 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,7 +1,103 @@ +[project] +name = 'safetensors' +requires-python = '>=3.7' +authors = [ + {name = 'Nicolas Patry', email = 'patry.nicolas@protonmail.com'} +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dynamic = [ + 'description', + 'license', + 'readme', + 'version' +] + +[project.urls] +Homepage = 'https://github.com/huggingface/safetensors' +Source = 'https://github.com/huggingface/safetensors' + +[project.optional-dependencies] +numpy = ["numpy>=1.21.6"] +torch = [ + "safetensors[numpy]", + "torch>=1.10", +] +tensorflow = [ + "safetensors[numpy]", + "tensorflow>=2.11.0", +] +# pinning tf version 2.11.0 for doc-builder +pinned-tf = [ + "safetensors[numpy]", + "tensorflow==2.11.0", +] +jax = [ + "safetensors[numpy]", + "flax>=0.6.3", + "jax>=0.3.25", + "jaxlib>=0.3.25", +] +paddlepaddle = [ + "safetensors[numpy]", + "paddlepaddle>=2.4.1", +] +quality = [ + "black==22.3", # after updating to black 2023, also update Python version in pyproject.toml to 3.7 + "click==8.0.4", + "isort>=5.5.4", + "flake8>=3.8.3", +] +testing = [ + "safetensors[numpy]", + "h5py>=3.7.0", + "huggingface_hub>=0.12.1", + "setuptools_rust>=1.5.2", + "pytest>=7.2.0", + "pytest-benchmark>=4.0.0", + "python-afl>=0.7.3", + "hypothesis>=6.70.2", +] +all = [ + "safetensors[torch]", + "safetensors[numpy]", + "safetensors[pinned-tf]", + "safetensors[jax]", + "safetensors[paddlepaddle]", + "safetensors[quality]", + "safetensors[testing]", +] +dev = [ + "safetensors[all]", +] + + [build-system] -requires = ["setuptools", "wheel", "setuptools-rust"] -build-backend = "setuptools.build_meta" +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +python-source = "py_src" +module-name = "safetensors.safetensors_rust" +bindings = 'pyo3' +features = ["pyo3/extension-module"] [tool.black] line-length = 119 target-version = ['py35'] + +[tool.setuptools.dynamic] +version = {attr = "safetensors.__version__"} +readme = {file = ["README.rst"]} diff --git a/bindings/python/setup.py b/bindings/python/setup.py deleted file mode 100644 index 324632c8..00000000 --- a/bindings/python/setup.py +++ /dev/null @@ -1,97 +0,0 @@ -import re - -from setuptools import setup -from setuptools_rust import Binding, RustExtension - - -# IMPORTANT: -# 1. all dependencies should be listed here with their version requirements if any -_deps = [ - "black==22.3", # after updating to black 2023, also update Python version in pyproject.toml to 3.7 - "click==8.0.4", - "flake8>=3.8.3", - "flax>=0.6.3", - "h5py>=3.7.0", - "huggingface_hub>=0.12.1", - "isort>=5.5.4", - "jax>=0.3.25", - "jaxlib>=0.3.25", - "numpy>=1.21.6", - "setuptools_rust>=1.5.2", - "pytest>=7.2.0", - "pytest-benchmark>=4.0.0", - "tensorflow>=2.11.0", - "torch>=1.10", - "paddlepaddle>=2.4.1", - "python-afl>=0.7.3", - "hypothesis>=6.70.2", -] - - -deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)} - - -def deps_list(*pkgs): - return [deps[pkg] for pkg in pkgs] - - -extras = {} -extras["numpy"] = deps_list("numpy") -extras["torch"] = deps_list("torch") + extras["numpy"] -extras["tensorflow"] = deps_list("tensorflow") + extras["numpy"] -extras["jax"] = deps_list("jax", "flax", "jaxlib") + extras["numpy"] -extras["paddlepaddle"] = deps_list("paddlepaddle") + extras["numpy"] -extras["quality"] = deps_list("black", "isort", "flake8", "click") -extras["testing"] = ( - deps_list("setuptools_rust", "huggingface_hub", "pytest", "pytest-benchmark", "h5py") + extras["numpy"] -) -# pinning tf version 2.11.0 for doc-builder -extras["pinned-tf"] = ["tensorflow==2.11.0"] -extras["all"] = ( - extras["torch"] - + extras["numpy"] - + extras["pinned-tf"] - + extras["jax"] - + extras["paddlepaddle"] - + extras["quality"] - + extras["testing"] -) -extras["dev"] = extras["all"] - -with open("py_src/safetensors/__init__.py", "r") as f: - version = f.readline().split("=")[-1].strip().strip('"') - -setup( - name="safetensors", - version=version, - description="Fast and Safe Tensor serialization", - long_description=open("README.md", "r", encoding="utf-8").read(), - long_description_content_type="text/markdown", - keywords="", - author="", - author_email="", - url="https://github.com/huggingface/safetensors", - license="Apache License 2.0", - rust_extensions=[RustExtension("safetensors._safetensors_rust", binding=Binding.PyO3, debug=False)], - extras_require=extras, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - ], - package_dir={"": "py_src"}, - packages=[ - "safetensors", - ], - package_data={}, - zip_safe=False, -)