diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml deleted file mode 100644 index 969c791f..00000000 --- a/.azure-pipelines/azure-pipelines.yml +++ /dev/null @@ -1,133 +0,0 @@ -trigger: - branches: - include: - - '*' - tags: - include: - - '*' - -stages: -- stage: static - displayName: Static Analysis - jobs: - - job: checks - displayName: static code analysis - pool: - vmImage: ubuntu-20.04 - steps: - # Use Python >=3.8 for syntax validation - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.8 - - # Run syntax validation on a shallow clone - - bash: | - python .azure-pipelines/syntax-validation.py - displayName: Syntax validation - - # Run flake8 validation on a shallow clone - - bash: | - pip install flake8 - python .azure-pipelines/flake8-validation.py - displayName: Flake8 validation - -- stage: build - displayName: Build - dependsOn: - jobs: - - job: build - displayName: build package - pool: - vmImage: ubuntu-20.04 - steps: - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.9 - - - bash: | - pip install -U pip - pip install collective.checkdocs wheel - displayName: Install dependencies - - - bash: | - set -ex - python setup.py sdist bdist_wheel - mkdir -p dist/pypi - shopt -s extglob - mv -v dist/!(pypi) dist/pypi - git archive HEAD | gzip > dist/repo-source.tar.gz - ls -laR dist - displayName: Build python package - - - task: PublishBuildArtifacts@1 - displayName: Store artifact - inputs: - pathToPublish: dist/ - artifactName: package - - - bash: python setup.py checkdocs - displayName: Check package description - -- stage: tests - displayName: Run unit tests - dependsOn: - - static - - build - jobs: - - job: linux - pool: - vmImage: ubuntu-20.04 - strategy: - matrix: - python38: - PYTHON_VERSION: 3.8 - python39: - PYTHON_VERSION: 3.9 - python310: - PYTHON_VERSION: 3.10 - python311: - PYTHON_VERSION: 3.11 - steps: - - template: ci.yml - -- stage: deploy - displayName: Publish release - dependsOn: - - tests - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) - jobs: - - job: pypi - displayName: Publish pypi release - pool: - vmImage: ubuntu-20.04 - steps: - - checkout: none - - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.8 - - - task: DownloadBuildArtifacts@0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - - - script: | - pip install -U pip - pip install twine - displayName: Install twine - - - task: TwineAuthenticate@1 - displayName: Set up credentials - inputs: - pythonUploadServiceConnection: pypi-nexgen - - - bash: | - python -m twine upload -r pypi-nexgen --config-file $(PYPIRC_PATH) $(System.ArtifactsDirectory)/package/pypi/*.tar.gz $(System.ArtifactsDirectory)/package/pypi/*.whl - displayName: Publish package diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml deleted file mode 100644 index 52555002..00000000 --- a/.azure-pipelines/ci.yml +++ /dev/null @@ -1,43 +0,0 @@ -steps: -- checkout: none - -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(PYTHON_VERSION)' - displayName: 'Use Python $(PYTHON_VERSION)' - -- task: DownloadBuildArtifacts@0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - -- task: ExtractFiles@1 - displayName: Checkout sources - inputs: - archiveFilePatterns: "$(System.ArtifactsDirectory)/package/repo-source.tar.gz" - destinationFolder: "$(Pipeline.Workspace)/src" - -- script: | - pip install -U pip - pip install -r "$(Pipeline.Workspace)/src/requirements_dev.txt" - pip install -e "$(Pipeline.Workspace)/src" - displayName: Install package - -- script: | - pip install pytest - pip install pytest-cov - python -m pytest -ra --cov=nexgen --cov-report=xml --cov-branch - displayName: Run tests - workingDirectory: $(Pipeline.Workspace)/src - -- bash: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -B $(Build.SourceBranchName) -C $(Build.SourceVersion) -t $(CODECOV_TOKEN) -n "Python $(PYTHON_VERSION) $(Agent.OS)" - displayName: 'Publish coverage stats' - continueOnError: True - workingDirectory: $(Pipeline.Workspace)/src - timeoutInMinutes: 2 diff --git a/.azure-pipelines/flake8-validation.py b/.azure-pipelines/flake8-validation.py deleted file mode 100644 index 89594a95..00000000 --- a/.azure-pipelines/flake8-validation.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import subprocess - -# Flake8 validation -failures = 0 -try: - flake8 = subprocess.run( - [ - "flake8", - "--exit-zero", - "--max-line-length=88", - "--select=E401,E711,E712,E713,E714,E721,E722,E901,F401,F402,F403,F405,F631,F632,F633,F811,F812,F821,F822,F841,F901,W191,W291,W292,W293,W602,W603,W604,W605,W606", - ], - capture_output=True, - check=True, - encoding="latin-1", - timeout=300, - ) -except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - print( - "##vso[task.logissue type=error;]flake8 validation failed with", - str(e.__class__.__name__), - ) - print(e.stdout) - print(e.stderr) - print("##vso[task.complete result=Failed;]flake8 validation failed") - exit() -for line in flake8.stdout.split("\n"): - if ":" not in line: - continue - filename, lineno, column, error = line.split(":", maxsplit=3) - errcode, error = error.strip().split(" ", maxsplit=1) - filename = os.path.normpath(filename) - failures += 1 - print( - f"##vso[task.logissue type=error;sourcepath={filename};" - f"linenumber={lineno};columnnumber={column};code={errcode};]" + error - ) - -if failures: - print(f"##vso[task.logissue type=warning]Found {failures} flake8 violation(s)") - print(f"##vso[task.complete result=Failed;]Found {failures} flake8 violation(s)") diff --git a/.azure-pipelines/syntax-validation.py b/.azure-pipelines/syntax-validation.py deleted file mode 100644 index 9359ad84..00000000 --- a/.azure-pipelines/syntax-validation.py +++ /dev/null @@ -1,30 +0,0 @@ -import ast -import os -import sys - -print("Python", sys.version, "\n") - -failures = 0 - -for base, _, files in os.walk("."): - for f in files: - if not f.endswith(".py"): - continue - filename = os.path.normpath(os.path.join(base, f)) - try: - with open(filename) as fh: - ast.parse(fh.read()) - except SyntaxError as se: - failures += 1 - print( - f"##vso[task.logissue type=error;sourcepath={filename};" - f"linenumber={se.lineno};columnnumber={se.offset};]" - f"SyntaxError: {se.msg}" - ) - print(" " + se.text + " " * se.offset + "^") - print(f"SyntaxError: {se.msg} in {filename} line {se.lineno}") - print() - -if failures: - print(f"##vso[task.logissue type=warning]Found {failures} syntax error(s)") - print(f"##vso[task.complete result=Failed;]Found {failures} syntax error(s)") diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 77030f3d..a147a746 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -3,9 +3,9 @@ current_version = 0.7.2 commit = True tag = True -[bumpversion:file:setup.cfg] -search = version = {current_version} -replace = version = {new_version} +[bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" [bumpversion:file:src/nexgen/__init__.py] search = __version__ = "{current_version}" diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml new file mode 100644 index 00000000..42ede5d7 --- /dev/null +++ b/.github/actions/install_requirements/action.yml @@ -0,0 +1,57 @@ +name: Install requirements +description: Run pip install with requirements and upload resulting requirements +inputs: + requirements_file: + description: Name of requirements file to use and upload + required: true + install_options: + description: Parameters to pass to pip install + required: true + python_version: + description: Python version to install + default: "3.x" + +runs: + using: composite + + steps: + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Pip install + run: | + touch ${{ inputs.requirements_file }} + # -c uses requirements.txt as constraints, see 'Validate requirements file' + pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }} + shell: bash + + - name: Create lockfile + run: | + mkdir -p lockfiles + pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} + # delete the self referencing line and make sure it isn't blank + sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }} + shell: bash + + - name: Upload lockfiles + uses: actions/upload-artifact@v3 + with: + name: lockfiles + path: lockfiles + + # This eliminates the class of problems where the requirements being given no + # longer match what the packages themselves dictate. E.g. In the rare instance + # where I install some-package which used to depend on vulnerable-dependency + # but now uses good-dependency (despite being nominally the same version) + # pip will install both if given a requirements file with -r + - name: If requirements file exists, check it matches pip installed packages + run: | + if [ -s ${{ inputs.requirements_file }} ]; then + if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then + echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive" + exit 1 + fi + fi + shell: bash \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..752a1b3f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml new file mode 100644 index 00000000..b1ef4a31 --- /dev/null +++ b/.github/workflows/code.yml @@ -0,0 +1,153 @@ +name: Code CI + +on: + push: + pull_request: + schedule: + # Run weekly to check latest versions of dependencies + - cron: "0 8 * * WED" + +jobs: + lint: + # pull requests are a duplicate of a branch push if within the same repo. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + + steps: + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + architecture: x64 + + - name: Checkout nexgen + uses: actions/checkout@v3 + + - name: Install python packages + uses: ./.github/actions/install_requirements + with: + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] + + - name: Install ruff + run: pip install ruff + + - name: Run ruff + run: ruff . + + test: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] # can add windows-latest, macos-latest + python: ["3.9", "3.10", "3.11"] + install: ["-e .[dev]"] + # Make one version be non-editable to test both paths of version code + include: + - os: "ubuntu-latest" + python: "3.8" + install: ".[dev]" + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Install python packages + uses: ./.github/actions/install_requirements + with: + python_version: ${{ matrix.python }} + requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt + install_options: ${{ matrix.install }} + + - name: List dependency tree + run: pipdeptree + + - name: Run tests + run: pytest + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + name: ${{ matrix.python }}/${{ matrix.os }} + files: cov.xml + + dist: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: "ubuntu-latest" + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Build sdist and wheel + run: | + export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + pipx run build + + - name: Upload sdist and wheel as artifacts + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Check for packaging errors + run: pipx run twine check --strict dist/* + + - name: Install python packages + uses: ./.github/actions/install_requirements + with: + python_version: '3.10' + requirements_file: requirements.txt + install_options: dist/*.whl + + - name: Test module --version works using the installed wheel + # If more than one module in src/ replace with module name to test + run: python -m nexgen --version + + release: + # upload to PyPI and make a release on every tag + needs: [lint, dist, test] + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} + runs-on: ubuntu-latest + env: + HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }} + + environment: + name: pypi + url: https://pypi.org/p/nexgen + + steps: + - uses: actions/download-artifact@v3 + + - name: Fixup blank lockfiles + # Github release artifacts can't be blank + run: for f in lockfiles/*; do [ -s $f ] || echo '# No requirements' >> $f; done + + - name: Github Release + # We pin to the SHA, not the tag, for security reasons. + # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 + with: + prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} + files: | + dist/* + lockfiles/* + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish to PyPI + if: ${{ env.HAS_PYPI_TOKEN }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + \ No newline at end of file diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml new file mode 100644 index 00000000..fa7ba779 --- /dev/null +++ b/.github/workflows/linkcheck.yml @@ -0,0 +1,24 @@ +name: Link Check + +on: + workflow_dispatch: + schedule: + # Run weekly to check URL links still resolve + - cron: "0 8 * * WED" + +jobs: + docs: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install python packages + uses: ./.github/actions/install_requirements + with: + requirements_file: requirements_dev.txt + install_options: -e .[dev] + + - name: Check links + run: tox -e docs build -- -b linkcheck \ No newline at end of file diff --git a/.gitignore b/.gitignore index fa12309e..0530b3f2 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,13 @@ docs/_build/ # PyBuilder target/ +# likely venv names +.venv* +venv* + +# further build artifacts +lockfiles/ + # Jupyter Notebook .ipynb_checkpoints @@ -134,3 +141,6 @@ dmypy.json # JetBrains PyCharm project files /.idea/* + +# ruff cache +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5820957d..527dd8cd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,21 +7,17 @@ repos: - id: black args: [--safe, --quiet] -# Sort imports -- repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - name: isort (python) - args: ['--profile=black', '--add_imports="from __future__ import annotations'] -# Linting -- repo: https://github.com/pycqa/flake8 - rev: 5.0.4 +# Linter +- repo: local hooks: - - id: flake8 - additional_dependencies: ['flake8-comprehensions==3.8.0'] - args: ['--max-line-length=88', '--select=E401,E711,E712,E713,E714,E721,E722,E901,F401,F402,F403,F405,F631,F632,F633,F811,F812,F821,F822,F841,F901,W191,W291,W292,W293,W602,W603,W604,W605,W606', '--ignore=E203,E266,E402,E501,W503,E741'] + - id: ruff + name: Run ruff + stages: [commit] + language: system + entry: ruff + types: [python] + # Other syntax checks - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/docs/conf.py b/docs/conf.py index 2f681ddb..9d237ce0 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,11 +20,10 @@ import os import sys -sys.path.insert(0, os.path.abspath("..")) +import nexgen -import sphinx_rtd_theme # noqa; F401 - install theme +sys.path.insert(0, os.path.abspath("..")) -import nexgen # -- General configuration --------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 61356e3a..09a2db8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,126 @@ [build-system] -requires = ["setuptools >= 40.6.0", "wheel"] +requires = ["setuptools>=64", "setuptools_scm[toml]>=8.0.1", "wheel"] build-backend = "setuptools.build_meta" -[tool.isort] -profile = "black" \ No newline at end of file +[project] +name = "nexgen" +version = "0.7.2" +description = "Next Generation Nexus Generator" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +keywords = ["nexus", "NXmx"] +dependencies = [ + "freephil", + "h5py", + "hdf5plugin>=4.0.1", + "numpy", + "pint", + "importlib_resources>=1.1", + "scanspec", + "dataclasses-json", + "pydantic<2.0", +] +license.file = "LICENSE" +readme = "README.rst" +requires-python = ">=3.8" + +[project.optional-dependencies] +dev = [ + "black", + "ruff", + "pytest-cov", + "pytest-random-order", + "sphinx-autobuild", + "pipdeptree", + "ipython", + "mockito", + "pre-commit", + "mypy", + "tox", + "build", + "types-mock", +] + +[project.urls] +GitHub = "https://github.com/dials/nexgen" +Documentation = "https://nexgen.readthedocs.io/" +Bug-Tracker = "https://github.com/dials/nexgen/issues" + +#[[project.authors]] +#email = "data_analysis@diamond.ac.uk" +#name = "Diamond Light Source - Scientific Software" + +[project.scripts] +nexgen = "nexgen.__main__:main" +generate_nexus = "nexgen.command_line.nexus_generator:main" +copy_nexus = "nexgen.command_line.copy_nexus:main" +nexgen_phil = "nexgen.command_line.phil_files_cli:main" +I19_nexus = "nexgen.command_line.I19_2_cli:main" +ED_nexus = "nexgen.command_line.ED_nexus:main" +SSX_nexus = "nexgen.command_line.SSX_cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +src = ["src", "tests"] +line-length = 88 +extend-ignore = [ + "E501", # Line too long + "F811", # support typing.overload decorator + "E203", + "E266", + "E402", + "E741", +] +select = [ + "C4", # flake8-comprehensions - https://beta.ruff.rs/docs/rules/#flake8-comprehensions-c4 + "E", # pycodestyle errors - https://beta.ruff.rs/docs/rules/#error-e + "F", # pyflakes rules - https://beta.ruff.rs/docs/rules/#pyflakes-f + "W", # pycodestyle warnings - https://beta.ruff.rs/docs/rules/#warning-w + "I001", # isort +] + +[tool.mypy] +plugins = ["pydantic.mypy"] +ignore_missing_imports = true # Ignore missing stubs in imported modules + +[tool.pytest.ini_options] +addopts = """ + -ra + --cov=nexgen --cov-report=xml --cov-branch + """ +junit_family = "xunit2" +testpaths = "src tests" + +# tox must currently be configured via an embedded ini string +# See: https://github.com/tox-dev/tox/issues/999 +[tool.tox] +legacy_tox_ini = """ +[tox] +skipsdist=True + +[testenv:{pre-commit,mypy,pytest,docs}] +# Don't create a virtualenv for the command, requires tox-direct plugin +direct = True +passenv = * +allowlist_externals = + pytest + pre-commit + mypy + sphinx-build + sphinx-autobuild +commands = + pytest: pytest {posargs} + mypy: mypy src tests --ignore-missing-imports --no-strict-optional {posargs} + pre-commit: pre-commit run --all-files {posargs} + docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html +""" \ No newline at end of file diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b4efef80..00000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = -ra -junit_family = xunit2 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8432eeb3..00000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -freephil==0.2.1 -h5py==3.9.0 -hdf5plugin==4.1.3 -numpy==1.24.3 -pint==0.22 -scanspec==0.6.1 -pydantic==1.10.12 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 46889e3c..66d5d253 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,6 +5,7 @@ numpy==1.24.3 pint==0.22 scanspec==0.6.1 pydantic==1.10.12 +importlib_resources==6.0.1 pytest==7.4.0 -pytest-cov==3.11.1 +pytest-cov==4.1.0 pytest-random-order==1.1.0 diff --git a/requirements_doc.txt b/requirements_doc.txt index 6fa1db7b..5a8bc8e8 100644 --- a/requirements_doc.txt +++ b/requirements_doc.txt @@ -5,6 +5,6 @@ pint==0.22 h5py==3.9.0 hdf5plugin==4.1.3 freephil==0.2.1 -importlib_resources==5.6.0 +importlib_resources==6.0.1 scanspec==0.6.1 pydantic==1.10.12 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 94342c1a..00000000 --- a/setup.cfg +++ /dev/null @@ -1,76 +0,0 @@ -[metadata] -name = nexgen -version = 0.7.2 -description = Next Generation Nexus Generator -long_description = file: README.rst -author = Diamond Light Source - Scientific Software -author_email = data_analysis@diamond.ac.uk -license = BSD 3-Clause License -license_file = LICENSE -classifiers = - Development Status :: 3 - Alpha - Intended Audience :: Developers - License :: OSI Approved :: BSD License - Natural Language :: English - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 -keywords = - dials - nexus -project-urls = - Documentation = https://nexgen.readthedocs.io/ - GitHub = https://github.com/dials/nexgen - Bug-Tracker = https://github.com/dials/nexgen/issues - -[options] -include_package_data = True -install_requires = - freephil - h5py - hdf5plugin>=4.0.1 - numpy - pint - importlib_resources>=1.1 - scanspec - dataclasses-json - pydantic<2.0 -packages = find: -package_dir = - =src -python_requires = >=3.8 -zip_safe = False - -[options.extras_require] -dev = - black - pytest-cov - pytest-random-order - ipython - pre-commit - flake8 - build - -[options.packages.find] -where = src - -[isort] -profile=black -float_to_top=true - -[flake8] -max-line-length = 88 -extend-ignore = E203,E266,E402,E501,W503,E741 - -[options.entry_points] -libtbx.precommit = - nexgen = nexgen -console_scripts = - generate_nexus = nexgen.command_line.nexus_generator:main - copy_nexus = nexgen.command_line.copy_nexus:main - nexgen_phil = nexgen.command_line.phil_files_cli:main - I19_nexus = nexgen.command_line.I19_2_cli:main - ED_nexus = nexgen.command_line.ED_nexus:main - SSX_nexus = nexgen.command_line.SSX_cli:main \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index bac24a43..00000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -import setuptools - -if __name__ == "__main__": - setuptools.setup() diff --git a/src/nexgen/__main__.py b/src/nexgen/__main__.py new file mode 100644 index 00000000..d6c70251 --- /dev/null +++ b/src/nexgen/__main__.py @@ -0,0 +1,10 @@ +from .command_line import version_parser + + +def main(args=None): + version_parser.parse_args(args) + + +# Test with python -m nexgen +if __name__ == "__main__": + main() diff --git a/src/nexgen/beamlines/GDAtools/GDAjson2params.py b/src/nexgen/beamlines/GDAtools/GDAjson2params.py index 90701801..5e541bf5 100644 --- a/src/nexgen/beamlines/GDAtools/GDAjson2params.py +++ b/src/nexgen/beamlines/GDAtools/GDAjson2params.py @@ -38,7 +38,7 @@ def get_goniometer_axes_from_file(self) -> List[Axis]: """Read the axes information from the GDA-supplied json file.""" axes_list = [] for v in self.params.values(): - if type(v) is dict and v["location"] == "sample": + if isinstance(v, dict) and v["location"] == "sample": ax_depends = self._find_axis_depends_on(v["depends_on"]) ax_type = ( TransformationType.ROTATION @@ -54,7 +54,7 @@ def get_detector_axes_from_file(self) -> List[Axis]: """Read the detector axes information from the GDA-supplied json file.""" axes_list = [] for v in self.params.values(): - if type(v) is dict and v["location"] == "detector": + if isinstance(v, dict) and v["location"] == "detector": ax_type = ( TransformationType.ROTATION if v["type"] == "rotation" diff --git a/src/nexgen/command_line/cli_utils.py b/src/nexgen/command_line/cli_utils.py index d9190d24..62613b29 100644 --- a/src/nexgen/command_line/cli_utils.py +++ b/src/nexgen/command_line/cli_utils.py @@ -285,9 +285,9 @@ def calculate_scan_range( Dict[str, ArrayLike]: A dictionary of ("axis_name": axis_range) key-value pairs. """ if ( - type(axes_names) is not list - or type(axes_starts) is not list - or type(axes_ends) is not list + not isinstance(axes_names, list) + or not isinstance(axes_starts, list) + or not isinstance(axes_ends, list) ): raise TypeError("Input values for axes must be passed as lists.") @@ -416,7 +416,7 @@ def ScanReader( transl_start = [goniometer["starts"][i] for i in transl_idx] transl_end = [goniometer["ends"][i] for i in transl_idx] transl_increment = [goniometer["increments"][i] for i in transl_idx] - if n_images and type(n_images) is int: + if n_images and isinstance(n_images, int): TRANSL = calculate_scan_range( transl_axes, transl_start, @@ -550,7 +550,7 @@ def call_writers( datafiles = [Path(f).expanduser().resolve() for f in datafiles] if metafile: - if type(metafile) is str: + if isinstance(metafile, str): metafile = Path(metafile).expanduser().resolve() # NXdata: entry/data diff --git a/src/nexgen/command_line/nexus_generator.py b/src/nexgen/command_line/nexus_generator.py index cb7d9e53..355ff54f 100644 --- a/src/nexgen/command_line/nexus_generator.py +++ b/src/nexgen/command_line/nexus_generator.py @@ -34,7 +34,10 @@ phil2dict, version_parser, ) -from .cli_utils import ScanReader, call_writers # write_nexus_demo, write_nexus +from .cli_utils import ( + ScanReader, # write_nexus_demo, write_nexus + call_writers, +) # Define a logger object logger = logging.getLogger("nexgen.NeXusGenerator") diff --git a/src/nexgen/command_line/phil_files_cli.py b/src/nexgen/command_line/phil_files_cli.py index 4a5ddf44..53078031 100644 --- a/src/nexgen/command_line/phil_files_cli.py +++ b/src/nexgen/command_line/phil_files_cli.py @@ -7,19 +7,19 @@ import logging import shutil import sys +from pathlib import Path import freephil +from .. import log, templates +from . import config_parser, nexus_parser, version_parser + try: from importlib.resources import files except ImportError: # Python < 3.9 compatibility from importlib_resources import files -from pathlib import Path - -from .. import log, templates -from . import config_parser, nexus_parser, version_parser # Define a logger object logger = logging.getLogger("nexgen.NeXusGenerator") diff --git a/src/nexgen/nxs_copy/copy_utils.py b/src/nexgen/nxs_copy/copy_utils.py index 2ec82a6a..c5800566 100644 --- a/src/nexgen/nxs_copy/copy_utils.py +++ b/src/nexgen/nxs_copy/copy_utils.py @@ -141,7 +141,7 @@ def convert_scan_axis(nxsample: h5py.Group, nxdata: h5py.Group, ax: str): def check_and_fix_det_axis(nxs_in: h5py.File): det_z_grp = nxs_in["/entry/instrument/detector/transformations/detector_z"] det_z = det_z_grp["det_z"] - if type(det_z[()]) is bytes or type(det_z[()]) is str: + if isinstance(det_z[()], bytes) or isinstance(det_z[()], str): det_z_attrs = {} for k, v in det_z.attrs.items(): det_z_attrs[k] = v diff --git a/src/nexgen/nxs_write/NXclassWriters.py b/src/nexgen/nxs_write/NXclassWriters.py index cd7da6ae..e4d0b05f 100644 --- a/src/nexgen/nxs_write/NXclassWriters.py +++ b/src/nexgen/nxs_write/NXclassWriters.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, get_args +import h5py # isort: skip import numpy as np from numpy.typing import ArrayLike @@ -30,9 +31,6 @@ # from hdf5plugin import Bitshuffle # noqa: F401 -import h5py # isort: skip - - NXclass_logger = logging.getLogger("nexgen.NXclass_writers") NXclass_logger.setLevel(logging.DEBUG) @@ -334,7 +332,7 @@ def write_NXsample( if sample_details: for k, v in sample_details.items(): - if type(v) is str: + if isinstance(v, str): v = np.string_(v) nxsample.create_dataset(k, data=v) @@ -534,7 +532,7 @@ def write_NXdetector( nxdetector["flatfield"] = h5py.ExternalLink(flatfield.name, image_key) else: # Flatfield - if type(detector["flatfield"]) is str: + if isinstance(detector["flatfield"], str): nxdetector.create_dataset( "flatfield_applied", data=detector["flatfield_applied"] ) @@ -550,7 +548,7 @@ def write_NXdetector( ) write_compressed_copy(nxdetector, "flatfield", data=detector["flatfield"]) # Bad pixel mask - if type(detector["pixel_mask"]) is str: + if isinstance(detector["pixel_mask"], str): nxdetector.create_dataset( "pixel_mask_applied", data=detector["pixel_mask_applied"] ) @@ -691,7 +689,7 @@ def write_NXdetector( if nxdetector.__contains__(dset) is False and dset in detector.keys(): val = ( np.string_(detector[dset]) - if type(detector[dset]) is str + if isinstance(detector[dset], str) else detector[dset] ) if val is not None: # FIXME bit of a gorilla here, for bit_depth_readout @@ -939,7 +937,7 @@ def write_NXnote(nxsfile: h5py.File, loc: str, info: Dict): # Write datasets for k, v in info.items(): if v: # Just in case one value is not recorded and set as None - if type(v) is str: + if isinstance(v, str): v = np.string_(v) nxnote.create_dataset(k, data=v) NXclass_logger.info(f"{k} dataset written in {loc}.") diff --git a/src/nexgen/nxs_write/write_utils.py b/src/nexgen/nxs_write/write_utils.py index 92ea9251..b1c1f8bc 100644 --- a/src/nexgen/nxs_write/write_utils.py +++ b/src/nexgen/nxs_write/write_utils.py @@ -10,12 +10,11 @@ from pathlib import Path from typing import List, Literal, Tuple +import h5py # isort: skip import numpy as np from hdf5plugin import Bitshuffle, Blosc from numpy.typing import ArrayLike -import h5py # isort: skip - # Define Timestamp dataset names TSdset = Literal["start_time", "end_time", "end_time_estimated"] diff --git a/src/nexgen/tools/MetaReader.py b/src/nexgen/tools/MetaReader.py index e45ae071..352e5f36 100644 --- a/src/nexgen/tools/MetaReader.py +++ b/src/nexgen/tools/MetaReader.py @@ -44,7 +44,7 @@ def overwrite_beam(meta_file: h5py.File, name: str, beam: Dict | ScopeExtract): # If value exists, overwrite. Otherwise, create. overwrite_logger.warning("Wavelength will be overwritten.") overwrite_logger.info(f"Value for wavelength found in meta file: {wl}") - if type(beam) is dict: + if isinstance(beam, dict): beam["wavelength"] = wl else: try: @@ -72,7 +72,7 @@ def overwrite_detector( """ new_values = {} link_list = [[], []] - if type(detector) is dict: + if isinstance(detector, dict): detector_name = detector["description"].lower() else: detector_name = detector.description.lower() @@ -158,7 +158,7 @@ def overwrite_detector( del new_values[i] for k, v in new_values.items(): - if type(detector) is dict: + if isinstance(detector, dict): detector[k] = v else: try: diff --git a/src/nexgen/tools/VDS_tools.py b/src/nexgen/tools/VDS_tools.py index 9b1de44a..b173bd4b 100644 --- a/src/nexgen/tools/VDS_tools.py +++ b/src/nexgen/tools/VDS_tools.py @@ -119,13 +119,13 @@ def split_datasets( if start_idx < 0: raise ValueError("Start index must be positive") - if type(data_shape[0]) is not int: + if not isinstance(data_shape[0], int): vds_logger.warning("Datashape not passed as int, will attempt to cast") - if type(start_idx) is not int: + if not isinstance(start_idx, int): vds_logger.warning("VDS start index not passed as int, will attempt to cast") - if vds_shape and type(vds_shape[0]) is not int: + if vds_shape and not isinstance(vds_shape[0], int): vds_logger.warning("VDS start index not passed as int, will attempt to cast") if vds_shape is None: diff --git a/tests/beamlines/test_GDA_tools.py b/tests/beamlines/test_GDA_tools.py index 486166b9..1ee6ef4b 100644 --- a/tests/beamlines/test_GDA_tools.py +++ b/tests/beamlines/test_GDA_tools.py @@ -12,14 +12,14 @@ def test_get_coordinate_frame_from_json(dummy_geometry_json): def test_get_gonio_axes_from_json(dummy_geometry_json): gonio_axes = JSONParamsIO(dummy_geometry_json.name).get_goniometer_axes_from_file() - assert type(gonio_axes) is list and len(gonio_axes) == 3 + assert isinstance(gonio_axes, list) and len(gonio_axes) == 3 assert gonio_axes[0].name == "omega" and gonio_axes[1].name == "sam_x" assert gonio_axes[2].name == "phi" and gonio_axes[2].depends == "sam_x" def test_get_detector_axes_from_json(dummy_geometry_json): det_axes = JSONParamsIO(dummy_geometry_json.name).get_detector_axes_from_file() - assert type(det_axes) is list and len(det_axes) == 1 + assert isinstance(det_axes, list) and len(det_axes) == 1 assert det_axes[0].name == "det_z" @@ -51,7 +51,7 @@ def test_read_scan_from_xml(dummy_xml_file): scan_axis, pos, num = read_scan_from_xml(test_ecr) assert scan_axis == test_ecr.getAxisChoice() assert num == 10 - assert type(pos) is dict and len(pos) == 6 # gonio axes on I19-2 + assert isinstance(pos, dict) and len(pos) == 6 # gonio axes on I19-2 assert pos["omega"] == (-180.0, -160.0, 2) assert pos["phi"] == (*2 * (test_ecr.getOtherAxis(),), 0.0) diff --git a/tests/beamlines/test_SSX_chip.py b/tests/beamlines/test_SSX_chip.py index 599da1b4..759ab549 100644 --- a/tests/beamlines/test_SSX_chip.py +++ b/tests/beamlines/test_SSX_chip.py @@ -29,20 +29,20 @@ def test_chip_windows(): def test_chip_size(): size = test_chip.chip_size() - assert type(size) is tuple + assert isinstance(size, tuple) assert size == (6.35, 6.35) def test_chip_types(): - assert type(test_chip.num_steps[0]) is int - assert type(test_chip.step_size[0]) is float - assert type(test_chip.num_blocks[0]) is int - assert type(test_chip.block_size[0]) is float + assert isinstance(test_chip.num_steps[0], int) + assert isinstance(test_chip.step_size[0], float) + assert isinstance(test_chip.num_blocks[0], int) + assert isinstance(test_chip.block_size[0], float) def test_no_chip_map_passed_returns_fullchip(): res = read_chip_map(None, 1, 1) - assert type(res) is dict + assert isinstance(res, dict) assert list(res.values())[0] == "fullchip" diff --git a/tests/beamlines/test_SSX_expt.py b/tests/beamlines/test_SSX_expt.py index 4535d142..675b651c 100644 --- a/tests/beamlines/test_SSX_expt.py +++ b/tests/beamlines/test_SSX_expt.py @@ -35,7 +35,7 @@ def test_run_extruder(): assert gonio[idx].num_steps == 10 assert list(osc.keys()) == ["omega"] assert_array_equal(osc["omega"], np.zeros(10)) - assert type(info) is dict + assert isinstance(info, dict) assert info["pump_exposure"] == 0.1 and info["pump_delay"] is None diff --git a/tests/nxs_utils/test_detector.py b/tests/nxs_utils/test_detector.py index cd3b449a..d177a317 100644 --- a/tests/nxs_utils/test_detector.py +++ b/tests/nxs_utils/test_detector.py @@ -44,7 +44,7 @@ def test_jungfrau_detector(): assert test_jungfrau.sensor_material == "Si" assert test_jungfrau.sensor_thickness == "0.320mm" assert test_jungfrau.hasMeta is False - assert type(test_jungfrau.constants) is dict + assert isinstance(test_jungfrau.constants, dict) def test_singla_detector(): @@ -58,7 +58,7 @@ def test_detector_axes(): det = Detector( test_eiger, det_axes, [100, 200], 0.1, [(0, 0, 1), Point3D(0, -1, 0)] ) - assert type(det.detector_axes) is list + assert isinstance(det.detector_axes, list) assert det.detector_axes[0].name == "two_theta" assert det.detector_axes[1].name == "det_z" assert [det.detector_axes[0].depends, det.detector_axes[1].depends] == [ @@ -105,7 +105,7 @@ def test_detector_to_module_dict(): mod = Detector( test_eiger, det_axes, [100, 200], 0.1, [(0, 0, 1), Point3D(0, -1, 0)] ).to_module_dict() - assert type(mod) is dict + assert isinstance(mod, dict) assert mod["module_offset"] == "1" assert_array_equal(mod["fast_axis"], [0, 0, 1]) assert_array_equal(mod["slow_axis"], [0, -1, 0]) diff --git a/tests/nxs_utils/test_goniometer.py b/tests/nxs_utils/test_goniometer.py index 64b7e8db..90f4bb99 100644 --- a/tests/nxs_utils/test_goniometer.py +++ b/tests/nxs_utils/test_goniometer.py @@ -15,7 +15,7 @@ def test_goniometer_to_dict(): gonio = Goniometer(axes_list[:2]).to_dict() - assert type(gonio) is dict + assert isinstance(gonio, dict) assert gonio["axes"] == ["omega", "sam_z"] assert gonio["depends"] == [".", "omega"] assert gonio["vectors"] == [(0, 0, -1), (0, 0, 1)]