From babefdbc67f1dd66e3586acb577b2b7954627678 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Sun, 12 May 2024 00:12:58 +0200 Subject: [PATCH 01/24] feat: init lint workflow --- .github/workflows/lint.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..0441e6978 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,36 @@ +name: Linters +on: + pull_request: + paths: + - "**.py" + - ".github/workflows/lint.yml" + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup python 3.8 + uses: actions/setup-python@v5 + with: + python-version: "3.8" + - name: Get changed services + id: changed-services + uses: tj-actions/changed-files@v44 + run: | + changed_services=() + for file in ${{ steps.changed-services.outputs.all_changed_files }}; do + changed_service=$(echo "$file" | cut -d'/' -f1) + changed_services+=("changed_service") + done + # Output the unique services + echo "name=services::$(printf '%s\n' "${changed_services[@]}" | sort -u | tr '\n' ',')" >> "$GITHUB_OUTPUT" + - name: Install python dependencies + run: + python -m pip install --upgrade pip + pip install black==24.4.2 pylint==3.1.0 mypy==1.10.0 + - name: Run Pylint for ${{ steps.changed-services.outputs.services }} + run: | + pylint --max-line-length=79 --errors-only ${{ steps.changed-services.outputs.services }} + # TODO: add mypy and black, specify config files, not hardcoded values \ No newline at end of file From 41c824ad757826354acac589d4a984caf817164a Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Sun, 12 May 2024 00:16:06 +0200 Subject: [PATCH 02/24] fix: separate get changed files and get services --- .github/workflows/lint.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0441e6978..4fc960cde 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,12 +15,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.8" + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v44 - name: Get changed services id: changed-services - uses: tj-actions/changed-files@v44 run: | changed_services=() - for file in ${{ steps.changed-services.outputs.all_changed_files }}; do + for file in ${{ steps.changed-files.outputs.all_changed_files }}; do changed_service=$(echo "$file" | cut -d'/' -f1) changed_services+=("changed_service") done From 4425423b7a99c492931eb1d398062eb6fbb32dae Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Mon, 13 May 2024 14:24:54 +0200 Subject: [PATCH 03/24] fix: check only changed files; add mypy and black --- .github/workflows/lint.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4fc960cde..6f59be2a7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,21 +18,20 @@ jobs: - name: Get changed files id: changed-files uses: tj-actions/changed-files@v44 - - name: Get changed services - id: changed-services + - name: List changed files run: | - changed_services=() - for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - changed_service=$(echo "$file" | cut -d'/' -f1) - changed_services+=("changed_service") - done - # Output the unique services - echo "name=services::$(printf '%s\n' "${changed_services[@]}" | sort -u | tr '\n' ',')" >> "$GITHUB_OUTPUT" + echo Changed files: ${{ steps.changed-files.outputs.all_changed_files }} - name: Install python dependencies - run: + run: | python -m pip install --upgrade pip pip install black==24.4.2 pylint==3.1.0 mypy==1.10.0 - - name: Run Pylint for ${{ steps.changed-services.outputs.services }} + # TODO: specify config files, not hardcoded values + - name: Run Pylint + run: | + pylint --max-line-length=79 --errors-only ${{ steps.changed-files.outputs.all_changed_files }} + - name: Run Mypy + run: | + mypy --strict {{ steps.changed-files.outputs.all_changed_files }} + - name: Run Black run: | - pylint --max-line-length=79 --errors-only ${{ steps.changed-services.outputs.services }} - # TODO: add mypy and black, specify config files, not hardcoded values \ No newline at end of file + black --line-length=79 --diff --check ${{ steps.changed-files.outputs.all_changed_files }} From bef81cfa322e505e61e9f0e94ecf3a5b02cceba9 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Mon, 13 May 2024 14:55:11 +0200 Subject: [PATCH 04/24] fix: trigger ci only on changes of py files --- .github/workflows/lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6f59be2a7..f1ee9e6b8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,7 +3,6 @@ on: pull_request: paths: - "**.py" - - ".github/workflows/lint.yml" jobs: lint: From 1d2f07f9f0bf7e293c02f904a180fd6dcad0bcd9 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Mon, 13 May 2024 15:07:00 +0200 Subject: [PATCH 05/24] feat: add isort --- .github/workflows/lint.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f1ee9e6b8..4f4bd89a9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,14 +23,18 @@ jobs: - name: Install python dependencies run: | python -m pip install --upgrade pip - pip install black==24.4.2 pylint==3.1.0 mypy==1.10.0 + pip install black==24.4.2 pylint==3.1.0 mypy==1.10.0 isort==5.13.2 # TODO: specify config files, not hardcoded values + # TODO: move deps versions somewhere else (pyproject.toml in the root) - name: Run Pylint run: | pylint --max-line-length=79 --errors-only ${{ steps.changed-files.outputs.all_changed_files }} - name: Run Mypy run: | mypy --strict {{ steps.changed-files.outputs.all_changed_files }} + - name: Run Isort + run: | + isort --profile=black --line-length=79 --check-only {{ steps.changed-files.outputs.all_changed_files }} - name: Run Black run: | black --line-length=79 --diff --check ${{ steps.changed-files.outputs.all_changed_files }} From fb7a1e19efa4b8e19dc4a1dad520f3cbf9831380 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Tue, 14 May 2024 11:12:49 +0300 Subject: [PATCH 06/24] fix: specify config for linters --- .github/workflows/lint.yml | 21 ++++++++++++--------- pyproject.toml | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4f4bd89a9..23a1bdef1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,19 +1,24 @@ -name: Linters +name: Lint on: pull_request: paths: - "**.py" +env: + PYTHON_VERSION: 3.8 + LINT_CONFIG: pyproject.toml + + jobs: lint: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup python 3.8 + - name: Setup python $PYTHON_VERSION uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: $PYTHON_VERSION - name: Get changed files id: changed-files uses: tj-actions/changed-files@v44 @@ -24,17 +29,15 @@ jobs: run: | python -m pip install --upgrade pip pip install black==24.4.2 pylint==3.1.0 mypy==1.10.0 isort==5.13.2 - # TODO: specify config files, not hardcoded values - # TODO: move deps versions somewhere else (pyproject.toml in the root) - name: Run Pylint run: | - pylint --max-line-length=79 --errors-only ${{ steps.changed-files.outputs.all_changed_files }} + pylint ${{ steps.changed-files.outputs.all_changed_files }} - name: Run Mypy run: | - mypy --strict {{ steps.changed-files.outputs.all_changed_files }} + mypy --config-file $LINT_CONFIG {{ steps.changed-files.outputs.all_changed_files }} - name: Run Isort run: | - isort --profile=black --line-length=79 --check-only {{ steps.changed-files.outputs.all_changed_files }} + isort --config $LINT_CONFIG --check-only {{ steps.changed-files.outputs.all_changed_files }} - name: Run Black run: | - black --line-length=79 --diff --check ${{ steps.changed-files.outputs.all_changed_files }} + black --config $LINT_CONFIG --diff --check ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..c16327f18 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[tool.black] +line_length = 79 + +[tool.isort] +profile = "black" +line_length = 79 + +[tool.pylint.MAIN] +errors-only = "True" +max-line-length = 79 + +[tool.pylint.FORMAT] +errors-only = "True" +max-line-length = 79 + +[tool.pylint."MESSAGES CONTROL"] +disable = "E0401,E0611" + +[tool.mypy] +ignore_missing_imports = true +scripts_are_modules = true +allow_untyped_decorators = true +strict = true +implicit_reexport = true From 61bc3c3b61156bdb4307a57da248934faf2b23d6 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Wed, 15 May 2024 19:13:56 +0300 Subject: [PATCH 07/24] fix: keep only black and isort; run linters in parallel --- .github/workflows/lint.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 23a1bdef1..9ec39b65f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,6 +12,9 @@ env: jobs: lint: runs-on: ubuntu-latest + strategy: + matrix: + tool: [ black, isort ] steps: - name: Checkout uses: actions/checkout@v4 @@ -28,16 +31,12 @@ jobs: - name: Install python dependencies run: | python -m pip install --upgrade pip - pip install black==24.4.2 pylint==3.1.0 mypy==1.10.0 isort==5.13.2 - - name: Run Pylint - run: | - pylint ${{ steps.changed-files.outputs.all_changed_files }} - - name: Run Mypy + pip install black==24.4.2 isort==5.13.2 + - name: Run Black + if: ${{ matrix.tool }} == black run: | - mypy --config-file $LINT_CONFIG {{ steps.changed-files.outputs.all_changed_files }} + black --config $LINT_CONFIG --diff --check ${{ steps.changed-files.outputs.all_changed_files }} - name: Run Isort + if: ${{ matrix.tool }} == isort run: | isort --config $LINT_CONFIG --check-only {{ steps.changed-files.outputs.all_changed_files }} - - name: Run Black - run: | - black --config $LINT_CONFIG --diff --check ${{ steps.changed-files.outputs.all_changed_files }} From 1f8378dc1b22807d17ca9ee862675a939aa5a05d Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Wed, 15 May 2024 19:19:37 +0300 Subject: [PATCH 08/24] fix: remove excess lint configs --- pyproject.toml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c16327f18..11e5f5f74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,21 +4,3 @@ line_length = 79 [tool.isort] profile = "black" line_length = 79 - -[tool.pylint.MAIN] -errors-only = "True" -max-line-length = 79 - -[tool.pylint.FORMAT] -errors-only = "True" -max-line-length = 79 - -[tool.pylint."MESSAGES CONTROL"] -disable = "E0401,E0611" - -[tool.mypy] -ignore_missing_imports = true -scripts_are_modules = true -allow_untyped_decorators = true -strict = true -implicit_reexport = true From 2df60c4d3a53ff934d9c82e1224297863da2cb5c Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 15:58:21 +0300 Subject: [PATCH 09/24] feat: add root poetry --- .github/workflows/lint.yml | 13 +-- poetry.lock | 162 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 18 +++++ 3 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 poetry.lock diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9ec39b65f..dee39c6cf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,12 +31,13 @@ jobs: - name: Install python dependencies run: | python -m pip install --upgrade pip - pip install black==24.4.2 isort==5.13.2 - - name: Run Black - if: ${{ matrix.tool }} == black - run: | - black --config $LINT_CONFIG --diff --check ${{ steps.changed-files.outputs.all_changed_files }} + pip install poetry + poetry install --only dev - name: Run Isort if: ${{ matrix.tool }} == isort run: | - isort --config $LINT_CONFIG --check-only {{ steps.changed-files.outputs.all_changed_files }} + poetry run isort --config $LINT_CONFIG --check-only {{ steps.changed-files.outputs.all_changed_files }} + - name: Run Black + if: ${{ matrix.tool }} == black + run: | + poetry run black --config $LINT_CONFIG --diff --check ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 000000000..208ce4b3a --- /dev/null +++ b/poetry.lock @@ -0,0 +1,162 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "black" +version = "24.4.2" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-24.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce"}, + {file = "black-24.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021"}, + {file = "black-24.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063"}, + {file = "black-24.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:7768a0dbf16a39aa5e9a3ded568bb545c8c2727396d063bbaf847df05b08cd96"}, + {file = "black-24.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474"}, + {file = "black-24.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c"}, + {file = "black-24.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb"}, + {file = "black-24.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1"}, + {file = "black-24.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d"}, + {file = "black-24.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04"}, + {file = "black-24.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc"}, + {file = "black-24.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0"}, + {file = "black-24.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf10f7310db693bb62692609b397e8d67257c55f949abde4c67f9cc574492cc7"}, + {file = "black-24.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:98e123f1d5cfd42f886624d84464f7756f60ff6eab89ae845210631714f6db94"}, + {file = "black-24.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48a85f2cb5e6799a9ef05347b476cce6c182d6c71ee36925a6c194d074336ef8"}, + {file = "black-24.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:b1530ae42e9d6d5b670a34db49a94115a64596bc77710b1d05e9801e62ca0a7c"}, + {file = "black-24.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1"}, + {file = "black-24.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da33a1a5e49c4122ccdfd56cd021ff1ebc4a1ec4e2d01594fef9b6f267a9e741"}, + {file = "black-24.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e"}, + {file = "black-24.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:b9176b9832e84308818a99a561e90aa479e73c523b3f77afd07913380ae2eab7"}, + {file = "black-24.4.2-py3-none-any.whl", hash = "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c"}, + {file = "black-24.4.2.tar.gz", hash = "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "24.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.11.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8.0" +content-hash = "7c7eb9f9087e03a1105bbe9ec8b8a84b3a932e84b27afa8efeb052e4b3922c5d" diff --git a/pyproject.toml b/pyproject.toml index 11e5f5f74..4600b81de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,21 @@ +[tool.poetry] +name = "badgerdoc" +version = "1.8.0" +description = "" +authors = ["BadgerDoc team"] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.8.0" + +[tool.poetry.group.dev.dependencies] +black = "^24.4.2" +isort = "^5.13.2" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + [tool.black] line_length = 79 From 0bfee7a00ae2e9b224fc144adae2d135810a7fc2 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:03:46 +0300 Subject: [PATCH 10/24] feat: add input to lint workflow for optional directory/files --- .github/workflows/lint.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index dee39c6cf..ebba22079 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,6 +3,12 @@ on: pull_request: paths: - "**.py" +workflow_dispatch: + inputs: + files: + description: 'Files to lint [optional]' + required: false + default: '' env: PYTHON_VERSION: 3.8 @@ -28,6 +34,14 @@ jobs: - name: List changed files run: | echo Changed files: ${{ steps.changed-files.outputs.all_changed_files }} + - name: Determine files to lint + id: determine-files + run: | + if [ "${{ github.event.inputs.files }}" ]; then + echo "name=files::${{ github.event.inputs.files }}" >> "$GITHUB_OUTPUT" + else + echo "name=files::${{ steps.changed-files.outputs.all_changed_files }}" >> "$GITHUB_OUTPUT" + fi - name: Install python dependencies run: | python -m pip install --upgrade pip @@ -36,8 +50,8 @@ jobs: - name: Run Isort if: ${{ matrix.tool }} == isort run: | - poetry run isort --config $LINT_CONFIG --check-only {{ steps.changed-files.outputs.all_changed_files }} + poetry run isort --config $LINT_CONFIG --check-only {{ steps.determine-files.outputs.files }} - name: Run Black if: ${{ matrix.tool }} == black run: | - poetry run black --config $LINT_CONFIG --diff --check ${{ steps.changed-files.outputs.all_changed_files }} + poetry run black --config $LINT_CONFIG --diff --check ${{ steps.determine-files.outputs.files }} From 90756f3a08c779f66e04ccd9cf1f98e78d389b1c Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:10:46 +0300 Subject: [PATCH 11/24] test lint --- annotation/annotation/categories/resources.py | 1 + 1 file changed, 1 insertion(+) diff --git a/annotation/annotation/categories/resources.py b/annotation/annotation/categories/resources.py index 40f411677..6c708da02 100644 --- a/annotation/annotation/categories/resources.py +++ b/annotation/annotation/categories/resources.py @@ -24,6 +24,7 @@ add_category_db, delete_category_db, fetch_category_db, + filter_category_db, insert_category_tree, recursive_subcategory_search, From 219562c9f1dbe1a819422b2fd0fdeadccf3fb8c8 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:12:37 +0300 Subject: [PATCH 12/24] fix: input --- .github/workflows/lint.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ebba22079..4efe23d03 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,12 +3,12 @@ on: pull_request: paths: - "**.py" -workflow_dispatch: - inputs: - files: - description: 'Files to lint [optional]' - required: false - default: '' + workflow_dispatch: + inputs: + files: + description: 'Files to lint [optional]' + required: false + default: '' env: PYTHON_VERSION: 3.8 From 6b9577f2c418a8ec136a0449f65514780aeb066d Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:32:01 +0300 Subject: [PATCH 13/24] fix: simplify workflow --- .github/workflows/lint.yml | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4efe23d03..ad560b767 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,12 +3,7 @@ on: pull_request: paths: - "**.py" - workflow_dispatch: - inputs: - files: - description: 'Files to lint [optional]' - required: false - default: '' + - ".github/workflows/lint.yml" env: PYTHON_VERSION: 3.8 @@ -18,40 +13,21 @@ env: jobs: lint: runs-on: ubuntu-latest - strategy: - matrix: - tool: [ black, isort ] steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup python $PYTHON_VERSION + - name: Setup python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 with: - python-version: $PYTHON_VERSION - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v44 - - name: List changed files - run: | - echo Changed files: ${{ steps.changed-files.outputs.all_changed_files }} - - name: Determine files to lint - id: determine-files - run: | - if [ "${{ github.event.inputs.files }}" ]; then - echo "name=files::${{ github.event.inputs.files }}" >> "$GITHUB_OUTPUT" - else - echo "name=files::${{ steps.changed-files.outputs.all_changed_files }}" >> "$GITHUB_OUTPUT" - fi + python-version: ${{ env.PYTHON_VERSION }} - name: Install python dependencies run: | python -m pip install --upgrade pip pip install poetry poetry install --only dev - name: Run Isort - if: ${{ matrix.tool }} == isort run: | - poetry run isort --config $LINT_CONFIG --check-only {{ steps.determine-files.outputs.files }} + poetry run isort --config $LINT_CONFIG --check-only / - name: Run Black - if: ${{ matrix.tool }} == black run: | - poetry run black --config $LINT_CONFIG --diff --check ${{ steps.determine-files.outputs.files }} + poetry run black --config $LINT_CONFIG --diff --check / From 9e1be87d19f8e2d19e14b094ea059d7daef14332 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:37:37 +0300 Subject: [PATCH 14/24] fix --- .github/workflows/lint.yml | 2 +- annotation/annotation/categories/resources.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ad560b767..3320e95c2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,7 +27,7 @@ jobs: poetry install --only dev - name: Run Isort run: | - poetry run isort --config $LINT_CONFIG --check-only / + poetry run isort --config $LINT_CONFIG --diff --check-only / - name: Run Black run: | poetry run black --config $LINT_CONFIG --diff --check / diff --git a/annotation/annotation/categories/resources.py b/annotation/annotation/categories/resources.py index 6c708da02..40f411677 100644 --- a/annotation/annotation/categories/resources.py +++ b/annotation/annotation/categories/resources.py @@ -24,7 +24,6 @@ add_category_db, delete_category_db, fetch_category_db, - filter_category_db, insert_category_tree, recursive_subcategory_search, From d4e188bc7b4412afa364601ee1e47cc31c6fb7ec Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:42:14 +0300 Subject: [PATCH 15/24] fix: remove explicit isort config --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3320e95c2..35eaf179c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,7 +27,7 @@ jobs: poetry install --only dev - name: Run Isort run: | - poetry run isort --config $LINT_CONFIG --diff --check-only / + poetry run isort --diff --check-only . - name: Run Black run: | - poetry run black --config $LINT_CONFIG --diff --check / + poetry run black --config $LINT_CONFIG --diff --check From 2a29cb78e6ddda0bab7db6bc28eaa0a0cba56054 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:45:02 +0300 Subject: [PATCH 16/24] fix: isort issues --- airflow/dags/print_operator.py | 2 +- annotation/alembic/env.py | 2 +- ...72a5043_add_categories_editor_url_data_.py | 3 +- ..._alter_categories_id_integer_to_varchar.py | 3 +- ...3136551008d8_agreement_metrics_relation.py | 3 +- .../36bff2d016f7_expand_validationschema.py | 3 +- .../versions/3a083a1fbba0_first_revision.py | 3 +- ...0dc1_add_name_and_job_type_to_job_model.py | 3 +- .../versions/4272d0a43ff1_agreement_score.py | 3 +- .../4cd10ac49cc2_expand_annotated_doc.py | 3 +- .../versions/615379303da2_added_links_json.py | 3 +- ...bd1429_change_links_json_server_default.py | 3 +- .../66cd6054c2d0_add_categories_tree.py | 1 - ...fb3e0d231ff_create_document_links_table.py | 1 - ...8e6343_add_extensive_coverage_parameter.py | 2 +- .../7cc1ed83c309_compare_agreement_scores.py | 1 - ...4_create_association_doc_category_table.py | 1 - ..._category_change_editor_data_attributes.py | 3 +- ...e_make_document_categories_an_array_of_.py | 1 - .../9c07a25ca06f_expand_file_model.py | 3 +- ...4_adds_overall_load_field_not_negative_.py | 1 - .../versions/c053ae380212_expand_job_model.py | 3 +- .../c06c594c7435_change_file_statuses.py | 3 +- .../cf633ca94498_add_statuses_to_job.py | 3 +- ...adds_not_nullable_tenant_field_to_jobs_.py | 1 - ...963_drop_is_link_add_type_to_categories.py | 3 +- annotation/tests/conftest.py | 58 +++++++++---------- .../tests/test_annotators_overall_load.py | 6 +- annotation/tests/test_assets_communication.py | 2 +- annotation/tests/test_category_crud.py | 4 +- annotation/tests/test_cross_validation.py | 2 +- annotation/tests/test_delete_batch_tasks.py | 8 +-- annotation/tests/test_distribution.py | 2 +- annotation/tests/test_finish_task.py | 4 +- .../tests/test_get_accumulated_revisions.py | 6 +- ..._get_annotation_for_particular_revision.py | 4 +- annotation/tests/test_get_entities_status.py | 2 +- annotation/tests/test_get_job.py | 4 +- annotation/tests/test_get_job_files.py | 2 +- annotation/tests/test_get_job_progress.py | 4 +- .../tests/test_get_jobs_info_by_files.py | 4 +- annotation/tests/test_get_pages_info.py | 4 +- annotation/tests/test_get_revisions.py | 4 +- .../test_get_revisions_without_annotation.py | 4 +- annotation/tests/test_get_unassigned_files.py | 2 +- annotation/tests/test_get_users_for_job.py | 2 +- annotation/tests/test_job_categories.py | 6 +- annotation/tests/test_post.py | 2 +- annotation/tests/test_post_job.py | 6 +- annotation/tests/test_search_kafka.py | 2 +- annotation/tests/test_start_job.py | 2 +- annotation/tests/test_tasks_crud_cr.py | 6 +- annotation/tests/test_tasks_crud_ud.py | 6 +- annotation/tests/test_validation.py | 8 +-- assets/alembic/env.py | 6 +- ..._add_original_ext_column_to_files_table.py | 4 +- .../versions/9e837ea0c11d_image_pages.py | 4 +- .../versions/afa33cc83d57_new_fields.py | 4 +- .../versions/fe5926249504_count_datasets.py | 4 +- assets/assets/db/models.py | 3 +- assets/assets/db/service.py | 7 ++- assets/assets/main.py | 6 +- assets/assets/routers/bonds_router.py | 1 + assets/assets/routers/datasets_router.py | 1 + assets/assets/routers/s3_router.py | 1 + assets/assets/schemas.py | 3 +- assets/assets/utils/convert_service_utils.py | 1 + assets/assets/utils/s3_utils.py | 1 + assets/tests/conftest.py | 18 +++--- assets/tests/test_helpers.py | 3 +- assets/tests/test_models.py | 1 + jobs/alembic/versions/13ac4bb3abd2_.py | 3 +- ...120_add_available_annotation_types_and_.py | 3 +- jobs/alembic/versions/3f5b2d199d38_.py | 1 - jobs/alembic/versions/7511c6790067_.py | 3 +- jobs/alembic/versions/83694c0b2df6_.py | 1 - jobs/alembic/versions/86f432539475_.py | 1 - jobs/alembic/versions/9229e70d2791_.py | 3 +- ...add_start_manual_job_automatically_flag.py | 1 - jobs/alembic/versions/d1ddce2d5352_.py | 1 - ...0dd492b17f_add_extensive_coverage_param.py | 1 - .../test_change_job-proxy_to_annotation.py | 1 - .../test_API_functions/test_search_jobs.py | 1 - lib/filter_lib/tests/test_pagination.py | 1 - lib/filter_lib/tests/test_query_modifier.py | 1 - lib/filter_lib/tests/test_schema_generator.py | 1 - lib/filter_lib/usage_example/app.py | 5 +- .../filter_lib/tests/test_pagination.py | 1 - .../filter_lib/tests/test_query_modifier.py | 1 - .../filter_lib/tests/test_schema_generator.py | 1 - .../filter_lib/usage_example/app.py | 5 +- processing/alembic/env.py | 6 +- .../alembic/versions/52af1473946f_init.py | 1 - .../versions/8e973b70b26f_noneasnull.py | 1 - .../versions/f637b13c744d_renamed_column.py | 1 - processing/tests/conftest.py | 8 +-- processing/tests/test_assets_status.py | 1 + search/search/harvester.py | 5 +- search/search/main.py | 4 +- search/tests/conftest.py | 6 +- search/tests/test_harvester.py | 1 + taxonomy/alembic/env.py | 4 +- ...ecbed_add_association_taxonomy_category.py | 1 - .../versions/bdea8a93cafe_first_revision.py | 1 - .../d3ba69ca9d97_change_category_linking.py | 1 - taxonomy/tests/conftest.py | 4 +- taxonomy/tests/test_taxon_crud.py | 4 +- taxonomy/tests/test_taxonomy_router.py | 4 +- 108 files changed, 175 insertions(+), 209 deletions(-) diff --git a/airflow/dags/print_operator.py b/airflow/dags/print_operator.py index 2c6562bba..04ab937a4 100644 --- a/airflow/dags/print_operator.py +++ b/airflow/dags/print_operator.py @@ -4,7 +4,7 @@ from datetime import datetime from pprint import pprint -from airflow.decorators import task, dag +from airflow.decorators import dag, task @dag(schedule="@daily", start_date=datetime(2021, 12, 1), catchup=False, dag_id="print") diff --git a/annotation/alembic/env.py b/annotation/alembic/env.py index 1ad908137..5ef55fa55 100644 --- a/annotation/alembic/env.py +++ b/annotation/alembic/env.py @@ -1,9 +1,9 @@ import os from logging.config import fileConfig +from alembic import context # type: ignore from sqlalchemy import engine_from_config, pool -from alembic import context # type: ignore from annotation.database import SQLALCHEMY_DATABASE_URL from annotation.utils import get_test_db_url diff --git a/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py b/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py index 4b2c7754e..c1c31a5ee 100644 --- a/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py +++ b/annotation/alembic/versions/1edef72a5043_add_categories_editor_url_data_.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "1edef72a5043" diff --git a/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py b/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py index 11122ce27..d08e1effc 100644 --- a/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py +++ b/annotation/alembic/versions/2b3ed53127ea_alter_categories_id_integer_to_varchar.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.engine.reflection import Inspector - from alembic import op +from sqlalchemy.engine.reflection import Inspector # revision identifiers, used by Alembic. revision = "2b3ed53127ea" diff --git a/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py b/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py index 6e818bd84..56374c21d 100644 --- a/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py +++ b/annotation/alembic/versions/3136551008d8_agreement_metrics_relation.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "3136551008d8" diff --git a/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py b/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py index 81faf2828..91f830f50 100644 --- a/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py +++ b/annotation/alembic/versions/36bff2d016f7_expand_validationschema.py @@ -9,9 +9,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "36bff2d016f7" diff --git a/annotation/alembic/versions/3a083a1fbba0_first_revision.py b/annotation/alembic/versions/3a083a1fbba0_first_revision.py index f96568a1f..3bbc99da4 100644 --- a/annotation/alembic/versions/3a083a1fbba0_first_revision.py +++ b/annotation/alembic/versions/3a083a1fbba0_first_revision.py @@ -7,11 +7,10 @@ """ import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql from sqlalchemy.engine.reflection import Inspector -from alembic import op - # revision identifiers, used by Alembic. revision = "3a083a1fbba0" down_revision = None diff --git a/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py b/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py index 92a94c9ed..1fa9e9f12 100644 --- a/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py +++ b/annotation/alembic/versions/416952520dc1_add_name_and_job_type_to_job_model.py @@ -9,9 +9,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "416952520dc1" diff --git a/annotation/alembic/versions/4272d0a43ff1_agreement_score.py b/annotation/alembic/versions/4272d0a43ff1_agreement_score.py index c5a1693e9..e97ab6e03 100644 --- a/annotation/alembic/versions/4272d0a43ff1_agreement_score.py +++ b/annotation/alembic/versions/4272d0a43ff1_agreement_score.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "4272d0a43ff1" diff --git a/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py b/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py index d17001704..caf1e4013 100644 --- a/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py +++ b/annotation/alembic/versions/4cd10ac49cc2_expand_annotated_doc.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "4cd10ac49cc2" diff --git a/annotation/alembic/versions/615379303da2_added_links_json.py b/annotation/alembic/versions/615379303da2_added_links_json.py index a04267153..c51859e21 100644 --- a/annotation/alembic/versions/615379303da2_added_links_json.py +++ b/annotation/alembic/versions/615379303da2_added_links_json.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "615379303da2" diff --git a/annotation/alembic/versions/62470ebd1429_change_links_json_server_default.py b/annotation/alembic/versions/62470ebd1429_change_links_json_server_default.py index 487d15fe1..c4aee8537 100644 --- a/annotation/alembic/versions/62470ebd1429_change_links_json_server_default.py +++ b/annotation/alembic/versions/62470ebd1429_change_links_json_server_default.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "62470ebd1429" diff --git a/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py b/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py index 6aab0b789..d31c2c49b 100644 --- a/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py +++ b/annotation/alembic/versions/66cd6054c2d0_add_categories_tree.py @@ -8,7 +8,6 @@ import sqlalchemy as sa import sqlalchemy_utils - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py b/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py index 340f61d58..ee820ebc0 100644 --- a/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py +++ b/annotation/alembic/versions/6fb3e0d231ff_create_document_links_table.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py b/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py index 237e2a52e..ed67e7201 100644 --- a/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py +++ b/annotation/alembic/versions/71095b8e6343_add_extensive_coverage_parameter.py @@ -7,8 +7,8 @@ """ import sqlalchemy as sa - from alembic import op + from annotation.models import ValidationSchema # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py b/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py index e5918ef61..191057052 100644 --- a/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py +++ b/annotation/alembic/versions/7cc1ed83c309_compare_agreement_scores.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py b/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py index efa5d355f..87f7c530c 100644 --- a/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py +++ b/annotation/alembic/versions/8ea2ff0fea64_create_association_doc_category_table.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py b/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py index c8c9bd4f0..1aad0a12a 100644 --- a/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py +++ b/annotation/alembic/versions/8fbac489cef2_category_change_editor_data_attributes.py @@ -8,9 +8,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "8fbac489cef2" diff --git a/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py b/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py index 081f8a2d9..dabd3c965 100644 --- a/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py +++ b/annotation/alembic/versions/9083dea3783e_make_document_categories_an_array_of_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py b/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py index 79c313a33..15dca9471 100644 --- a/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py +++ b/annotation/alembic/versions/9c07a25ca06f_expand_file_model.py @@ -9,9 +9,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "9c07a25ca06f" diff --git a/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py b/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py index b375ec4b4..283d22383 100644 --- a/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py +++ b/annotation/alembic/versions/bda0eac5ce64_adds_overall_load_field_not_negative_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/c053ae380212_expand_job_model.py b/annotation/alembic/versions/c053ae380212_expand_job_model.py index 40f917d4f..46287cf1b 100644 --- a/annotation/alembic/versions/c053ae380212_expand_job_model.py +++ b/annotation/alembic/versions/c053ae380212_expand_job_model.py @@ -9,9 +9,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "c053ae380212" diff --git a/annotation/alembic/versions/c06c594c7435_change_file_statuses.py b/annotation/alembic/versions/c06c594c7435_change_file_statuses.py index 23f777dd9..d6cd6897c 100644 --- a/annotation/alembic/versions/c06c594c7435_change_file_statuses.py +++ b/annotation/alembic/versions/c06c594c7435_change_file_statuses.py @@ -9,9 +9,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "c06c594c7435" diff --git a/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py b/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py index a70df2a16..713aad0d5 100644 --- a/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py +++ b/annotation/alembic/versions/cf633ca94498_add_statuses_to_job.py @@ -9,9 +9,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "cf633ca94498" diff --git a/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py b/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py index ee3c15c6f..be198484a 100644 --- a/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py +++ b/annotation/alembic/versions/d30649e367e0_adds_not_nullable_tenant_field_to_jobs_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py b/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py index a8b2e1e10..fccaf67cb 100644 --- a/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py +++ b/annotation/alembic/versions/f44cabeef963_drop_is_link_add_type_to_categories.py @@ -9,9 +9,8 @@ from enum import Enum import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "f44cabeef963" diff --git a/annotation/tests/conftest.py b/annotation/tests/conftest.py index ee9c0b319..5ad09c601 100644 --- a/annotation/tests/conftest.py +++ b/annotation/tests/conftest.py @@ -9,40 +9,16 @@ import pytest import sqlalchemy import sqlalchemy_utils -from moto import mock_s3 -from sqlalchemy.engine import create_engine -from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.orm import Session, sessionmaker -from sqlalchemy.orm.exc import FlushError - import tests.test_get_accumulated_revisions as accumulated_revs import tests.test_get_jobs_info_by_files as jobs_info_by_files import tests.test_validation as validation from alembic import command from alembic.config import Config -from annotation.annotations import MANIFEST, S3_START_PATH -from annotation.categories import cache -from annotation.database import SQLALCHEMY_DATABASE_URL, Base -from annotation.jobs import update_user_overall_load -from annotation.models import ( - AnnotatedDoc, - Category, - DocumentLinks, - File, - Job, - ManualAnnotationTask, - User, -) -from annotation.schemas import ( - AnnotationStatisticsInputSchema, - CategoryTypeSchema, - FileStatusEnumSchema, - JobStatusEnumSchema, - TaskStatusEnumSchema, - ValidationSchema, -) -from annotation.tasks import add_task_stats_record -from annotation.utils import get_test_db_url +from moto import mock_s3 +from sqlalchemy.engine import create_engine +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm.exc import FlushError from tests.override_app_dependency import TEST_TENANT from tests.test_annotators_overall_load import ( OVERALL_LOAD_CREATED_TASKS, @@ -172,6 +148,30 @@ UPDATE_USER_NO_JOBS, ) +from annotation.annotations import MANIFEST, S3_START_PATH +from annotation.categories import cache +from annotation.database import SQLALCHEMY_DATABASE_URL, Base +from annotation.jobs import update_user_overall_load +from annotation.models import ( + AnnotatedDoc, + Category, + DocumentLinks, + File, + Job, + ManualAnnotationTask, + User, +) +from annotation.schemas import ( + AnnotationStatisticsInputSchema, + CategoryTypeSchema, + FileStatusEnumSchema, + JobStatusEnumSchema, + TaskStatusEnumSchema, + ValidationSchema, +) +from annotation.tasks import add_task_stats_record +from annotation.utils import get_test_db_url + DEFAULT_REGION = "us-east-1" alembic_cfg = Config("alembic.ini") diff --git a/annotation/tests/test_annotators_overall_load.py b/annotation/tests/test_annotators_overall_load.py index 378eb787e..6af0b3fb3 100644 --- a/annotation/tests/test_annotators_overall_load.py +++ b/annotation/tests/test_annotators_overall_load.py @@ -5,6 +5,9 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import CRUD_TASKS_PATH, FINISH_TASK_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT +from tests.test_tasks_crud_ud import construct_path from annotation.jobs import update_user_overall_load from annotation.main import app @@ -24,9 +27,6 @@ JobTypeEnumSchema, ValidationSchema, ) -from tests.consts import CRUD_TASKS_PATH, FINISH_TASK_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT -from tests.test_tasks_crud_ud import construct_path client = TestClient(app) diff --git a/annotation/tests/test_assets_communication.py b/annotation/tests/test_assets_communication.py index 21d809a48..f8070fc97 100644 --- a/annotation/tests/test_assets_communication.py +++ b/annotation/tests/test_assets_communication.py @@ -4,6 +4,7 @@ import responses from fastapi import HTTPException from requests import ConnectionError, RequestException, Timeout +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN from annotation.microservice_communication.assets_communication import ( get_dataset_info, @@ -11,7 +12,6 @@ get_file_path_and_bucket, get_files_info, ) -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN FILES = [ { diff --git a/annotation/tests/test_category_crud.py b/annotation/tests/test_category_crud.py index c69afccfd..307556176 100644 --- a/annotation/tests/test_category_crud.py +++ b/annotation/tests/test_category_crud.py @@ -8,11 +8,11 @@ from fastapi.testclient import TestClient from pytest import fixture, mark from sqlalchemy.exc import IntegrityError, SQLAlchemyError - -from annotation.models import Category from tests.consts import CATEGORIES_PATH from tests.override_app_dependency import TEST_HEADERS, app +from annotation.models import Category + client = TestClient(app) ATTRIBUTES_NOT_IN_CATEGORY_MODEL = ("is_leaf",) diff --git a/annotation/tests/test_cross_validation.py b/annotation/tests/test_cross_validation.py index 5ce0e3653..34214485e 100644 --- a/annotation/tests/test_cross_validation.py +++ b/annotation/tests/test_cross_validation.py @@ -2,6 +2,7 @@ from uuid import UUID import pytest +from tests.test_distribution import JOB_ID from annotation.distribution import ( distribute_validation_partial_files, @@ -10,7 +11,6 @@ from annotation.errors import FieldConstraintError from annotation.jobs import check_annotators, check_validators from annotation.schemas import TaskStatusEnumSchema, ValidationSchema -from tests.test_distribution import JOB_ID TASKS_STATUS = TaskStatusEnumSchema.pending VALIDATION_TYPE = ValidationSchema.cross diff --git a/annotation/tests/test_delete_batch_tasks.py b/annotation/tests/test_delete_batch_tasks.py index 256143df7..0ee570472 100644 --- a/annotation/tests/test_delete_batch_tasks.py +++ b/annotation/tests/test_delete_batch_tasks.py @@ -3,6 +3,10 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from tests.consts import CRUD_TASKS_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from tests.test_post import check_files_distributed_pages +from tests.test_tasks_crud_ud import BAD_ID, NOT_EXISTING_ID from annotation.annotations import row_to_dict from annotation.models import Category, File, Job, ManualAnnotationTask, User @@ -11,10 +15,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.consts import CRUD_TASKS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app -from tests.test_post import check_files_distributed_pages -from tests.test_tasks_crud_ud import BAD_ID, NOT_EXISTING_ID client = TestClient(app) diff --git a/annotation/tests/test_distribution.py b/annotation/tests/test_distribution.py index 8f7f39b04..158a662a5 100644 --- a/annotation/tests/test_distribution.py +++ b/annotation/tests/test_distribution.py @@ -3,6 +3,7 @@ from typing import List import pytest +from tests.override_app_dependency import TEST_TENANT from annotation.distribution import ( add_unassigned_file, @@ -24,7 +25,6 @@ ) from annotation.models import File from annotation.schemas import FileStatusEnumSchema, TaskStatusEnumSchema -from tests.override_app_dependency import TEST_TENANT JOB_ID = 1 ANNOTATORS = [ diff --git a/annotation/tests/test_finish_task.py b/annotation/tests/test_finish_task.py index 38c3bc104..1693fc45c 100644 --- a/annotation/tests/test_finish_task.py +++ b/annotation/tests/test_finish_task.py @@ -10,6 +10,8 @@ from sqlalchemy import asc, not_ from sqlalchemy.exc import DBAPIError, SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import FINISH_TASK_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.annotations import accumulate_pages_info, row_to_dict from annotation.models import ( @@ -30,8 +32,6 @@ ValidationSchema, ) from annotation.tasks import get_task_revisions -from tests.consts import FINISH_TASK_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app client = TestClient(app) diff --git a/annotation/tests/test_get_accumulated_revisions.py b/annotation/tests/test_get_accumulated_revisions.py index e879f3391..e133be8c9 100644 --- a/annotation/tests/test_get_accumulated_revisions.py +++ b/annotation/tests/test_get_accumulated_revisions.py @@ -3,6 +3,9 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from tests.consts import ANNOTATION_PATH +from tests.override_app_dependency import TEST_TOKEN, app +from tests.test_post_annotation import POST_ANNOTATION_PG_DOC from annotation.annotations import LATEST from annotation.microservice_communication.search import ( @@ -11,9 +14,6 @@ HEADER_TENANT, ) from annotation.models import AnnotatedDoc, User -from tests.consts import ANNOTATION_PATH -from tests.override_app_dependency import TEST_TOKEN, app -from tests.test_post_annotation import POST_ANNOTATION_PG_DOC client = TestClient(app) diff --git a/annotation/tests/test_get_annotation_for_particular_revision.py b/annotation/tests/test_get_annotation_for_particular_revision.py index 16231727d..fe3e10ec3 100644 --- a/annotation/tests/test_get_annotation_for_particular_revision.py +++ b/annotation/tests/test_get_annotation_for_particular_revision.py @@ -3,6 +3,8 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from tests.consts import ANNOTATION_PATH +from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app from annotation.microservice_communication.search import ( AUTHORIZATION, @@ -10,8 +12,6 @@ HEADER_TENANT, ) from annotation.models import AnnotatedDoc, User -from tests.consts import ANNOTATION_PATH -from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app client = TestClient(app) diff --git a/annotation/tests/test_get_entities_status.py b/annotation/tests/test_get_entities_status.py index dd96fd74e..0040fce04 100644 --- a/annotation/tests/test_get_entities_status.py +++ b/annotation/tests/test_get_entities_status.py @@ -1,8 +1,8 @@ import pytest from fastapi.testclient import TestClient +from tests.override_app_dependency import TEST_HEADERS, app from annotation.schemas import EntitiesStatusesSchema, TaskStatusEnumSchema -from tests.override_app_dependency import TEST_HEADERS, app client = TestClient(app) diff --git a/annotation/tests/test_get_job.py b/annotation/tests/test_get_job.py index 1d72db5e3..becffdcd0 100644 --- a/annotation/tests/test_get_job.py +++ b/annotation/tests/test_get_job.py @@ -4,6 +4,8 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import ANNOTATION_PATH +from tests.override_app_dependency import TEST_TOKEN, app from annotation.jobs import collect_job_names from annotation.microservice_communication.search import ( @@ -13,8 +15,6 @@ ) from annotation.models import Category, File, Job, User from annotation.schemas import FileStatusEnumSchema, ValidationSchema -from tests.consts import ANNOTATION_PATH -from tests.override_app_dependency import TEST_TOKEN, app client = TestClient(app) diff --git a/annotation/tests/test_get_job_files.py b/annotation/tests/test_get_job_files.py index 1822e1f60..bcbb88f3d 100644 --- a/annotation/tests/test_get_job_files.py +++ b/annotation/tests/test_get_job_files.py @@ -4,6 +4,7 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.override_app_dependency import TEST_TOKEN, app from annotation.microservice_communication.search import ( AUTHORIZATION, @@ -16,7 +17,6 @@ FileStatusEnumSchema, ValidationSchema, ) -from tests.override_app_dependency import TEST_TOKEN, app client = TestClient(app) diff --git a/annotation/tests/test_get_job_progress.py b/annotation/tests/test_get_job_progress.py index 2e55e5c22..ae2192bb4 100644 --- a/annotation/tests/test_get_job_progress.py +++ b/annotation/tests/test_get_job_progress.py @@ -1,5 +1,7 @@ import pytest from fastapi.testclient import TestClient +from tests.consts import POST_JOBS_PATH +from tests.override_app_dependency import TEST_TOKEN, app from annotation.microservice_communication.search import ( AUTHORIZATION, @@ -13,8 +15,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import TEST_TOKEN, app client = TestClient(app) diff --git a/annotation/tests/test_get_jobs_info_by_files.py b/annotation/tests/test_get_jobs_info_by_files.py index c6de4938e..b6ffe1abd 100644 --- a/annotation/tests/test_get_jobs_info_by_files.py +++ b/annotation/tests/test_get_jobs_info_by_files.py @@ -3,12 +3,12 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from tests.consts import POST_JOBS_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.jobs.services import get_jobs_by_files from annotation.models import File, Job, User from annotation.schemas import JobStatusEnumSchema, ValidationSchema -from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app client = TestClient(app) diff --git a/annotation/tests/test_get_pages_info.py b/annotation/tests/test_get_pages_info.py index 89de5c9bb..be1554bb1 100644 --- a/annotation/tests/test_get_pages_info.py +++ b/annotation/tests/test_get_pages_info.py @@ -4,6 +4,8 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import CRUD_TASKS_PATH +from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app from annotation.annotations import accumulate_pages_info from annotation.microservice_communication.search import ( @@ -14,8 +16,6 @@ from annotation.models import AnnotatedDoc, File, Job, ManualAnnotationTask, User from annotation.schemas import TaskStatusEnumSchema, ValidationSchema from annotation.tasks import get_task_revisions -from tests.consts import CRUD_TASKS_PATH -from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app client = TestClient(app) diff --git a/annotation/tests/test_get_revisions.py b/annotation/tests/test_get_revisions.py index 1bc35daa6..3d468d690 100644 --- a/annotation/tests/test_get_revisions.py +++ b/annotation/tests/test_get_revisions.py @@ -6,11 +6,11 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import ANNOTATION_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.annotations import S3_START_PATH from annotation.models import DocumentLinks -from tests.consts import ANNOTATION_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app JOBS_IDS = ( 10, diff --git a/annotation/tests/test_get_revisions_without_annotation.py b/annotation/tests/test_get_revisions_without_annotation.py index 0a393e808..abba7ace0 100644 --- a/annotation/tests/test_get_revisions_without_annotation.py +++ b/annotation/tests/test_get_revisions_without_annotation.py @@ -1,6 +1,8 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from tests.consts import ANNOTATION_PATH +from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app from annotation.microservice_communication.search import ( AUTHORIZATION, @@ -13,8 +15,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.consts import ANNOTATION_PATH -from tests.override_app_dependency import TEST_TENANT, TEST_TOKEN, app client = TestClient(app) diff --git a/annotation/tests/test_get_unassigned_files.py b/annotation/tests/test_get_unassigned_files.py index 6cf9457d6..c39d2e48f 100644 --- a/annotation/tests/test_get_unassigned_files.py +++ b/annotation/tests/test_get_unassigned_files.py @@ -4,10 +4,10 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.models import Category, File, Job, User from annotation.schemas import CategoryTypeSchema, ValidationSchema -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app client = TestClient(app) diff --git a/annotation/tests/test_get_users_for_job.py b/annotation/tests/test_get_users_for_job.py index 88ea95e66..1e22fc7d2 100644 --- a/annotation/tests/test_get_users_for_job.py +++ b/annotation/tests/test_get_users_for_job.py @@ -1,10 +1,10 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.models import Job, User from annotation.schemas import ValidationSchema -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app client = TestClient(app) diff --git a/annotation/tests/test_job_categories.py b/annotation/tests/test_job_categories.py index 86027745a..430753616 100644 --- a/annotation/tests/test_job_categories.py +++ b/annotation/tests/test_job_categories.py @@ -5,6 +5,9 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.orm import Session +from tests.consts import POST_JOBS_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app +from tests.test_category_crud import prepare_category_body from annotation.microservice_communication.search import ( AUTHORIZATION, @@ -13,9 +16,6 @@ ) from annotation.models import Category, Job from annotation.schemas import JobTypeEnumSchema, ValidationSchema -from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app -from tests.test_category_crud import prepare_category_body JOBS_PATH = "/jobs" MOCK_ID = 1 diff --git a/annotation/tests/test_post.py b/annotation/tests/test_post.py index 8e0c64314..fc05a1766 100644 --- a/annotation/tests/test_post.py +++ b/annotation/tests/test_post.py @@ -6,11 +6,11 @@ from sqlalchemy import not_ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.microservice_communication.assets_communication import ASSETS_URL from annotation.models import Category, File, Job, ManualAnnotationTask, User from annotation.schemas import CategoryTypeSchema, ValidationSchema -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app client = TestClient(app) diff --git a/annotation/tests/test_post_job.py b/annotation/tests/test_post_job.py index 00d415261..756a66b0a 100644 --- a/annotation/tests/test_post_job.py +++ b/annotation/tests/test_post_job.py @@ -6,6 +6,9 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.declarative import DeclarativeMeta from sqlalchemy.orm import Session +from tests.consts import POST_JOBS_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from tests.test_post import check_files_distributed_pages from annotation.annotations import row_to_dict from annotation.jobs import get_job_attributes_for_post @@ -27,9 +30,6 @@ JobTypeEnumSchema, ValidationSchema, ) -from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app -from tests.test_post import check_files_distributed_pages client = TestClient(app) diff --git a/annotation/tests/test_search_kafka.py b/annotation/tests/test_search_kafka.py index 5c0e58954..21cd85b9f 100644 --- a/annotation/tests/test_search_kafka.py +++ b/annotation/tests/test_search_kafka.py @@ -4,6 +4,7 @@ import responses from fastapi.testclient import TestClient from kafka.errors import NoBrokersAvailable +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.annotations import add_search_annotation_producer from annotation.kafka_client import producers @@ -15,7 +16,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from .consts import ANNOTATION_PATH diff --git a/annotation/tests/test_start_job.py b/annotation/tests/test_start_job.py index 39baf6124..9e95a2692 100644 --- a/annotation/tests/test_start_job.py +++ b/annotation/tests/test_start_job.py @@ -6,6 +6,7 @@ from requests.exceptions import RequestException from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from annotation.annotations import row_to_dict from annotation.jobs import update_inner_job_status @@ -16,7 +17,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app client = TestClient(app) diff --git a/annotation/tests/test_tasks_crud_cr.py b/annotation/tests/test_tasks_crud_cr.py index 9046e291b..89da82cbe 100644 --- a/annotation/tests/test_tasks_crud_cr.py +++ b/annotation/tests/test_tasks_crud_cr.py @@ -9,6 +9,9 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import CRUD_TASKS_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from tests.test_post import check_files_distributed_pages from annotation.microservice_communication.assets_communication import ASSETS_FILES_URL from annotation.microservice_communication.jobs_communication import JOBS_SEARCH_URL @@ -16,9 +19,6 @@ from annotation.microservice_communication.user import USERS_GET_USER_URL from annotation.models import Category, File, Job, ManualAnnotationTask, User from annotation.schemas import CategoryTypeSchema, ValidationSchema -from tests.consts import CRUD_TASKS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app -from tests.test_post import check_files_distributed_pages client = TestClient(app) diff --git a/annotation/tests/test_tasks_crud_ud.py b/annotation/tests/test_tasks_crud_ud.py index cd40cec61..4fb75f8f1 100644 --- a/annotation/tests/test_tasks_crud_ud.py +++ b/annotation/tests/test_tasks_crud_ud.py @@ -1,6 +1,9 @@ import pytest from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from tests.consts import CRUD_TASKS_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from tests.test_post import check_files_distributed_pages from annotation.annotations import row_to_dict from annotation.models import Category, File, Job, ManualAnnotationTask, User @@ -9,9 +12,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.consts import CRUD_TASKS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app -from tests.test_post import check_files_distributed_pages client = TestClient(app) diff --git a/annotation/tests/test_validation.py b/annotation/tests/test_validation.py index 4ff45384b..69e83dd5f 100644 --- a/annotation/tests/test_validation.py +++ b/annotation/tests/test_validation.py @@ -5,6 +5,10 @@ from fastapi import HTTPException from fastapi.testclient import TestClient from sqlalchemy import or_ +from tests.consts import FINISH_TASK_PATH +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app +from tests.test_finish_task import check_files_finished_pages +from tests.test_post import check_files_distributed_pages from annotation.annotations import row_to_dict from annotation.models import AnnotatedDoc, File, Job, ManualAnnotationTask, User @@ -25,10 +29,6 @@ find_initial_annotators, get_annotators_revisions, ) -from tests.consts import FINISH_TASK_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app -from tests.test_finish_task import check_files_finished_pages -from tests.test_post import check_files_distributed_pages client = TestClient(app) diff --git a/assets/alembic/env.py b/assets/alembic/env.py index 84aca146d..a5742cff3 100644 --- a/assets/alembic/env.py +++ b/assets/alembic/env.py @@ -1,11 +1,11 @@ import os from logging.config import fileConfig -from assets.config import settings -from assets.db.utils import get_test_db_url +from alembic import context from sqlalchemy import engine_from_config, pool -from alembic import context +from assets.config import settings +from assets.db.utils import get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py b/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py index 7f186790e..9817226a6 100644 --- a/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py +++ b/assets/alembic/versions/0f6c859c1d1c_add_original_ext_column_to_files_table.py @@ -7,12 +7,12 @@ """ import sqlalchemy as sa -from assets.db.models import FileObject # noqa +from alembic import op # revision identifiers, used by Alembic. from sqlalchemy.orm import Session # noqa -from alembic import op +from assets.db.models import FileObject # noqa revision = "0f6c859c1d1c" down_revision = "fe5926249504" diff --git a/assets/alembic/versions/9e837ea0c11d_image_pages.py b/assets/alembic/versions/9e837ea0c11d_image_pages.py index db6947c97..c797ade45 100644 --- a/assets/alembic/versions/9e837ea0c11d_image_pages.py +++ b/assets/alembic/versions/9e837ea0c11d_image_pages.py @@ -6,10 +6,10 @@ """ -from assets.db.models import FileObject +from alembic import op from sqlalchemy.orm import Session -from alembic import op +from assets.db.models import FileObject # revision identifiers, used by Alembic. diff --git a/assets/alembic/versions/afa33cc83d57_new_fields.py b/assets/alembic/versions/afa33cc83d57_new_fields.py index 02a8e91f0..66c25fecc 100644 --- a/assets/alembic/versions/afa33cc83d57_new_fields.py +++ b/assets/alembic/versions/afa33cc83d57_new_fields.py @@ -7,10 +7,10 @@ """ import sqlalchemy as sa -from assets.db.models import TSVector - from alembic import op +from assets.db.models import TSVector + # revision identifiers, used by Alembic. revision = "afa33cc83d57" down_revision = None diff --git a/assets/alembic/versions/fe5926249504_count_datasets.py b/assets/alembic/versions/fe5926249504_count_datasets.py index 775d1b8d2..054458651 100644 --- a/assets/alembic/versions/fe5926249504_count_datasets.py +++ b/assets/alembic/versions/fe5926249504_count_datasets.py @@ -7,10 +7,10 @@ """ import sqlalchemy as sa -from assets.db.models import Association, Datasets, FileObject +from alembic import op from sqlalchemy.orm import Session -from alembic import op +from assets.db.models import Association, Datasets, FileObject # revision identifiers, used by Alembic. revision = "fe5926249504" diff --git a/assets/assets/db/models.py b/assets/assets/db/models.py index 764f26030..0276b694b 100644 --- a/assets/assets/db/models.py +++ b/assets/assets/db/models.py @@ -2,7 +2,6 @@ from typing import Any, Dict, Optional import sqlalchemy as sa -from assets.config import settings from filter_lib import create_filter_model from sqlalchemy.dialects.postgresql import TSVECTOR from sqlalchemy.engine.default import DefaultExecutionContext @@ -10,6 +9,8 @@ from sqlalchemy.orm import relationship, sessionmaker from sqlalchemy.types import TypeDecorator +from assets.config import settings + Base = declarative_base() engine = sa.create_engine( settings.database_url, diff --git a/assets/assets/db/service.py b/assets/assets/db/service.py index 4492c48d7..a6620896d 100644 --- a/assets/assets/db/service.py +++ b/assets/assets/db/service.py @@ -1,12 +1,13 @@ from typing import Any, Dict, Optional, Tuple -from assets.db.models import Association, Datasets, FileObject, SessionLocal -from assets.logger import get_logger -from assets.schemas import FileProcessingStatusForUpdate from filter_lib import PaginationParams, form_query, map_request_to_filter from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Query, Session, load_only, selectinload +from assets.db.models import Association, Datasets, FileObject, SessionLocal +from assets.logger import get_logger +from assets.schemas import FileProcessingStatusForUpdate + logger = get_logger(__name__) diff --git a/assets/assets/main.py b/assets/assets/main.py index e6b37d248..527ebc599 100644 --- a/assets/assets/main.py +++ b/assets/assets/main.py @@ -1,10 +1,12 @@ import os -from assets import routers -from assets.config import settings + from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from tenant_dependency import get_tenant_info +from assets import routers +from assets.config import settings + tenant = get_tenant_info(url=settings.keycloak_host, algorithm="RS256") diff --git a/assets/assets/routers/bonds_router.py b/assets/assets/routers/bonds_router.py index b8f3e6cad..38bfb6c0c 100644 --- a/assets/assets/routers/bonds_router.py +++ b/assets/assets/routers/bonds_router.py @@ -4,6 +4,7 @@ import fastapi import filter_lib import sqlalchemy.orm + from assets import db, schemas, utils router = fastapi.APIRouter(prefix="/datasets/bonds", tags=["bonds"]) diff --git a/assets/assets/routers/datasets_router.py b/assets/assets/routers/datasets_router.py index 2094cdf45..f6ffa3e89 100644 --- a/assets/assets/routers/datasets_router.py +++ b/assets/assets/routers/datasets_router.py @@ -6,6 +6,7 @@ import sqlalchemy.exc import sqlalchemy.orm import sqlalchemy_filters.exceptions + from assets import db, schemas router = fastapi.APIRouter(prefix="/datasets", tags=["datasets"]) diff --git a/assets/assets/routers/s3_router.py b/assets/assets/routers/s3_router.py index 1fd31c624..049849f19 100644 --- a/assets/assets/routers/s3_router.py +++ b/assets/assets/routers/s3_router.py @@ -4,6 +4,7 @@ import minio import sqlalchemy.orm import urllib3.exceptions + from assets import db, exceptions, schemas, utils router = fastapi.APIRouter(prefix="/s3_upload", tags=["s_3"]) diff --git a/assets/assets/schemas.py b/assets/assets/schemas.py index b0cfacc76..8d9d7e7c1 100644 --- a/assets/assets/schemas.py +++ b/assets/assets/schemas.py @@ -2,9 +2,10 @@ import enum from typing import List, Optional -from assets.db.models import Datasets from pydantic import BaseModel, validator +from assets.db.models import Datasets + class MinioObjects(BaseModel): objects: List[int] diff --git a/assets/assets/utils/convert_service_utils.py b/assets/assets/utils/convert_service_utils.py index c406c0331..7f6f0f7d0 100644 --- a/assets/assets/utils/convert_service_utils.py +++ b/assets/assets/utils/convert_service_utils.py @@ -1,4 +1,5 @@ import requests + from assets import logger from assets.config import settings diff --git a/assets/assets/utils/s3_utils.py b/assets/assets/utils/s3_utils.py index dbe1d7f05..df5a514ef 100644 --- a/assets/assets/utils/s3_utils.py +++ b/assets/assets/utils/s3_utils.py @@ -3,6 +3,7 @@ import boto3 import urllib3.exceptions + from assets import exceptions, logger from assets.config import settings diff --git a/assets/tests/conftest.py b/assets/tests/conftest.py index 6a01349aa..81eb81313 100644 --- a/assets/tests/conftest.py +++ b/assets/tests/conftest.py @@ -10,13 +10,8 @@ import boto3 import pytest import urllib3 -from assets.config import settings -from assets.db.models import Base, FileObject -from assets.db.service import session_scope_for_dependency -from assets.db.utils import get_test_db_url -from assets.main import app, tenant -from assets.utils import minio_utils -from assets.utils.common_utils import FileConverter +from alembic import command +from alembic.config import Config from botocore.config import Config as BotoConfig from botocore.exceptions import ClientError from fastapi.testclient import TestClient @@ -26,8 +21,13 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy_utils import create_database, database_exists -from alembic import command -from alembic.config import Config +from assets.config import settings +from assets.db.models import Base, FileObject +from assets.db.service import session_scope_for_dependency +from assets.db.utils import get_test_db_url +from assets.main import app, tenant +from assets.utils import minio_utils +from assets.utils.common_utils import FileConverter BUCKET_TESTS = "tests" + uuid.uuid4().hex diff --git a/assets/tests/test_helpers.py b/assets/tests/test_helpers.py index 6ca6c3b9f..e50b6fcfc 100644 --- a/assets/tests/test_helpers.py +++ b/assets/tests/test_helpers.py @@ -2,6 +2,8 @@ from unittest.mock import Mock, patch import pytest +from fastapi import HTTPException + from assets.db.models import FileObject from assets.db.service import ( delete_file_from_db, @@ -10,7 +12,6 @@ ) from assets.schemas import FileProcessingStatus from assets.utils.minio_utils import check_bucket, delete_one_from_minio -from fastapi import HTTPException @pytest.fixture diff --git a/assets/tests/test_models.py b/assets/tests/test_models.py index b6510109f..edcc71e94 100644 --- a/assets/tests/test_models.py +++ b/assets/tests/test_models.py @@ -1,6 +1,7 @@ import uuid import pytest + from assets.db.models import Association, Datasets, FileObject diff --git a/jobs/alembic/versions/13ac4bb3abd2_.py b/jobs/alembic/versions/13ac4bb3abd2_.py index 8db2dcf3d..557d9c2a5 100644 --- a/jobs/alembic/versions/13ac4bb3abd2_.py +++ b/jobs/alembic/versions/13ac4bb3abd2_.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "13ac4bb3abd2" diff --git a/jobs/alembic/versions/1408c6c00120_add_available_annotation_types_and_.py b/jobs/alembic/versions/1408c6c00120_add_available_annotation_types_and_.py index 0944eac1d..7bda11c31 100644 --- a/jobs/alembic/versions/1408c6c00120_add_available_annotation_types_and_.py +++ b/jobs/alembic/versions/1408c6c00120_add_available_annotation_types_and_.py @@ -8,9 +8,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "1408c6c00120" diff --git a/jobs/alembic/versions/3f5b2d199d38_.py b/jobs/alembic/versions/3f5b2d199d38_.py index 8b1872cd8..e7d0884d9 100644 --- a/jobs/alembic/versions/3f5b2d199d38_.py +++ b/jobs/alembic/versions/3f5b2d199d38_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/7511c6790067_.py b/jobs/alembic/versions/7511c6790067_.py index 2857a099a..d17007933 100644 --- a/jobs/alembic/versions/7511c6790067_.py +++ b/jobs/alembic/versions/7511c6790067_.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "7511c6790067" diff --git a/jobs/alembic/versions/83694c0b2df6_.py b/jobs/alembic/versions/83694c0b2df6_.py index 64f769bc2..1398e1840 100644 --- a/jobs/alembic/versions/83694c0b2df6_.py +++ b/jobs/alembic/versions/83694c0b2df6_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/86f432539475_.py b/jobs/alembic/versions/86f432539475_.py index ec341ec58..18e95a46d 100644 --- a/jobs/alembic/versions/86f432539475_.py +++ b/jobs/alembic/versions/86f432539475_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/9229e70d2791_.py b/jobs/alembic/versions/9229e70d2791_.py index 19b0c9c87..d9223fd51 100644 --- a/jobs/alembic/versions/9229e70d2791_.py +++ b/jobs/alembic/versions/9229e70d2791_.py @@ -7,9 +7,8 @@ """ import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "9229e70d2791" diff --git a/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py b/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py index 78c880883..d7932406b 100644 --- a/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py +++ b/jobs/alembic/versions/b4afb5ae8923_add_start_manual_job_automatically_flag.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/d1ddce2d5352_.py b/jobs/alembic/versions/d1ddce2d5352_.py index 1e1ea086b..2b2c28312 100644 --- a/jobs/alembic/versions/d1ddce2d5352_.py +++ b/jobs/alembic/versions/d1ddce2d5352_.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py b/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py index b9b29334d..2ea35925b 100644 --- a/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py +++ b/jobs/alembic/versions/f60dd492b17f_add_extensive_coverage_param.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/tests/test_API_functions/test_change_job-proxy_to_annotation.py b/jobs/tests/test_API_functions/test_change_job-proxy_to_annotation.py index 9b312acc4..c7232a7e7 100644 --- a/jobs/tests/test_API_functions/test_change_job-proxy_to_annotation.py +++ b/jobs/tests/test_API_functions/test_change_job-proxy_to_annotation.py @@ -2,7 +2,6 @@ from unittest.mock import patch import pytest - from tests.test_db import create_mock_annotation_job_in_db diff --git a/jobs/tests/test_API_functions/test_search_jobs.py b/jobs/tests/test_API_functions/test_search_jobs.py index 0990fd596..353213cf8 100644 --- a/jobs/tests/test_API_functions/test_search_jobs.py +++ b/jobs/tests/test_API_functions/test_search_jobs.py @@ -1,5 +1,4 @@ import pytest - from tests.test_db import ( create_mock_annotation_job_in_db, create_mock_extraction_job_in_db, diff --git a/lib/filter_lib/tests/test_pagination.py b/lib/filter_lib/tests/test_pagination.py index 592497d2e..ab227813e 100644 --- a/lib/filter_lib/tests/test_pagination.py +++ b/lib/filter_lib/tests/test_pagination.py @@ -1,6 +1,5 @@ import pytest from pydantic import ValidationError - from src.pagination import ( PaginationParams, _calculate_num_pages, diff --git a/lib/filter_lib/tests/test_query_modifier.py b/lib/filter_lib/tests/test_query_modifier.py index da0dba643..83c1cacb2 100644 --- a/lib/filter_lib/tests/test_query_modifier.py +++ b/lib/filter_lib/tests/test_query_modifier.py @@ -1,5 +1,4 @@ import pytest - from src.enum_generator import get_enum_from_orm from src.pagination import PaginationParams from src.query_modificator import ( diff --git a/lib/filter_lib/tests/test_schema_generator.py b/lib/filter_lib/tests/test_schema_generator.py index 872f3c046..5c27eeaee 100644 --- a/lib/filter_lib/tests/test_schema_generator.py +++ b/lib/filter_lib/tests/test_schema_generator.py @@ -1,6 +1,5 @@ import pytest from pydantic import ValidationError - from src.schema_generator import ( Page, Pagination, diff --git a/lib/filter_lib/usage_example/app.py b/lib/filter_lib/usage_example/app.py index f8de34a8b..19882eb70 100644 --- a/lib/filter_lib/usage_example/app.py +++ b/lib/filter_lib/usage_example/app.py @@ -2,9 +2,6 @@ from db_example import Address, User, get_db from fastapi import Depends, FastAPI -from pydantic import BaseModel -from sqlalchemy.orm import Session - from filter_lib import ( # type: ignore Page, create_filter_model, @@ -12,6 +9,8 @@ map_request_to_filter, paginate, ) +from pydantic import BaseModel +from sqlalchemy.orm import Session app = FastAPI() diff --git a/lib/python3.12/filter_lib/tests/test_pagination.py b/lib/python3.12/filter_lib/tests/test_pagination.py index 6badf9d59..8732f68f3 100644 --- a/lib/python3.12/filter_lib/tests/test_pagination.py +++ b/lib/python3.12/filter_lib/tests/test_pagination.py @@ -1,6 +1,5 @@ import pytest from pydantic import ValidationError - from src.pagination import ( PaginationParams, _calculate_num_pages, diff --git a/lib/python3.12/filter_lib/tests/test_query_modifier.py b/lib/python3.12/filter_lib/tests/test_query_modifier.py index da0dba643..83c1cacb2 100644 --- a/lib/python3.12/filter_lib/tests/test_query_modifier.py +++ b/lib/python3.12/filter_lib/tests/test_query_modifier.py @@ -1,5 +1,4 @@ import pytest - from src.enum_generator import get_enum_from_orm from src.pagination import PaginationParams from src.query_modificator import ( diff --git a/lib/python3.12/filter_lib/tests/test_schema_generator.py b/lib/python3.12/filter_lib/tests/test_schema_generator.py index 9cd0ab065..5c366a508 100644 --- a/lib/python3.12/filter_lib/tests/test_schema_generator.py +++ b/lib/python3.12/filter_lib/tests/test_schema_generator.py @@ -1,6 +1,5 @@ import pytest from pydantic import ValidationError - from src.schema_generator import ( Page, Pagination, diff --git a/lib/python3.12/filter_lib/usage_example/app.py b/lib/python3.12/filter_lib/usage_example/app.py index f8de34a8b..19882eb70 100644 --- a/lib/python3.12/filter_lib/usage_example/app.py +++ b/lib/python3.12/filter_lib/usage_example/app.py @@ -2,9 +2,6 @@ from db_example import Address, User, get_db from fastapi import Depends, FastAPI -from pydantic import BaseModel -from sqlalchemy.orm import Session - from filter_lib import ( # type: ignore Page, create_filter_model, @@ -12,6 +9,8 @@ map_request_to_filter, paginate, ) +from pydantic import BaseModel +from sqlalchemy.orm import Session app = FastAPI() diff --git a/processing/alembic/env.py b/processing/alembic/env.py index c7680ea88..1ffbfd1a2 100644 --- a/processing/alembic/env.py +++ b/processing/alembic/env.py @@ -1,12 +1,12 @@ import os from logging.config import fileConfig +from alembic import context +from sqlalchemy import engine_from_config, pool + from processing.config import settings from processing.db.models import Base from processing.db.service import get_test_db_url -from sqlalchemy import engine_from_config, pool - -from alembic import context # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/processing/alembic/versions/52af1473946f_init.py b/processing/alembic/versions/52af1473946f_init.py index 6c9e83f49..1b834ea23 100644 --- a/processing/alembic/versions/52af1473946f_init.py +++ b/processing/alembic/versions/52af1473946f_init.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/processing/alembic/versions/8e973b70b26f_noneasnull.py b/processing/alembic/versions/8e973b70b26f_noneasnull.py index 387914de4..5b4d8fc1b 100644 --- a/processing/alembic/versions/8e973b70b26f_noneasnull.py +++ b/processing/alembic/versions/8e973b70b26f_noneasnull.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/processing/alembic/versions/f637b13c744d_renamed_column.py b/processing/alembic/versions/f637b13c744d_renamed_column.py index 6b30da0ba..bf91e03c1 100644 --- a/processing/alembic/versions/f637b13c744d_renamed_column.py +++ b/processing/alembic/versions/f637b13c744d_renamed_column.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/processing/tests/conftest.py b/processing/tests/conftest.py index 89ad287ec..138182aed 100644 --- a/processing/tests/conftest.py +++ b/processing/tests/conftest.py @@ -3,15 +3,15 @@ from unittest.mock import patch import pytest -from processing.config import settings -from processing.db.service import get_test_db_url +from alembic import command +from alembic.config import Config from sqlalchemy import create_engine from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import sessionmaker from sqlalchemy_utils import create_database, database_exists, drop_database -from alembic import command -from alembic.config import Config +from processing.config import settings +from processing.db.service import get_test_db_url pytest_plugins = ["docker_compose"] diff --git a/processing/tests/test_assets_status.py b/processing/tests/test_assets_status.py index 1792faaf1..5d7e5a981 100644 --- a/processing/tests/test_assets_status.py +++ b/processing/tests/test_assets_status.py @@ -1,6 +1,7 @@ from unittest.mock import patch import pytest + from processing.config import settings from processing.tasks import PreprocessingTask diff --git a/search/search/harvester.py b/search/search/harvester.py index 289261d04..df95b1291 100644 --- a/search/search/harvester.py +++ b/search/search/harvester.py @@ -2,10 +2,11 @@ from typing import Iterator, Optional import boto3 -import search.es as es -import search.schemas as schemas from botocore.errorfactory import ClientError from elasticsearch import helpers + +import search.es as es +import search.schemas as schemas from search.config import settings from search.logger import logger diff --git a/search/search/main.py b/search/search/main.py index a9800d940..d77306417 100644 --- a/search/search/main.py +++ b/search/search/main.py @@ -1,11 +1,11 @@ -import os import asyncio +import os from typing import Optional import fastapi -from fastapi.middleware.cors import CORSMiddleware from botocore.exceptions import BotoCoreError from elasticsearch.exceptions import ElasticsearchException +from fastapi.middleware.cors import CORSMiddleware from tenant_dependency import TenantData, get_tenant_info import search.es as es diff --git a/search/tests/conftest.py b/search/tests/conftest.py index 7a1d2436c..08e3c69cc 100644 --- a/search/tests/conftest.py +++ b/search/tests/conftest.py @@ -6,9 +6,6 @@ from elasticsearch import AsyncElasticsearch from kafka.errors import TopicAlreadyExistsError from moto import mock_s3 - -from search.config import settings -from search.es import INDEX_SETTINGS from tests.test_get import CHILD_CATEGORIES_DATA, TEST_DATA from tests.test_harvester import ( DOCS_IN_ES, @@ -18,6 +15,9 @@ S3_PAGES, ) +from search.config import settings +from search.es import INDEX_SETTINGS + BUCKET_NAME = INDEX_NAME diff --git a/search/tests/test_harvester.py b/search/tests/test_harvester.py index 029d486c9..44395b7aa 100644 --- a/search/tests/test_harvester.py +++ b/search/tests/test_harvester.py @@ -2,6 +2,7 @@ from unittest.mock import Mock import pytest + from search.harvester import parse_json, start_harvester from .override_app_dependency import TEST_TENANT diff --git a/taxonomy/alembic/env.py b/taxonomy/alembic/env.py index 35c9c241d..48fae5ba8 100644 --- a/taxonomy/alembic/env.py +++ b/taxonomy/alembic/env.py @@ -1,10 +1,10 @@ import os from logging.config import fileConfig +from alembic import context # type: ignore from sqlalchemy import engine_from_config, pool -from taxonomy.database import SQLALCHEMY_DATABASE_URL, get_test_db_url -from alembic import context # type: ignore +from taxonomy.database import SQLALCHEMY_DATABASE_URL, get_test_db_url # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py b/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py index 0a5a8062f..25bc3778c 100644 --- a/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py +++ b/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py b/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py index 3adec516a..5ece0c493 100644 --- a/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py +++ b/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa import sqlalchemy_utils - from alembic import op # revision identifiers, used by Alembic. diff --git a/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py b/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py index 1dafe6073..8e37244bf 100644 --- a/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py +++ b/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py @@ -6,7 +6,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/taxonomy/tests/conftest.py b/taxonomy/tests/conftest.py index f0e6345ca..17763523d 100644 --- a/taxonomy/tests/conftest.py +++ b/taxonomy/tests/conftest.py @@ -7,14 +7,14 @@ import pytest import sqlalchemy +from alembic import command +from alembic.config import Config from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_utils import create_database, database_exists, drop_database -from alembic import command -from alembic.config import Config from taxonomy.database import ( SQLALCHEMY_DATABASE_URL, Base, diff --git a/taxonomy/tests/test_taxon_crud.py b/taxonomy/tests/test_taxon_crud.py index 9c925b97b..722579700 100644 --- a/taxonomy/tests/test_taxon_crud.py +++ b/taxonomy/tests/test_taxon_crud.py @@ -3,10 +3,10 @@ from typing import Any, List, Optional import pytest -from taxonomy.models import Taxon - from tests.override_app_dependency import TEST_HEADER +from taxonomy.models import Taxon + TAXON_PATH = "/taxons" diff --git a/taxonomy/tests/test_taxonomy_router.py b/taxonomy/tests/test_taxonomy_router.py index 1a1597b2a..80cee30ca 100644 --- a/taxonomy/tests/test_taxonomy_router.py +++ b/taxonomy/tests/test_taxonomy_router.py @@ -1,12 +1,12 @@ from typing import Tuple import pytest +from tests.override_app_dependency import TEST_HEADER, TEST_TENANTS + from taxonomy.models import Taxonomy from taxonomy.schemas import CategoryLinkSchema from taxonomy.taxonomy import services -from tests.override_app_dependency import TEST_HEADER, TEST_TENANTS - @pytest.mark.integration @pytest.mark.skip(reason="tests refactoring") From 3d1e79ca5f54443a859ae60d8981c0e218d507e1 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:52:06 +0300 Subject: [PATCH 17/24] trigger workflow --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 35eaf179c..15e710bb4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,7 +25,7 @@ jobs: python -m pip install --upgrade pip pip install poetry poetry install --only dev - - name: Run Isort + - name: Run isort run: | poetry run isort --diff --check-only . - name: Run Black From 8aea6f9c816fdcd30b18a17d19a0d804e2801596 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:58:28 +0300 Subject: [PATCH 18/24] fix: isort issues --- .../annotation/annotations/resources.py | 19 ++++++++++++++++--- annotation/annotation/categories/resources.py | 4 +++- .../annotation/distribution/resources.py | 10 ++++++++-- annotation/annotation/filters.py | 8 +++++++- annotation/annotation/jobs/resources.py | 17 ++++++++++++++--- annotation/annotation/jobs/services.py | 14 +++++++++++--- annotation/annotation/metadata/resources.py | 4 +++- .../microservice_communication/task.py | 5 ++++- annotation/annotation/revisions/resources.py | 4 +++- annotation/annotation/tasks/resources.py | 5 ++++- annotation/annotation/tasks/services.py | 10 ++++++++-- annotation/tests/conftest.py | 13 +++++++++++-- .../tests/test_annotators_overall_load.py | 4 +++- annotation/tests/test_get_child_categories.py | 17 ++++++++++++----- annotation/tests/test_get_pages_info.py | 8 +++++++- annotation/tests/test_job_categories.py | 7 ++++++- annotation/tests/test_microservices_search.py | 7 +++++-- annotation/tests/test_post.py | 4 +++- annotation/tests/test_post_annotation.py | 15 +++++++++++---- annotation/tests/test_post_next_task.py | 13 ++++++++++--- .../tests/test_post_unassgined_files.py | 9 +++++++-- annotation/tests/test_search_kafka.py | 4 +++- annotation/tests/test_tasks_crud_cr.py | 8 ++++++-- annotation/tests/test_update_job.py | 9 +++++++-- annotation/tests/test_validation.py | 8 +++++++- jobs/alembic/env.py | 2 +- .../ef5c796d3c00_add_pipeline_engine.py | 1 - jobs/tests/conftest.py | 4 ++-- .../test_API_functions/test_change_job.py | 4 ++-- .../test_other_API_functions.py | 4 ++-- jobs/tests/test_utils.py | 2 +- 31 files changed, 187 insertions(+), 56 deletions(-) diff --git a/annotation/annotation/annotations/resources.py b/annotation/annotation/annotations/resources.py index bb7722379..1b3e6fd81 100644 --- a/annotation/annotation/annotations/resources.py +++ b/annotation/annotation/annotations/resources.py @@ -2,12 +2,23 @@ from typing import Dict, List, Optional, Set from uuid import UUID -from fastapi import APIRouter, Depends, HTTPException, Path, Query, Response, status +from fastapi import ( + APIRouter, + Depends, + HTTPException, + Path, + Query, + Response, + status, +) from sqlalchemy import and_, desc from sqlalchemy.orm import Session from tenant_dependency import TenantData -from annotation.categories.services import combine_categories, fetch_bunch_categories_db +from annotation.categories.services import ( + combine_categories, + fetch_bunch_categories_db, +) from annotation.database import get_db from annotation.errors import NoSuchRevisionsError from annotation.jobs.services import update_jobs_categories @@ -15,7 +26,9 @@ from annotation.microservice_communication.assets_communication import ( get_file_path_and_bucket, ) -from annotation.microservice_communication.search import X_CURRENT_TENANT_HEADER +from annotation.microservice_communication.search import ( + X_CURRENT_TENANT_HEADER, +) from annotation.schemas import ( AnnotatedDocSchema, BadRequestErrorSchema, diff --git a/annotation/annotation/categories/resources.py b/annotation/annotation/categories/resources.py index 40f411677..8efeb687f 100644 --- a/annotation/annotation/categories/resources.py +++ b/annotation/annotation/categories/resources.py @@ -8,7 +8,9 @@ from annotation.database import get_db from annotation.errors import NoSuchCategoryError from annotation.filters import CategoryFilter -from annotation.microservice_communication.search import X_CURRENT_TENANT_HEADER +from annotation.microservice_communication.search import ( + X_CURRENT_TENANT_HEADER, +) from annotation.schemas import ( BadRequestErrorSchema, CategoryBaseSchema, diff --git a/annotation/annotation/distribution/resources.py b/annotation/annotation/distribution/resources.py index 452abce0f..a8ef4a6ff 100644 --- a/annotation/annotation/distribution/resources.py +++ b/annotation/annotation/distribution/resources.py @@ -6,7 +6,11 @@ from tenant_dependency import TenantData from annotation.database import get_db -from annotation.distribution import distribute, find_unassigned_files, prepare_response +from annotation.distribution import ( + distribute, + find_unassigned_files, + prepare_response, +) from annotation.errors import FieldConstraintError from annotation.jobs import ( check_annotators, @@ -17,7 +21,9 @@ get_files_info, prepare_files_for_distribution, ) -from annotation.microservice_communication.search import X_CURRENT_TENANT_HEADER +from annotation.microservice_communication.search import ( + X_CURRENT_TENANT_HEADER, +) from annotation.models import File, Job, User from annotation.schemas import ( BadRequestErrorSchema, diff --git a/annotation/annotation/filters.py b/annotation/annotation/filters.py index 9b2d62d18..c1c123a07 100644 --- a/annotation/annotation/filters.py +++ b/annotation/annotation/filters.py @@ -1,6 +1,12 @@ from filter_lib import create_filter_model -from annotation.models import AnnotatedDoc, Category, Job, ManualAnnotationTask, User +from annotation.models import ( + AnnotatedDoc, + Category, + Job, + ManualAnnotationTask, + User, +) ADDITIONAL_TASK_FIELDS = ["file_name", "job_name"] diff --git a/annotation/annotation/jobs/resources.py b/annotation/annotation/jobs/resources.py index d1c8a5e43..29e80c03c 100644 --- a/annotation/annotation/jobs/resources.py +++ b/annotation/annotation/jobs/resources.py @@ -15,12 +15,16 @@ from annotation.database import get_db from annotation.distribution import distribute, redistribute from annotation.filters import CategoryFilter -from annotation.microservice_communication.assets_communication import get_files_info +from annotation.microservice_communication.assets_communication import ( + get_files_info, +) from annotation.microservice_communication.jobs_communication import ( JobUpdateException, update_job_status, ) -from annotation.microservice_communication.search import X_CURRENT_TENANT_HEADER +from annotation.microservice_communication.search import ( + X_CURRENT_TENANT_HEADER, +) from annotation.schemas import ( BadRequestErrorSchema, CategoryResponseSchema, @@ -42,7 +46,14 @@ from annotation.tags import FILES_TAG, JOBS_TAG from annotation.token_dependency import TOKEN -from ..models import AnnotatedDoc, Category, File, Job, ManualAnnotationTask, User +from ..models import ( + AnnotatedDoc, + Category, + File, + Job, + ManualAnnotationTask, + User, +) from .services import ( add_users, collect_job_names, diff --git a/annotation/annotation/jobs/services.py b/annotation/annotation/jobs/services.py index 7e5eb4c86..354e2f31a 100644 --- a/annotation/annotation/jobs/services.py +++ b/annotation/annotation/jobs/services.py @@ -12,9 +12,17 @@ from annotation.categories import fetch_bunch_categories_db from annotation.categories.services import response_object_from_db from annotation.database import Base -from annotation.errors import EnumValidationError, FieldConstraintError, WrongJobError -from annotation.microservice_communication.assets_communication import get_files_info -from annotation.microservice_communication.jobs_communication import get_job_names +from annotation.errors import ( + EnumValidationError, + FieldConstraintError, + WrongJobError, +) +from annotation.microservice_communication.assets_communication import ( + get_files_info, +) +from annotation.microservice_communication.jobs_communication import ( + get_job_names, +) from annotation.models import ( Category, File, diff --git a/annotation/annotation/metadata/resources.py b/annotation/annotation/metadata/resources.py index be013589f..f64c2aab1 100644 --- a/annotation/annotation/metadata/resources.py +++ b/annotation/annotation/metadata/resources.py @@ -1,6 +1,8 @@ from fastapi import APIRouter, status -from annotation.microservice_communication.search import X_CURRENT_TENANT_HEADER +from annotation.microservice_communication.search import ( + X_CURRENT_TENANT_HEADER, +) from annotation.schemas import EntitiesStatusesSchema from annotation.tags import METADATA_TAG, TASKS_TAG diff --git a/annotation/annotation/microservice_communication/task.py b/annotation/annotation/microservice_communication/task.py index d2c9eb5f8..87283e2b9 100644 --- a/annotation/annotation/microservice_communication/task.py +++ b/annotation/annotation/microservice_communication/task.py @@ -11,7 +11,10 @@ BEARER, HEADER_TENANT, ) -from annotation.schemas import AgreementScoreServiceInput, AgreementScoreServiceResponse +from annotation.schemas import ( + AgreementScoreServiceInput, + AgreementScoreServiceResponse, +) load_dotenv(find_dotenv()) AGREEMENT_SCORE_SERVICE_HOST = os.environ.get("AGREEMENT_SCORE_SERVICE_HOST") diff --git a/annotation/annotation/revisions/resources.py b/annotation/annotation/revisions/resources.py index 6739fb119..2bc837d27 100644 --- a/annotation/annotation/revisions/resources.py +++ b/annotation/annotation/revisions/resources.py @@ -5,7 +5,9 @@ from starlette import status from annotation.database import get_db -from annotation.microservice_communication.search import X_CURRENT_TENANT_HEADER +from annotation.microservice_communication.search import ( + X_CURRENT_TENANT_HEADER, +) from annotation.models import AnnotatedDoc from annotation.schemas import AnnotatedDocSchema, ConnectionErrorSchema from annotation.tags import ANNOTATION_TAG, REVISION_TAG diff --git a/annotation/annotation/tasks/resources.py b/annotation/annotation/tasks/resources.py index 55544e31f..ad4e07283 100644 --- a/annotation/annotation/tasks/resources.py +++ b/annotation/annotation/tasks/resources.py @@ -76,7 +76,10 @@ ValidationSchema, ) from annotation.tags import REVISION_TAG, TASKS_TAG -from annotation.tasks.validation import create_annotation_tasks, create_validation_tasks +from annotation.tasks.validation import ( + create_annotation_tasks, + create_validation_tasks, +) from annotation.token_dependency import TOKEN from ..models import File, Job, ManualAnnotationTask diff --git a/annotation/annotation/tasks/services.py b/annotation/annotation/tasks/services.py index 9a2a82e16..d38bc2732 100644 --- a/annotation/annotation/tasks/services.py +++ b/annotation/annotation/tasks/services.py @@ -23,14 +23,20 @@ ) from annotation.errors import CheckFieldError, FieldConstraintError from annotation.filters import ADDITIONAL_TASK_FIELDS, TaskFilter -from annotation.jobs import get_jobs_by_name, update_files, update_user_overall_load +from annotation.jobs import ( + get_jobs_by_name, + update_files, + update_user_overall_load, +) from annotation.logger import Logger from annotation.microservice_communication.assets_communication import ( get_file_names_by_request, get_file_path_and_bucket, get_files_by_request, ) -from annotation.microservice_communication.search import get_user_names_by_request +from annotation.microservice_communication.search import ( + get_user_names_by_request, +) from annotation.microservice_communication.task import get_agreement_score from annotation.models import ( AgreementMetrics, diff --git a/annotation/tests/conftest.py b/annotation/tests/conftest.py index 5ad09c601..b73bcce37 100644 --- a/annotation/tests/conftest.py +++ b/annotation/tests/conftest.py @@ -87,7 +87,13 @@ TASKS_TEST_PROGRESS, ) from tests.test_get_pages_info import PAGES_INFO_ENTITIES -from tests.test_get_revisions import JOBS_IDS, PAGE, PAGES_PATHS, REVISIONS, USERS_IDS +from tests.test_get_revisions import ( + JOBS_IDS, + PAGE, + PAGES_PATHS, + REVISIONS, + USERS_IDS, +) from tests.test_get_revisions_without_annotation import ( REV_WITHOUT_ANNOTATION_DOC_1, REV_WITHOUT_ANNOTATION_DOC_2, @@ -96,7 +102,10 @@ REV_WITHOUT_ANNOTATION_TASK, ) from tests.test_get_unassigned_files import UNASSIGNED_FILES_ENTITIES -from tests.test_get_users_for_job import USERS_FOR_JOB_ANNOTATORS, USERS_FOR_JOB_JOBS +from tests.test_get_users_for_job import ( + USERS_FOR_JOB_ANNOTATORS, + USERS_FOR_JOB_JOBS, +) from tests.test_job_categories import CATEGORIES_USERS, MOCK_ID from tests.test_post import POST_JOBS, TEST_POST_USERS from tests.test_post_annotation import ( diff --git a/annotation/tests/test_annotators_overall_load.py b/annotation/tests/test_annotators_overall_load.py index 6af0b3fb3..7b4515a76 100644 --- a/annotation/tests/test_annotators_overall_load.py +++ b/annotation/tests/test_annotators_overall_load.py @@ -11,7 +11,9 @@ from annotation.jobs import update_user_overall_load from annotation.main import app -from annotation.microservice_communication.assets_communication import ASSETS_FILES_URL +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, +) from annotation.models import ( AnnotatedDoc, Category, diff --git a/annotation/tests/test_get_child_categories.py b/annotation/tests/test_get_child_categories.py index 20a419a11..a256c1ff7 100644 --- a/annotation/tests/test_get_child_categories.py +++ b/annotation/tests/test_get_child_categories.py @@ -6,8 +6,19 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import CATEGORIES_PATH, POST_JOBS_PATH +from tests.override_app_dependency import ( + TEST_HEADERS, + TEST_TENANT, + TEST_TOKEN, + app, +) +from tests.test_job_categories import prepare_job_body +from tests.test_post_next_task import ASSETS_RESPONSE -from annotation.microservice_communication.assets_communication import ASSETS_FILES_URL +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, +) from annotation.microservice_communication.search import ( AUTHORIZATION, BEARER, @@ -15,10 +26,6 @@ ) from annotation.models import Category from annotation.schemas import CategoryTypeSchema -from tests.consts import CATEGORIES_PATH, POST_JOBS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app -from tests.test_job_categories import prepare_job_body -from tests.test_post_next_task import ASSETS_RESPONSE # Cyclic categories have tree hierarchical structure of ids: # "1" -> "2" -> "4" -> "1" diff --git a/annotation/tests/test_get_pages_info.py b/annotation/tests/test_get_pages_info.py index be1554bb1..b48eee361 100644 --- a/annotation/tests/test_get_pages_info.py +++ b/annotation/tests/test_get_pages_info.py @@ -13,7 +13,13 @@ BEARER, HEADER_TENANT, ) -from annotation.models import AnnotatedDoc, File, Job, ManualAnnotationTask, User +from annotation.models import ( + AnnotatedDoc, + File, + Job, + ManualAnnotationTask, + User, +) from annotation.schemas import TaskStatusEnumSchema, ValidationSchema from annotation.tasks import get_task_revisions diff --git a/annotation/tests/test_job_categories.py b/annotation/tests/test_job_categories.py index 430753616..63d640108 100644 --- a/annotation/tests/test_job_categories.py +++ b/annotation/tests/test_job_categories.py @@ -6,7 +6,12 @@ from fastapi.testclient import TestClient from sqlalchemy.orm import Session from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app +from tests.override_app_dependency import ( + TEST_HEADERS, + TEST_TENANT, + TEST_TOKEN, + app, +) from tests.test_category_crud import prepare_category_body from annotation.microservice_communication.search import ( diff --git a/annotation/tests/test_microservices_search.py b/annotation/tests/test_microservices_search.py index 461cf7f8b..d475d10dd 100644 --- a/annotation/tests/test_microservices_search.py +++ b/annotation/tests/test_microservices_search.py @@ -2,6 +2,7 @@ import responses from fastapi import HTTPException from requests import ConnectionError, RequestException, Timeout +from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN from annotation.microservice_communication.search import ( PAGE_SIZE, @@ -11,8 +12,10 @@ get_response, ) from annotation.models import ManualAnnotationTask -from annotation.schemas import ExpandedManualAnnotationTaskSchema, TaskStatusEnumSchema -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN +from annotation.schemas import ( + ExpandedManualAnnotationTaskSchema, + TaskStatusEnumSchema, +) AMOUNT_OF_ELEMENTS = 150 diff --git a/annotation/tests/test_post.py b/annotation/tests/test_post.py index fc05a1766..4760ae5fd 100644 --- a/annotation/tests/test_post.py +++ b/annotation/tests/test_post.py @@ -8,7 +8,9 @@ from sqlalchemy.orm import Session from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app -from annotation.microservice_communication.assets_communication import ASSETS_URL +from annotation.microservice_communication.assets_communication import ( + ASSETS_URL, +) from annotation.models import Category, File, Job, ManualAnnotationTask, User from annotation.schemas import CategoryTypeSchema, ValidationSchema diff --git a/annotation/tests/test_post_annotation.py b/annotation/tests/test_post_annotation.py index c21761016..3dbd05a87 100644 --- a/annotation/tests/test_post_annotation.py +++ b/annotation/tests/test_post_annotation.py @@ -12,6 +12,14 @@ from requests import RequestException from sqlalchemy.exc import DBAPIError, SQLAlchemyError from sqlalchemy.orm import Session +from tests.consts import ANNOTATION_PATH +from tests.override_app_dependency import ( + TEST_HEADERS, + TEST_TENANT, + TEST_TOKEN, + app, +) +from tests.test_tasks_crud_ud import construct_path from annotation.annotations import ( MANIFEST, @@ -27,7 +35,9 @@ upload_pages_to_minio, ) from annotation.kafka_client import producers -from annotation.microservice_communication.assets_communication import ASSETS_FILES_URL +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, +) from annotation.microservice_communication.search import ( AUTHORIZATION, BEARER, @@ -49,9 +59,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.consts import ANNOTATION_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app -from tests.test_tasks_crud_ud import construct_path client = TestClient(app) diff --git a/annotation/tests/test_post_next_task.py b/annotation/tests/test_post_next_task.py index 2c3951ac3..18c7295c4 100644 --- a/annotation/tests/test_post_next_task.py +++ b/annotation/tests/test_post_next_task.py @@ -7,8 +7,17 @@ from requests import ConnectionError, RequestException, Timeout from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session +from tests.override_app_dependency import ( + TEST_HEADERS, + TEST_TENANT, + TEST_TOKEN, + app, +) +from tests.test_tasks_crud_cr import USERS_SEARCH_RESPONSE -from annotation.microservice_communication.assets_communication import ASSETS_FILES_URL +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, +) from annotation.microservice_communication.search import ( AUTHORIZATION, BEARER, @@ -21,8 +30,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app -from tests.test_tasks_crud_cr import USERS_SEARCH_RESPONSE client = TestClient(app) diff --git a/annotation/tests/test_post_unassgined_files.py b/annotation/tests/test_post_unassgined_files.py index 92da8ef47..fe70b2197 100644 --- a/annotation/tests/test_post_unassgined_files.py +++ b/annotation/tests/test_post_unassgined_files.py @@ -4,6 +4,13 @@ from fastapi.testclient import TestClient from sqlalchemy.exc import DBAPIError, SQLAlchemyError from sqlalchemy.sql.elements import not_ +from tests.override_app_dependency import ( + TEST_HEADERS, + TEST_TENANT, + TEST_TOKEN, + app, +) +from tests.test_post import check_files_distributed_pages from annotation.annotations import row_to_dict from annotation.microservice_communication.search import ( @@ -17,8 +24,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app -from tests.test_post import check_files_distributed_pages client = TestClient(app) diff --git a/annotation/tests/test_search_kafka.py b/annotation/tests/test_search_kafka.py index 21cd85b9f..e8a67eb88 100644 --- a/annotation/tests/test_search_kafka.py +++ b/annotation/tests/test_search_kafka.py @@ -8,7 +8,9 @@ from annotation.annotations import add_search_annotation_producer from annotation.kafka_client import producers -from annotation.microservice_communication.assets_communication import ASSETS_FILES_URL +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, +) from annotation.models import Category, File, Job, ManualAnnotationTask, User from annotation.schemas import ( CategoryTypeSchema, diff --git a/annotation/tests/test_tasks_crud_cr.py b/annotation/tests/test_tasks_crud_cr.py index 89da82cbe..023d801e9 100644 --- a/annotation/tests/test_tasks_crud_cr.py +++ b/annotation/tests/test_tasks_crud_cr.py @@ -13,8 +13,12 @@ from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, app from tests.test_post import check_files_distributed_pages -from annotation.microservice_communication.assets_communication import ASSETS_FILES_URL -from annotation.microservice_communication.jobs_communication import JOBS_SEARCH_URL +from annotation.microservice_communication.assets_communication import ( + ASSETS_FILES_URL, +) +from annotation.microservice_communication.jobs_communication import ( + JOBS_SEARCH_URL, +) from annotation.microservice_communication.search import USERS_SEARCH_URL from annotation.microservice_communication.user import USERS_GET_USER_URL from annotation.models import Category, File, Job, ManualAnnotationTask, User diff --git a/annotation/tests/test_update_job.py b/annotation/tests/test_update_job.py index 78985b865..83bd769d6 100644 --- a/annotation/tests/test_update_job.py +++ b/annotation/tests/test_update_job.py @@ -6,6 +6,13 @@ from fastapi.testclient import TestClient from sqlalchemy import asc from sqlalchemy.exc import SQLAlchemyError +from tests.consts import POST_JOBS_PATH +from tests.override_app_dependency import ( + TEST_HEADERS, + TEST_TENANT, + TEST_TOKEN, + app, +) from annotation.annotations import row_to_dict from annotation.microservice_communication import jobs_communication @@ -28,8 +35,6 @@ TaskStatusEnumSchema, ValidationSchema, ) -from tests.consts import POST_JOBS_PATH -from tests.override_app_dependency import TEST_HEADERS, TEST_TENANT, TEST_TOKEN, app JOBS_SEARCH_URL = jobs_communication.JOBS_SEARCH_URL diff --git a/annotation/tests/test_validation.py b/annotation/tests/test_validation.py index 69e83dd5f..d402c62b9 100644 --- a/annotation/tests/test_validation.py +++ b/annotation/tests/test_validation.py @@ -11,7 +11,13 @@ from tests.test_post import check_files_distributed_pages from annotation.annotations import row_to_dict -from annotation.models import AnnotatedDoc, File, Job, ManualAnnotationTask, User +from annotation.models import ( + AnnotatedDoc, + File, + Job, + ManualAnnotationTask, + User, +) from annotation.schemas import ( AnnotationAndValidationActionsSchema, FileStatusEnumSchema, diff --git a/jobs/alembic/env.py b/jobs/alembic/env.py index d24e7391d..e4f7ea6cb 100644 --- a/jobs/alembic/env.py +++ b/jobs/alembic/env.py @@ -2,9 +2,9 @@ import os from logging.config import fileConfig +from alembic import context from sqlalchemy import engine_from_config, pool -from alembic import context from jobs.config import POSTGRESQL_JOBMANAGER_DATABASE_URI from jobs.models import Base from jobs.utils import get_test_db_url diff --git a/jobs/alembic/versions/ef5c796d3c00_add_pipeline_engine.py b/jobs/alembic/versions/ef5c796d3c00_add_pipeline_engine.py index a4e6f9e8b..0f0d952a5 100644 --- a/jobs/alembic/versions/ef5c796d3c00_add_pipeline_engine.py +++ b/jobs/alembic/versions/ef5c796d3c00_add_pipeline_engine.py @@ -7,7 +7,6 @@ """ import sqlalchemy as sa - from alembic import op # revision identifiers, used by Alembic. diff --git a/jobs/tests/conftest.py b/jobs/tests/conftest.py index bddf4bbd2..b769e1f18 100644 --- a/jobs/tests/conftest.py +++ b/jobs/tests/conftest.py @@ -4,6 +4,8 @@ from unittest.mock import patch import pytest +from alembic import command +from alembic.config import Config from fastapi.testclient import TestClient from pydantic import BaseModel from sqlalchemy import create_engine # type: ignore @@ -17,8 +19,6 @@ import jobs.db_service as service import jobs.main as main -from alembic import command -from alembic.config import Config from jobs import pipeline pytest_plugins = [ diff --git a/jobs/tests/test_API_functions/test_change_job.py b/jobs/tests/test_API_functions/test_change_job.py index cc7dd1b88..3ad3601fb 100644 --- a/jobs/tests/test_API_functions/test_change_job.py +++ b/jobs/tests/test_API_functions/test_change_job.py @@ -2,13 +2,13 @@ from unittest.mock import patch import pytest - -import jobs.schemas as schemas from tests.test_db import ( create_mock_annotation_job_in_db, create_mock_extraction_job_in_db, ) +import jobs.schemas as schemas + @pytest.mark.skip(reason="tests refactoring") def test_change_job_status_with_validation_correct_jwt_provided( diff --git a/jobs/tests/test_API_functions/test_other_API_functions.py b/jobs/tests/test_API_functions/test_other_API_functions.py index f4d5a349a..4239bd2b4 100644 --- a/jobs/tests/test_API_functions/test_other_API_functions.py +++ b/jobs/tests/test_API_functions/test_other_API_functions.py @@ -2,13 +2,13 @@ from unittest.mock import patch import pytest - -import jobs.schemas as schemas from tests.test_db import ( create_mock_annotation_job_in_db, create_mock_extraction_job_in_db, ) +import jobs.schemas as schemas + @pytest.mark.skip(reason="tests refactoring") def test_get_all_jobs_endpoint( diff --git a/jobs/tests/test_utils.py b/jobs/tests/test_utils.py index ab4c98230..1369470b8 100644 --- a/jobs/tests/test_utils.py +++ b/jobs/tests/test_utils.py @@ -3,9 +3,9 @@ import aiohttp.client_exceptions import pytest from fastapi import HTTPException +from tests.conftest import FakePipeline, patched_create_pre_signed_s3_url import jobs.utils as utils -from tests.conftest import FakePipeline, patched_create_pre_signed_s3_url # --------------TEST get_files_data_from_datasets--------------- From 56546e0ec2fd9cf2221f34bde0e2e4a5f3b9eb8a Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 16:59:35 +0300 Subject: [PATCH 19/24] fix: black conf --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 15e710bb4..4158e5a8d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,9 +25,9 @@ jobs: python -m pip install --upgrade pip pip install poetry poetry install --only dev - - name: Run isort + - name: Run Isort run: | poetry run isort --diff --check-only . - name: Run Black run: | - poetry run black --config $LINT_CONFIG --diff --check + poetry run black --config $LINT_CONFIG --diff --check . From cd10945ad521c88f7d8750144c4f809153890402 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 17:01:06 +0300 Subject: [PATCH 20/24] fix: black issues --- assets/tests/test_utils.py | 1 - convert/convert/config.py | 1 + .../converters/base_format/models/tokens.py | 1 - jobs/jobs/s3.py | 14 ++-- jobs/tests/conftest.py | 6 +- lib/filter_lib/src/query_modificator.py | 8 ++- .../filter_lib/src/query_modificator.py | 8 ++- search/tests/conftest.py | 3 +- search/tests/override_app_dependency.py | 1 - ...ecbed_add_association_taxonomy_category.py | 1 + .../versions/bdea8a93cafe_first_revision.py | 1 + .../d3ba69ca9d97_change_category_linking.py | 1 + taxonomy/taxonomy/token_dependency.py | 1 + taxonomy/tests/override_app_dependency.py | 1 + taxonomy/tests/test_taxonomy_router.py | 5 +- users/tests/keycloak/test_query.py | 9 ++- users/tests/test_main.py | 72 +++++++++++++++---- 17 files changed, 96 insertions(+), 38 deletions(-) diff --git a/assets/tests/test_utils.py b/assets/tests/test_utils.py index a28b2574c..39b7d2cb7 100644 --- a/assets/tests/test_utils.py +++ b/assets/tests/test_utils.py @@ -496,7 +496,6 @@ def test_file_processor_conversion_error( assert converter.conversion_status == "conversion error" - @pytest.mark.skip(reason="tests refactoring") def test_convert_pdf(file_converter_service): with NamedTemporaryFile(suffix=".pdf", prefix="123") as tmp_file: diff --git a/convert/convert/config.py b/convert/convert/config.py index 3d4665e69..a1c10bd4d 100644 --- a/convert/convert/config.py +++ b/convert/convert/config.py @@ -25,6 +25,7 @@ DEFAULT_PDF_FONT_WIDTH = 7 DEFAULT_PDF_LINE_SPACING = 2 + def get_service_uri(prefix: str) -> str: # noqa service_scheme = os.getenv(f"{prefix}SERVICE_SCHEME") service_host = os.getenv(f"{prefix}SERVICE_HOST") diff --git a/convert/convert/converters/base_format/models/tokens.py b/convert/convert/converters/base_format/models/tokens.py index 0bd9044de..b15615f6e 100644 --- a/convert/convert/converters/base_format/models/tokens.py +++ b/convert/convert/converters/base_format/models/tokens.py @@ -25,7 +25,6 @@ class PageSize(BaseModel): class Page(BaseModel): - """A model for the field with bboxes.""" page_num: int = Field(..., example=1) diff --git a/jobs/jobs/s3.py b/jobs/jobs/s3.py index 022f04666..6e7f38a87 100644 --- a/jobs/jobs/s3.py +++ b/jobs/jobs/s3.py @@ -28,12 +28,14 @@ def create_boto3_config() -> Dict[str, Optional[str]]: "aws_access_key_id": config.S3_ACCESS_KEY, "aws_secret_access_key": config.S3_SECRET_KEY, "endpoint_url": ( - ("https" if config.S3_SECURE else "http") - + "://" - + config.S3_ENDPOINT - ) - if config.S3_ENDPOINT - else None, + ( + ("https" if config.S3_SECURE else "http") + + "://" + + config.S3_ENDPOINT + ) + if config.S3_ENDPOINT + else None + ), } ) elif config.S3_PROVIDER == S3Providers.AWS_IAM: diff --git a/jobs/tests/conftest.py b/jobs/tests/conftest.py index b769e1f18..0bb195812 100644 --- a/jobs/tests/conftest.py +++ b/jobs/tests/conftest.py @@ -141,9 +141,9 @@ def testing_app(testing_engine, testing_session, setup_tenant): ), ): main.app.dependency_overrides[main.tenant] = lambda: setup_tenant - main.app.dependency_overrides[ - service.get_session - ] = lambda: testing_session + main.app.dependency_overrides[service.get_session] = ( + lambda: testing_session + ) client = TestClient(main.app) yield client diff --git a/lib/filter_lib/src/query_modificator.py b/lib/filter_lib/src/query_modificator.py index 6ab5d25ff..ff1f41cd7 100644 --- a/lib/filter_lib/src/query_modificator.py +++ b/lib/filter_lib/src/query_modificator.py @@ -18,9 +18,11 @@ def splint_to_distinct_and_not( ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: distinct_filters, non_distinct_filters = [], [] for fil in filters: - distinct_filters.append(fil) if _op_is_distinct( - fil - ) else non_distinct_filters.append(fil) + ( + distinct_filters.append(fil) + if _op_is_distinct(fil) + else non_distinct_filters.append(fil) + ) return distinct_filters, non_distinct_filters diff --git a/lib/python3.12/filter_lib/src/query_modificator.py b/lib/python3.12/filter_lib/src/query_modificator.py index 900906d0b..e72c443b5 100644 --- a/lib/python3.12/filter_lib/src/query_modificator.py +++ b/lib/python3.12/filter_lib/src/query_modificator.py @@ -18,9 +18,11 @@ def splint_to_distinct_and_not( ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: distinct_filters, non_distinct_filters = [], [] for fil in filters: - distinct_filters.append(fil) if _op_is_distinct( - fil - ) else non_distinct_filters.append(fil) + ( + distinct_filters.append(fil) + if _op_is_distinct(fil) + else non_distinct_filters.append(fil) + ) return distinct_filters, non_distinct_filters diff --git a/search/tests/conftest.py b/search/tests/conftest.py index 08e3c69cc..efb5a6315 100644 --- a/search/tests/conftest.py +++ b/search/tests/conftest.py @@ -179,8 +179,7 @@ def __init__(self): async def __aenter__(self): return self - async def __aexit__(self, exc_type, exc_val, exc_tb): - ... + async def __aexit__(self, exc_type, exc_val, exc_tb): ... def __aiter__(self): return self diff --git a/search/tests/override_app_dependency.py b/search/tests/override_app_dependency.py index b4141c9fe..dd3df2a6d 100644 --- a/search/tests/override_app_dependency.py +++ b/search/tests/override_app_dependency.py @@ -8,7 +8,6 @@ there are necessary tenants in token. """ - from tenant_dependency import TenantData from search.main import TOKEN, app diff --git a/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py b/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py index 25bc3778c..2d6d30521 100644 --- a/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py +++ b/taxonomy/alembic/versions/48dc50decbed_add_association_taxonomy_category.py @@ -5,6 +5,7 @@ Create Date: 2022-12-02 15:04:55.726594 """ + import sqlalchemy as sa from alembic import op diff --git a/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py b/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py index 5ece0c493..546af4e6b 100644 --- a/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py +++ b/taxonomy/alembic/versions/bdea8a93cafe_first_revision.py @@ -5,6 +5,7 @@ Create Date: 2022-11-24 16:36:12.022247 """ + import sqlalchemy as sa import sqlalchemy_utils from alembic import op diff --git a/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py b/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py index 8e37244bf..b0ccabd8d 100644 --- a/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py +++ b/taxonomy/alembic/versions/d3ba69ca9d97_change_category_linking.py @@ -5,6 +5,7 @@ Create Date: 2022-12-26 04:40:50.608253 """ + import sqlalchemy as sa from alembic import op diff --git a/taxonomy/taxonomy/token_dependency.py b/taxonomy/taxonomy/token_dependency.py index da0642b7b..dac783099 100644 --- a/taxonomy/taxonomy/token_dependency.py +++ b/taxonomy/taxonomy/token_dependency.py @@ -2,6 +2,7 @@ Dependency, that will validate X-Current-Tenant and Authorization token """ + import os from tenant_dependency import TenantData, get_tenant_info diff --git a/taxonomy/tests/override_app_dependency.py b/taxonomy/tests/override_app_dependency.py index 03df08db0..f47df368b 100644 --- a/taxonomy/tests/override_app_dependency.py +++ b/taxonomy/tests/override_app_dependency.py @@ -7,6 +7,7 @@ that given token is valid, not expired and there are necessary tenants in token. """ + from tenant_dependency import TenantData HEADER_TENANT = "X-Current-Tenant" diff --git a/taxonomy/tests/test_taxonomy_router.py b/taxonomy/tests/test_taxonomy_router.py index 80cee30ca..c987d3027 100644 --- a/taxonomy/tests/test_taxonomy_router.py +++ b/taxonomy/tests/test_taxonomy_router.py @@ -300,7 +300,10 @@ def test_should_delete_latest_taxonomy_from_db( db_session, ): # given - new_latest_taxonomy, taxonomy_to_delete, = sorted( + ( + new_latest_taxonomy, + taxonomy_to_delete, + ) = sorted( prepare_two_taxonomy_records_with_same_id_in_db, key=lambda x: x.version, ) diff --git a/users/tests/keycloak/test_query.py b/users/tests/keycloak/test_query.py index 9ccb6bacd..9c6f14f3e 100644 --- a/users/tests/keycloak/test_query.py +++ b/users/tests/keycloak/test_query.py @@ -1,4 +1,5 @@ """Testing src/keycloak/query.py.""" + from unittest.mock import patch import pytest @@ -31,7 +32,9 @@ async def test_get_token_v2(request_mock): @pytest.mark.asyncio -async def test_introspect_token_test(request_mock, mocked_token1, mocked_token1_data): +async def test_introspect_token_test( + request_mock, mocked_token1, mocked_token1_data +): request_mock.return_value.__aenter__.return_value.json.return_value = ( mocked_token1_data ) @@ -40,7 +43,9 @@ async def test_introspect_token_test(request_mock, mocked_token1, mocked_token1_ @pytest.mark.asyncio -async def test_get_master_realm_auth_data(request_mock, mocked_admin_auth_data): +async def test_get_master_realm_auth_data( + request_mock, mocked_admin_auth_data +): request_mock.return_value.__aenter__.return_value.json.return_value = ( mocked_admin_auth_data ) diff --git a/users/tests/test_main.py b/users/tests/test_main.py index 96d5d19e4..31a2312e6 100644 --- a/users/tests/test_main.py +++ b/users/tests/test_main.py @@ -96,7 +96,9 @@ def does_not_raise(): roles=["admin"], tenants=["tenant"], ), - TenantData(token="token", user_id="user_id", roles=[], tenants=["tenant"]), + TenantData( + token="token", user_id="user_id", roles=[], tenants=["tenant"] + ), ], ) def test_check_authorization_role_is_missing(mock_tenant_data): @@ -210,7 +212,9 @@ def test_login_status_code(token_schema, request_body, status_code): class TestGetUserGWT: def test_get_user_jwt_body(self, mock_user, user_representation): response = client.get("/users/current") - assert response.json() == user_representation(user_id="1", user_name="user") + assert response.json() == user_representation( + user_id="1", user_name="user" + ) def test_get_user_jwt_status_code(self, mock_user): response = client.get("/users/current") @@ -222,7 +226,9 @@ class TestGetUser: @pytest.mark.skip(reason="tests refactoring") def test_get_user_body(self, mock_user, user_representation): response = client.get("/users/user-id") - assert response.json() == user_representation(user_id="1", user_name="user") + assert response.json() == user_representation( + user_id="1", user_name="user" + ) @pytest.mark.skip(reason="tests refactoring") def test_get_user_status_code(self, mock_user): @@ -230,7 +236,9 @@ def test_get_user_status_code(self, mock_user): assert response.status_code == 200 -def test_get_user_info_from_token_introspection(mocked_token1, mocked_token1_data): +def test_get_user_info_from_token_introspection( + mocked_token1, mocked_token1_data +): with patch( "users.keycloak.query.introspect_token", return_value=mocked_token1_data, @@ -338,7 +346,9 @@ def test_add_user_to_tenant2( ("group_1", {"detail": "User has been removed from the tenant"}), ], ) -def test_remove_user_from_tenant_body(mock_user, update_user, tenant, expected_result): +def test_remove_user_from_tenant_body( + mock_user, update_user, tenant, expected_result +): response = client.delete(f"/tenants/{tenant}/users/user_1") assert response.json() == expected_result @@ -359,7 +369,9 @@ def test_remove_user_from_tenant_status_code( @patch("users.keycloak.query.get_users_v2", return_value=mock_all_users) -@patch("users.keycloak.query.get_users_by_role", return_value=mock_users_with_role) +@patch( + "users.keycloak.query.get_users_by_role", return_value=mock_users_with_role +) class TestUsersSearch: @pytest.mark.parametrize("request_body", [{}, {"filters": []}]) def test_get_all_users_body( @@ -388,7 +400,11 @@ def test_filter_users_by_name_body( ): response = client.post( "/users/search", - json={"filters": [{"field": "name", "operator": "like", "value": "r"}]}, + json={ + "filters": [ + {"field": "name", "operator": "like", "value": "r"} + ] + }, ) assert response.json() == [ user_representation(user_id="1", user_name="user"), @@ -400,7 +416,11 @@ def test_filter_users_by_name_status_code( ): response = client.post( "/users/search", - json={"filters": [{"field": "name", "operator": "like", "value": "r"}]}, + json={ + "filters": [ + {"field": "name", "operator": "like", "value": "r"} + ] + }, ) assert response.status_code == 200 @@ -443,14 +463,20 @@ def test_filter_users_by_empty_name_status_code( ): response = client.post( "/users/search", - json={"filters": [{"field": "name", "operator": "like", "value": ""}]}, + json={ + "filters": [{"field": "name", "operator": "like", "value": ""}] + }, ) assert response.status_code == 422 @pytest.mark.parametrize( "request_body", [ - {"filters": [{"field": "id", "operator": "in", "value": ["1", "2"]}]}, + { + "filters": [ + {"field": "id", "operator": "in", "value": ["1", "2"]} + ] + }, { "filters": [ { @@ -482,7 +508,11 @@ def test_filter_users_by_id_body( @pytest.mark.parametrize( "request_body", [ - {"filters": [{"field": "id", "operator": "in", "value": ["1", "2"]}]}, + { + "filters": [ + {"field": "id", "operator": "in", "value": ["1", "2"]} + ] + }, { "filters": [ { @@ -511,7 +541,11 @@ def test_filter_users_by_id_status_code( "request_body", [ {"filters": [{"field": "id", "operator": "in", "value": []}]}, - {"filters": [{"field": "id", "operator": "in", "value": ["wrong_id"]}]}, + { + "filters": [ + {"field": "id", "operator": "in", "value": ["wrong_id"]} + ] + }, ], ) def test_filter_users_by_wrong_or_empty_id_body( @@ -531,7 +565,11 @@ def test_filter_users_by_wrong_or_empty_id_body( "request_body", [ {"filters": [{"field": "id", "operator": "in", "value": []}]}, - {"filters": [{"field": "id", "operator": "in", "value": ["wrong_id"]}]}, + { + "filters": [ + {"field": "id", "operator": "in", "value": ["wrong_id"]} + ] + }, ], ) def test_filter_users_by_wrong_or_empty_id_status_code( @@ -544,7 +582,9 @@ def test_filter_users_by_wrong_or_empty_id_status_code( response = client.post( "/users/search", json={ - "filters": [{"field": "id", "operator": "in", "value": ["wrong_id"]}] + "filters": [ + {"field": "id", "operator": "in", "value": ["wrong_id"]} + ] }, ) assert response.status_code == 200 @@ -592,7 +632,9 @@ def test_filter_users_by_wrong_role_body( response = client.post( "/users/search", json={ - "filters": [{"field": "role", "operator": "eq", "value": "wrong_role"}] + "filters": [ + {"field": "role", "operator": "eq", "value": "wrong_role"} + ] }, ) assert response.status_code == 422 From 5e64a00d3302cee67210b36e8cc3f054fb75ccd4 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 17:11:06 +0300 Subject: [PATCH 21/24] fix: remove black & isort from other workflows --- .github/workflows/annotation.yml | 11 ----------- .github/workflows/assets.yml | 11 ----------- .github/workflows/convert.yml | 5 +++-- .github/workflows/filter-lib.yml | 14 -------------- .github/workflows/jobs.yml | 4 +--- .github/workflows/processing.yml | 11 ----------- .github/workflows/taxonomy.yml | 4 +--- .github/workflows/tenants.yml | 14 -------------- .github/workflows/users.yml | 4 +--- 9 files changed, 6 insertions(+), 72 deletions(-) diff --git a/.github/workflows/annotation.yml b/.github/workflows/annotation.yml index 864256811..db7d4051e 100644 --- a/.github/workflows/annotation.yml +++ b/.github/workflows/annotation.yml @@ -19,17 +19,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Run isort - uses: isort/isort-action@v1.1.0 - with: - configuration: - --profile=black - --filter-files - --line-length=79 - - name: Black - uses: psf/black@stable - with: - options: "--line-length=79" - run: pip install flake8 - run: flake8 --extend-ignore=E203 annotation annotation-build: diff --git a/.github/workflows/assets.yml b/.github/workflows/assets.yml index f3302e629..8ac4d77ea 100644 --- a/.github/workflows/assets.yml +++ b/.github/workflows/assets.yml @@ -21,17 +21,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Run isort - uses: isort/isort-action@v1.1.0 - with: - configuration: - --profile=black - --filter-files - --line-length=79 - - name: Black - uses: psf/black@stable - with: - options: "--line-length=79" - run: pip install flake8 - run: flake8 --extend-ignore=E203 assets assets-build: diff --git a/.github/workflows/convert.yml b/.github/workflows/convert.yml index 37abfaeb2..2eb19818b 100644 --- a/.github/workflows/convert.yml +++ b/.github/workflows/convert.yml @@ -26,10 +26,11 @@ jobs: poetry install --all-extras poetry add --editable ../lib/filter_lib poetry add --editable ../lib/tenants - - name: Run linters and checkers [isort -> black -> mypy -> pylint] + - name: Run linters and checkers [mypy -> pylint] working-directory: ./convert run: | - git ls-files -- . | xargs pre-commit run --files + git ls-files -- . | xargs pre-commit run mypy --files + git ls-files -- . | xargs pre-commit run pylint --files - name: Run tests working-directory: ./convert run: | diff --git a/.github/workflows/filter-lib.yml b/.github/workflows/filter-lib.yml index 4de0337ea..3dc644491 100644 --- a/.github/workflows/filter-lib.yml +++ b/.github/workflows/filter-lib.yml @@ -21,20 +21,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Run isort - uses: isort/isort-action@v1.1.0 - with: - sort-paths: lib/filter_lib/* - configuration: - --check-only - --profile=black - --line-length=79 - --skip=tests - - name: Black - uses: psf/black@stable - with: - src: "lib/filter_lib" - options: '--check --line-length=79 --exclude="tests" --verbose' - run: pip install flake8 - run: flake8 --exclude=tests --extend-ignore=E203 lib/filter_lib filter-lib-build: diff --git a/.github/workflows/jobs.yml b/.github/workflows/jobs.yml index c873c30da..6699405b4 100644 --- a/.github/workflows/jobs.yml +++ b/.github/workflows/jobs.yml @@ -28,12 +28,10 @@ jobs: poetry install --no-root poetry add ../lib/filter_lib poetry add ../lib/tenants - - name: Run linters and checkers [flake8 -> isort -> black] + - name: Run linters and checkers [flake8] working-directory: ./jobs run: | poetry run flake8 --extend-ignore=E203 jobs/ - poetry run isort --profile=black --line-length=79 --check-only jobs/ - poetry run black --check --line-length=79 jobs/ - name: Run tests working-directory: ./jobs run: | diff --git a/.github/workflows/processing.yml b/.github/workflows/processing.yml index f0cbfca78..47a5eb002 100644 --- a/.github/workflows/processing.yml +++ b/.github/workflows/processing.yml @@ -21,17 +21,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Run isort - uses: isort/isort-action@v1.1.0 - with: - configuration: - --profile=black - --filter-files - --line-length=79 - - name: Black - uses: psf/black@stable - with: - options: "--line-length=79" - run: pip install flake8 - run: flake8 --extend-ignore=E203 processing diff --git a/.github/workflows/taxonomy.yml b/.github/workflows/taxonomy.yml index ae86b09eb..829699f84 100644 --- a/.github/workflows/taxonomy.yml +++ b/.github/workflows/taxonomy.yml @@ -28,12 +28,10 @@ jobs: poetry install --no-root poetry add ../lib/filter_lib poetry add ../lib/tenants - - name: Run linters and checkers [flake8 -> isort -> black] + - name: Run linters and checkers [flake8] working-directory: ./taxonomy run: | poetry run flake8 --extend-ignore=E203 taxonomy/ - poetry run isort --profile=black --line-length=79 --check-only taxonomy/ - poetry run black --check --line-length=79 taxonomy/ - name: Run tests working-directory: ./taxonomy run: | diff --git a/.github/workflows/tenants.yml b/.github/workflows/tenants.yml index 214e18d28..1773192f5 100644 --- a/.github/workflows/tenants.yml +++ b/.github/workflows/tenants.yml @@ -21,20 +21,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Run isort - uses: isort/isort-action@v1.1.0 - with: - sort-paths: lib/tenants/* - configuration: - --check-only - --profile=black - --line-length=79 - --skip=tests - - name: Black - uses: psf/black@stable - with: - src: "lib/tenants" - options: '--check --line-length=79 --exclude="tests" --verbose' - run: pip install flake8 - run: flake8 --exclude=tests --extend-ignore=E203 lib/tenants tenants-build: diff --git a/.github/workflows/users.yml b/.github/workflows/users.yml index 17bc1a89b..b1c4879c7 100644 --- a/.github/workflows/users.yml +++ b/.github/workflows/users.yml @@ -28,12 +28,10 @@ jobs: pip install poetry>=1.1.13 poetry install --no-root poetry add ../lib/tenants - - name: Run linters and checkers [flake8 -> isort -> black] + - name: Run linters and checkers [flake8] working-directory: ./users run: | poetry run flake8 --extend-ignore=E203 users/ - poetry run isort --profile=black --line-length=79 --check-only users/ - poetry run black --check --line-length=79 users/ - name: Run tests working-directory: ./users run: | From 160983e18c38b72a958f75ad8f29ff313e776706 Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 17:21:03 +0300 Subject: [PATCH 22/24] fix: ignore mypy convert issues --- convert/convert/config.py | 2 +- convert/convert/converters/coco/coco_export/convert.py | 2 +- convert/convert/converters/labelstudio/badgerdoc_downloader.py | 2 +- .../labelstudio/labelstudio_to_badgerdoc_converter.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/convert/convert/config.py b/convert/convert/config.py index a1c10bd4d..4c6f9068b 100644 --- a/convert/convert/config.py +++ b/convert/convert/config.py @@ -35,7 +35,7 @@ def get_service_uri(prefix: str) -> str: # noqa return "" -class Settings(BaseSettings): +class Settings(BaseSettings): # type: ignore """Base settings values""" s3_endpoint_url: Optional[str] = os.getenv("S3_ENDPOINT_URL") diff --git a/convert/convert/converters/coco/coco_export/convert.py b/convert/convert/converters/coco/coco_export/convert.py index 2e1e7d64b..484271911 100644 --- a/convert/convert/converters/coco/coco_export/convert.py +++ b/convert/convert/converters/coco/coco_export/convert.py @@ -321,7 +321,7 @@ def convert(self) -> ZipFile: coco_annotation.categories = sorted( list( { - v.dict()["name"]: v.dict() for v in coco_annotation.categories # type: ignore + v.dict()["name"]: v.dict() for v in coco_annotation.categories }.values() ), key=lambda x: x["id"], # type: ignore diff --git a/convert/convert/converters/labelstudio/badgerdoc_downloader.py b/convert/convert/converters/labelstudio/badgerdoc_downloader.py index fc26e1221..43e387bca 100644 --- a/convert/convert/converters/labelstudio/badgerdoc_downloader.py +++ b/convert/convert/converters/labelstudio/badgerdoc_downloader.py @@ -58,7 +58,7 @@ def get_manifest(self, tmp_dir: Path) -> Manifest: manifest_file = self.download_file_from_s3( self.s3_input_manifest, tmp_dir ) - return Manifest.parse_file(tmp_dir / manifest_file.name) + return Manifest.parse_file(tmp_dir / manifest_file.name) # type: ignore def get_annotations( self, manifest: Manifest, tmp_dir: Path diff --git a/convert/convert/converters/labelstudio/labelstudio_to_badgerdoc_converter.py b/convert/convert/converters/labelstudio/labelstudio_to_badgerdoc_converter.py index 907b7c233..cc1255d0f 100644 --- a/convert/convert/converters/labelstudio/labelstudio_to_badgerdoc_converter.py +++ b/convert/convert/converters/labelstudio/labelstudio_to_badgerdoc_converter.py @@ -181,7 +181,7 @@ def download( detail="Error during file fetching: file not found", ) from e - return LabelStudioModel.parse_file(input_file) + return LabelStudioModel.parse_file(input_file) # type: ignore def get_output_tokens_path(self, file_id_in_assets: int) -> str: return ( From 41a43ed7a4d9fff65fe90bfd868cb9ddd252a30e Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 17:22:09 +0300 Subject: [PATCH 23/24] fix: black issue --- convert/convert/converters/coco/coco_export/convert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert/convert/converters/coco/coco_export/convert.py b/convert/convert/converters/coco/coco_export/convert.py index 484271911..3246b5a89 100644 --- a/convert/convert/converters/coco/coco_export/convert.py +++ b/convert/convert/converters/coco/coco_export/convert.py @@ -321,7 +321,8 @@ def convert(self) -> ZipFile: coco_annotation.categories = sorted( list( { - v.dict()["name"]: v.dict() for v in coco_annotation.categories + v.dict()["name"]: v.dict() + for v in coco_annotation.categories }.values() ), key=lambda x: x["id"], # type: ignore From 128163ba1ec010b68f8accd2a4211fcab713dcce Mon Sep 17 00:00:00 2001 From: Denis Rybakov Date: Fri, 17 May 2024 18:02:19 +0300 Subject: [PATCH 24/24] fix: processing tests --- processing/.env | 1 + 1 file changed, 1 insertion(+) diff --git a/processing/.env b/processing/.env index 8bdf73b12..0b37b4a3a 100644 --- a/processing/.env +++ b/processing/.env @@ -14,6 +14,7 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres POSTGRES_DB=badgerdoc +ROOT_PATH="" PROCESSING_SERVICE_HOST=processing ASSETS_SERVICE_HOST=assets MODELS_SERVICE_HOST=models