test(jobs, taxonomy): fix failing tests (#836)

.github/workflows/jobs.yml

@@ -12,19 +12,29 @@ on:
 jobs:
    jobs-linters-and-tests:
      runs-on: ubuntu-latest
+     strategy:
+       matrix:
+         python-version: [ "3.8.15" ]
      steps:
-       - uses: actions/checkout@v2
-       - name: update docker compose
-         run: pip install --upgrade docker compose
-       - name: create base image
-         run: cd infra/docker/python_base && make build image_name=818863528939.dkr.ecr.eu-central-1.amazonaws.com/badgerdoc/python_base:0.1.7
-       - name: run docker compose up
-         run: cd jobs && docker compose up -d --build
-       - name: run tests
-         run: docker exec jobs_web_app bash -c "poetry run pytest"
-       - name: check with flake8
-         run: docker exec jobs_web_app bash -c "flake8 --exclude=alembic,tests --extend-ignore=E203"
-       - name: check with isort
-         run: docker exec jobs_web_app bash -c "isort --profile=black --line-length=79 --check-only --skip=alembic --skip=tests ."
-       - name: check with black
-         run: docker exec jobs_web_app bash -c 'black --check --line-length=79 --exclude="(alembic|tests)" .'
+       - uses: actions/checkout@v4
+       - uses: actions/setup-python@v4
+         with:
+           python-version: ${{ matrix.python-version }}
+       - name: Install dependencies
+         working-directory: ./jobs
+         run: |
+           python -m pip install --upgrade pip
+           pip install poetry>=1.1.13
+           poetry install --no-root
+           poetry add ../lib/filter_lib
+           poetry add ../lib/tenants
+       - name: Run linters and checkers [flake8 -> isort -> black]
+         working-directory: ./jobs
+         run: |
+           poetry run flake8 --extend-ignore=E203 jobs/
+           poetry run isort --profile=black --line-length=79 --check-only jobs/
+           poetry run black --check --line-length=79 jobs/
+       - name: Run tests
+         working-directory: ./jobs
+         run: |
+           poetry run pytest tests/

.github/workflows/taxonomy.yml

@@ -12,19 +12,29 @@ on:
 jobs:
    taxonomy-linters-and-tests:
      runs-on: ubuntu-latest
+     strategy:
+       matrix:
+         python-version: [ "3.8.15" ]
      steps:
-       - uses: actions/checkout@v2
-       - name: update docker-compose
-         run: pip install --upgrade docker-compose
-       - name: create base image
-         run: cd infra/docker/python_base && make build image_name=818863528939.dkr.ecr.eu-central-1.amazonaws.com/badgerdoc/python_base:0.1.7
-       - name: run docker-compose up
-         run: cd taxonomy && docker-compose up -d --build
-       - name: run tests
-         run: docker exec taxonomy_web_app bash -c "poetry run pytest"
-       - name: check with flake8
-         run: docker exec taxonomy_web_app bash -c "flake8 --exclude=alembic,tests,poetry --extend-ignore=E203"
-       - name: check with isort
-         run: docker exec taxonomy_web_app bash -c "isort --profile=black --line-length=79 --check-only --skip=alembic --skip=tests --skip=poetry ."
-       - name: check with black
-         run: docker exec taxonomy_web_app bash -c 'black --check --line-length=79 --exclude="(alembic|tests|poetry)" .'
+       - uses: actions/checkout@v4
+       - uses: actions/setup-python@v4
+         with:
+           python-version: ${{ matrix.python-version }}
+       - name: Install dependencies
+         working-directory: ./taxonomy
+         run: |
+           python -m pip install --upgrade pip
+           pip install poetry>=1.1.13
+           poetry install --no-root
+           poetry add ../lib/filter_lib
+           poetry add ../lib/tenants
+       - name: Run linters and checkers [flake8 -> isort -> black]
+         working-directory: ./taxonomy
+         run: |
+           poetry run flake8 --extend-ignore=E203 taxonomy/
+           poetry run isort --profile=black --line-length=79 --check-only taxonomy/
+           poetry run black --check --line-length=79 taxonomy/
+       - name: Run tests
+         working-directory: ./taxonomy
+         run: |
+           poetry run pytest tests/

jobs/jobs/config.py

@@ -16,6 +16,7 @@ def get_version() -> str:
 
     return default
 
+
 def get_service_uri(prefix: str) -> str:  # noqa
     service_scheme = os.getenv(f"{prefix}SERVICE_SCHEME")
     service_host = os.getenv(f"{prefix}SERVICE_HOST")
@@ -42,13 +43,13 @@ def get_service_uri(prefix: str) -> str:  # noqa
 )
 
 KEYCLOAK_HOST = os.environ.get("KEYCLOAK_HOST", "")
-USERS_HOST = get_service_uri('USERS_')
+USERS_HOST = get_service_uri("USERS_")
 
-PIPELINES_SERVICE_HOST = get_service_uri('PIPELINES_')
-ASSETS_SERVICE_HOST = get_service_uri('ASSETS_')
-ANNOTATION_SERVICE_HOST = get_service_uri('ANNOTATION_')
-TAXONOMY_SERVICE_HOST = get_service_uri('TAXONOMY_')
-JOBS_SERVICE_HOST = get_service_uri('JOBS_')
+PIPELINES_SERVICE_HOST = get_service_uri("PIPELINES_")
+ASSETS_SERVICE_HOST = get_service_uri("ASSETS_")
+ANNOTATION_SERVICE_HOST = get_service_uri("ANNOTATION_")
+TAXONOMY_SERVICE_HOST = get_service_uri("TAXONOMY_")
+JOBS_SERVICE_HOST = get_service_uri("JOBS_")
 
 PAGINATION_THRESHOLD = 7
 PROVIDE_JWT_IF_NO_ANY = True

jobs/jobs/create_job_funcs.py

@@ -58,7 +58,7 @@ async def create_extraction_job(
 
     pipeline_id = (
         extraction_job_input.pipeline_name
-        if extraction_job_input.pipeline_name.endswith(':airflow')
+        if extraction_job_input.pipeline_name.endswith(":airflow")
         else pipeline_instance.get("id")
     )
     pipeline_categories = pipeline_instance.get("meta", {}).get(
@@ -137,7 +137,7 @@ async def create_extraction_annotation_job(
     )
     pipeline_id = (
         extraction_annotation_job_input.pipeline_name
-        if extraction_annotation_job_input.pipeline_name.endswith(':airflow')
+        if extraction_annotation_job_input.pipeline_name.endswith(":airflow")
         else pipeline_instance.get("id")
     )
 

jobs/jobs/main.py

@@ -1,5 +1,5 @@
-import os
 import asyncio
+import os
 from typing import Any, Dict, List, Optional, Union
 
 from fastapi import Depends, FastAPI, Header, HTTPException, status

jobs/jobs/run_job_funcs.py

@@ -19,7 +19,9 @@ async def run_extraction_job(
     """Runs ExtractionJob - creates init_args
     and sends it to the Pipeline Manager"""
 
-    if isinstance(job_to_run.pipeline_id, str) and not job_to_run.pipeline_id.endswith(':airflow'):
+    if isinstance(
+        job_to_run.pipeline_id, str
+    ) and not job_to_run.pipeline_id.endswith(":airflow"):
         raise HTTPException(
             status_code=400,
             detail="Wrong pipeline value.",

jobs/jobs/utils.py

@@ -8,11 +8,11 @@
 from jobs.config import (
     ANNOTATION_SERVICE_HOST,
     ASSETS_SERVICE_HOST,
-    PIPELINES_SERVICE_HOST,
-    TAXONOMY_SERVICE_HOST,
     JOBS_SERVICE_HOST,
     PAGINATION_THRESHOLD,
+    PIPELINES_SERVICE_HOST,
     ROOT_PATH,
+    TAXONOMY_SERVICE_HOST,
     USERS_HOST,
 )
 from jobs.logger import logger