diff --git a/.env.docker b/.env.docker deleted file mode 100644 index 274cd73..0000000 --- a/.env.docker +++ /dev/null @@ -1,2 +0,0 @@ -# Common -ENV=dev diff --git a/.env.template b/.env.template index b498546..982a47f 100644 --- a/.env.template +++ b/.env.template @@ -1,20 +1,14 @@ -POSTGIS_HOST=localhost -POSTGIS_PORT=9232 -POSTGIS_DATABASE_NAME=NieuweWarmteNu -POSTGIS_ROOT_USER=root -POSTGIS_ROOT_PASSWORD=1234 +RABBITMQ_HOSTNAME=localhost +RABBITMQ_PORT=5672 +RABBITMQ_USERNAME=omotes +RABBITMQ_PASSWORD=somepass1 +RABBITMQ_VIRTUALHOST=omotes -NWN_POSTGRES_HOST=localhost -NWN_POSTGRES_PORT=6432 -NWN_POSTGRES_DATABASE_NAME=nieuwewarmtenu -NWN_POSTGRES_ROOT_USER=root -NWN_POSTGRES_ROOT_PASSWORD=1234 - -NWN_RABBITMQ_HOST=localhost -NWN_RABBITMQ_PORT=5672 -NWN_RABBITMQ_EXCHANGE=nwn -NWN_RABBITMQ_ROOT_USER=root -NWN_RABBITMQ_ROOT_PASSWORD=5678 -NWN_RABBITMQ_HIPE_COMPILE=1 - -#FLASK_APP=src.flask_rest_api.main +POSTGRES_ROOT_USER=root +POSTGRES_ROOT_PASSWORD=1234 +POSTGRES_DEV_PORT=7432 +POSTGRESQL_HOST=localhost +POSTGRESQL_PORT=7432 +POSTGRESQL_DATABASE=omotes +POSTGRESQL_USERNAME=omotes_rest +POSTGRESQL_PASSWORD=somepass3 diff --git a/Dockerfile b/Dockerfile index 5f291df..bb32bde 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.10-slim ENV ENV=prod -ENV FLASK_APP=tno/mapeditor_dispatcher/main.py +ENV FLASK_APP=src/omotes_rest/main.py RUN apt-get -y update RUN pip install --upgrade pip @@ -18,4 +18,4 @@ COPY . /code RUN pip install -e . -CMD gunicorn src.mapeditor_dispatcher.main:app -t 300 -w 1 -b :9200 +CMD gunicorn omotes_rest.main:app -t 300 -w 1 -b :9200 --pythonpath src diff --git a/Dockerfile-uwsgi b/Dockerfile-uwsgi index 9616c91..92ba99d 100644 --- a/Dockerfile-uwsgi +++ b/Dockerfile-uwsgi @@ -1,7 +1,7 @@ FROM python:3.8-slim ENV ENV=prod -ENV FLASK_APP=tno/mapeditor_dispatcher/main.py +ENV FLASK_APP=src/omotes_rest/main.py RUN apt-get -y update RUN apt-get install -y gcc libpcre3 libpcre3-dev zlib1g zlib1g-dev libssl-dev diff --git a/docker-compose.infra.yml b/docker-compose.infra.yml index 8d12f5a..7fa4405 100644 --- a/docker-compose.infra.yml +++ b/docker-compose.infra.yml @@ -1,7 +1,7 @@ version: "3.8" networks: - omotes: + computation-engine_omotes: volumes: db-data: @@ -13,7 +13,7 @@ services: volumes: - "db-data:/var/lib/postgresql/data/" networks: - - omotes + - computation-engine_omotes environment: PGDATA: /var/lib/postgresql/data POSTGRES_DB: omotes @@ -24,8 +24,8 @@ services: interval: 5s timeout: 5s retries: 10 - ports: - - "${POSTGRESQL_PORT}:5432" +# ports: +# - "${POSTGRESQL_PORT}:5432" # Skipped in 'docker-compose up', only used to develop database revisions. rest_postgres_db_dev: @@ -39,7 +39,7 @@ services: rest_postgres_db_upgrade: build: postgres_db_upgrade networks: - - omotes + - computation-engine_omotes environment: POSTGRES_ROOT_USER: ${POSTGRES_ROOT_USER} POSTGRES_ROOT_PASSWORD: ${POSTGRES_ROOT_PASSWORD} diff --git a/docker-compose.yml b/docker-compose.yml index 377fae9..2723385 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,37 @@ version: "3.8" +networks: + computation-engine_omotes: + external: true + +volumes: + db-data: + services: + rest_postgres_db: + extends: + file: docker-compose.infra.yml + service: rest_postgres_db + omotes-rest: - build: . - image: mvrijlandt/mapeditor_nwn_dispatcher:0.0.2 +# build: . + image: ghcr.io/project-omotes/omotes_rest:0.0.1 + networks: + - computation-engine_omotes ports: - "9200:9200" - env_file: - - ".env" + environment: + RABBITMQ_HOSTNAME: rabbitmq + RABBITMQ_PORT: 5672 + RABBITMQ_USERNAME: ${RABBITMQ_USERNAME} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD} + RABBITMQ_VIRTUALHOST: omotes + + POSTGRESQL_HOST: rest_postgres_db + POSTGRESQL_PORT: 5432 + POSTGRESQL_DATABASE: omotes + POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} + POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} + depends_on: + rest_postgres_db: + condition: service_healthy diff --git a/esdl-mapeditor-config/esdl_config.py b/esdl-mapeditor-config/esdl_config.py index b24cd8a..95f207b 100644 --- a/esdl-mapeditor-config/esdl_config.py +++ b/esdl-mapeditor-config/esdl_config.py @@ -13,7 +13,7 @@ # TNO import os -from src.settings import heatnetwork_dispatcher_config +from settings import heatnetwork_dispatcher_config EPS_WEB_HOST = os.getenv("EPS_WEB_HOST", "http://epsweb:3401") ESDL_AGGREGATOR_HOST = os.getenv("ESDL_AGGREGATOR_HOST", "http://esdl-aggregator:3490") diff --git a/postgres_db_upgrade/README.md b/postgres_db_upgrade/README.md index ea09cad..c333329 100644 --- a/postgres_db_upgrade/README.md +++ b/postgres_db_upgrade/README.md @@ -1,6 +1,6 @@ # How to work with alembic to make database revisions -The following commands should be run from this folder. (`cd rest_postgres_db_upgrade`) +The following commands should be run from this folder. (`cd postgres_db_upgrade`) - Setup the virtual environment: `./scripts/create_venv.sh` - Start postgres in setup mode to expose the port to the host: `./scripts/start_postgres_in_setup_mode.sh` diff --git a/postgres_db_upgrade/alembic/versions/2024_03_15_2214-50f731d37156_initial_commit.py b/postgres_db_upgrade/alembic/versions/2024_03_18_1020-0b2b6ee9274d_initial_commit.py similarity index 87% rename from postgres_db_upgrade/alembic/versions/2024_03_15_2214-50f731d37156_initial_commit.py rename to postgres_db_upgrade/alembic/versions/2024_03_18_1020-0b2b6ee9274d_initial_commit.py index 6653191..b4bcdbf 100644 --- a/postgres_db_upgrade/alembic/versions/2024_03_15_2214-50f731d37156_initial_commit.py +++ b/postgres_db_upgrade/alembic/versions/2024_03_18_1020-0b2b6ee9274d_initial_commit.py @@ -1,8 +1,8 @@ """initial commit -Revision ID: 50f731d37156 +Revision ID: 0b2b6ee9274d Revises: -Create Date: 2024-03-15 22:14:08.152971 +Create Date: 2024-03-18 10:20:54.447282 """ from typing import Sequence, Union @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. -revision: str = '50f731d37156' +revision: str = '0b2b6ee9274d' down_revision: Union[str, None] = None branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -24,7 +24,7 @@ def upgrade() -> None: sa.Column('job_id', sa.UUID(), nullable=False), sa.Column('job_name', sa.String(), nullable=False), sa.Column('work_flow_name', sa.Enum('GROW_OPTIMIZER', 'GROW_SIMULATOR', 'GROW_OPTIMIZER_NO_HEAT_LOSSES', 'GROW_OPTIMIZER_NO_HEAT_LOSSES_DISCOUNTED_CAPEX', 'SIMULATOR', name='workflowname'), nullable=False), - sa.Column('status', sa.Enum('REGISTERED', 'ENQUEUED', 'RUNNING', 'SUCCEEDED', 'CANCELLED', 'TIMEOUT', 'ERROR', name='jobreststatus'), nullable=False), + sa.Column('status', sa.Enum('REGISTERED', 'RUNNING', 'SUCCEEDED', 'CANCELLED', 'TIMEOUT', 'ERROR', name='jobreststatus'), nullable=False), sa.Column('progress_fraction', sa.Float(), nullable=False), sa.Column('progress_message', sa.String(), nullable=False), sa.Column('registered_at', sa.DateTime(timezone=True), nullable=False), diff --git a/pyproject.toml b/pyproject.toml index f20cf9b..b0581cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "Flask-Cors ~= 4.0.0", "Flask-DotEnv ~= 0.1.2", "flask-smorest ~= 0.42.0", + "psycopg2-binary==2.9.3", "gunicorn ~= 21.2.0", "marshmallow ~= 3.20.1", "marshmallow-dataclass ~= 8.5.14", diff --git a/scripts/start.sh b/scripts/start.sh index 61e1849..1f59aaf 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,5 +2,5 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE -f ./docker-compose.infra.yml --profile=manual_dev --profile=manual_setup down -$DOCKER_COMPOSE -f ./docker-compose.infra.yml --profile mapeditor-dispatcher up -d +$DOCKER_COMPOSE -f ./docker-compose.yml --profile=manual_dev --profile=manual_setup down +$DOCKER_COMPOSE -f ./docker-compose.yml up -d diff --git a/src/omotes_rest/__init__.py b/src/omotes_rest/__init__.py index 27b761d..377b794 100644 --- a/src/omotes_rest/__init__.py +++ b/src/omotes_rest/__init__.py @@ -32,8 +32,8 @@ def create_app(object_name): api.init_app(app) # Register blueprints. - # from src.omotes_rest.apis.status import api as status_api - from src.omotes_rest.apis.job import api as job_api + # from omotes_rest.apis.status import api as status_api + from omotes_rest.apis.job import api as job_api # api.register_blueprint(status_api) api.register_blueprint(job_api) diff --git a/src/omotes_rest/apis/api_dataclasses.py b/src/omotes_rest/apis/api_dataclasses.py index bfb13d0..69e56aa 100644 --- a/src/omotes_rest/apis/api_dataclasses.py +++ b/src/omotes_rest/apis/api_dataclasses.py @@ -20,9 +20,7 @@ class JobRestStatus(Enum): this state is not defined in this `Enum`. """ REGISTERED = "registered" - """Job is registered but not yet submitted to Celery.""" - ENQUEUED = "enqueued" - """Job is submitted to Celery but not yet started.""" + """Job is registered but not yet running.""" RUNNING = "running" """Job is started and waiting to complete.""" SUCCEEDED = "succeeded" diff --git a/src/omotes_rest/apis/status.py b/src/omotes_rest/apis/status.py index 7cc3c9d..b871dde 100644 --- a/src/omotes_rest/apis/status.py +++ b/src/omotes_rest/apis/status.py @@ -1,6 +1,6 @@ # from flask_smorest import Blueprint # from flask.views import MethodView -# from src.shared.log import get_logger +# from shared.log import get_logger # # logger = get_logger("omotes_rest") # diff --git a/src/omotes_rest/database.py b/src/omotes_rest/database.py deleted file mode 100644 index 50a98f0..0000000 --- a/src/omotes_rest/database.py +++ /dev/null @@ -1,69 +0,0 @@ -from contextlib import contextmanager -from typing import Generator - -from sqlalchemy import create_engine, orm -from sqlalchemy.engine import URL -from sqlalchemy.orm import Session as SQLSession - -from src.omotes_rest.settings import EnvSettings -from omotes_rest.log import get_logger - -logger = get_logger("omotes_rest") - -session_factory = orm.sessionmaker() -Session = orm.scoped_session(session_factory) - - -@contextmanager -def session_scope(bind=None) -> Generator[SQLSession, None, None]: - """Provide a transactional scope around a series of operations. Ensures that the session is - committed and closed. Exceptions raised within the 'with' block using this contextmanager - should be handled in the with block itself. They will not be caught by the 'except' here.""" - try: - if bind: - yield Session(bind=bind) - yield Session() - Session.commit() - except Exception: - # Only the exceptions raised by session.commit above are caught here - Session.rollback() - raise - finally: - Session.remove() - - -def initialize_db(application_name: str): - """ - Initialize the database connection by creating the engine and configuring - the default session maker. - """ - logger.info( - "Connecting to PostgresDB at %s:%s as user %s", - EnvSettings.postgis_host(), - EnvSettings.postgis_port(), - EnvSettings.postgis_user(), - ) - url = URL.create( - "postgresql+psycopg2", - username=EnvSettings.postgis_user(), - password=EnvSettings.postgis_password(), - host=EnvSettings.postgis_host(), - port=EnvSettings.postgis_port(), - database=EnvSettings.postgis_database_name(), - ) - - engine = create_engine( - url, - pool_size=20, - max_overflow=5, - echo=True, - connect_args={ - "application_name": application_name, - "options": "-c lock_timeout=30000 -c statement_timeout=300000", # 5 minutes - }, - ) - - # Bind the global session to the actual engine. - Session.configure(bind=engine) - - return engine diff --git a/src/omotes_rest/log.py b/src/omotes_rest/log.py index aa8f363..f784612 100644 --- a/src/omotes_rest/log.py +++ b/src/omotes_rest/log.py @@ -2,7 +2,7 @@ from typing import List, Any import structlog -from src.omotes_rest.settings import EnvSettings +from omotes_rest.settings import EnvSettings from structlog.threadlocal import merge_threadlocal timestamper = structlog.processors.TimeStamper(fmt="iso") diff --git a/src/omotes_rest/main.py b/src/omotes_rest/main.py index 7a8d523..093fc0e 100644 --- a/src/omotes_rest/main.py +++ b/src/omotes_rest/main.py @@ -2,9 +2,8 @@ from time import strftime from flask import request, send_from_directory -from src.omotes_rest import create_app -# from src.omotes_rest.database import initialize_db -from src.omotes_rest.settings import EnvSettings +from omotes_rest import create_app +from omotes_rest.settings import EnvSettings from omotes_rest.log import get_logger from werkzeug.exceptions import HTTPException @@ -81,7 +80,6 @@ def handle_500(e): def main() -> None: - # initialize_db("nwn") app.run( host=EnvSettings.flask_server_host(), port=EnvSettings.flask_server_port(), diff --git a/src/omotes_rest/omotes_actions.py b/src/omotes_rest/omotes_actions.py index 86ad9e8..56d87b6 100644 --- a/src/omotes_rest/omotes_actions.py +++ b/src/omotes_rest/omotes_actions.py @@ -31,7 +31,7 @@ def handle_on_job_finished(job: Job, result: JobResult) -> None: postgres_if.set_job_status( job_id=job.id, new_status=3, # FINISHED - result_type=result.result_type, + result=result, ) diff --git a/src/omotes_rest/postgres_interface.py b/src/omotes_rest/postgres_interface.py index d582f6c..9dc5d4e 100644 --- a/src/omotes_rest/postgres_interface.py +++ b/src/omotes_rest/postgres_interface.py @@ -4,6 +4,7 @@ import logging from typing import Generator, Optional +from omotes_sdk_protocol.job_pb2 import JobResult from sqlalchemy import select, update, delete, create_engine, orm from sqlalchemy.orm import Session as SQLSession from sqlalchemy.engine import Engine, URL @@ -53,27 +54,32 @@ def initialize_db(application_name: str, config: PostgreSQLConfig) -> Engine: :param config: Configuration on how to connect to the SQL database. """ LOGGER.info( - "Connecting to PostgresDB at %s:%s as user %s", config.host, config.port, config.username - ) - url = URL.create( - "postgresql+psycopg2", - username=config.username, - password=config.password, - host=config.host, - port=config.port, - database=config.database, + "Connecting to PostgresDB at %s:%s as user %s to db %s", config.host, config.port, + config.username, config.database ) - engine = create_engine( - url, - pool_size=20, - max_overflow=5, - echo=False, - connect_args={ - "application_name": application_name, - "options": "-c lock_timeout=30000 -c statement_timeout=300000", # 5 minutes - }, - ) + try: + url = URL.create( + "postgresql+psycopg2", + username=config.username, + password=config.password, + host=config.host, + port=config.port, + database=config.database, + ) + + engine = create_engine( + url, + pool_size=20, + max_overflow=5, + echo=False, + connect_args={ + "application_name": application_name, + "options": "-c lock_timeout=30000 -c statement_timeout=300000", # 5 minutes + }, + ) + except Exception as e: + LOGGER.error(e) # Bind the global session to the actual engine. Session.configure(bind=engine) @@ -136,12 +142,12 @@ def put_new_job( session.add(new_job) LOGGER.debug("Job %s is submitted as new job in database", job_id) - def set_job_status(self, job_id: uuid.UUID, new_status: int, result_type: int = None) -> None: + def set_job_status(self, job_id: uuid.UUID, new_status: int, result: JobResult = None) -> None: """Set the status of the job to SUBMITTED. :param job_id: Job to set the status to SUBMITTED. :param new_status: new status 'int' identifier from protobuf message. - :param result_type: optional result type 'int' identifier from protobuf message. + :param result: optional result containing the result type, logs and output esdl. """ LOGGER.debug("For job '%s' received new status '%s'", job_id, new_status) @@ -171,23 +177,34 @@ def set_job_status(self, job_id: uuid.UUID, new_status: int, result_type: int = ) ) elif new_status == 3 or new_status == 4: # FINISHED or CANCELLED - if result_type is None: # Should this happen? + if result is None: # Should this happen? final_status = JobRestStatus.ERROR - elif result_type == 0: # SUCCEEDED + elif result.result_type == 0: # SUCCEEDED final_status = JobRestStatus.SUCCEEDED - elif result_type == 1: # TIMEOUT + elif result.result_type == 1: # TIMEOUT final_status = JobRestStatus.TIMEOUT - elif result_type == 2: # ERROR + elif result.result_type == 2: # ERROR final_status = JobRestStatus.ERROR - elif result_type == 3: # CANCELLED + elif result.result_type == 3: # CANCELLED final_status = JobRestStatus.CANCELLED - stmnt = ( - update(JobRest) - .where(JobRest.job_id == job_id) - .values( - status=final_status, stopped_at=datetime.now() + + if result: + stmnt = ( + update(JobRest) + .where(JobRest.job_id == job_id) + .values( + status=final_status, stopped_at=datetime.now(), logs=result.logs, + output_esdl=result.output_esdl + ) + ) + else: + stmnt = ( + update(JobRest) + .where(JobRest.job_id == job_id) + .values( + status=final_status, stopped_at=datetime.now() + ) ) - ) session.execute(stmnt) if result_type: diff --git a/src/omotes_rest/settings.py b/src/omotes_rest/settings.py index a7b15af..61cbddb 100644 --- a/src/omotes_rest/settings.py +++ b/src/omotes_rest/settings.py @@ -1,6 +1,5 @@ import os import secrets -from pathlib import Path from dotenv import load_dotenv @@ -24,70 +23,6 @@ def flask_server_port() -> int: def is_production(): return EnvSettings.env() == "prod" - @staticmethod - def postgis_host() -> str: - return os.getenv("POSTGIS_HOST", "localhost") - - @staticmethod - def postgis_port() -> int: - return int(os.getenv("POSTGIS_PORT", "9232")) - - @staticmethod - def postgis_database_name() -> str: - return os.getenv("POSTGIS_DATABASE_NAME", "NieuweWarmteNu") - - @staticmethod - def postgis_user() -> str: - return os.getenv("POSTGIS_ROOT_USER", "root") - - @staticmethod - def postgis_password() -> str: - return os.getenv("POSTGIS_ROOT_PASSWORD", "1234") - - @staticmethod - def nwn_postgres_host() -> str: - return os.getenv("NWN_POSTGRES_HOST", "localhost") - - @staticmethod - def nwn_postgres_port() -> int: - return int(os.getenv("NWN_POSTGRES_PORT", "6432")) - - @staticmethod - def nwn_postgres_database_name() -> str: - return os.getenv("NWN_POSTGRES_DATABASE_NAME", "nieuwewarmtenu") - - @staticmethod - def nwn_postgres_user() -> str: - return os.getenv("NWN_POSTGRES_ROOT_USER", "root") - - @staticmethod - def nwn_postgres_password() -> str: - return os.getenv("NWN_POSTGRES_ROOT_PASSWORD", "1234") - - @staticmethod - def nwn_rabbitmq_host() -> str: - return os.getenv("NWN_RABBITMQ_HOST", "localhost") - - @staticmethod - def nwn_rabbitmq_port() -> int: - return int(os.getenv("NWN_RABBITMQ_PORT", "5672")) - - @staticmethod - def nwn_rabbitmq_exchange() -> str: - return os.getenv("NWN_RABBITMQ_EXCHANGE", "nwn") - - @staticmethod - def nwn_rabbitmq_user() -> str: - return os.getenv("NWN_RABBITMQ_ROOT_USER", "root") - - @staticmethod - def nwn_rabbitmq_password() -> str: - return os.getenv("NWN_RABBITMQ_ROOT_PASSWORD", "5678") - - @staticmethod - def nwn_rabbitmq_hipe_compile() -> int: - return int(os.getenv("NWN_RABBITMQ_HIPE_COMPILE", "1")) - class Config(object): """Generic config for all environments.""" diff --git a/uwsgi/uwsgi-long-polling.ini b/uwsgi/uwsgi-long-polling.ini index 659e27c..da10fd3 100644 --- a/uwsgi/uwsgi-long-polling.ini +++ b/uwsgi/uwsgi-long-polling.ini @@ -1,5 +1,5 @@ [uwsgi] -wsgi-file = tno/mapeditor_dispatcher/main.py +wsgi-file = src/omotes_rest/main.py callable = app #module = app:app