pollution v2: ci/cd: fix docker image name #51
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD pollution-v2 | |
| on: | |
| push: | |
| paths: | |
| - "pollution_v2/**" | |
| - ".github/workflows/ci-pollution-v2.yml" | |
| - "!*.md" | |
| env: | |
| WORKING_DIRECTORY: pollution_v2 | |
| jobs: | |
| # Deploy Test | |
| deploy-test-pollution-v2: | |
| runs-on: ubuntu-24.04 | |
| if: github.ref == 'refs/heads/main' | |
| concurrency: deploy-test-pollution-v2 | |
| env: | |
| KEYCLOAK_URL: https://auth.opendatahub.testingmachine.eu | |
| DOCKER_IMAGE: ghcr.io/${{ github.repository }}/odh-mobility-el-pollution-v2-test | |
| DOCKER_TAG: ${{ github.sha }}-test | |
| PROJECT_NAME: odh-mobility-el-pollution-v2-test | |
| steps: | |
| - name: Checkout source code | |
| uses: noi-techpark/github-actions/checkout@v2 | |
| - name: Create .env file | |
| uses: noi-techpark/github-actions/env-file@v2 | |
| with: | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| env: | |
| # General deployment options | |
| X_COMPOSE_PROJECT_NAME: ${{ env.PROJECT_NAME }} | |
| X_DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }} | |
| X_DOCKER_TAG: ${{ env.DOCKER_TAG }} | |
| X_LOG_LEVEL: INFO | |
| X_LOG_LEVEL_LIBS: INFO | |
| X_SERVER_PORT: 1097 | |
| X_pollution-v2_TASK_SCHEDULING_MINUTE: "*/10" | |
| X_pollution-v2_TASK_SCHEDULING_HOUR: "*" | |
| X_ODH_BASE_READER_URL: https://mobility.api.opendatahub.testingmachine.eu | |
| X_ODH_BASE_WRITER_URL: https://share.opendatahub.testingmachine.eu | |
| X_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/ | |
| X_ODH_CLIENT_ID: odh-a22-dataprocessor | |
| X_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_TEST }} | |
| X_ODH_GRANT_TYPE: client_credentials | |
| X_ODH_PAGINATION_SIZE: 10000 | |
| X_ODH_MAX_POST_BATCH_SIZE: 10000 | |
| X_PROVENANCE_NAME: ${{ env.PROJECT_NAME }} | |
| X_PROVENANCE_VERSION: ${{ github.sha }} | |
| X_AIRFLOW_PROJ_DIR: /opt/pollution-v2 | |
| X_AIRFLOW_UID: 5000 | |
| X_AIRFLOW__CORE__EXECUTOR: CeleryExecutor | |
| X_AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow | |
| X_AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow | |
| X_AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 | |
| X_AIRFLOW__CORE__FERNET_KEY: '' | |
| X_AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' | |
| X_AIRFLOW__CORE__LOAD_EXAMPLES: 'false' | |
| X_AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' | |
| # yamllint disable rule:line-length | |
| # Use simple http server on scheduler for health checks | |
| # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server | |
| # yamllint enable rule:line-length | |
| X_AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' | |
| # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks | |
| # for other purpose (development, test and especially production usage) build/extend Airflow image. | |
| X__PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} | |
| X_AIRFLOW_VAR_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/ | |
| X_AIRFLOW_VAR_ODH_BASE_READER_URL: 'https://mobility.api.opendatahub.testingmachine.eu' | |
| X_AIRFLOW_VAR_ODH_BASE_WRITER_URL: 'https://share.opendatahub.testingmachine.eu' | |
| X_AIRFLOW_VAR_ODH_CLIENT_ID: 'odh-a22-dataprocessor' | |
| X_AIRFLOW_VAR_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_TEST }} | |
| X_AIRFLOW_VAR_ODH_GRANT_TYPE: client_credentials | |
| X_AIRFLOW_VAR_ODH_MAX_POST_BATCH_SIZE: '10000' | |
| X_AIRFLOW_VAR_ODH_PAGINATION_SIZE: '10000' | |
| X_AIRFLOW_WWW_USER_USERNAME: ${{ secrets.AIRFLOW_WWW_USER_USERNAME }} | |
| X_AIRFLOW_WWW_USER_PASSWORD: ${{ secrets.AIRFLOW_WWW_USER_PASSWORD }} | |
| X_AIRFLOW_VAR_DATATYPE_PREFIX: '' | |
| X_AIRFLOW_VAR_DAG_POLLUTION_EXECUTION_CRONTAB: 0 4 * * * | |
| X_AIRFLOW_VAR_DAG_VALIDATION_EXECUTION_CRONTAB: 0 2 * * * | |
| # username/pass not needed with client_credentials | |
| X_AIRFLOW_VAR_ODH_PASSWORD: 'ODH_PASSWORD' | |
| X_AIRFLOW_VAR_ODH_USERNAME: 'ODH_USERNAME' | |
| X_AIRFLOW_VAR_ODH_MINIMUM_STARTING_DATE: "2017-01-01" | |
| X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_HOST: 'redis' | |
| X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_PORT: 6379 | |
| X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_DB: 3 | |
| # X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 4 | |
| X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 16 | |
| # Currently no logging enabled | |
| # X_AIRFLOW__CORE__REMOTE_LOGGING: '<your_choice_on_remote_logging>' | |
| # X_AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: '<your_bucket_for_remote_logging>' | |
| # X_AIRFLOW__CORE__REMOTE_LOG_CONN_ID: '<your_connection_id_for_remote_logging>' | |
| # X_AIRFLOW__CORE__ENCRYPT_S3_LOGS: '<your_choice_on_encrypt_logs>' | |
| # X_AIRFLOW_CONN_MINIO_S3_CONN: '<your_connection_details_for_remote_logging>' | |
| X_NO_PROXY: '*' | |
| - name: Build and push images | |
| uses: noi-techpark/github-actions/docker-build-and-push@v2 | |
| with: | |
| working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure | |
| docker-username: ${{ github.actor }} | |
| docker-password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Deploy application | |
| uses: noi-techpark/github-actions/docker-deploy@v2 | |
| with: | |
| working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure/ansible | |
| hosts: 'test' | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| docker-username: 'noi-techpark-bot' | |
| docker-password: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| project-name: ${{ env.PROJECT_NAME }} | |
| # Deploy Prod | |
| deploy-prod-pollution-v2: | |
| runs-on: ubuntu-24.04 | |
| if: github.ref == 'refs/heads/prod' | |
| concurrency: deploy-prod-pollution-v2 | |
| env: | |
| KEYCLOAK_URL: https://auth.opendatahub.com | |
| DOCKER_IMAGE: ghcr.io/${{ github.repository }}/odh-mobility-el-pollution-v2-prod | |
| DOCKER_TAG: ${{ github.sha }}-prod | |
| PROJECT_NAME: odh-mobility-el-pollution-v2-prod | |
| steps: | |
| - name: Checkout source code | |
| uses: noi-techpark/github-actions/checkout@v2 | |
| - name: Create .env file | |
| uses: noi-techpark/github-actions/env-file@v2 | |
| with: | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| env: | |
| # General deployment options | |
| X_COMPOSE_PROJECT_NAME: ${{ env.PROJECT_NAME }} | |
| X_DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }} | |
| X_DOCKER_TAG: ${{ env.DOCKER_TAG }} | |
| X_LOG_LEVEL: INFO | |
| X_LOG_LEVEL_LIBS: INFO | |
| X_SERVER_PORT: 1098 | |
| X_pollution-v2_TASK_SCHEDULING_MINUTE: "*/10" | |
| X_pollution-v2_TASK_SCHEDULING_HOUR: "*" | |
| X_ODH_BASE_READER_URL: https://mobility.api.opendatahub.com | |
| X_ODH_BASE_WRITER_URL: 'https://mobility.share.opendatahub.com' | |
| X_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/ | |
| X_ODH_CLIENT_ID: odh-a22-dataprocessor | |
| X_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_PROD }} | |
| X_ODH_GRANT_TYPE: client_credentials | |
| X_ODH_PAGINATION_SIZE: 10000 | |
| X_ODH_MAX_POST_BATCH_SIZE: 10000 | |
| X_PROVENANCE_NAME: ${{ env.PROJECT_NAME }} | |
| X_PROVENANCE_VERSION: ${{ github.sha }} | |
| X_AIRFLOW_PROJ_DIR: /opt/pollution-v2 | |
| X_AIRFLOW_UID: 5000 | |
| X_AIRFLOW__CORE__EXECUTOR: CeleryExecutor | |
| X_AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow | |
| X_AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow | |
| X_AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 | |
| X_AIRFLOW__CORE__FERNET_KEY: '' | |
| X_AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' | |
| X_AIRFLOW__CORE__LOAD_EXAMPLES: 'false' | |
| X_AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' | |
| # yamllint disable rule:line-length | |
| # Use simple http server on scheduler for health checks | |
| # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server | |
| # yamllint enable rule:line-length | |
| X_AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' | |
| # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks | |
| # for other purpose (development, test and especially production usage) build/extend Airflow image. | |
| X__PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} | |
| X_AIRFLOW_VAR_ODH_AUTHENTICATION_URL: ${{ env.KEYCLOAK_URL }}/auth/ | |
| X_AIRFLOW_VAR_ODH_BASE_READER_URL: 'https://mobility.api.opendatahub.com' | |
| X_AIRFLOW_VAR_ODH_BASE_WRITER_URL: 'https://mobility.share.opendatahub.com' | |
| X_AIRFLOW_VAR_ODH_CLIENT_ID: 'odh-a22-dataprocessor' | |
| X_AIRFLOW_VAR_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_PROD }} | |
| X_AIRFLOW_VAR_ODH_GRANT_TYPE: client_credentials | |
| X_AIRFLOW_VAR_ODH_MAX_POST_BATCH_SIZE: '10000' | |
| X_AIRFLOW_VAR_ODH_PAGINATION_SIZE: '10000' | |
| X_AIRFLOW_WWW_USER_USERNAME: ${{ secrets.AIRFLOW_WWW_USER_USERNAME }} | |
| X_AIRFLOW_WWW_USER_PASSWORD: ${{ secrets.AIRFLOW_WWW_USER_PASSWORD }} | |
| X_AIRFLOW_VAR_DATATYPE_PREFIX: '' | |
| X_AIRFLOW_VAR_DAG_POLLUTION_EXECUTION_CRONTAB: 0 4 * * * | |
| X_AIRFLOW_VAR_DAG_VALIDATION_EXECUTION_CRONTAB: 0 2 * * * | |
| # username/pass not needed with client_credentials | |
| X_AIRFLOW_VAR_ODH_PASSWORD: 'ODH_PASSWORD' | |
| X_AIRFLOW_VAR_ODH_USERNAME: 'ODH_USERNAME' | |
| X_AIRFLOW_VAR_ODH_MINIMUM_STARTING_DATE: "2017-01-01" | |
| X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_HOST: 'redis' | |
| X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_PORT: 6379 | |
| X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_DB: 3 | |
| # X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 4 | |
| X_AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 16 | |
| # Currently no logging enabled | |
| # X_AIRFLOW__CORE__REMOTE_LOGGING: '<your_choice_on_remote_logging>' | |
| # X_AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: '<your_bucket_for_remote_logging>' | |
| # X_AIRFLOW__CORE__REMOTE_LOG_CONN_ID: '<your_connection_id_for_remote_logging>' | |
| # X_AIRFLOW__CORE__ENCRYPT_S3_LOGS: '<your_choice_on_encrypt_logs>' | |
| # X_AIRFLOW_CONN_MINIO_S3_CONN: '<your_connection_details_for_remote_logging>' | |
| X_NO_PROXY: '*' | |
| - name: Build and push images | |
| uses: noi-techpark/github-actions/docker-build-and-push@v2 | |
| with: | |
| working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure | |
| docker-username: ${{ github.actor }} | |
| docker-password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Deploy application | |
| uses: noi-techpark/github-actions/docker-deploy@v2 | |
| with: | |
| working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure/ansible | |
| hosts: 'prod' | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| docker-username: 'noi-techpark-bot' | |
| docker-password: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| project-name: ${{ env.PROJECT_NAME }} |