pollution-v2: remove all airflow-common alias #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI/CD pollution-v2 | |
on: | |
push: | |
paths: | |
- "pollution_v2/**" | |
- ".github/workflows/ci-pollution-v2.yml" | |
- "!*.md" | |
env: | |
WORKING_DIRECTORY: pollution_v2 | |
PROJECT_NAME: odh-mobility-el-pollution-v2 | |
DOCKER_IMAGE: ghcr.io/${{ github.repository }}/odh-mobility-el-pollution-v2 | |
DOCKER_TAG: ${{ github.sha }} | |
jobs: | |
test: | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Checkout source code | |
uses: noi-techpark/github-actions/checkout@v2 | |
- name: REUSE Compliance Check | |
uses: fsfe/reuse-action@v1 | |
- name: install setuptools needed to use distutils with python > 3.12 | |
run: python3 -m pip install setuptools | |
- name: Test code | |
uses: noi-techpark/github-actions/pip-test@v2 | |
with: | |
working-directory: ${{ env.WORKING_DIRECTORY }} | |
test-command: | | |
echo "WARNING: NO TESTS FOUND!" | |
# Deploy Test | |
deploy-test-pollution-v2: | |
runs-on: ubuntu-22.04 | |
if: github.ref == 'refs/heads/main' | |
needs: test | |
concurrency: deploy-test-pollution-v2 | |
env: | |
KEYCLOAK_URL: https://auth.opendatahub.testingmachine.eu | |
steps: | |
- name: Checkout source code | |
uses: noi-techpark/github-actions/checkout@v2 | |
- name: Create .env file | |
uses: noi-techpark/github-actions/env-file@v2 | |
with: | |
working-directory: ${{ env.WORKING_DIRECTORY }} | |
env: | |
# General deployment options | |
X_COMPOSE_PROJECT_NAME: ${{ env.PROJECT_NAME }} | |
X_DOCKER_IMAGE: ${{ env.DOCKER_IMAGE }} | |
X_DOCKER_TAG: ${{ env.DOCKER_TAG }} | |
X_LOG_LEVEL: INFO | |
X_LOG_LEVEL_LIBS: INFO | |
X_CELERY_BROKER_URL: redis://pollution-v2-connector-redis:6379/1 | |
X_CELERY_BACKEND_URL: redis://pollution-v2-connector-redis:6379/2 | |
X_CELERY_RESULT_EXPIRATION_SECONDS: "604800" | |
X_pollution-v2_TASK_SCHEDULING_MINUTE: "*/10" | |
X_pollution-v2_TASK_SCHEDULING_HOUR: "*" | |
X_ODH_BASE_READER_URL: https://mobility.api.opendatahub.testingmachine.eu | |
X_ODH_BASE_WRITER_URL: https://share.opendatahub.testingmachine.eu | |
X_ODH_AUTHENTICATION_URL: https://auth.opendatahub.testingmachine.eu/auth/ | |
X_ODH_CLIENT_ID: odh-a22-dataprocessor | |
X_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_TEST }} | |
X_ODH_GRANT_TYPE: client_credentials | |
X_ODH_PAGINATION_SIZE: 10000 | |
X_ODH_MAX_POST_BATCH_SIZE: 10000 | |
X_PROVENANCE_NAME: ${{ env.PROJECT_NAME }} | |
X_PROVENANCE_VERSION: ${{ github.sha }} | |
X_AIRFLOW__CORE__EXECUTOR: CeleryExecutor | |
X_AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow | |
X_AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow | |
X_AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 | |
X_AIRFLOW__CORE__FERNET_KEY: '' | |
X_AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' | |
X_AIRFLOW__CORE__LOAD_EXAMPLES: 'false' | |
X_AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' | |
# yamllint disable rule:line-length | |
# Use simple http server on scheduler for health checks | |
# See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server | |
# yamllint enable rule:line-length | |
X_AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' | |
# WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks | |
# for other purpose (development, test and especially production usage) build/extend Airflow image. | |
X__PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} | |
X_AIRFLOW_VAR_ODH_AUTHENTICATION_URL: '<your_odh_auth_url>' | |
X_AIRFLOW_VAR_ODH_BASE_READER_URL: 'https://mobility.api.opendatahub.testingmachine.eu' | |
X_AIRFLOW_VAR_ODH_BASE_WRITER_URL: 'https://share.opendatahub.testingmachine.eu' | |
X_AIRFLOW_VAR_ODH_CLIENT_ID: 'odh-a22-dataprocessor' | |
X_AIRFLOW_VAR_ODH_CLIENT_SECRET: ${{ secrets.COMBINED_CLIENT_SECRET_TEST }} | |
X_AIRFLOW_VAR_ODH_GRANT_TYPE: client_credentials | |
X_AIRFLOW_VAR_ODH_MAX_POST_BATCH_SIZE: '10000' | |
X_AIRFLOW_VAR_ODH_PAGINATION_SIZE: '10000' | |
# username/pass not needed with client_credentials | |
# X_AIRFLOW_VAR_ODH_PASSWORD: '<your_odh_password>' | |
# X_AIRFLOW_VAR_ODH_USERNAME: '<your_odh_username>' | |
X_AIRFLOW_VAR_ODH_MINIMUM_STARTING_DATE: "2017-01-01 00:00:00+00:00" | |
X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_HOST: 'pollution-v2-connector-redis' | |
X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_PORT: 6379 | |
X_AIRFLOW_VAR_COMPUTATION_CHECKPOINT_REDIS_DB: 3 | |
# Currently no logging enabled | |
# X_AIRFLOW__CORE__REMOTE_LOGGING: '<your_choice_on_remote_logging>' | |
# X_AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: '<your_bucket_for_remote_logging>' | |
# X_AIRFLOW__CORE__REMOTE_LOG_CONN_ID: '<your_connection_id_for_remote_logging>' | |
# X_AIRFLOW__CORE__ENCRYPT_S3_LOGS: '<your_choice_on_encrypt_logs>' | |
# X_AIRFLOW_CONN_MINIO_S3_CONN: '<your_connection_details_for_remote_logging>' | |
X_NO_PROXY: '*' | |
- name: Build and push images | |
uses: noi-techpark/github-actions/docker-build-and-push@v2 | |
with: | |
working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure | |
docker-username: ${{ github.actor }} | |
docker-password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Deploy application | |
uses: noi-techpark/github-actions/docker-deploy@v2 | |
with: | |
working-directory: ${{ env.WORKING_DIRECTORY }}/infrastructure/ansible | |
hosts: 'test' | |
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
docker-username: 'noi-techpark-bot' | |
docker-password: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
project-name: ${{ env.PROJECT_NAME }} | |
# Deploy Prod | |
# TODO |