Skip to content

Commit

Permalink
dags: Add a "data consistency" DAG
Browse files Browse the repository at this point in the history
It runs consistency checks against production data.
  • Loading branch information
vperron committed Jun 27, 2023
1 parent fac6819 commit dbdbf13
Showing 1 changed file with 27 additions and 0 deletions.
27 changes: 27 additions & 0 deletions dags/data_consistency.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import airflow
from airflow.operators import bash, empty

from dags.common import db, dbt, default_dag_args, slack


dag_args = default_dag_args() | {"default_args": dbt.get_default_args()}

with airflow.DAG(
dag_id="data_consistency",
schedule_interval="@hourly",
**dag_args,
) as dag:
start = empty.EmptyOperator(task_id="start")

env_vars = db.connection_envvars()

dbt_test = bash.BashOperator(
task_id="dbt_test",
bash_command="dbt test",
env=env_vars,
append_env=True,
)

end = slack.success_notifying_task()

(start >> dbt_test >> end)

0 comments on commit dbdbf13

Please sign in to comment.