From 601cdb7dfb1b29f08ea7b013e84e6bb6cfe412f7 Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Fri, 23 Jun 2023 17:08:29 +0200 Subject: [PATCH] Install dbt-fal As of today, this is the only way to use Python models with the PostgreSQL datasource. --- profiles.yml | 5 ++- requirements-ci.txt | 95 ++++++++++++++++++++++++++++++++++----------- requirements.txt | 5 ++- 3 files changed, 80 insertions(+), 25 deletions(-) diff --git a/profiles.yml b/profiles.yml index b86cc8c0..c840a76c 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,5 +1,8 @@ dbt_pilotage: outputs: + main_fal: + type: fal + db_profile: main main: schema: public @@ -11,4 +14,4 @@ dbt_pilotage: port: "{{ env_var('PGPORT') | int }}" user: "{{ env_var('PGUSER') }}" - target: main + target: main_fal diff --git a/requirements-ci.txt b/requirements-ci.txt index 2609269f..1c8e6fbf 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -16,14 +16,21 @@ appdirs==1.4.4 argcomplete==3.1.1 asgiref==3.7.2 asn1crypto==1.5.1 +asttokens==2.2.1 async-timeout==4.0.2 attrs==23.1.0 +auth0-python==4.2.0 Babel==2.12.1 +backcall==0.2.0 +backoff==1.11.1 +backports.functools-lru-cache==1.6.4 beautifulsoup4==4.12.2 +betterproto==1.2.5 black==23.3.0 blinker==1.6.2 -boto3==1.26.156 -botocore==1.29.156 +boto3==1.26.159 +botocore==1.29.159 +build==0.10.0 cachelib==0.9.0 cattrs==23.1.2 certifi==2023.5.7 @@ -34,22 +41,29 @@ click==8.1.3 clickclick==20.10.2 colorama==0.4.6 colorlog==4.8.0 +commonmark==0.9.1 ConfigUpdater==3.1.1 connexion==2.14.2 cron-descriptor==1.4.0 croniter==1.4.1 cryptography==41.0.1 -dbt-core==1.5.1 +datadog-api-client==2.12.0 +dbt-core==1.5.2 dbt-extractor==0.4.1 +dbt-fal==1.5.3 dbt-postgres==1.5.1 decorator==5.1.1 Deprecated==1.2.14 -diff-cover==7.6.0 +diff-cover==7.5.0 dill==0.3.6 +distlib==0.3.6 dnspython==2.3.0 docutils==0.20.1 email-validator==1.3.1 -exceptiongroup==1.1.1 +exceptiongroup==1.1.0 +executing==1.2.0 +fal-serverless==0.6.35 +filelock==3.12.2 flake8==6.0.0 Flask==2.2.5 Flask-AppBuilder==4.3.1 @@ -61,24 +75,37 @@ Flask-Login==0.6.2 Flask-Session==0.5.0 Flask-SQLAlchemy==2.5.1 Flask-WTF==1.1.1 +freezegun==1.2.2 frozenlist==1.3.3 future==0.18.3 graphviz==0.20.1 greenlet==2.0.2 +grpc-interceptor==0.15.2 +grpcio==1.56.0 +grpclib==0.4.3 gunicorn==20.1.0 h11==0.14.0 +h2==4.1.0 hologram==0.0.16 +hpack==4.0.0 httpcore==0.17.2 httpx==0.24.1 +hyperframe==6.0.1 idna==3.4 +importlib-metadata==6.0.1 importlib-resources==5.12.0 inflection==0.5.1 iniconfig==2.0.0 +ipdb==0.13.13 +ipython==8.12.0 isodate==0.6.1 +isolate==0.12.2 +isolate-proto==0.0.31 isort==5.12.0 itsdangerous==2.1.2 +jedi==0.18.2 Jinja2==3.1.2 -jinja2-simple-tags==0.5.0 +jinja2-simple-tags==0.4.1 jmespath==1.0.1 jsonpath-ng==1.5.3 jsonschema==4.17.3 @@ -98,41 +125,57 @@ marshmallow-enum==1.5.1 marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.26.1 mashumaro==3.6 +matplotlib-inline==0.1.6 mccabe==0.7.0 mdit-py-plugins==0.4.0 mdurl==0.1.2 minimal-snowplow-tracker==0.0.2 +monotonic==1.6 msgpack==1.0.5 multidict==6.0.4 -mypy-boto3-appflow==1.26.145 +mypy-boto3-appflow==1.26.157 mypy-boto3-rds==1.26.144 mypy-boto3-redshift-data==1.26.109 mypy-boto3-s3==1.26.155 mypy-extensions==1.0.0 networkx==2.8.8 numpy==1.25.0 +opentelemetry-api==1.18.0 +opentelemetry-sdk==1.18.0 +opentelemetry-semantic-conventions==0.39b0 ordered-set==4.1.0 packaging==23.1 -pandas==2.0.2 +pandas==1.5.3 parsedatetime==2.4 -pathspec==0.9.0 +parso==0.8.3 +pathspec==0.11.1 pendulum==2.1.2 -platformdirs==3.6.0 -pluggy==1.0.0 +pexpect==4.8.0 +pickleshare==0.7.5 +pip-tools==6.13.0 +platformdirs==3.8.0 +pluggy==1.2.0 ply==3.11 +portalocker==2.7.0 +posthog==1.4.9 prison==0.2.1 +prompt-toolkit==3.0.38 protobuf==4.23.3 psutil==5.9.5 psycopg2-binary==2.9.6 +ptyprocess==0.7.0 +pure-eval==0.2.2 pycodestyle==2.10.0 pycparser==2.21 pydantic==1.10.9 pyflakes==3.0.1 Pygments==2.15.1 PyJWT==2.7.0 +pyparsing==3.0.9 +pyproject_hooks==1.0.0 pyrsistent==0.19.3 -pytest==7.3.2 -pytest-subtests==0.11.0 +pytest==7.3.1 +pytest-subtests==0.10.0 python-daemon==3.0.1 python-dateutil==2.8.2 python-nvd3==0.15.0 @@ -142,17 +185,18 @@ pytz==2023.3 pytzdata==2020.1 PyYAML==6.0 redshift-connector==2.0.911 -regex==2023.6.3 +regex==2023.3.23 requests==2.31.0 requests-toolbelt==1.0.0 rfc3339-validator==0.1.4 +rfc3986==1.5.0 rich==13.4.2 rich_argparse==1.1.1 -ruamel.yaml==0.17.32 +ruamel.yaml==0.17.21 ruamel.yaml.clib==0.2.7 s3transfer==0.6.1 scramp==1.4.4 -sentry-sdk==1.25.1 +sentry-sdk==1.26.0 setproctitle==1.3.2 six==1.16.0 slack-sdk==3.21.3 @@ -162,24 +206,31 @@ SQLAlchemy==1.4.48 SQLAlchemy-JSONField==1.0.1.post0 sqlalchemy-redshift==0.8.14 SQLAlchemy-Utils==0.41.1 -sqlfluff==2.1.1 -sqlfluff-templater-dbt==2.1.1 +sqlfluff==2.0.5 +sqlfluff-templater-dbt==2.0.5 sqlparse==0.4.3 +stack-data==0.6.2 +stringcase==1.2.0 +structlog==22.3.0 tabulate==0.9.0 -tblib==1.7.0 +tblib==2.0.0 tenacity==8.2.2 termcolor==2.3.0 text-unidecode==1.3 toml==0.10.2 tomli==2.0.1 tqdm==4.65.0 -typing_extensions==4.6.3 +traitlets==5.9.0 +typing_extensions==4.4.0 tzdata==2023.3 uc-micro-py==1.0.2 unicodecsv==0.14.1 -urllib3==1.26.16 +urllib3==2.0.3 +virtualenv==20.23.1 watchtower==2.0.1 -Werkzeug==2.2.3 +wcwidth==0.2.6 +Werkzeug==2.3.6 wrapt==1.15.0 WTForms==3.0.1 yarl==1.9.2 +zipp==3.15.0 diff --git a/requirements.txt b/requirements.txt index 364d106c..bec576b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -apache-airflow[amazon,sentry,slack] -dbt-postgres==1.5.1 +apache-airflow[amazon,sentry,slack]==2.6.2 +dbt-postgres==1.5.2 +dbt-fal[postgres] pandas