From 561f81b347007bf4fd390b692e33b9b228da1518 Mon Sep 17 00:00:00 2001 From: Lubomir Dolezal Date: Thu, 20 Jun 2024 19:21:27 +0200 Subject: [PATCH] first adaptations --- .gitignore | 170 ++++++++++++++++++++++++++++-- .pre-commit-config.yaml | 6 -- .vscode/settings.json | 8 +- README.md | 16 +++ pyproject.toml | 3 + requirements.txt | 7 +- ruff.toml | 8 ++ src/eodash_catalog/duration.py | 25 ++--- src/eodash_catalog/sh_endpoint.py | 1 + src/eodash_catalog/thumbnails.py | 49 +++++++++ tests/test_generate.py | 2 + 11 files changed, 260 insertions(+), 35 deletions(-) delete mode 100644 .pre-commit-config.yaml create mode 100644 ruff.toml create mode 100644 src/eodash_catalog/thumbnails.py create mode 100644 tests/test_generate.py diff --git a/.gitignore b/.gitignore index 3591a20..b5e520e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,167 @@ -.eggs -*.egg-info -*.pyc -.cache +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python build/ +develop-eggs/ dist/ -.pytest* -*.gfs -__pycache__ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ catalogs/ collections/ layers/ - +.venv +.pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 2ee05ed..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -repos: - - repo: https://github.com/ambv/black - rev: 23.3.0 - hooks: - - id: black - language_version: python3 diff --git a/.vscode/settings.json b/.vscode/settings.json index 6169866..848024c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,10 @@ { - "editor.formatOnSave": true, "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.codeActionsOnSave": { + "source.fixAll": "explicit", + "source.organizeImports": "explicit" + }, + "editor.formatOnSave": true, }, + "editor.defaultFormatter": "charliermarsh.ruff", } diff --git a/README.md b/README.md index 274eaf7..ad3d104 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,22 @@ pip install eodash_catalog ``` +## Testing + +Project uses pytest and runs it as part of CI: + +```bash +python -m pytest +``` + +## Testing + +Project uses ruff to perform checks on code style and formatting +```bash +ruff check . +``` + + ## Building and publishing To build and publish we use hatch, first bump the version in `src/eodash_catalog/__about__.py` then run diff --git a/pyproject.toml b/pyproject.toml index ecf248d..448e968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,3 +83,6 @@ tests = ["tests", "*/eodash_catalog/tests"] [tool.coverage.report] exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] + +[tool.mypy] +disable_error_code = ["import-untyped"] diff --git a/requirements.txt b/requirements.txt index e3bffa0..d94cf2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,11 @@ python-dateutil<3 swiftspec==0.0.2 mergedeep structlog<22.0 -OWSLib +OWSLib<1.0 spdx-lookup<=0.3.3 pystac[validation]<=1.8.3 + +# dev tooling +pytest==8.1.1 +pytest-watch==4.2.0 +ruff==0.4.2 diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..e5d3b81 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,8 @@ +exclude = ["venv"] + +line-length = 100 +target-version = "py312" + +[lint] +select = ["E", "F", "UP", "B", "SIM", "I", "W", "C90", "ASYNC", "A", "C4", "PERF", "RUF"] +ignore = ["B019", "C901"] diff --git a/src/eodash_catalog/duration.py b/src/eodash_catalog/duration.py index d533a6f..d86d9d7 100644 --- a/src/eodash_catalog/duration.py +++ b/src/eodash_catalog/duration.py @@ -4,8 +4,9 @@ The class Duration allows to define durations in years and months and can be used as limited replacement for timedelta objects. """ + from datetime import timedelta -from decimal import Decimal, ROUND_FLOOR +from decimal import ROUND_FLOOR, Decimal def fquotmod(val, low, high): @@ -82,9 +83,7 @@ def __init__( years = Decimal(str(years)) self.months = months self.years = years - self.tdelta = timedelta( - days, seconds, microseconds, milliseconds, minutes, hours, weeks - ) + self.tdelta = timedelta(days, seconds, microseconds, milliseconds, minutes, hours, weeks) def __getstate__(self): return self.__dict__ @@ -167,13 +166,8 @@ def __add__(self, other): carry, newmonth = fquotmod(newmonth, 1, 13) newyear = other.year + self.years + carry maxdays = max_days_in_month(newyear, newmonth) - if other.day > maxdays: - newday = maxdays - else: - newday = other.day - newdt = other.replace( - year=int(newyear), month=int(newmonth), day=int(newday) - ) + newday = maxdays if other.day > maxdays else other.day + newdt = other.replace(year=int(newyear), month=int(newmonth), day=int(newday)) # does a timedelta + date/datetime return self.tdelta + newdt except AttributeError: @@ -252,13 +246,8 @@ def __rsub__(self, other): carry, newmonth = fquotmod(newmonth, 1, 13) newyear = other.year - self.years + carry maxdays = max_days_in_month(newyear, newmonth) - if other.day > maxdays: - newday = maxdays - else: - newday = other.day - newdt = other.replace( - year=int(newyear), month=int(newmonth), day=int(newday) - ) + newday = maxdays if other.day > maxdays else other.day + newdt = other.replace(year=int(newyear), month=int(newmonth), day=int(newday)) return newdt - self.tdelta except AttributeError: # other probably was not compatible with data/datetime diff --git a/src/eodash_catalog/sh_endpoint.py b/src/eodash_catalog/sh_endpoint.py index 041ac14..5dab676 100644 --- a/src/eodash_catalog/sh_endpoint.py +++ b/src/eodash_catalog/sh_endpoint.py @@ -1,4 +1,5 @@ import os + from oauthlib.oauth2 import BackendApplicationClient from requests_oauthlib import OAuth2Session diff --git a/src/eodash_catalog/thumbnails.py b/src/eodash_catalog/thumbnails.py new file mode 100644 index 0000000..46422f9 --- /dev/null +++ b/src/eodash_catalog/thumbnails.py @@ -0,0 +1,49 @@ +import os +import re +from pathlib import Path + +import requests + +from eodash_catalog.endpoints import generate_veda_cog_link + + +def fetch_and_save_thumbnail(data, url): + collection_path = "../thumbnails/{}_{}/".format(data["EodashIdentifier"], data["Name"]) + Path(collection_path).mkdir(parents=True, exist_ok=True) + image_path = f"{collection_path}/thumbnail.png" + if not os.path.exists(image_path): + data = requests.get(url).content + with open(image_path, "wb") as f: + f.write(data) + + +def generate_thumbnail(stac_object, data, endpoint, file_url=None, time=None, styles=None): + if endpoint["Name"] == "Sentinel Hub" or endpoint["Name"] == "WMS": + instanceId = os.getenv("SH_INSTANCE_ID") + if "InstanceId" in endpoint: + instanceId = endpoint["InstanceId"] + # Build example url + wms_config = ( + "REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image/png&STYLES=&TRANSPARENT=true" + ) + bbox_s = stac_object.bbox + bbox = f"{bbox_s[1]},{bbox_s[0]},{bbox_s[3]},{bbox_s[2]}" + output_format = f"format=image/png&WIDTH=256&HEIGHT=128&CRS=EPSG:4326&BBOX={bbox}" + item_datetime = stac_object.get_datetime() + # it is possible for datetime to be null, + # if it is start and end datetime have to exist + if item_datetime: + time = item_datetime.isoformat()[:-6] + "Z" + url = "https://services.sentinel-hub.com/ogc/wms/{}?{}&layers={}&time={}&{}".format( + instanceId, + wms_config, + endpoint["LayerId"], + time, + output_format, + ) + fetch_and_save_thumbnail(data, url) + elif endpoint["Name"] == "VEDA": + target_url = generate_veda_cog_link(endpoint, file_url) + # set to get 0/0/0 tile + url = re.sub(r"\{.\}", "0", target_url) + fetch_and_save_thumbnail(data, url) diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..9b75c2b --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,2 @@ +def test_create_test_run(): + assert True