From f413b032fa8c80047e3ff7869a68680689b9c7d0 Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Thu, 11 Apr 2024 15:25:29 -0600 Subject: [PATCH 01/10] Copied xpublish examples from nextgen-ioos-2023 --- examples/README.md | 63 ++++++++++++++++++++++++++++ examples/demo.py | 41 ++++++++++++++++++ examples/environment.yaml | 26 ++++++++++++ examples/lme.py | 87 +++++++++++++++++++++++++++++++++++++++ examples/mean.py | 31 ++++++++++++++ 5 files changed, 248 insertions(+) create mode 100644 examples/README.md create mode 100644 examples/demo.py create mode 100644 examples/environment.yaml create mode 100644 examples/lme.py create mode 100644 examples/mean.py diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..4bb97288 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,63 @@ +# XPublish Exploring Setup + +## Using Mamba + +In many cases, conda times out while trying to create this environment. In that case, install [Micromamba](https://mamba.readthedocs.io/en/latest/micromamba-installation.html) first and then run these commands. + +```bash +micromamba env create -f environment.yaml +micromamba activate xpublish-exploring +``` + +## Using conda + +> Note: This method may time out + +Create the environment and activate: + +```bash +conda env create -f environment.yaml +conda activate xpublish-exploring +``` + +## Starting the Server + +Execute `demo.py` and you should see similar output indicating success: + +`Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit)` + +After seeing this message, you can then navigate to this address in your browser: + +http://localhost:9000/docs + +Here you should see a web page that allows you to inspect and test the available endpoints from Xpublish. + +# XPublish Tutorial + +## Exploring available datasets + +Navigate to http://localhost:9000/datasets to return a list of available datasets running on your xpublish host. These are defined in the `TutorialsDataset` plugin class in `demo.py`. + +Inspect one of those datasets by appending the dataset name to the URL. For example, http://localhost:9000/datasets/air_temperature/ + +This returns an xarray model view of that dataset. + +## Calculating the mean value + +XPublish's plugin system allows custom operations to be run against the data on the server. In the example `mean.py`, a custom plugin has been defined that allows the mean of a specific variable to be calculated. The power of this ecosystem is that any dataset in xpublish is interoperable with any sensible plugin. + +For the end-user, you can choose which variable to calculate using the URL syntax `/[dataset]/[variable]/mean`. For example, http://localhost:9000/datasets/air_temperature/air/mean + +## Working with Custom Bounding Boxes + +Another custom plugin is defined in `lme.py`. You can access this plugin at http://localhost:9000/lme/ + +The plugin defines several regions of interest (not actual LME's, but several areas of interest from GMRI). The key or `lmi_ID` references a bounding box which can be used to subset any of the datasets, assuming those overlap. + +This plugin will provide a subset of data based on the bounding box defined for each LME. The URL structure is `/[dataset]/lme/[lme_ID]` for example, http://localhost:9000/datasets/air_temperature/lme/EC/ + +> Note: This will currently return "NaN" when the regions don't overlap. This is something we can work on as part of this breakout session. + +## Combining the Plugins + +Plugins can be combined by adding their URLs together. For example, to find the mean of a specific region, you could access http://localhost:9000/datasets/air_temperature/lme/EC/air/mean adding the variable and mean calculation to the end. \ No newline at end of file diff --git a/examples/demo.py b/examples/demo.py new file mode 100644 index 00000000..6e28a75f --- /dev/null +++ b/examples/demo.py @@ -0,0 +1,41 @@ +import cf_xarray +import xarray as xr +from requests import HTTPError + +from xpublish import Plugin, Rest, hookimpl + + +class TutorialDataset(Plugin): + name: str = "xarray-tutorial-datasets" + + @hookimpl + def get_datasets(self): + return list(xr.tutorial.file_formats) + + @hookimpl + def get_dataset(self, dataset_id: str): + try: + ds = xr.tutorial.open_dataset(dataset_id) + if ds.cf.coords['longitude'].dims[0] == 'longitude': + ds = ds.assign_coords(longitude=(((ds.longitude + 180) % 360) - 180)).sortby('longitude') + # TODO: Yeah this should not be assumed... but for regular grids we will viz with rioxarray so for now we will assume + ds = ds.rio.write_crs(4326) + return ds + except HTTPError: + return None + + + + + +rest = Rest({}) +rest.register_plugin(TutorialDataset()) + +### For this tutorial, you can uncomment the following lines to activate the other plugins: + +from mean import MeanPlugin +from lme import LmeSubsetPlugin +rest.register_plugin(MeanPlugin()) +rest.register_plugin(LmeSubsetPlugin()) + +rest.serve() diff --git a/examples/environment.yaml b/examples/environment.yaml new file mode 100644 index 00000000..4b2375b4 --- /dev/null +++ b/examples/environment.yaml @@ -0,0 +1,26 @@ +name: xpublish-exploring +channels: + - conda-forge +dependencies: + - python=3.11 + - xarray-datatree + - xpublish>=0.3.1 + - xpublish-edr + - xpublish-opendap + - dask + - distributed + - netcdf4 + - zarr + - s3fs + - fsspec + - cf_xarray + - kerchunk + - h5py + - intake-xarray + - h5netcdf + - pydap + - h5pyd + - regionmask + - ipykernel + - pooch + \ No newline at end of file diff --git a/examples/lme.py b/examples/lme.py new file mode 100644 index 00000000..3949e4e1 --- /dev/null +++ b/examples/lme.py @@ -0,0 +1,87 @@ +from typing import Sequence + +from fastapi import APIRouter +from xpublish import Plugin, Dependencies, hookimpl + +regions = { + "GB": {"bbox": [-69.873, -65.918, 40.280, 42.204], "name": "Georges Bank"}, + "GOM": {"bbox": [-70.975, -65.375, 40.375, 45.125], "name": "Gulf Of Maine"}, + "MAB": { + "bbox": [-77.036, -70.005, 35.389, 41.640], + "name": "MidAtlantic Bight", + }, + "NESHELF": { + "bbox": [-77.45, -66.35, 34.50, 44.50], + "name": "North East Shelf", + }, + "SS": {"bbox": [-66.775, -65.566, 41.689, 45.011], "name": "Scotian Shelf"}, + "EC": {"bbox": [-81.75, -65.375, 25.000, 45.125], "name": "East Coast"}, + "NEC": {"bbox": [-81.45, -63.30, 28.70, 44.80], "name": "Northeast Coast"}, +} + +DEFAULT_TAGS = ['lme', 'large marine ecosystem', 'subset'] + + +class LmeSubsetPlugin(Plugin): + name: str = "lme-subset-plugin" + + app_router_prefix: str = "/lme" + app_router_tags: Sequence[str] = DEFAULT_TAGS + + dataset_router_prefix: str = '/lme' + dataset_router_tags: Sequence[str] = DEFAULT_TAGS + + @hookimpl + def app_router(self): + router = APIRouter(prefix=self.app_router_prefix, tags=list(self.app_router_tags)) + + @router.get("/") + def get_lme_regions(): + return {key: value["name"] for key, value in regions.items()} + + return router + + # @router.get("/{lme_name}") + # def get_lme_regions(lme_name: str): + # return regions[lme_name].bbox + + # return router + + + + @hookimpl + def dataset_router(self, deps: Dependencies): + router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) + + def get_region_dataset(dataset_id: str, region_id: str): + region = regions[region_id] + bbox = region['bbox'] + + # lat_slice = slice(bbox[2], bbox[3]) + lat_slice = slice(bbox[3], bbox[2]) # air_temperature lats are descending + # lon_slice = slice(bbox[0], bbox[1]) + + # print(lat_slice, lon_slice) + + dataset = deps.dataset(dataset_id) + + sliced = dataset.cf.sel(latitude=lat_slice) + + return sliced + + region_deps = Dependencies( + dataset_ids=deps.dataset_ids, + dataset=get_region_dataset, + cache=deps.cache, + plugins=deps.plugins, + plugin_manager=deps.plugin_manager, + ) + + all_plugins = list(deps.plugin_manager().get_plugins()) + this_plugin = [p for p in all_plugins if p.name == self.name] + + for new_router in deps.plugin_manager().subset_hook_caller('dataset_router', + remove_plugins=this_plugin)(deps=region_deps): + router.include_router(new_router, prefix="/{region_id}") + + return router diff --git a/examples/mean.py b/examples/mean.py new file mode 100644 index 00000000..8c4eeeda --- /dev/null +++ b/examples/mean.py @@ -0,0 +1,31 @@ +from typing import Sequence + +from fastapi import APIRouter, Depends, HTTPException + +from xpublish import Plugin, hookimpl, Dependencies + + +class MeanPlugin(Plugin): + name: str = 'mean' + + dataset_router_prefix: str = '' + dataset_router_tags: Sequence[str] = ['mean'] + + @hookimpl + def dataset_router(self, deps: Dependencies): + router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) + + @router.get('/{var_name}/mean') + def get_mean(var_name: str, dataset=Depends(deps.dataset)): + if var_name not in dataset.variables: + raise HTTPException( + status_code=404, + detail=f"Variable '{var_name}' not found in dataset", + ) + + mean = dataset[var_name].mean() + if mean.isnull(): + return "NaN" + return float(mean) + + return router From 8758d27de270e34056d6fc77242295d2be16d455 Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Mon, 20 May 2024 11:20:11 -0600 Subject: [PATCH 02/10] Exclude examples from distribution --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 435f85dc..1ea4c77b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ line-length = 100 exclude = [ "tests/", "docs/", + "examples/" ] [tool.ruff.mccabe] From 1e0db876afcd4a43504da97acdbd19692b897c7e Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Mon, 20 May 2024 11:30:02 -0600 Subject: [PATCH 03/10] Prune examples from distro. Corrected pyproject.toml back. --- MANIFEST.in | 1 + pyproject.toml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7038e37a..074f775c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ graft xpublish prune docs prune tests prune notebooks +prune examples prune *.egg-info global-exclude *.nc diff --git a/pyproject.toml b/pyproject.toml index aae3b14e..9c585ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,8 +92,7 @@ ignore = [ ] exclude = [ "tests/", - "docs/", - "examples/" + "docs/" ] [tool.ruff.lint.per-file-ignores] From 7275d2172ea470c550eb307099419e327db98fca Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Tue, 21 May 2024 13:11:35 -0600 Subject: [PATCH 04/10] Documented classes and functions for demo --- examples/demo.py | 10 ++++++---- examples/lme.py | 51 +++++++++++++++++++++++++++++++++++------------- examples/mean.py | 17 ++++++++++++---- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/examples/demo.py b/examples/demo.py index 6e28a75f..f15ce549 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -6,14 +6,20 @@ class TutorialDataset(Plugin): + """ + Demonstrates how to create a plugin to load a dataset for demo purposes. + This uses the default xarray tutorial datasets. + """ name: str = "xarray-tutorial-datasets" @hookimpl def get_datasets(self): + """Returns a list of available datasets""" return list(xr.tutorial.file_formats) @hookimpl def get_dataset(self, dataset_id: str): + """Returns a dataset specified by dataset_id""" try: ds = xr.tutorial.open_dataset(dataset_id) if ds.cf.coords['longitude'].dims[0] == 'longitude': @@ -24,10 +30,6 @@ def get_dataset(self, dataset_id: str): except HTTPError: return None - - - - rest = Rest({}) rest.register_plugin(TutorialDataset()) diff --git a/examples/lme.py b/examples/lme.py index 3949e4e1..b95ebbdf 100644 --- a/examples/lme.py +++ b/examples/lme.py @@ -23,6 +23,16 @@ class LmeSubsetPlugin(Plugin): + """ + The LmeSubsetPlugin class is a FastAPI plugin that provides an API for retrieving information about Large Marine Ecosystems (LMEs) and generating datasets for specific LME regions. + + The plugin defines two routers: + - The `app_router` provides a GET endpoint at `/lme` that returns a dictionary of LME names and their IDs. + - The `dataset_router` provides a GET endpoint at `/lme/{region_id}` that takes a dataset ID and a region ID, and returns a subset of the dataset for the specified region. + + The `get_region_dataset` function is used to generate the dataset subset by slicing the dataset along the latitude dimension based on the bounding box of the specified region. + """ + name: str = "lme-subset-plugin" app_router_prefix: str = "/lme" @@ -33,32 +43,45 @@ class LmeSubsetPlugin(Plugin): @hookimpl def app_router(self): - router = APIRouter(prefix=self.app_router_prefix, tags=list(self.app_router_tags)) + """ + Provides an API router for retrieving a list of LME regions. + + The `app_router` function returns an instance of `APIRouter` with the following configuration: + - Prefix: The value of `self.app_router_prefix` + - Tags: A list of values from `self.app_router_tags` + + The router includes a single GET endpoint at the root path ("/") that returns a dictionary mapping region keys to their names. + """ + router = APIRouter(prefix=self.app_router_prefix, + tags=list(self.app_router_tags)) @router.get("/") def get_lme_regions(): return {key: value["name"] for key, value in regions.items()} return router - - # @router.get("/{lme_name}") - # def get_lme_regions(lme_name: str): - # return regions[lme_name].bbox - - # return router - - @hookimpl def dataset_router(self, deps: Dependencies): - router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) + """ + Defines a dataset router that allows accessing datasets for specific regions. + + The `dataset_router` function creates a FastAPI router that provides an endpoint for retrieving a dataset for a specific region. The region is identified by its `region_id`, and the dataset is identified by its `dataset_id`. + + The function uses the `Dependencies` object to access the dataset and perform the necessary slicing operations to extract the data for the specified region. The `get_region_dataset` function is defined within the `dataset_router` function and is responsible for the actual data retrieval and slicing. + + The router is then populated with the necessary routes and returned for inclusion in the main application. + """ + router = APIRouter(prefix=self.dataset_router_prefix, + tags=list(self.dataset_router_tags)) def get_region_dataset(dataset_id: str, region_id: str): region = regions[region_id] bbox = region['bbox'] # lat_slice = slice(bbox[2], bbox[3]) - lat_slice = slice(bbox[3], bbox[2]) # air_temperature lats are descending + # air_temperature lats are descending + lat_slice = slice(bbox[3], bbox[2]) # lon_slice = slice(bbox[0], bbox[1]) # print(lat_slice, lon_slice) @@ -68,7 +91,7 @@ def get_region_dataset(dataset_id: str, region_id: str): sliced = dataset.cf.sel(latitude=lat_slice) return sliced - + region_deps = Dependencies( dataset_ids=deps.dataset_ids, dataset=get_region_dataset, @@ -80,8 +103,8 @@ def get_region_dataset(dataset_id: str, region_id: str): all_plugins = list(deps.plugin_manager().get_plugins()) this_plugin = [p for p in all_plugins if p.name == self.name] - for new_router in deps.plugin_manager().subset_hook_caller('dataset_router', - remove_plugins=this_plugin)(deps=region_deps): + for new_router in deps.plugin_manager().subset_hook_caller('dataset_router', + remove_plugins=this_plugin)(deps=region_deps): router.include_router(new_router, prefix="/{region_id}") return router diff --git a/examples/mean.py b/examples/mean.py index 8c4eeeda..31785bd7 100644 --- a/examples/mean.py +++ b/examples/mean.py @@ -1,11 +1,19 @@ from typing import Sequence - from fastapi import APIRouter, Depends, HTTPException - from xpublish import Plugin, hookimpl, Dependencies class MeanPlugin(Plugin): + """ + Provides a plugin that adds a dataset router for computing the mean of variables in a dataset. + + The `MeanPlugin` class defines the following: + - `name`: The name of the plugin, set to 'mean'. + - `dataset_router_prefix`: The prefix for the dataset router, set to an empty string. + - `dataset_router_tags`: The tags for the dataset router, set to ['mean']. + + The `dataset_router` method creates an APIRouter with the defined prefix and tags, and adds a GET endpoint for computing the mean of a variable in the dataset. If the variable is not found in the dataset, an HTTPException is raised with a 404 status code. + """ name: str = 'mean' dataset_router_prefix: str = '' @@ -13,7 +21,8 @@ class MeanPlugin(Plugin): @hookimpl def dataset_router(self, deps: Dependencies): - router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) + router = APIRouter(prefix=self.dataset_router_prefix, + tags=list(self.dataset_router_tags)) @router.get('/{var_name}/mean') def get_mean(var_name: str, dataset=Depends(deps.dataset)): @@ -22,7 +31,7 @@ def get_mean(var_name: str, dataset=Depends(deps.dataset)): status_code=404, detail=f"Variable '{var_name}' not found in dataset", ) - + mean = dataset[var_name].mean() if mean.isnull(): return "NaN" From e44ce860fe29bd2da400f0b794e18fd342c3bee2 Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Tue, 21 May 2024 13:24:15 -0600 Subject: [PATCH 05/10] Reformatted files using ruff --- examples/demo.py | 11 ++++++++--- examples/lme.py | 17 ++++++++--------- examples/mean.py | 14 +++++++------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/examples/demo.py b/examples/demo.py index f15ce549..b3d344b1 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -10,26 +10,30 @@ class TutorialDataset(Plugin): Demonstrates how to create a plugin to load a dataset for demo purposes. This uses the default xarray tutorial datasets. """ + name: str = "xarray-tutorial-datasets" @hookimpl def get_datasets(self): """Returns a list of available datasets""" return list(xr.tutorial.file_formats) - + @hookimpl def get_dataset(self, dataset_id: str): """Returns a dataset specified by dataset_id""" try: ds = xr.tutorial.open_dataset(dataset_id) - if ds.cf.coords['longitude'].dims[0] == 'longitude': - ds = ds.assign_coords(longitude=(((ds.longitude + 180) % 360) - 180)).sortby('longitude') + if ds.cf.coords["longitude"].dims[0] == "longitude": + ds = ds.assign_coords(longitude=(((ds.longitude + 180) % 360) - 180)).sortby( + "longitude" + ) # TODO: Yeah this should not be assumed... but for regular grids we will viz with rioxarray so for now we will assume ds = ds.rio.write_crs(4326) return ds except HTTPError: return None + rest = Rest({}) rest.register_plugin(TutorialDataset()) @@ -37,6 +41,7 @@ def get_dataset(self, dataset_id: str): from mean import MeanPlugin from lme import LmeSubsetPlugin + rest.register_plugin(MeanPlugin()) rest.register_plugin(LmeSubsetPlugin()) diff --git a/examples/lme.py b/examples/lme.py index b95ebbdf..6c1c4bcb 100644 --- a/examples/lme.py +++ b/examples/lme.py @@ -19,7 +19,7 @@ "NEC": {"bbox": [-81.45, -63.30, 28.70, 44.80], "name": "Northeast Coast"}, } -DEFAULT_TAGS = ['lme', 'large marine ecosystem', 'subset'] +DEFAULT_TAGS = ["lme", "large marine ecosystem", "subset"] class LmeSubsetPlugin(Plugin): @@ -38,7 +38,7 @@ class LmeSubsetPlugin(Plugin): app_router_prefix: str = "/lme" app_router_tags: Sequence[str] = DEFAULT_TAGS - dataset_router_prefix: str = '/lme' + dataset_router_prefix: str = "/lme" dataset_router_tags: Sequence[str] = DEFAULT_TAGS @hookimpl @@ -52,8 +52,7 @@ def app_router(self): The router includes a single GET endpoint at the root path ("/") that returns a dictionary mapping region keys to their names. """ - router = APIRouter(prefix=self.app_router_prefix, - tags=list(self.app_router_tags)) + router = APIRouter(prefix=self.app_router_prefix, tags=list(self.app_router_tags)) @router.get("/") def get_lme_regions(): @@ -72,12 +71,11 @@ def dataset_router(self, deps: Dependencies): The router is then populated with the necessary routes and returned for inclusion in the main application. """ - router = APIRouter(prefix=self.dataset_router_prefix, - tags=list(self.dataset_router_tags)) + router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) def get_region_dataset(dataset_id: str, region_id: str): region = regions[region_id] - bbox = region['bbox'] + bbox = region["bbox"] # lat_slice = slice(bbox[2], bbox[3]) # air_temperature lats are descending @@ -103,8 +101,9 @@ def get_region_dataset(dataset_id: str, region_id: str): all_plugins = list(deps.plugin_manager().get_plugins()) this_plugin = [p for p in all_plugins if p.name == self.name] - for new_router in deps.plugin_manager().subset_hook_caller('dataset_router', - remove_plugins=this_plugin)(deps=region_deps): + for new_router in deps.plugin_manager().subset_hook_caller( + "dataset_router", remove_plugins=this_plugin + )(deps=region_deps): router.include_router(new_router, prefix="/{region_id}") return router diff --git a/examples/mean.py b/examples/mean.py index 31785bd7..4ed55e09 100644 --- a/examples/mean.py +++ b/examples/mean.py @@ -1,6 +1,6 @@ from typing import Sequence from fastapi import APIRouter, Depends, HTTPException -from xpublish import Plugin, hookimpl, Dependencies +from xpublish import Plugin, hookimpl, Dependencies class MeanPlugin(Plugin): @@ -14,17 +14,17 @@ class MeanPlugin(Plugin): The `dataset_router` method creates an APIRouter with the defined prefix and tags, and adds a GET endpoint for computing the mean of a variable in the dataset. If the variable is not found in the dataset, an HTTPException is raised with a 404 status code. """ - name: str = 'mean' - dataset_router_prefix: str = '' - dataset_router_tags: Sequence[str] = ['mean'] + name: str = "mean" + + dataset_router_prefix: str = "" + dataset_router_tags: Sequence[str] = ["mean"] @hookimpl def dataset_router(self, deps: Dependencies): - router = APIRouter(prefix=self.dataset_router_prefix, - tags=list(self.dataset_router_tags)) + router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) - @router.get('/{var_name}/mean') + @router.get("/{var_name}/mean") def get_mean(var_name: str, dataset=Depends(deps.dataset)): if var_name not in dataset.variables: raise HTTPException( From ec146c160db008db328499b31fd5803568398d48 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 19:25:47 +0000 Subject: [PATCH 06/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/README.md | 4 +-- examples/demo.py | 12 ++++----- examples/environment.yaml | 1 - examples/lme.py | 52 +++++++++++++++++++-------------------- examples/mean.py | 17 +++++++------ pyproject.toml | 4 +-- 6 files changed, 43 insertions(+), 47 deletions(-) diff --git a/examples/README.md b/examples/README.md index 4bb97288..65825d75 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,7 +13,7 @@ micromamba activate xpublish-exploring > Note: This method may time out -Create the environment and activate: +Create the environment and activate: ```bash conda env create -f environment.yaml @@ -60,4 +60,4 @@ This plugin will provide a subset of data based on the bounding box defined for ## Combining the Plugins -Plugins can be combined by adding their URLs together. For example, to find the mean of a specific region, you could access http://localhost:9000/datasets/air_temperature/lme/EC/air/mean adding the variable and mean calculation to the end. \ No newline at end of file +Plugins can be combined by adding their URLs together. For example, to find the mean of a specific region, you could access http://localhost:9000/datasets/air_temperature/lme/EC/air/mean adding the variable and mean calculation to the end. diff --git a/examples/demo.py b/examples/demo.py index b3d344b1..32370c0b 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -1,4 +1,3 @@ -import cf_xarray import xarray as xr from requests import HTTPError @@ -6,12 +5,11 @@ class TutorialDataset(Plugin): - """ - Demonstrates how to create a plugin to load a dataset for demo purposes. + """Demonstrates how to create a plugin to load a dataset for demo purposes. This uses the default xarray tutorial datasets. """ - name: str = "xarray-tutorial-datasets" + name: str = 'xarray-tutorial-datasets' @hookimpl def get_datasets(self): @@ -23,9 +21,9 @@ def get_dataset(self, dataset_id: str): """Returns a dataset specified by dataset_id""" try: ds = xr.tutorial.open_dataset(dataset_id) - if ds.cf.coords["longitude"].dims[0] == "longitude": + if ds.cf.coords['longitude'].dims[0] == 'longitude': ds = ds.assign_coords(longitude=(((ds.longitude + 180) % 360) - 180)).sortby( - "longitude" + 'longitude' ) # TODO: Yeah this should not be assumed... but for regular grids we will viz with rioxarray so for now we will assume ds = ds.rio.write_crs(4326) @@ -39,8 +37,8 @@ def get_dataset(self, dataset_id: str): ### For this tutorial, you can uncomment the following lines to activate the other plugins: -from mean import MeanPlugin from lme import LmeSubsetPlugin +from mean import MeanPlugin rest.register_plugin(MeanPlugin()) rest.register_plugin(LmeSubsetPlugin()) diff --git a/examples/environment.yaml b/examples/environment.yaml index 4b2375b4..d5310f78 100644 --- a/examples/environment.yaml +++ b/examples/environment.yaml @@ -23,4 +23,3 @@ dependencies: - regionmask - ipykernel - pooch - \ No newline at end of file diff --git a/examples/lme.py b/examples/lme.py index 6c1c4bcb..21152874 100644 --- a/examples/lme.py +++ b/examples/lme.py @@ -1,30 +1,30 @@ from typing import Sequence from fastapi import APIRouter -from xpublish import Plugin, Dependencies, hookimpl + +from xpublish import Dependencies, Plugin, hookimpl regions = { - "GB": {"bbox": [-69.873, -65.918, 40.280, 42.204], "name": "Georges Bank"}, - "GOM": {"bbox": [-70.975, -65.375, 40.375, 45.125], "name": "Gulf Of Maine"}, - "MAB": { - "bbox": [-77.036, -70.005, 35.389, 41.640], - "name": "MidAtlantic Bight", + 'GB': {'bbox': [-69.873, -65.918, 40.280, 42.204], 'name': 'Georges Bank'}, + 'GOM': {'bbox': [-70.975, -65.375, 40.375, 45.125], 'name': 'Gulf Of Maine'}, + 'MAB': { + 'bbox': [-77.036, -70.005, 35.389, 41.640], + 'name': 'MidAtlantic Bight', }, - "NESHELF": { - "bbox": [-77.45, -66.35, 34.50, 44.50], - "name": "North East Shelf", + 'NESHELF': { + 'bbox': [-77.45, -66.35, 34.50, 44.50], + 'name': 'North East Shelf', }, - "SS": {"bbox": [-66.775, -65.566, 41.689, 45.011], "name": "Scotian Shelf"}, - "EC": {"bbox": [-81.75, -65.375, 25.000, 45.125], "name": "East Coast"}, - "NEC": {"bbox": [-81.45, -63.30, 28.70, 44.80], "name": "Northeast Coast"}, + 'SS': {'bbox': [-66.775, -65.566, 41.689, 45.011], 'name': 'Scotian Shelf'}, + 'EC': {'bbox': [-81.75, -65.375, 25.000, 45.125], 'name': 'East Coast'}, + 'NEC': {'bbox': [-81.45, -63.30, 28.70, 44.80], 'name': 'Northeast Coast'}, } -DEFAULT_TAGS = ["lme", "large marine ecosystem", "subset"] +DEFAULT_TAGS = ['lme', 'large marine ecosystem', 'subset'] class LmeSubsetPlugin(Plugin): - """ - The LmeSubsetPlugin class is a FastAPI plugin that provides an API for retrieving information about Large Marine Ecosystems (LMEs) and generating datasets for specific LME regions. + """The LmeSubsetPlugin class is a FastAPI plugin that provides an API for retrieving information about Large Marine Ecosystems (LMEs) and generating datasets for specific LME regions. The plugin defines two routers: - The `app_router` provides a GET endpoint at `/lme` that returns a dictionary of LME names and their IDs. @@ -33,18 +33,17 @@ class LmeSubsetPlugin(Plugin): The `get_region_dataset` function is used to generate the dataset subset by slicing the dataset along the latitude dimension based on the bounding box of the specified region. """ - name: str = "lme-subset-plugin" + name: str = 'lme-subset-plugin' - app_router_prefix: str = "/lme" + app_router_prefix: str = '/lme' app_router_tags: Sequence[str] = DEFAULT_TAGS - dataset_router_prefix: str = "/lme" + dataset_router_prefix: str = '/lme' dataset_router_tags: Sequence[str] = DEFAULT_TAGS @hookimpl def app_router(self): - """ - Provides an API router for retrieving a list of LME regions. + """Provides an API router for retrieving a list of LME regions. The `app_router` function returns an instance of `APIRouter` with the following configuration: - Prefix: The value of `self.app_router_prefix` @@ -54,16 +53,15 @@ def app_router(self): """ router = APIRouter(prefix=self.app_router_prefix, tags=list(self.app_router_tags)) - @router.get("/") + @router.get('/') def get_lme_regions(): - return {key: value["name"] for key, value in regions.items()} + return {key: value['name'] for key, value in regions.items()} return router @hookimpl def dataset_router(self, deps: Dependencies): - """ - Defines a dataset router that allows accessing datasets for specific regions. + """Defines a dataset router that allows accessing datasets for specific regions. The `dataset_router` function creates a FastAPI router that provides an endpoint for retrieving a dataset for a specific region. The region is identified by its `region_id`, and the dataset is identified by its `dataset_id`. @@ -75,7 +73,7 @@ def dataset_router(self, deps: Dependencies): def get_region_dataset(dataset_id: str, region_id: str): region = regions[region_id] - bbox = region["bbox"] + bbox = region['bbox'] # lat_slice = slice(bbox[2], bbox[3]) # air_temperature lats are descending @@ -102,8 +100,8 @@ def get_region_dataset(dataset_id: str, region_id: str): this_plugin = [p for p in all_plugins if p.name == self.name] for new_router in deps.plugin_manager().subset_hook_caller( - "dataset_router", remove_plugins=this_plugin + 'dataset_router', remove_plugins=this_plugin )(deps=region_deps): - router.include_router(new_router, prefix="/{region_id}") + router.include_router(new_router, prefix='/{region_id}') return router diff --git a/examples/mean.py b/examples/mean.py index 4ed55e09..06a138cf 100644 --- a/examples/mean.py +++ b/examples/mean.py @@ -1,11 +1,12 @@ from typing import Sequence + from fastapi import APIRouter, Depends, HTTPException -from xpublish import Plugin, hookimpl, Dependencies + +from xpublish import Dependencies, Plugin, hookimpl class MeanPlugin(Plugin): - """ - Provides a plugin that adds a dataset router for computing the mean of variables in a dataset. + """Provides a plugin that adds a dataset router for computing the mean of variables in a dataset. The `MeanPlugin` class defines the following: - `name`: The name of the plugin, set to 'mean'. @@ -15,16 +16,16 @@ class MeanPlugin(Plugin): The `dataset_router` method creates an APIRouter with the defined prefix and tags, and adds a GET endpoint for computing the mean of a variable in the dataset. If the variable is not found in the dataset, an HTTPException is raised with a 404 status code. """ - name: str = "mean" + name: str = 'mean' - dataset_router_prefix: str = "" - dataset_router_tags: Sequence[str] = ["mean"] + dataset_router_prefix: str = '' + dataset_router_tags: Sequence[str] = ['mean'] @hookimpl def dataset_router(self, deps: Dependencies): router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) - @router.get("/{var_name}/mean") + @router.get('/{var_name}/mean') def get_mean(var_name: str, dataset=Depends(deps.dataset)): if var_name not in dataset.variables: raise HTTPException( @@ -34,7 +35,7 @@ def get_mean(var_name: str, dataset=Depends(deps.dataset)): mean = dataset[var_name].mean() if mean.isnull(): - return "NaN" + return 'NaN' return float(mean) return router diff --git a/pyproject.toml b/pyproject.toml index 9c585ca6..41ebb9af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,8 +91,8 @@ ignore = [ "C901", ] exclude = [ - "tests/", - "docs/" + "tests/", + "docs/", ] [tool.ruff.lint.per-file-ignores] From 19cd1f899cfe10ec090a858798b3bdb4be1f5abb Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Tue, 21 May 2024 13:52:46 -0600 Subject: [PATCH 07/10] Fixed whitespace and documentation --- examples/demo.py | 19 +++++++++++++++---- examples/mean.py | 13 +++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/examples/demo.py b/examples/demo.py index 32370c0b..9e5392ea 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -6,6 +6,7 @@ class TutorialDataset(Plugin): """Demonstrates how to create a plugin to load a dataset for demo purposes. + This uses the default xarray tutorial datasets. """ @@ -13,17 +14,27 @@ class TutorialDataset(Plugin): @hookimpl def get_datasets(self): - """Returns a list of available datasets""" + """Returns a list of available datasets. + + This function returns a list of the available datasets that can be loaded using the xarray.tutorial.file_formats module. + """ return list(xr.tutorial.file_formats) @hookimpl def get_dataset(self, dataset_id: str): - """Returns a dataset specified by dataset_id""" + """Retrieves a dataset from the xarray tutorial dataset by the given dataset ID. + + Args: + dataset_id (str): The ID of the dataset to retrieve. + + Returns: + xarray.Dataset: The retrieved dataset, or None if the dataset could not be loaded. + """ try: ds = xr.tutorial.open_dataset(dataset_id) - if ds.cf.coords['longitude'].dims[0] == 'longitude': + if ds.cf.coords["longitude"].dims[0] == "longitude": ds = ds.assign_coords(longitude=(((ds.longitude + 180) % 360) - 180)).sortby( - 'longitude' + "longitude" ) # TODO: Yeah this should not be assumed... but for regular grids we will viz with rioxarray so for now we will assume ds = ds.rio.write_crs(4326) diff --git a/examples/mean.py b/examples/mean.py index 06a138cf..d91383e7 100644 --- a/examples/mean.py +++ b/examples/mean.py @@ -23,9 +23,18 @@ class MeanPlugin(Plugin): @hookimpl def dataset_router(self, deps: Dependencies): + """Provides a route to retrieve the mean value of a variable in a dataset. + + Args: + var_name (str): The name of the variable to retrieve the mean for. + dataset (Dataset): The dataset containing the variable. + + Returns: + float: The mean value of the variable, or 'NaN' if the mean is null. + """ router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) - @router.get('/{var_name}/mean') + @router.get("/{var_name}/mean") def get_mean(var_name: str, dataset=Depends(deps.dataset)): if var_name not in dataset.variables: raise HTTPException( @@ -35,7 +44,7 @@ def get_mean(var_name: str, dataset=Depends(deps.dataset)): mean = dataset[var_name].mean() if mean.isnull(): - return 'NaN' + return "NaN" return float(mean) return router From 69e8676d90339ce5304fa6f2bcdde6e7e856e560 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 20:31:00 +0000 Subject: [PATCH 08/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/demo.py | 4 ++-- examples/mean.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/demo.py b/examples/demo.py index 9e5392ea..c834d605 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -32,9 +32,9 @@ def get_dataset(self, dataset_id: str): """ try: ds = xr.tutorial.open_dataset(dataset_id) - if ds.cf.coords["longitude"].dims[0] == "longitude": + if ds.cf.coords['longitude'].dims[0] == 'longitude': ds = ds.assign_coords(longitude=(((ds.longitude + 180) % 360) - 180)).sortby( - "longitude" + 'longitude' ) # TODO: Yeah this should not be assumed... but for regular grids we will viz with rioxarray so for now we will assume ds = ds.rio.write_crs(4326) diff --git a/examples/mean.py b/examples/mean.py index d91383e7..e31942fa 100644 --- a/examples/mean.py +++ b/examples/mean.py @@ -34,7 +34,7 @@ def dataset_router(self, deps: Dependencies): """ router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) - @router.get("/{var_name}/mean") + @router.get('/{var_name}/mean') def get_mean(var_name: str, dataset=Depends(deps.dataset)): if var_name not in dataset.variables: raise HTTPException( @@ -44,7 +44,7 @@ def get_mean(var_name: str, dataset=Depends(deps.dataset)): mean = dataset[var_name].mean() if mean.isnull(): - return "NaN" + return 'NaN' return float(mean) return router From 62efbb45575ef1e3eaf753d4afb864938e09d397 Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Wed, 22 May 2024 07:31:48 -0600 Subject: [PATCH 09/10] Documented function correctly --- examples/mean.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/mean.py b/examples/mean.py index d91383e7..3b2b00f8 100644 --- a/examples/mean.py +++ b/examples/mean.py @@ -26,11 +26,7 @@ def dataset_router(self, deps: Dependencies): """Provides a route to retrieve the mean value of a variable in a dataset. Args: - var_name (str): The name of the variable to retrieve the mean for. - dataset (Dataset): The dataset containing the variable. - - Returns: - float: The mean value of the variable, or 'NaN' if the mean is null. + deps (Dependencies): The dependencies for plugin routers """ router = APIRouter(prefix=self.dataset_router_prefix, tags=list(self.dataset_router_tags)) From 7bc4689131f4f558f0f19fa5ca55f0f82591a5b6 Mon Sep 17 00:00:00 2001 From: Jonathan Joyce Date: Wed, 22 May 2024 09:59:06 -0600 Subject: [PATCH 10/10] Ignore examples and docs for code coverage --- codecov.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codecov.yml b/codecov.yml index f07c1c5b..94d671bb 100644 --- a/codecov.yml +++ b/codecov.yml @@ -21,3 +21,7 @@ coverage: default: threshold: 0% if_not_found: success + +ignore: + - "examples" + - "docs"