From 83c642dd9631f979d52668546546a9d7d81ad1bf Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 8 Dec 2023 12:19:41 +0100 Subject: [PATCH 1/4] Add `export_collection`, `export_workspace`, `stac_update`; `save_results` returns the STAC resource instead of boolean `true` https://github.com/Open-EO/openeo-api/issues/376 --- CHANGELOG.md | 7 +++++ meta/subtype-schemas.json | 24 ++++++++++++++++ proposals/export_collection.json | 45 ++++++++++++++++++++++++++++++ proposals/export_workspace.json | 48 ++++++++++++++++++++++++++++++++ proposals/stac_update.json | 41 +++++++++++++++++++++++++++ save_result.json | 9 +++--- 6 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 proposals/export_collection.json create mode 100644 proposals/export_workspace.json create mode 100644 proposals/stac_update.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c2a4f1d7..a964c3c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased / Draft +### Added + +- `export_collection` +- `export_workspace` +- `stac_update` + ### Changed - `clip`: Throw an exception if min > max [#472](https://github.com/Open-EO/openeo-processes/issues/472) +- `save_results`: Returns the STAC resource instead of boolean `true` [API#376](https://github.com/Open-EO/openeo-api/issues/376) ### Fixed diff --git a/meta/subtype-schemas.json b/meta/subtype-schemas.json index 347df234..f113ea24 100644 --- a/meta/subtype-schemas.json +++ b/meta/subtype-schemas.json @@ -278,6 +278,23 @@ "description": "A raster data cube, which is a data cube with two dimension of type spatial (x and y). This has been deprecated in favour of `datacube`.", "deprecated": true }, + "stac": { + "type": "object", + "subtype": "stac", + "title": "STAC resource", + "description": "A STAC Catalog, Collection, or Item, as defined by the [STAC specification](https://stacspec.org) version 0.9.0 or later.", + "oneOf": [ + { + "$ref": "http://schemas.stacspec.org/v1.0.0/catalog-spec/json-schema/catalog.json" + }, + { + "$ref": "http://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json" + }, + { + "$ref": "http://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json" + } + ] + }, "temporal-interval": { "type": "array", "subtype": "temporal-interval", @@ -413,6 +430,13 @@ "title": "WKT2 definition", "description": "Specifies details about cartographic projections as WKT2 string. Refers to the latest WKT2 version (currently [WKT2:2018](http://docs.opengeospatial.org/is/18-010r7/18-010r7.html) / ISO 19162:2018) unless otherwise stated by the process." }, + "workspace-id": { + "type": "string", + "subtype": "workspace-id", + "title": "Workspace ID", + "description": "A workspace identifier from the list of available workspaces.", + "pattern": "^[\\w\\-\\.~]+$" + }, "year": { "type": "integer", "subtype": "year", diff --git a/proposals/export_collection.json b/proposals/export_collection.json new file mode 100644 index 00000000..e5014811 --- /dev/null +++ b/proposals/export_collection.json @@ -0,0 +1,45 @@ +{ + "id": "export_collection", + "summary": "Export data to an openEO collection", + "description": "Exports the given processing results made available through a STAC resource (e.g., a STAC Collection or Item) to the given openEO collection. The STAC resource itself is exported with all STAC resources and assets underneath.", + "categories": [ + "export", + "stac" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The data to export to the openEO collection.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + { + "name": "collection", + "description": "The identifier of the collection to export to. If the collection **exists**, the data will be added to the collection. If the collection **does not exist** yet, it will be created based on the given STAC metadata.", + "schema": [ + { + "title": "New Collection", + "type": "string", + "pattern": "^[A-Za-z0-9_\\-\\.~/]+$" + }, + { + "title": "Existing Collection", + "type": "string", + "subtype": "collection-id", + "pattern": "^[A-Za-z0-9_\\-\\.~/]+$" + } + ] + } + ], + "returns": { + "description": "Returns the collection identifier.", + "schema": { + "type": "string", + "subtype": "collection-id", + "pattern": "^[A-Za-z0-9_\\-\\.~/]+$" + } + } +} diff --git a/proposals/export_workspace.json b/proposals/export_workspace.json new file mode 100644 index 00000000..c469254f --- /dev/null +++ b/proposals/export_workspace.json @@ -0,0 +1,48 @@ +{ + "id": "export_workspace", + "summary": "Export data to a cloud user workspace", + "description": "Exports the given processing results made available through a STAC resource (e.g., a STAC Collection) to the given user workspace. The STAC resource itself is exported with all STAC resources and assets underneath.", + "categories": [ + "export", + "stac" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The data to export to the user workspace as a STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + { + "name": "workspace", + "description": "The identifier of the workspace to export to.", + "schema": { + "type": "string", + "pattern": "^[\\w\\-\\.~]+$", + "subtype": "workspace-id" + } + }, + { + "name": "merge", + "description": "Provides a cloud-specific path identifier to a STAC resource to merge the given STAC resource into. If not provided, the STAC resource is kept separate from any other STAC resources in the workspace.", + "schema": { + "type": [ + "string", + "null" + ] + }, + "optional": true, + "default": null + } + ], + "returns": { + "description": "Returns the potentially updated STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + } +} diff --git a/proposals/stac_update.json b/proposals/stac_update.json new file mode 100644 index 00000000..7df3a29b --- /dev/null +++ b/proposals/stac_update.json @@ -0,0 +1,41 @@ +{ + "id": "stac_update", + "summary": "Export data to an openEO collection", + "description": "Exports the given processing results made available through a STAC resource (e.g., a STAC Collection or Item) to the given openEO collection. The STAC resource itself is exported with all STAC resources and assets underneath.", + "categories": [ + "stac" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The existing STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + { + "name": "merge", + "description": "A potentially incomplete STAC resource that should be merged with the existing STAC resource. It follows the [RFC 7386: JSON Merge Path](https://tools.ietf.org/html/rfc7386) specification.", + "schema": { + "type": "object" + } + } + ], + "returns": { + "description": "Returns the updated STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + "links": [ + { + "href": "https://www.rfc-editor.org/rfc/rfc7386.html", + "title": "RFC 7386: JSON Merge Path", + "type": "text/html", + "rel": "about" + } + ] +} diff --git a/save_result.json b/save_result.json index 7b952ead..418bafc7 100644 --- a/save_result.json +++ b/save_result.json @@ -4,7 +4,8 @@ "description": "Makes the processed data available in the given file format to the corresponding medium that is relevant for the context this processes is applied in:\n\n* For **batch jobs** the data is stored on the back-end. STAC-compatible metadata is usually made available with the processed data.\n* For **synchronous processing** the data is sent to the client as a direct response to the request.\n* **Secondary web services** are provided with the processed data so that it can make use of it (e.g., visualize it). Web service may require the data in a certain format. Please refer to the documentation of the individual service types for details.", "categories": [ "cubes", - "export" + "export", + "stac" ], "parameters": [ { @@ -35,10 +36,10 @@ } ], "returns": { - "description": "Always returns `true` as in case of an error an exception is thrown which aborts the execution of the process.", + "description": "Returns the STAC resource that was created in the process of saving the result.", "schema": { - "type": "boolean", - "const": true + "type": "object", + "subtype": "stac" } }, "exceptions": { From a208cc104f69eff8088ba92c08137a91286da15d Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 11 Dec 2023 13:24:06 +0100 Subject: [PATCH 2/4] Update stac_update/modify --- CHANGELOG.md | 2 +- proposals/{stac_update.json => stac_modify.json} | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) rename proposals/{stac_update.json => stac_modify.json} (59%) diff --git a/CHANGELOG.md b/CHANGELOG.md index a964c3c1..559e73d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `export_collection` - `export_workspace` -- `stac_update` +- `stac_modify` ### Changed diff --git a/proposals/stac_update.json b/proposals/stac_modify.json similarity index 59% rename from proposals/stac_update.json rename to proposals/stac_modify.json index 7df3a29b..8377c8a8 100644 --- a/proposals/stac_update.json +++ b/proposals/stac_modify.json @@ -1,7 +1,7 @@ { - "id": "stac_update", - "summary": "Export data to an openEO collection", - "description": "Exports the given processing results made available through a STAC resource (e.g., a STAC Collection or Item) to the given openEO collection. The STAC resource itself is exported with all STAC resources and assets underneath.", + "id": "stac_modify", + "summary": "Updates an existing STAC resource", + "description": "Modifies the given STAC resource (e.g., a STAC Collection or Item) based on the given changeset.", "categories": [ "stac" ], @@ -16,15 +16,15 @@ } }, { - "name": "merge", - "description": "A potentially incomplete STAC resource that should be merged with the existing STAC resource. It follows the [RFC 7386: JSON Merge Path](https://tools.ietf.org/html/rfc7386) specification.", + "name": "changes", + "description": "A potentially incomplete STAC resource that should be merged with the existing STAC resource. It follows the [RFC 7386: JSON Merge Patch](https://www.rfc-editor.org/rfc/rfc7386.html) specification.", "schema": { "type": "object" } } ], "returns": { - "description": "Returns the updated STAC resource.", + "description": "Returns the modified STAC resource.", "schema": { "type": "object", "subtype": "stac" @@ -33,7 +33,7 @@ "links": [ { "href": "https://www.rfc-editor.org/rfc/rfc7386.html", - "title": "RFC 7386: JSON Merge Path", + "title": "RFC 7386: JSON Merge Patch", "type": "text/html", "rel": "about" } From 3ed4396e9859acb772b40ccc549ab648d1f77417 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 11 Dec 2023 13:32:23 +0100 Subject: [PATCH 3/4] Added details about STAC support. --- meta/implementation.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/meta/implementation.md b/meta/implementation.md index 65a24430..e1d5dde9 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -226,3 +226,24 @@ We have found some libraries that can be used for an implementation: - Julia: [Statistics.quantile](https://docs.julialang.org/en/v1/stdlib/Statistics/#Statistics.quantile!), type 7 is the default. - Python: [numpy](https://numpy.org/doc/stable/reference/generated/numpy.quantile.html), [pandas](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.quantile.html), [xarray](http://xarray.pydata.org/en/stable/generated/xarray.DataArray.quantile.html) - type 7 (called 'linear' for the interpolation parameter) is the default for all of them. - R: [quantile](https://stat.ethz.ch/R-manual/R-patched/library/stats/html/quantile.html) - type 7 is the default. + +## STAC support + +The subtype `stac` is an abstract type that refers to a STAC resource of any type (Catalog, Collection, or Item). +It can refer to: +- static STAC resources, e.g. hosted on cloud storage +- "dynamic" STAC resources made available via a STAC API +- a STAC JSON representation embedded as an argument into an openEO user-defined process + +### stac_modify + +The process `stac_modify` updates a given STAC resource based on +[RFC 7386: JSON Merge Patch](https://www.rfc-editor.org/rfc/rfc7386.html). +For static STAC resources, the content of the JSON files shall be updated according to RFC 7386. + +If the underlying STAC resource is part of an API, the following HTTP endpoints shall be used for the updates: +- For STAC Items: `PATCH /collections/{collectionId}/items/{featureId}` + according to the [Transaction Extension](https://github.com/stac-api-extensions/transaction) +- For STAC Collections: `PATCH /collections/{collectionId}` + according to the [Collection Transaction Extension](https://github.com/stac-api-extensions/collection-transaction) +- For STAC Catalogs there is no API support for updates. \ No newline at end of file From 58a3c61f06f918f9bc9d4aa75aca70dc5b87ea6a Mon Sep 17 00:00:00 2001 From: Michele Claus <31700619+clausmichele@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:26:49 +0200 Subject: [PATCH 4/4] Update meta/implementation.md Co-authored-by: Matthias Mohr --- meta/implementation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/implementation.md b/meta/implementation.md index ac424a2f..ccdb8ed3 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -236,7 +236,7 @@ We have found some libraries that can be used for an implementation: The subtype `stac` is an abstract type that refers to a STAC resource of any type (Catalog, Collection, or Item). It can refer to: - static STAC resources, e.g. hosted on cloud storage -- "dynamic" STAC resources made available via a STAC API +- dynamic STAC resources made available via a STAC API - a STAC JSON representation embedded as an argument into an openEO user-defined process ### stac_modify