Merge pull request #562 from crim-ca/cli-provider-package

CHANGES.rst

@@ -12,7 +12,17 @@ Changes
 
 Changes:
 --------
-- No change.
+- Add ``GET /providers/{provider_id}/processes/{process_id}/package`` endpoint that allows retrieval of the `CWL`
+  `Application Package` definition generated for the specific `Provider`'s `Process` definition.
+- Add `CLI` ``package`` operation to request the remote `Provider` or local `Process` `CWL` `Application Package`.
+- Add `CLI` output reporting of performed HTTP requests details when using the ``--debug/-d`` option.
+- Modify default behavior of ``visibility`` field (under ``processDescription`` or ``processDescription.process``)
+  to employ the expected functionality by native `OGC API - Processes` clients that do not support this option
+  (i.e.: ``public`` by default), and to align resolution strategy with deployments by direct `CWL` payload which do not
+  include this feature either. A `Process` deployment that desires to employ this feature (``visibility: private``) will
+  have to provide the value explicitly, or update the deployed `Process` definition afterwards with the relevant
+  ``PUT`` request. Since ``public`` will now be used by default, the `CLI` will not automatically inject the value
+  in the payload anymore when omitted.
 
 Fixes:
 ------
@@ -24,6 +34,10 @@ Fixes:
 - Fix ``get_cwl_io_type`` function that would modify the I/O definition passed as argument, which could lead to failing
   `CWL` ``class`` reference resolutions later on due to different ``type`` with ``org.w3id.cwl.cwl`` prefix simplified
   before ``cwltool`` had the chance to resolve them.
+- Fix ``links`` listing duplicated in response from `Process` deployment.
+  Links will only be listed within the returned ``processSummary`` to respect the `OGC API - Processes` schema.
+- Fix `CLI` not removing embedded ``links`` in ``processSummary`` from ``deploy`` operation response
+  when ``-nL``/``--no-links`` option is specified.
 
 .. _changes_4.31.0:
 

tests/functional/test_cli.py

@@ -102,6 +102,8 @@ def tearDownClass(cls):
 
 
 class TestWeaverClient(TestWeaverClientBase):
+    test_tmp_dir = None  # type: str
+
     @classmethod
     def setUpClass(cls):
         super(TestWeaverClient, cls).setUpClass()
@@ -342,6 +344,40 @@ def test_deploy_with_undeploy(self):
         assert result.success
         assert "undefined" not in result.message
 
+    def test_deploy_private_process_description(self):
+        test_id = f"{self.test_process_prefix}private-process-description"
+        payload = self.retrieve_payload("Echo", "deploy", local=True)
+        package = self.retrieve_payload("Echo", "package", local=True)
+        payload.pop("executionUnit", None)
+        process = payload["processDescription"].pop("process")
+        payload["processDescription"].update(process)
+        payload["processDescription"]["visibility"] = Visibility.PRIVATE
+
+        result = mocked_sub_requests(self.app, self.client.deploy, test_id, payload, package)
+        assert result.success
+        assert "processSummary" in result.body
+        assert result.body["processSummary"]["id"] == test_id
+
+        result = mocked_sub_requests(self.app, self.client.describe, test_id)
+        assert not result.success
+        assert result.code == 403
+
+    def test_deploy_private_process_nested(self):
+        test_id = f"{self.test_process_prefix}private-process-nested"
+        payload = self.retrieve_payload("Echo", "deploy", local=True)
+        package = self.retrieve_payload("Echo", "package", local=True)
+        payload.pop("executionUnit", None)
+        payload["processDescription"]["process"]["visibility"] = Visibility.PRIVATE
+
+        result = mocked_sub_requests(self.app, self.client.deploy, test_id, payload, package)
+        assert result.success
+        assert "processSummary" in result.body
+        assert result.body["processSummary"]["id"] == test_id
+
+        result = mocked_sub_requests(self.app, self.client.describe, test_id)
+        assert not result.success
+        assert result.code == 403
+
     def test_undeploy(self):
         # deploy a new process to leave the test one available
         other_payload = copy.deepcopy(self.test_payload["Echo"])
@@ -771,6 +807,7 @@ def test_deploy_no_process_id_option(self):
                 "-u", self.url,
                 "--body", payload,  # no --process/--id, but available through --body
                 "--cwl", package,
+                "-D",  # avoid conflict just in case
             ],
             trim=False,
             entrypoint=weaver_cli,
@@ -779,6 +816,28 @@ def test_deploy_no_process_id_option(self):
         assert any("\"id\": \"Echo\"" in line for line in lines)
         assert any("\"deploymentDone\": true" in line for line in lines)
 
+    def test_deploy_no_links(self):
+        payload = self.retrieve_payload("Echo", "deploy", local=True, ref_found=True)
+        package = self.retrieve_payload("Echo", "package", local=True, ref_found=True)
+        lines = mocked_sub_requests(
+            self.app, run_command,
+            [
+                # "weaver",
+                "deploy",
+                "-u", self.url,
+                "--body", payload,
+                "--cwl", package,
+                "-nL",
+                "-D",
+            ],
+            trim=False,
+            entrypoint=weaver_cli,
+            only_local=True,
+        )
+        # ignore indents of fields from formatted JSON content
+        assert any("\"id\": \"Echo\"" in line for line in lines)
+        assert all("\"links\":" not in line for line in lines)
+
     def test_deploy_docker_auth_help(self):
         """
         Validate some special handling to generate special combinations of help argument details.
@@ -1255,6 +1314,49 @@ def test_describe_no_links(self):
         assert any("\"outputs\": {" in line for line in lines)
         assert all("\"links\":" not in line for line in lines)
 
+    def test_package_process(self):
+        payload = self.retrieve_payload("Echo", "deploy", local=True, ref_found=True)
+        package = self.retrieve_payload("Echo", "package", local=True)
+        lines = mocked_sub_requests(
+            self.app, run_command,
+            [
+                # weaver
+                "deploy",
+                "-u", self.url,
+                "--body", payload,
+                "--cwl", package,
+                "--id", "test-echo-get-package"
+            ],
+            trim=False,
+            entrypoint=weaver_cli,
+            only_local=True,
+        )
+        assert any("\"id\": \"test-echo-get-package\"" in line for line in lines)
+
+        lines = mocked_sub_requests(
+            self.app, run_command,
+            [
+                # weaver
+                "package",
+                "-u", self.url,
+                "-p", "test-echo-get-package"
+            ],
+            trim=False,
+            entrypoint=weaver_cli,
+            only_local=True,
+        )
+        assert any("\"cwlVersion\"" in line for line in lines)
+        cwl = json.loads("".join(lines))
+
+        # package not 100% the same, but equivalent definitions
+        # check that what is returned is at least relatively equal
+        cwl.pop("$id", None)
+        cwl.pop("$schema", None)
+        pkg = package.copy()
+        pkg["inputs"] = [{"id": key, **val} for key, val in package["inputs"].items()]  # pylint: disable=E1136
+        pkg["outputs"] = [{"id": key, **val} for key, val in package["outputs"].items()]  # pylint: disable=E1136
+        assert cwl == pkg
+
     def test_execute_inputs_capture(self):
         """
         Verify that specified inputs are captured for a limited number of 1 item per ``-I`` option.

tests/functional/utils.py

@@ -38,6 +38,9 @@
     from typing import Any, Dict, Iterable, Optional, Tuple, Union
     from typing_extensions import Literal
 
+    from pyramid.config import Configurator
+    from webtest import TestApp
+
     from weaver.typedefs import (
         AnyRequestMethod,
         AnyResponseType,
@@ -305,7 +308,10 @@ class WpsConfigBase(unittest.TestCase):
     json_headers = {"Accept": ContentType.APP_JSON, "Content-Type": ContentType.APP_JSON}
     monitor_timeout = 30
     monitor_interval = 1
-    settings = {}  # type: SettingsType
+    settings = {}   # type: SettingsType
+    config = None   # type: Configurator
+    app = None      # type: TestApp
+    url = None      # type: str
 
     def __init__(self, *args, **kwargs):
         # won't run this as a test suite, only its derived classes

weaver/cli.py

@@ -7,13 +7,15 @@
 import os
 import re
 import sys
+import textwrap
 import time
 from typing import TYPE_CHECKING
 from urllib.parse import urlparse
 
 import yaml
 from pyramid.httpexceptions import HTTPNotImplemented
 from requests.auth import AuthBase, HTTPBasicAuth
+from requests.sessions import Session
 from requests.structures import CaseInsensitiveDict
 from webob.headers import ResponseHeaders
 from yaml.scanner import ScannerError
@@ -22,7 +24,7 @@
 from weaver.datatype import AutoBase
 from weaver.exceptions import PackageRegistrationError
 from weaver.execute import ExecuteMode, ExecuteResponse, ExecuteTransmissionMode
-from weaver.formats import ContentType, OutputFormat, get_content_type, get_format
+from weaver.formats import ContentType, OutputFormat, get_content_type, get_format, repr_json
 from weaver.processes.constants import ProcessSchema
 from weaver.processes.convert import (
     convert_input_values_schema,
@@ -51,7 +53,6 @@
     request_extra,
     setup_loggers
 )
-from weaver.visibility import Visibility
 from weaver.wps_restapi import swagger_definitions as sd
 
 if TYPE_CHECKING:
@@ -348,7 +349,6 @@ def __init__(self, url=None, auth=None):
     def _request(self,
                  method,                # type: AnyRequestMethod
                  url,                   # type: str
-                 *args,                 # type: Any
                  headers=None,          # type: Optional[AnyHeadersContainer]
                  x_headers=None,        # type: Optional[AnyHeadersContainer]
                  request_timeout=None,  # type: Optional[int]
@@ -370,7 +370,20 @@ def _request(self,
         if isinstance(request_retries, int) and request_retries > 0:
             kwargs.setdefault("retries", request_retries)
 
-        return request_extra(method, url, *args, headers=headers, **kwargs)
+        if LOGGER.isEnabledFor(logging.DEBUG):
+            fields = set(inspect.signature(Session.request).parameters) - {"params", "url", "method", "json", "body"}
+            options = {opt: val for opt, val in kwargs.items() if opt in fields}
+            tab = "  "
+            LOGGER.debug(
+                f"Request:\n{tab}%s %s\n{tab}Queries:\n%s\n{tab}Headers:\n%s\n{tab}Content:\n%s\n{tab}Options:\n%s",
+                method,
+                url,
+                textwrap.indent(repr_json(kwargs.get("params") or {}, indent=len(tab)), tab * 2),
+                textwrap.indent(repr_json(headers or {}, indent=len(tab)), tab * 2),
+                textwrap.indent(repr_json(kwargs.get("json") or kwargs.get("body") or {}, indent=len(tab)), tab * 2),
+                textwrap.indent(repr_json(options, indent=len(tab)), tab * 2),
+            )
+        return request_extra(method, url, headers=headers, **kwargs)
 
     def _get_url(self, url):
         # type: (Optional[str]) -> str
@@ -421,6 +434,8 @@ def _parse_result(response,             # type: Union[Response, OperationResult]
                             for item in nested:
                                 if isinstance(item, dict):
                                     item.pop("links", None)
+                        elif isinstance(nested, dict):
+                            nested.pop("links", None)
                     body.pop("links", None)
                 msg = body.get("description", body.get("message", "undefined"))
             if code >= 400:
@@ -470,7 +485,6 @@ def _parse_deploy_body(body, process_id):
                 desc = data.get("processDescription", {}).get("process", {}) or data.get("processDescription", {})
                 desc["id"] = process_id
             data.setdefault("processDescription", desc)  # already applied if description was found/updated at any level
-            desc["visibility"] = Visibility.PUBLIC
         except (ValueError, TypeError, ScannerError) as exc:  # pragma: no cover
             return OperationResult(False, f"Failed resolution of body definition: [{exc!s}]", body)
         return OperationResult(True, "", data)
@@ -538,6 +552,9 @@ def register(self,
         """
         Registers a remote :term:`Provider` using specified references.
 
+        .. note::
+            This operation is specific to `Weaver`. It is not supported by standard :term:`OGC API - Processes`.
+
         :param provider_id: Identifier to employ for registering the new :term:`Provider`.
         :param provider_url: Endpoint location to register the new remote :term:`Provider`.
         :param url: Instance URL if not already provided during client creation.
@@ -576,6 +593,9 @@ def unregister(self,
         """
         Unregisters a remote :term:`Provider` using the specified identifier.
 
+        .. note::
+            This operation is specific to `Weaver`. It is not supported by standard :term:`OGC API - Processes`.
+
         :param provider_id: Identifier to employ for unregistering the :term:`Provider`.
         :param url: Instance URL if not already provided during client creation.
         :param auth:
@@ -629,8 +649,13 @@ def deploy(self,
         If the reference is resolved to be a :term:`Workflow`, all its underlying :term:`Process` steps must be
         available under the same URL that this client was initialized with.
 
+        .. note::
+            This is only supported by :term:`OGC API - Processes` instances that support
+            the `Deploy, Replace, Undeploy` (DRU) extension.
+
         .. seealso::
-            :ref:`proc_op_deploy`
+            - :ref:`proc_op_deploy`
+            - |ogc-api-proc-part2|_
 
         :param process_id:
             Desired process identifier.
@@ -686,7 +711,8 @@ def deploy(self,
         resp = self._request("POST", path, json=data,
                              headers=req_headers, x_headers=headers, settings=self._settings, auth=auth,
                              request_timeout=request_timeout, request_retries=request_retries)
-        return self._parse_result(resp, with_links=with_links, with_headers=with_headers, output_format=output_format)
+        return self._parse_result(resp, with_links=with_links, nested_links="processSummary",
+                                  with_headers=with_headers, output_format=output_format)
 
     def undeploy(self,
                  process_id,            # type: str
@@ -856,6 +882,48 @@ def _get_process_url(self, url, process_id, provider_id=None):
             path = f"{base}/processes/{process_id}"
         return path
 
+    def package(self,
+                process_id,                 # type: str
+                provider_id=None,           # type: Optional[str]
+                url=None,                   # type: Optional[str]
+                auth=None,                  # type: Optional[AuthHandler]
+                headers=None,               # type: Optional[AnyHeadersContainer]
+                with_links=True,            # type: bool
+                with_headers=False,         # type: bool
+                request_timeout=None,       # type: Optional[int]
+                request_retries=None,       # type: Optional[int]
+                output_format=None,         # type: Optional[AnyOutputFormat]
+                ):                          # type: (...) -> OperationResult
+        """
+        Retrieve the :term:`Application Package` definition of the specified :term:`Process`.
+
+        .. note::
+            This operation is specific to `Weaver`. It is not supported by standard :term:`OGC API - Processes`.
+
+        :param process_id: Identifier of the local or remote process to describe.
+        :param provider_id: Identifier of the provider from which to locate a remote process to describe.
+        :param url: Instance URL if not already provided during client creation.
+        :param auth:
+            Instance authentication handler if not already created during client creation.
+            Should perform required adjustments to request to allow access control of protected contents.
+        :param headers:
+            Additional headers to employ when sending request.
+            Note that this can break functionalities if expected headers are overridden. Use with care.
+        :param with_links: Indicate if ``links`` section should be preserved in returned result body.
+        :param with_headers: Indicate if response headers should be returned in result output.
+        :param request_timeout: Maximum timout duration (seconds) to wait for a response when performing HTTP requests.
+        :param request_retries: Amount of attempt to retry HTTP requests in case of failure.
+        :param output_format: Select an alternate output representation of the result body contents.
+        :returns: Results of the operation.
+        """
+        path = self._get_process_url(url, process_id, provider_id)
+        path = f"{path}/package"
+        resp = self._request("GET", path,
+                             headers=self._headers, x_headers=headers, settings=self._settings, auth=auth,
+                             request_timeout=request_timeout, request_retries=request_retries)
+        return self._parse_result(resp, message="Retrieving process Application Package.",
+                                  output_format=output_format, with_links=with_links, with_headers=with_headers)
+
     @staticmethod
     def _parse_inputs(inputs):
         # type: (Optional[Union[str, JSON]]) -> Union[OperationResult, ExecutionInputsMap]
@@ -2365,6 +2433,17 @@ def make_parser():
         help="Representation schema of the returned process description (default: %(default)s, case-insensitive)."
     )
 
+    op_package = WeaverArgumentParser(
+        "package",
+        description="Obtain the Application Package definition of an existing process.",
+        formatter_class=ParagraphFormatter,
+    )
+    set_parser_sections(op_package)
+    add_url_param(op_package)
+    add_shared_options(op_package)
+    add_process_param(op_package)
+    add_provider_param(op_package, required=False)
+
     op_execute = WeaverArgumentParser(
         "execute",
         description="Submit a job execution for an existing process.",
@@ -2606,6 +2685,7 @@ def make_parser():
         op_unregister,
         op_capabilities,
         op_describe,
+        op_package,
         op_execute,
         op_jobs,
         op_monitor,

weaver/datatype.py

@@ -2247,7 +2247,7 @@ def estimator(self, estimator):
     @property
     def visibility(self):
         # type: () -> Visibility
-        return Visibility.get(self.get("visibility"), Visibility.PRIVATE)
+        return Visibility.get(self.get("visibility"), Visibility.PUBLIC)
 
     @visibility.setter
     def visibility(self, visibility):