Merge pull request #543 from crim-ca/fix-docker-worker

CHANGES.rst

@@ -16,7 +16,14 @@ Changes:
 
 Fixes:
 ------
-- No change.
+- Fix broken Docker build of ``weaver-worker`` image due to unresolved ``docker-ce-cli`` package.
+  Installation is updated according to the reference documentation (https://docs.docker.com/engine/install/debian/).
+- Fix incorrect stream reader type (``bytes`` instead of ``str``) for some handlers in ``open_module_resource_file``.
+- Fix invalid ``jsonschema.validators.RefResolver`` reference in ``jsonschema>=4.18.0`` caused by refactor
+  (see https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst#v4180,
+  https://python-jsonschema.readthedocs.io/en/v4.18.0/api/jsonschema/validators/#jsonschema.validators._RefResolver
+  and `python-jsonschema/jsonschema#1049 <https://github.com/python-jsonschema/jsonschema/pull/1049>`_).
+- Fix multiple linting checks, documentation dependencies and link references.
 
 .. _changes_4.30.0:
 

docker/Dockerfile-worker

@@ -7,14 +7,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         gnupg \
         gnupg-agent \
         software-properties-common \
-    # NOTE: Only 'worker' image should be using docker, 'manager' is only for API.
-    && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
-    && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" \
-    && apt update \
+    # NOTE: Only 'worker' image should be using docker, 'manager' is only for API. \
+    && install -m 0755 -d /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && chmod a+r /etc/apt/keyrings/docker.gpg \
+    && echo "\
+      deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
+      https://download.docker.com/linux/debian \
+      "$(. /etc/os-release && echo "${VERSION_CODENAME}")" stable" | \
+      tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get update \
     # NOTE:
     #   Only install CLI package, 'docker-ce' and 'containerd.io' not required as they should be provided by host.
     #   Docker sibling execution is expected. See 'docker/docker-compose.yml.example' for details.
-    && apt install --no-install-recommends docker-ce-cli \
+    && apt-get install --no-install-recommends docker-ce-cli \
     && rm -rf /var/lib/apt/lists/*
 
 # run app

docs/source/cli.rst

@@ -130,7 +130,7 @@ Below are examples of possible commands:
 
     weaver capabilities \
         -u ${WEAVER_URL} \
-        -aH requests_magpie.MagpieAuth \
+        -aC requests_magpie.MagpieAuth \
         -aU ${MAGPIE_URL} \
         -aI ${MAGPIE_USERNAME} \
         -aP ${MAGPIE_PASSWORD}

docs/source/conf.py

@@ -432,8 +432,6 @@ def doc_redirect_include(file_path):
     # following have sporadic downtimes
     "https://esgf-data.dkrz.de/",
     "https://indico.egi.eu/",
-    # ignore anchors not found although valid
-    "https://spec.openapis.org/oas/v3.1.0/*#*",
     ".*docker-registry.crim.ca.*",  # protected
     # might not exist yet (we are generating it!)
     "https://pavics-weaver.readthedocs.io/en/latest/api.html",
@@ -444,6 +442,12 @@ def doc_redirect_include(file_path):
     "https://ogc-ems.crim.ca/.*",
     "https://ogc-ades.crim.ca/.*",
     "https://ogc.crim.ca/.*",
+    "https://github.com/.*\\.rst#.*",
+]
+linkcheck_anchors_ignore = [
+    "xml-object",  # https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md
+    "data-types",  # https://spec.openapis.org/oas/v3.1.0
+    "defusedxmllxml",  # https://github.com/tiran/defusedxml/tree/main
 ]
 
 linkcheck_timeout = 30

docs/source/package.rst

@@ -579,10 +579,9 @@ In the :term:`WPS` context, three data types exist, namely ``Literal``, ``Boundi
     As of the current version of `Weaver`, :term:`WPS` data type ``BoundingBox`` is not completely supported.
     The schema definition exists in :term:`WPS` and :term:`OAS` contexts but is not handled by any :term:`CWL` type
     conversion yet. This feature is reflected by issue `#51 <https://github.com/crim-ca/weaver/issues/51>`_.
-    It is possible to use a ``Literal`` data of type ``string`` corresponding to :term:`WKT` [#]_, [#]_ in the meantime.
+    It is possible to use a ``Literal`` data of type ``string`` corresponding to :term:`WKT` [#]_ in the meantime.
 
 .. [#] |wkt-example|_
-.. [#] |wkt-format|_
 
 As presented in previous examples, :term:`I/O` in the :term:`WPS` context does not require an explicit indication of
 which data type from one of ``Literal``, ``BoundingBox`` and ``Complex`` to apply. Instead, :term:`WPS` type can be

docs/source/references.rst

@@ -102,8 +102,6 @@
 .. _pywps-multi-output: https://pywps.readthedocs.io/en/master/process.html#returning-multiple-files
 .. |wkt-example| replace:: WKT Examples
 .. _wkt-example: https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry
-.. |wkt-format| replace:: WKT Formats
-.. _wkt-format: https://docs.geotools.org/stable/javadocs/org/opengis/referencing/doc-files/WKT.html
 .. |weaver-issues| replace:: Weaver issues
 .. _weaver-issues: https://github.com/crim-ca/weaver/issues
 .. |submit-issue| replace:: submit a new issue

requirements-doc.txt

@@ -5,7 +5,11 @@
 -r requirements.txt
 cloud_sptheme
 jinja2<3.1  # fix sphinx failing, see: https://github.com/sphinx-doc/sphinx/issues/10291
-sphinx>=3.5,<6
+sphinx>=3.5,<6; python_version <= "3.7"
+# sphinx>=7 blocked by 'sphinx_rtd_theme'
+# - https://github.com/readthedocs/sphinx_rtd_theme/issues/1463
+# - https://github.com/readthedocs/sphinx_rtd_theme/pull/1464
+sphinx>=6,<7; python_version >= "3.8"
 sphinx-argparse
 sphinx-autoapi>=1.7.0
 sphinx-paramlinks>=0.4.1

requirements.txt

@@ -36,8 +36,6 @@ cryptography
 # (https://github.com/common-workflow-language/common-workflow-language/issues/587)
 ### git+https://github.com/crim-ca/cwltool@docker-gpu#egg=cwltool; python_version >= "3"
 cwltool==3.1.20230213100550
-# defused required for json2xml
-defusedxml
 docker
 duration
 esgf-compute-api @ git+https://github.com/ESGF/esgf-compute-api.git@v2.3.7

setup.cfg

@@ -89,7 +89,7 @@ exclude = *.egg-info,build,dist,env,tests,./tests,test_*
 targets = .
 
 [flake8]
-ignore = E126,E226,E402,F401,W503,W504
+ignore = E126,E226,E402,F401,W503,W504,B007,B009,B010,B023
 max-line-length = 120
 exclude = 
 	src,
@@ -99,6 +99,7 @@ exclude =
 	build,
 	dist,
 	eggs,
+	env,
 	parts,
 	examples,
 

tests/functional/test_quoting.py

@@ -259,7 +259,7 @@ def setup_docker(cls):
         if not image:
             pytest.fail("Cannot run test without quotation estimator docker image.")
         if image == "mock":
-            client = docker.from_env()
+            client = docker.from_env()  # pylint: disable=I1101
             path = os.path.join(WEAVER_ROOT_DIR, "tests/quotation")
             image = "weaver-tests/mock-quotation-estimator:latest"
             result = client.api.build(path, tag=image, rm=True, nocache=True)

tests/utils.py

@@ -1086,7 +1086,8 @@ def mocked_dismiss_process():
         with mock_celery_revoke:
             yield   # for direct use by context or decorator
     finally:
-        return mock_celery_revoke  # for use by combined ExitStack context  # pylint: disable=W0150.lost-exception
+        # used by ExitStack context, which would handle the exception appropriately
+        return mock_celery_revoke  # noqa: B012  # pylint: disable=W0150,lost-exception
 
 
 def mocked_process_job_runner(job_task_id="mocked-job-id"):

weaver/database/mongodb.py

@@ -141,8 +141,9 @@ def get_store(self, store_type, *store_args, **store_kwargs):
                     if "settings" not in store_kwargs:
                         store_kwargs["settings"] = self._settings
                     self._stores[store_type] = store(
+                        *store_args,
                         collection=getattr(self.get_session(), store_type),
-                        *store_args, **store_kwargs
+                        **store_kwargs,
                     )
                 return self._stores[store_type]
         raise NotImplementedError(f"Database '{self.type}' cannot find matching store '{store_type}'.")

weaver/datatype.py

@@ -24,7 +24,7 @@
 import requests.exceptions
 from cryptography.fernet import Fernet
 from dateutil.parser import parse as dt_parse
-from docker.auth import decode_auth
+from docker.auth import decode_auth  # pylint: disable=E0611
 from owslib.util import ServiceException as OWSServiceException
 from owslib.wps import Process as ProcessOWS, WPSException
 from pywps import Process as ProcessWPS

weaver/formats.py

@@ -102,7 +102,7 @@ class OutputFormat(Constants):
     """
     JSON = classproperty(fget=lambda self: "json", doc="""
     Representation as :term:`JSON` (object), which can still be manipulated in code.
-    """)
+    """)  # noqa: F811  # false-positive redefinition of JSON typing
 
     JSON_STR = classproperty(fget=lambda self: "json+str", doc="""
     Representation as :term:`JSON` content formatted as string with indentation and newlines.

weaver/owsexceptions.py

@@ -121,7 +121,7 @@ def json_formatter(status, body, title, environ):  # noqa
         return body
 
     def prepare(self, environ):
-        if not self.body:
+        if not self.body:  # pylint: disable=E0203,W0201
             accept_value = environ.get("HTTP_ACCEPT", "")
             accept = create_accept_header(accept_value)
 
@@ -140,7 +140,7 @@ def prepare(self, environ):
 
                 # json exception response should not have status 200
                 if self.status_code == HTTPOk.code:
-                    self.status = HTTPInternalServerError.code
+                    self.status = HTTPInternalServerError.code  # pylint: disable=E0203,W0201
 
                 class JsonPageTemplate(object):
                     def __init__(self, excobj):
@@ -170,8 +170,8 @@ def substitute(self, code, locator, message):
             page = page_template.substitute(**args)
             if isinstance(page, str):
                 page = page.encode(self.charset if self.charset else "UTF-8")
-            self.app_iter = [page]
-            self.body = page
+            self.app_iter = [page]  # pylint: disable=E0203,W0201
+            self.body = page  # pylint: disable=E0203,W0201
 
     @property
     def wsgi_response(self):

weaver/processes/utils.py

@@ -10,7 +10,7 @@
 import colander
 import docker
 import yaml
-from docker.errors import ImageNotFound
+from docker.errors import ImageNotFound  # pylint: disable=E0611
 from pyramid.httpexceptions import (
     HTTPBadRequest,
     HTTPConflict,
@@ -1233,7 +1233,7 @@ def pull_docker(docker_auth, logger=LOGGER):
     ref = docker_auth.reference
     try:
         # load from env is the same as CLI call
-        client = docker.from_env()
+        client = docker.from_env()  # pylint: disable=I1101
         # following login does not update '~/.docker/config.json' by design, but can use it if available
         # session remains active only within the client
         # Note:

weaver/processes/wps_process_base.py

@@ -159,7 +159,7 @@ def execute(self, workflow_inputs, out_dir, expected_outputs):
         self.update_status("Execution of remote process execution completed successfully.",
                            RemoteJobProgress.COMPLETED, Status.SUCCEEDED)
 
-    def prepare(self):
+    def prepare(self):  # noqa: B027  # intentionally not an abstract method to allow no-op
         # type: () -> None
         """
         Implementation dependent operations to prepare the :term:`Process` for :term:`Job` execution.

weaver/utils.py

@@ -4,6 +4,7 @@
 import functools
 import importlib.util
 import inspect
+import io
 import logging
 import os
 import posixpath
@@ -33,7 +34,6 @@
 from botocore.config import Config as S3Config
 from bs4 import BeautifulSoup
 from celery.app import Celery
-from jsonschema.validators import RefResolver as JsonSchemaRefResolver
 from mypy_boto3_s3.literals import RegionName
 from pyramid.config import Configurator
 from pyramid.exceptions import ConfigurationError
@@ -67,6 +67,11 @@
 from weaver.warning import TimeZoneInfoAlreadySetWarning
 from weaver.xml_util import HTML_TREE_BUILDER, XML
 
+try:  # refactor in jsonschema==4.18.0
+    from jsonschema.validators import _RefResolver as JsonSchemaRefResolver  # pylint: disable=E0611
+except ImportError:
+    from jsonschema.validators import RefResolver as JsonSchemaRefResolver  # pylint: disable=E0611
+
 if TYPE_CHECKING:
     import importlib.abc
     from types import FrameType, ModuleType
@@ -1055,7 +1060,7 @@ def import_target(target, default_root=None):
 
 
 def open_module_resource_file(module, file_path):
-    # type: (Union[str, ModuleType], str) -> IO[bytes]
+    # type: (Union[str, ModuleType], str) -> IO[str]
     """
     Opens a resource (data file) from an installed module.
 
@@ -1070,7 +1075,8 @@ def open_module_resource_file(module, file_path):
             reader = loader.get_resource_reader()  # type: importlib.abc.ResourceReader  # noqa
         except AttributeError:
             reader = loader  # noqa
-        return reader.open_resource(file_path)
+        buffer = reader.open_resource(file_path)
+        return io.TextIOWrapper(buffer, encoding="utf-8")
     except AttributeError:
         path = os.path.join(module.__path__[0], file_path)
         return open(path, mode="r", encoding="utf-8")

weaver/wps/service.py

@@ -307,7 +307,7 @@ def execute_job(self,
         execution = WPSExecution(version="2.0", url="localhost")
         xml_request = execution.buildRequest(process_id, wps_inputs, wps_outputs, mode=job.execution_mode, lineage=True)
         wps_request = WorkerRequest(http_headers=headers)
-        wps_request.identifier = process_id
+        wps_request.identifier = process_id  # pylint: disable=W0201
         wps_request.check_and_set_language(job.accept_language)
         wps_request.set_version("2.0.0")
         request_parser = wps_request._post_request_parser(wps_request.WPS.Execute().tag)  # noqa: W0212
@@ -317,7 +317,7 @@ def execute_job(self,
         #  Setting 'status = false' will disable async execution of 'pywps.app.Process.Process'
         #  but this is needed since this job is running within Celery worker already async
         #  (daemon process can't have children processes).
-        wps_request.status = "false"
+        wps_request.status = "false"  # pylint: disable=W0201
 
         # When 'execute' is called, pywps will in turn call 'prepare_process_for_execution',
         # which then setups and retrieves currently loaded 'local' processes.

weaver/wps_restapi/colander_extras.py

@@ -578,7 +578,7 @@ class XMLObject(object):
     The value of ``title`` provided as option or
 
     .. seealso::
-        - https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.3.md#user-content-xml-object
+        - https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#xml-object
         - https://swagger.io/docs/specification/data-models/representing-xml/
     """
     attribute = None    # define the corresponding node object as attribute instead of field
@@ -614,7 +614,7 @@ class ExtendedSchemaMeta(colander._SchemaMeta):
     pass
 
 
-class ExtendedSchemaBase(colander.SchemaNode, metaclass=ExtendedSchemaMeta):
+class ExtendedSchemaBase(colander.SchemaNode, metaclass=ExtendedSchemaMeta):  # pylint: disable=E1139
     """
     Utility base node definition that initializes additional parameters at creation time of any other extended schema.
 
@@ -638,6 +638,8 @@ def schema_type():
         raise NotImplementedError("Using SchemaNode for a field requires 'schema_type' definition.")
 
     def __init__(self, *args, **kwargs):
+        # pylint: disable=E0203
+
         schema_name = _get_node_name(self, schema_name=True)
         schema_type = _get_schema_type(self, check=True)
         if isinstance(self, XMLObject):
@@ -2310,7 +2312,7 @@ def convert_type(self, schema_node):
                 # fields that are shared across all the oneOf sub-items
                 # pass down the original title of that object to refer to that schema reference
                 obj_shared = ExtendedMappingSchema(title=shared_title)
-                obj_shared.children = schema_node.children
+                obj_shared.children = schema_node.children  # pylint: disable=W0201
                 obj_one_of = item_obj.clone()
                 obj_one_of.title = one_of_title
                 all_of = AllOfKeywordSchema(title=obj_req_title, _all_of=[obj_shared, obj_one_of])

weaver/xml_util.py

@@ -5,7 +5,7 @@
 instead, because that package's extension with :mod:`lxml` is marked as deprecated.
 
 .. seealso::
-    https://pypi.org/project/defusedxml/#defusedxml-lxml
+    https://github.com/tiran/defusedxml/tree/main#defusedxmllxml
 
 To use the module, import is as if importing :mod:`lxml.etree`: