Add support for castable integer and port number types in config sche…

…mas (#372)

* add CastableInt and PortNumber with tests

* typo fix

* delete accidentally committed test

CHANGELOG.md

@@ -12,6 +12,14 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+
+## 7.4.9
+
+### Added
+
+ * `CastableInt` class the represents an interger to be used in config schema definitions. The difference from using `int` is that the field of this type in the yaml file can be either a string or a number, while a field of type `int` must be a number in yaml.
+ * `PortNumber` class that represents a valid port number to be used in config schema definitions. Just like `CastableInt` it can be a string or a number in the yaml file. This allows for example setting a port number using an environment variable.
+
 ## 7.4.8
 
 ### Fixed

cognite/extractorutils/__init__.py

@@ -16,5 +16,5 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.4.8"
+__version__ = "7.4.9"
 from .base import Extractor

cognite/extractorutils/configtools/__init__.py

@@ -90,6 +90,7 @@ class MyConfig(BaseConfig):
 from .elements import (
     AuthenticatorConfig,
     BaseConfig,
+    CastableInt,
     CertificateConfig,
     CogniteConfig,
     ConfigType,
@@ -99,6 +100,7 @@ class MyConfig(BaseConfig):
     LocalStateStoreConfig,
     LoggingConfig,
     MetricsConfig,
+    PortNumber,
     RawDestinationConfig,
     RawStateStoreConfig,
     StateStoreConfig,

cognite/extractorutils/configtools/elements.py

@@ -744,3 +744,43 @@ def __post_init__(self) -> None:
             _logger.warning("'options' is preferred over 'flags' as this may be removed in a future release")
             self.options = self.flags
             self.flags = None
+
+
+class CastableInt(int):
+    """
+    Represents an integer in a config schema. Difference from regular int is that the
+    value if this type can be  either a string or an integer in the yaml file.
+    """
+
+    def __new__(cls, value: Any) -> "CastableInt":
+        """
+        Returns value as is if it's int. If it's str or bytes try to convert to int.
+        Raises ValueError if conversion is unsuccessful or value is of not supported type.
+
+        Type check is required to avoid unexpected behaviour, such as implictly casting booleans,
+        floats and other types supported by standard int.
+        """
+
+        if not isinstance(value, (int, str, bytes)):
+            raise ValueError(f"CastableInt cannot be created form value {value!r} of type {type(value)!r}.")
+
+        return super().__new__(cls, value)
+
+
+class PortNumber(CastableInt):
+    """
+    A subclass of int to be used in config schemas. It represents a valid port number (0 to 65535) and allows the value
+    to be of either str or int type. If the value is not a valid port number raises a ValueError at instantiation.
+    """
+
+    def __new__(cls, value: Any) -> "PortNumber":
+        """
+        Try to convert the `value` to int. If successful, check if it's within a valid range for a port number.
+        Raises ValueError if conversion to int or validation is unsuccessful.
+        """
+        value = super().__new__(cls, value)
+
+        if not (0 <= value <= 65535):
+            raise ValueError(f"Port number must be between 0 and 65535. Got: {value}.")
+
+        return value

cognite/extractorutils/configtools/loaders.py

@@ -36,8 +36,10 @@
 from cognite.extractorutils.configtools._util import _to_snake_case
 from cognite.extractorutils.configtools.elements import (
     BaseConfig,
+    CastableInt,
     ConfigType,
     IgnorePattern,
+    PortNumber,
     TimeIntervalConfig,
     _BaseConfig,
 )
@@ -224,7 +226,7 @@ def _load_yaml(
         config = dacite.from_dict(
             data=config_dict,
             data_class=config_type,
-            config=dacite.Config(strict=True, cast=[Enum, TimeIntervalConfig, Path]),
+            config=dacite.Config(strict=True, cast=[Enum, TimeIntervalConfig, Path, CastableInt, PortNumber]),
         )
     except dacite.UnexpectedDataError as e:
         unknowns = [f'"{k.replace("_", "-") if case_style == "hyphen" else k}"' for k in e.keys]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-extractor-utils"
-version = "7.4.8"
+version = "7.4.9"
 description = "Utilities for easier development of extractors for CDF"
 authors = ["Mathias Lohne <mathias.lohne@cognite.com>"]
 license = "Apache-2.0"

tests/tests_unit/test_configtools.py

@@ -34,7 +34,13 @@
     load_yaml,
 )
 from cognite.extractorutils.configtools._util import _to_snake_case
-from cognite.extractorutils.configtools.elements import AuthenticatorConfig, IgnorePattern, RegExpFlag
+from cognite.extractorutils.configtools.elements import (
+    AuthenticatorConfig,
+    CastableInt,
+    IgnorePattern,
+    PortNumber,
+    RegExpFlag,
+)
 from cognite.extractorutils.configtools.loaders import (
     ConfigResolver,
     compile_patterns,
@@ -571,3 +577,37 @@ def test_ignore_pattern() -> None:
 
     with pytest.raises(ValueError, match=r"Only one of either 'options' or 'flags' can be specified."):
         IgnorePattern("g*i", [RegExpFlag.IC], [RegExpFlag.IC])
+
+
+def test_castable_int_parsing(monkeypatch):
+    monkeypatch.setenv("PORT_NUMBER", "8080")
+
+    config = """
+    host: 'localhost'
+    port: ${PORT_NUMBER}
+    connections: 4
+    batch-size: ' 1000 '
+    """
+
+    @dataclass
+    class DbConfigStd:
+        host: str
+        port: int
+        connections: int
+        batch_size: int
+
+    @dataclass
+    class DbConfigCastable:
+        host: str
+        port: PortNumber
+        connections: CastableInt
+        batch_size: CastableInt
+
+    with pytest.raises(InvalidConfigError):
+        load_yaml(config, DbConfigStd)
+
+    parsed: DbConfigCastable = load_yaml(config, DbConfigCastable)
+    assert parsed.host == "localhost"
+    assert parsed.port == 8080
+    assert parsed.connections == 4
+    assert parsed.batch_size == 1000