From 286d4ed5fd4f88c010c051598c5c8407a9aa742a Mon Sep 17 00:00:00 2001 From: Dmytro Donukis <110604171+ddonukis@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:30:44 +0200 Subject: [PATCH] Add support for castable integer and port number types in config schemas (#372) * add CastableInt and PortNumber with tests * typo fix * delete accidentally committed test --- CHANGELOG.md | 8 ++++ cognite/extractorutils/__init__.py | 2 +- .../extractorutils/configtools/__init__.py | 2 + .../extractorutils/configtools/elements.py | 40 ++++++++++++++++++ cognite/extractorutils/configtools/loaders.py | 4 +- pyproject.toml | 2 +- tests/tests_unit/test_configtools.py | 42 ++++++++++++++++++- 7 files changed, 96 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9286d501..f6614544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ Changes are grouped as follows - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. + +## 7.4.9 + +### Added + + * `CastableInt` class the represents an interger to be used in config schema definitions. The difference from using `int` is that the field of this type in the yaml file can be either a string or a number, while a field of type `int` must be a number in yaml. + * `PortNumber` class that represents a valid port number to be used in config schema definitions. Just like `CastableInt` it can be a string or a number in the yaml file. This allows for example setting a port number using an environment variable. + ## 7.4.8 ### Fixed diff --git a/cognite/extractorutils/__init__.py b/cognite/extractorutils/__init__.py index 0932ec73..4b0cc600 100644 --- a/cognite/extractorutils/__init__.py +++ b/cognite/extractorutils/__init__.py @@ -16,5 +16,5 @@ Cognite extractor utils is a Python package that simplifies the development of new extractors. """ -__version__ = "7.4.8" +__version__ = "7.4.9" from .base import Extractor diff --git a/cognite/extractorutils/configtools/__init__.py b/cognite/extractorutils/configtools/__init__.py index 92c7928f..aa88e283 100644 --- a/cognite/extractorutils/configtools/__init__.py +++ b/cognite/extractorutils/configtools/__init__.py @@ -90,6 +90,7 @@ class MyConfig(BaseConfig): from .elements import ( AuthenticatorConfig, BaseConfig, + CastableInt, CertificateConfig, CogniteConfig, ConfigType, @@ -99,6 +100,7 @@ class MyConfig(BaseConfig): LocalStateStoreConfig, LoggingConfig, MetricsConfig, + PortNumber, RawDestinationConfig, RawStateStoreConfig, StateStoreConfig, diff --git a/cognite/extractorutils/configtools/elements.py b/cognite/extractorutils/configtools/elements.py index a98e36e7..a6a3f431 100644 --- a/cognite/extractorutils/configtools/elements.py +++ b/cognite/extractorutils/configtools/elements.py @@ -744,3 +744,43 @@ def __post_init__(self) -> None: _logger.warning("'options' is preferred over 'flags' as this may be removed in a future release") self.options = self.flags self.flags = None + + +class CastableInt(int): + """ + Represents an integer in a config schema. Difference from regular int is that the + value if this type can be either a string or an integer in the yaml file. + """ + + def __new__(cls, value: Any) -> "CastableInt": + """ + Returns value as is if it's int. If it's str or bytes try to convert to int. + Raises ValueError if conversion is unsuccessful or value is of not supported type. + + Type check is required to avoid unexpected behaviour, such as implictly casting booleans, + floats and other types supported by standard int. + """ + + if not isinstance(value, (int, str, bytes)): + raise ValueError(f"CastableInt cannot be created form value {value!r} of type {type(value)!r}.") + + return super().__new__(cls, value) + + +class PortNumber(CastableInt): + """ + A subclass of int to be used in config schemas. It represents a valid port number (0 to 65535) and allows the value + to be of either str or int type. If the value is not a valid port number raises a ValueError at instantiation. + """ + + def __new__(cls, value: Any) -> "PortNumber": + """ + Try to convert the `value` to int. If successful, check if it's within a valid range for a port number. + Raises ValueError if conversion to int or validation is unsuccessful. + """ + value = super().__new__(cls, value) + + if not (0 <= value <= 65535): + raise ValueError(f"Port number must be between 0 and 65535. Got: {value}.") + + return value diff --git a/cognite/extractorutils/configtools/loaders.py b/cognite/extractorutils/configtools/loaders.py index 47ee2888..5bf5d2bd 100644 --- a/cognite/extractorutils/configtools/loaders.py +++ b/cognite/extractorutils/configtools/loaders.py @@ -36,8 +36,10 @@ from cognite.extractorutils.configtools._util import _to_snake_case from cognite.extractorutils.configtools.elements import ( BaseConfig, + CastableInt, ConfigType, IgnorePattern, + PortNumber, TimeIntervalConfig, _BaseConfig, ) @@ -224,7 +226,7 @@ def _load_yaml( config = dacite.from_dict( data=config_dict, data_class=config_type, - config=dacite.Config(strict=True, cast=[Enum, TimeIntervalConfig, Path]), + config=dacite.Config(strict=True, cast=[Enum, TimeIntervalConfig, Path, CastableInt, PortNumber]), ) except dacite.UnexpectedDataError as e: unknowns = [f'"{k.replace("_", "-") if case_style == "hyphen" else k}"' for k in e.keys] diff --git a/pyproject.toml b/pyproject.toml index 0bdc4274..2b005d63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cognite-extractor-utils" -version = "7.4.8" +version = "7.4.9" description = "Utilities for easier development of extractors for CDF" authors = ["Mathias Lohne "] license = "Apache-2.0" diff --git a/tests/tests_unit/test_configtools.py b/tests/tests_unit/test_configtools.py index 91da75d5..fa7d3fb2 100644 --- a/tests/tests_unit/test_configtools.py +++ b/tests/tests_unit/test_configtools.py @@ -34,7 +34,13 @@ load_yaml, ) from cognite.extractorutils.configtools._util import _to_snake_case -from cognite.extractorutils.configtools.elements import AuthenticatorConfig, IgnorePattern, RegExpFlag +from cognite.extractorutils.configtools.elements import ( + AuthenticatorConfig, + CastableInt, + IgnorePattern, + PortNumber, + RegExpFlag, +) from cognite.extractorutils.configtools.loaders import ( ConfigResolver, compile_patterns, @@ -571,3 +577,37 @@ def test_ignore_pattern() -> None: with pytest.raises(ValueError, match=r"Only one of either 'options' or 'flags' can be specified."): IgnorePattern("g*i", [RegExpFlag.IC], [RegExpFlag.IC]) + + +def test_castable_int_parsing(monkeypatch): + monkeypatch.setenv("PORT_NUMBER", "8080") + + config = """ + host: 'localhost' + port: ${PORT_NUMBER} + connections: 4 + batch-size: ' 1000 ' + """ + + @dataclass + class DbConfigStd: + host: str + port: int + connections: int + batch_size: int + + @dataclass + class DbConfigCastable: + host: str + port: PortNumber + connections: CastableInt + batch_size: CastableInt + + with pytest.raises(InvalidConfigError): + load_yaml(config, DbConfigStd) + + parsed: DbConfigCastable = load_yaml(config, DbConfigCastable) + assert parsed.host == "localhost" + assert parsed.port == 8080 + assert parsed.connections == 4 + assert parsed.batch_size == 1000