From 80838f70ae71896f305aa9b3099e5207048027b7 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Thu, 3 Aug 2023 23:59:58 +0100 Subject: [PATCH 1/2] Working feature to expose env vars as parameters to the stack to allow change --- .../compose/compose_services/__init__.py | 1 + ecs_composex/ecs/ecs_family/__init__.py | 11 +++ ecs_composex/ecs/ecs_family/family_helpers.py | 67 ++++++++++++++++++- ecs_composex/ecs_composex.py | 1 + ecs_composex/specs/compose-spec.json | 3 + .../specs/services.x-environment.spec.json | 36 ++++++++++ 6 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 ecs_composex/specs/services.x-environment.spec.json diff --git a/ecs_composex/compose/compose_services/__init__.py b/ecs_composex/compose/compose_services/__init__.py index 71c6c90d..5de29eb4 100644 --- a/ecs_composex/compose/compose_services/__init__.py +++ b/ecs_composex/compose/compose_services/__init__.py @@ -160,6 +160,7 @@ def __init__( self.set_container_definition() self.links = set_else_none("links", definition) self.family_links: list = [] + self.x_environment: dict = set_else_none("x-environment", definition, {}) def __repr__(self): return self.name diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index cb3340cf..d573338b 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -656,6 +656,17 @@ def validate_compute_configuration_for_task(self, settings): validate_compute_configuration_for_task(self, settings) + def x_environment_processing(self): + """ + Checks for each service if `x-environment` was set + """ + from .family_helpers import swap_environment_value_with_parameter + + for service in self.ordered_services: + if not service.x_environment: + continue + swap_environment_value_with_parameter(self, service) + class ServiceStack(ComposeXStack): """ diff --git a/ecs_composex/ecs/ecs_family/family_helpers.py b/ecs_composex/ecs/ecs_family/family_helpers.py index ee503894..66b97846 100644 --- a/ecs_composex/ecs/ecs_family/family_helpers.py +++ b/ecs_composex/ecs/ecs_family/family_helpers.py @@ -4,26 +4,30 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Union if TYPE_CHECKING: from ecs_composex.common.settings import ComposeXSettings from ecs_composex.ecs.ecs_family import ComposeFamily from ecs_composex.common.stacks import ComposeXStack + from ecs_composex.compose.compose_services import ComposeService from troposphere.iam import Role from troposphere import ( AWS_ACCOUNT_ID, AWS_PARTITION, AWS_REGION, + MAX_PARAMETERS, FindInMap, GetAtt, NoValue, Ref, Sub, ) +from troposphere.ecs import Environment from troposphere.iam import Policy, PolicyType +from ecs_composex.common import NONALPHANUM from ecs_composex.common.cfn_params import Parameter from ecs_composex.common.logging import LOG from ecs_composex.common.troposphere_tools import add_parameters @@ -312,3 +316,64 @@ def set_service_dependency_on_all_iam_policies(family: ComposeFamily) -> None: else: setattr(family.ecs_service.ecs_service, "DependsOn", policies) LOG.debug(family.ecs_service.ecs_service.DependsOn) + + +def update_env_var_to_parameter( + family: ComposeFamily, + service: ComposeService, + env_var: Environment, + set_as_params: Union[list, dict], +) -> None: + """ + Function that will replace a user-defined environment variable with a Template Parameter + If the SetAsParameter is a list, goes through them and generates the CFN Parameter properties + If SetAsParameter is a dict, it will import the user-defined Parameter settings. + """ + type_to_param_type: dict = {str: "String", int: "Number", float: "Number"} + for var_name in set_as_params: + if env_var.Name != var_name: + continue + if env_var.Name not in service.environment: + continue + parameter_title: str = NONALPHANUM.sub("", var_name) + if isinstance(set_as_params, list): + env_var_param = Parameter( + parameter_title, + group_label="User Defined Service Variable", + Type=type_to_param_type[type(service.environment[var_name])], + ) + elif isinstance(set_as_params, dict): + env_var_param = Parameter( + parameter_title, + group_label="User Defined Service Variable", + **set_as_params[var_name], + ) + else: + raise TypeError( + "services.{} - Value for x-environment.SetAsParameter must be either a list or mapping/dict. Got", + type(set_as_params), + ) + add_parameters(family.template, [env_var_param]) + family.stack.Parameters.update( + {env_var_param.title: service.environment[var_name]} + ) + setattr(env_var, "Value", Ref(env_var_param)) + + +def swap_environment_value_with_parameter( + family: ComposeFamily, service: ComposeService +) -> None: + set_as_params = service.x_environment["SetAsParameter"] + for env_var in service.cfn_environment: + if len(family.stack.Parameters) > MAX_PARAMETERS: + print("Too many parameters already set") + break + if not isinstance(env_var, Environment): + continue + if not isinstance(env_var.Value, (str, int, float)): + print( + f"Env var {env_var.Name} is not str or int. Cannot convert", + type(env_var.Value), + ) + continue + update_env_var_to_parameter(family, service, env_var, set_as_params) diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index c2c95515..13483007 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -303,6 +303,7 @@ def generate_full_template(settings: ComposeXSettings): family.finalize_family_settings() map_resource_return_value_to_services_command(family, settings) family.state_facts() + family.x_environment_processing() set_ecs_cluster_identifier(settings.root_stack, settings) add_all_tags(settings.root_stack.stack_template, settings) diff --git a/ecs_composex/specs/compose-spec.json b/ecs_composex/specs/compose-spec.json index 688e6bb7..5ba50ceb 100644 --- a/ecs_composex/specs/compose-spec.json +++ b/ecs_composex/specs/compose-spec.json @@ -102,6 +102,9 @@ "x-docker_opts": { "$ref": "services.x-docker_opts.spec.json" }, + "x-environment": { + "$ref": "services.x-environment.spec.json" + }, "x-prometheus": { "$ref": "services.x-prometheus.spec.json" }, diff --git a/ecs_composex/specs/services.x-environment.spec.json b/ecs_composex/specs/services.x-environment.spec.json new file mode 100644 index 00000000..9805f5c5 --- /dev/null +++ b/ecs_composex/specs/services.x-environment.spec.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "id": "services.x-environment", + "$id": "services.x-environment.spec.json", + "type": "object", + "title": "services.x-environment specification", + "description": "The services.x-environment specification for ComposeX", + "additionalProperties": false, + "required": [ + "SetAsParameter" + ], + "properties": { + "SetAsParameter": { + "oneOf": [ + { + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "patternProperties": { + "^x-": {}, + "^[a-zA-Z0-9_]+$": { + "type": "object" + } + } + }, + { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + ] + } + } +} From ad3924520a4e1b619e7c15fb8ec6853919fe686c Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Fri, 4 Aug 2023 10:19:13 +0100 Subject: [PATCH 2/2] Logging and handling types better --- ecs_composex/ecs/ecs_family/family_helpers.py | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/ecs_composex/ecs/ecs_family/family_helpers.py b/ecs_composex/ecs/ecs_family/family_helpers.py index 66b97846..0af47b1a 100644 --- a/ecs_composex/ecs/ecs_family/family_helpers.py +++ b/ecs_composex/ecs/ecs_family/family_helpers.py @@ -336,6 +336,12 @@ def update_env_var_to_parameter( if env_var.Name not in service.environment: continue parameter_title: str = NONALPHANUM.sub("", var_name) + if type(service.environment[var_name]): + LOG.warning( + "{}.{} - Env var values have to be string. Value will be cast with Sub".format( + family.name, service.name + ) + ) if isinstance(set_as_params, list): env_var_param = Parameter( parameter_title, @@ -343,6 +349,15 @@ def update_env_var_to_parameter( Type=type_to_param_type[type(service.environment[var_name])], ) elif isinstance(set_as_params, dict): + if ( + "Type" in set_as_params[var_name] + and set_as_params[var_name]["Type"].find("List") >= 0 + ): + raise ValueError( + "{}.{} - For environment variables, Parameter property Type cannot be a List. Got {}".format( + family.name, service.name, set_as_params[var_name]["Type"] + ) + ) env_var_param = Parameter( parameter_title, group_label="User Defined Service Variable", @@ -357,7 +372,7 @@ def update_env_var_to_parameter( family.stack.Parameters.update( {env_var_param.title: service.environment[var_name]} ) - setattr(env_var, "Value", Ref(env_var_param)) + setattr(env_var, "Value", Sub(env_var_param.title)) def swap_environment_value_with_parameter( @@ -366,14 +381,19 @@ def swap_environment_value_with_parameter( set_as_params = service.x_environment["SetAsParameter"] for env_var in service.cfn_environment: if len(family.stack.Parameters) > MAX_PARAMETERS: - print("Too many parameters already set") + LOG.warning( + "{}.{} - Too many parameters already set for this stack".format( + family.name, service.name + ) + ) break if not isinstance(env_var, Environment): continue if not isinstance(env_var.Value, (str, int, float)): - print( - f"Env var {env_var.Name} is not str or int. Cannot convert", - type(env_var.Value), + LOG.debug( + "Env var {} is not str or int. Cannot convert. Got {}".format( + env_var.Name, type(env_var.Value) + ), ) continue update_env_var_to_parameter(family, service, env_var, set_as_params)