From ef4ab316cb315cfc00d042792a60cb6d4a4b8ca0 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Sun, 24 Mar 2024 19:58:58 +0000 Subject: [PATCH 1/7] Working service connect entry creation --- ecs_composex/cloudmap/cloudmap_params.py | 9 ++ ecs_composex/cloudmap/cloudmap_stack.py | 7 ++ ecs_composex/common/settings.py | 3 +- .../compose/compose_services/__init__.py | 1 + ecs_composex/ecs/ecs_family/__init__.py | 20 +++- .../ecs/service_networking/__init__.py | 12 ++- .../ecs/service_networking/ingress_helpers.py | 98 ++++++++++++++++++- ecs_composex/ecs_composex.py | 2 +- .../specs/services.x-network.spec.json | 23 +++-- use-cases/warpstream.yaml | 11 +++ 10 files changed, 163 insertions(+), 23 deletions(-) diff --git a/ecs_composex/cloudmap/cloudmap_params.py b/ecs_composex/cloudmap/cloudmap_params.py index ef863ecf..2b21af7b 100644 --- a/ecs_composex/cloudmap/cloudmap_params.py +++ b/ecs_composex/cloudmap/cloudmap_params.py @@ -48,6 +48,15 @@ AllowedPattern=ZONES_PATTERN.pattern, ) +PRIVATE_NAMESPACE_ARN_T: str = "PrivateNamespaceArn" +PRIVATE_NAMESPACE_ARN: Parameter = Parameter( + PRIVATE_NAMESPACE_ARN_T, + group_label=LABEL, + return_value="Arn", + Type="String", +) + + ECS_SERVICE_NAMESPACE_SERVICE_ID_T = "EcsCloudMapServiceName" ECS_SERVICE_NAMESPACE_SERVICE_ID = Parameter( ECS_SERVICE_NAMESPACE_SERVICE_ID_T, diff --git a/ecs_composex/cloudmap/cloudmap_stack.py b/ecs_composex/cloudmap/cloudmap_stack.py index 961313f1..0a312d18 100644 --- a/ecs_composex/cloudmap/cloudmap_stack.py +++ b/ecs_composex/cloudmap/cloudmap_stack.py @@ -43,6 +43,7 @@ MOD_KEY, PRIVATE_DNS_ZONE_ID, PRIVATE_DNS_ZONE_NAME, + PRIVATE_NAMESPACE_ARN, PRIVATE_NAMESPACE_ID, ) from .cloudmap_x_resources import handle_resource_cloudmap_settings @@ -99,6 +100,12 @@ def init_outputs(self): self.zone_name, False, ), + PRIVATE_NAMESPACE_ARN: ( + f"{self.logical_name}{PRIVATE_NAMESPACE_ARN.return_value}", + self.cfn_resource, + GetAtt, + PRIVATE_NAMESPACE_ARN.return_value, + ), } @property diff --git a/ecs_composex/common/settings.py b/ecs_composex/common/settings.py index b57e5c58..f2303c45 100644 --- a/ecs_composex/common/settings.py +++ b/ecs_composex/common/settings.py @@ -280,7 +280,8 @@ def get_resource_attribute(self, compose_resource_arn: str) -> tuple: parts.group("return_value") ] return resource, parameter - except LookupError: + except LookupError as error: + print(f"Not found {compose_resource_arn}") return None, None @property diff --git a/ecs_composex/compose/compose_services/__init__.py b/ecs_composex/compose/compose_services/__init__.py index 0c2e8058..c610bce2 100644 --- a/ecs_composex/compose/compose_services/__init__.py +++ b/ecs_composex/compose/compose_services/__init__.py @@ -105,6 +105,7 @@ def __init__( self.x_scaling = set_else_none("x-scaling", self.definition, None, False) self.x_network = set_else_none("x-network", self.definition, None, False) self.x_cloudmap = set_else_none("x-cloudmap", self.x_network, None, False) + self.x_ecs_connect = set_else_none("x-ecs_connect", self.x_network, None) self.x_ecs = set_else_none("x-ecs", self.definition, {}) self.ecr_config = set_else_none("x-ecr", self.definition, None) self.x_ecr = set_else_none("x-ecr", self.definition, {}) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 180d134c..b8b79417 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from troposphere.ecs import Service as CfnService from ecs_composex.common.settings import ComposeXSettings + from ecs_composex.ecs.ecs_service import EcsService import re from itertools import chain @@ -72,8 +73,8 @@ class ComposeFamily: """ def __init__(self, services: list[ComposeService], family_name): - self._compose_services = services - self.ordered_services = services + self._compose_services: list[ComposeService] = services + self.ordered_services: list[ComposeService] = services self.managed_sidecars = [] self.name = family_name self.family_hostname = self.name.replace("_", "-").lower() @@ -92,7 +93,7 @@ def __init__(self, services: list[ComposeService], family_name): self.task_definition = None self.service_tags = None self.enable_execute_command = False - self.ecs_service = None + self.ecs_service: EcsService = None self.runtime_cpu_arch = None self.runtime_os_family = None self.outputs = [] @@ -103,7 +104,7 @@ def __init__(self, services: list[ComposeService], family_name): self.iam_manager = TaskIam(self) self.iam_manager.init_update_policies() self.service_scaling = None - self.service_networking = None + self.service_networking: ServiceNetworking | None = None self.task_compute = None self.service_compute = ServiceCompute(self) self.set_enable_execute_command() @@ -447,13 +448,22 @@ def init_network_settings( self.service_networking.ingress.associate_ext_ingress_rules(self.template) self.service_networking.add_self_ingress() - def finalize_family_settings(self): + def finalize_family_settings(self, settings: ComposeXSettings): """ Once all services have been added, we add the sidecars and deal with appropriate permissions and settings Will add xray / prometheus sidecars """ from .family_helpers import set_service_dependency_on_all_iam_policies + self.service_networking.set_ecs_connect(settings) + if self.service_networking.ecs_connect_config and self.ecs_service: + print("ASSIGNING SERVICE CONNECT") + setattr( + self.ecs_service.ecs_service, + "ServiceConnectConfiguration", + self.service_networking.ecs_connect_config, + ) + self.add_containers_images_cfn_parameters() self.task_compute.set_task_compute_parameter() self.task_compute.unlock_compute_for_main_container() diff --git a/ecs_composex/ecs/service_networking/__init__.py b/ecs_composex/ecs/service_networking/__init__.py index 4af0644a..2fabc7a4 100644 --- a/ecs_composex/ecs/service_networking/__init__.py +++ b/ecs_composex/ecs/service_networking/__init__.py @@ -16,6 +16,7 @@ XStack as EcsIngressStack, ServiceSecurityGroup, ) + from ecs_composex.common.settings import ComposeXSettings from itertools import chain @@ -38,6 +39,7 @@ from ecs_composex.ecs.ecs_conditions import use_external_lt_con from ecs_composex.ecs.ecs_params import NETWORK_MODE, SERVICE_NAME from ecs_composex.ecs.service_networking.ingress_helpers import ( + import_set_ecs_connect_settings, merge_cloudmap_settings, merge_family_services_networking, ) @@ -83,6 +85,7 @@ def __init__(self, family: ComposeFamily, families_sg_stack: EcsIngressStack): self.cloudmap_config = ( merge_cloudmap_settings(family, self.ports) if self.ports else {} ) + self.ecs_connect_config: ServiceConnectConfiguration | None = None self.ingress = Ingress(self.definition[Ingress.master_key], self.ports) @property @@ -219,11 +222,7 @@ def merge_networks(self): self.networks.update(svc.networks) def merge_services_ports(self): - """ - Function to merge two sections of ports - - :return: - """ + """Function to merge two sections of ports""" source_ports = [ service.ports for service in chain( @@ -242,6 +241,9 @@ def merge_services_ports(self): if s_port["target"] not in f_overide_ports_targets: self.ports.append(s_port) + def set_ecs_connect(self, settings: ComposeXSettings): + self.ecs_connect_config = import_set_ecs_connect_settings(self.family, settings) + def add_self_ingress(self) -> None: """ Method to allow communications internally to the group on set ports diff --git a/ecs_composex/ecs/service_networking/ingress_helpers.py b/ecs_composex/ecs/service_networking/ingress_helpers.py index bb0d6659..ff4207e3 100644 --- a/ecs_composex/ecs/service_networking/ingress_helpers.py +++ b/ecs_composex/ecs/service_networking/ingress_helpers.py @@ -17,8 +17,13 @@ from json import dumps from compose_x_common.compose_x_common import keyisset, keypresent, set_else_none -from troposphere import AWS_ACCOUNT_ID, GetAtt, Ref, Sub +from troposphere import AWS_ACCOUNT_ID, GetAtt, NoValue, Ref, Sub from troposphere.ec2 import SecurityGroupIngress +from troposphere.ecs import ( + ServiceConnectClientAlias, + ServiceConnectConfiguration, + ServiceConnectService, +) from ecs_composex.cloudmap.cloudmap_params import RES_KEY as CLOUDMAP_KEY from ecs_composex.common.cfn_params import Parameter @@ -26,6 +31,7 @@ from ecs_composex.common.troposphere_tools import add_parameters, add_resource from ecs_composex.ecs.ecs_params import SERVICE_NAME from ecs_composex.ingress_settings import Ingress +from ecs_composex.resources_import import import_record_properties from ecs_composex.vpc.vpc_params import SG_ID_TYPE @@ -301,3 +307,93 @@ def merge_cloudmap_settings(family: ComposeFamily, ports: list) -> dict: elif isinstance(cloudmap_config, dict): handle_dict_cloudmap_config(family, family_mappings, cloudmap_config, ports) return family_mappings + + +def find_namespace( + family: ComposeFamily, namespace_id: str, settings: ComposeXSettings +): + """Finds the x-cloudmap: namespace and returns the identifier to use for it""" + x_resource_attribute: str = f"x-cloudmap::{namespace_id}::Arn" + print("RESOURCES?", settings.x_resources, namespace_id, x_resource_attribute) + namespace, parameter = settings.get_resource_attribute(x_resource_attribute) + value, params_to_add = namespace.get_resource_attribute_value(parameter, family) + return value + + +def set_ecs_connect_from_macro( + family: ComposeFamily, + service: ComposeService, + macro: dict, + settings: ComposeXSettings, +) -> ServiceConnectConfiguration: + """ + Based on the MacroParameters, creates the ServiceConnectConfiguration object. + Configuration is in the `macro` parameter + """ + port_name = macro["PortName"] + for the_port in family.service_networking.ports: + if the_port["name"] == port_name: + break + else: + raise AttributeError( + f"No port called {port_name} in family {family.name}", + [_port["name"] for _port in family.service_networking.ports], + ) + + dns_name = set_else_none("DnsName", macro, None) + client_aliases = NoValue + if dns_name: + client_aliases = [ + ServiceConnectClientAlias(DnsName=dns_name, Port=the_port["target"]) + ] + services_props: dict = { + "DiscoveryName": set_else_none("CloudMapServiceName", macro, family.name), + "PortName": port_name, + "Timeout": set_else_none("Timeout", macro, NoValue), + "IngressPortOverride": set_else_none("IngressPortOverride", macro, NoValue), + "ClientAliases": client_aliases, + } + props: dict = { + "Enabled": True, + "Namespace": find_namespace(family, macro["x-cloudmap"], settings), + "Services": [ServiceConnectService(**services_props)], + } + return ServiceConnectConfiguration(**props) + + +def process_ecs_connect_settings( + family: ComposeFamily, service: ComposeService, settings: ComposeXSettings +) -> ServiceConnectConfiguration: + """Determines whether to create the ECS Service connect from the Properties or MacroParameters""" + if keyisset("Properties", service.x_ecs_connect): + props = import_record_properties( + service.x_ecs_connect["Properties"], ServiceConnectConfiguration + ) + return ServiceConnectConfiguration(**props) + elif keyisset("MacroParameters", service.x_ecs_connect): + return set_ecs_connect_from_macro( + family, service, service.x_ecs_connect["MacroParameters"], settings + ) + else: + raise KeyError( + f"{family.name} - x-network.x-ecs_connect is not set correctly. " + "One of Properties or MacroParameters is required" + ) + + +def import_set_ecs_connect_settings( + family: ComposeFamily, settings: ComposeXSettings +) -> ServiceConnectConfiguration | None: + if not family.service_networking.ports: + LOG.warning(f"services.{family.name} - No ports defined: ignoring ECS Connect.") + return + x_ecs_configs: list[Service] = [ + service for service in family.ordered_services if service.x_ecs_connect + ] + if not x_ecs_configs: + return None + if len(x_ecs_configs) > 1: + raise ValueError( + f"{family.name} - x-network.x-ecs_connect can only be set once for all the services of the family." + ) + return process_ecs_connect_settings(family, x_ecs_configs[0], settings) diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index 85cf17cc..1ae56dd6 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -301,7 +301,7 @@ def generate_full_template(settings: ComposeXSettings): mesh.render_mesh_template(mesh.stack, settings) for family in settings.families.values(): - family.finalize_family_settings() + family.finalize_family_settings(settings) map_resource_return_value_to_services_command(family, settings) family.state_facts() family.x_environment_processing() diff --git a/ecs_composex/specs/services.x-network.spec.json b/ecs_composex/specs/services.x-network.spec.json index 53551c49..54a5d993 100644 --- a/ecs_composex/specs/services.x-network.spec.json +++ b/ecs_composex/specs/services.x-network.spec.json @@ -53,21 +53,24 @@ "MacroParameters": { "type": "object", "description": "ECS Compose-X Shorthand syntax to configure ECS Connect.", - "additionalProperties": false, + "additionalProperties": true, + "required": [ + "PortName", + "x-cloudmap" + ], "properties": { - "ServiceAlias": { + "DnsName": { "type": "string", - "description": "DNS Alias to use for this service with ECS Connect" + "description": "DNS name for the clients to find this service" }, - "ServicePort": { - "type": "number", - "minimum": 0, - "maximum": 65535, - "description": "The port to use for registration. If not set, uses the first port in the ports list" + "CloudMapServiceName": { + "type": "string", + "maxLength": 64, + "description": "Optional - Set the name of the service as it appears in the CloudMap namespace" }, - "ServicePortName": { + "PortName": { "type": "string", - "description": "Name of the port. Must be the same as ports[].name. If not specified, uses generated port name of the first port." + "description": "Name of the port. Must be the same as ports[].name. If not specified, uses generated port name of the first port in the service." }, "x-cloudmap": { "type": "string", diff --git a/use-cases/warpstream.yaml b/use-cases/warpstream.yaml index 11ccc6fd..efe8ca1e 100644 --- a/use-cases/warpstream.yaml +++ b/use-cases/warpstream.yaml @@ -20,6 +20,12 @@ services: - 9999/tcp - 8080/tcp x-network: + x-ecs_connect: + MacroParameters: + DnsName: wapstream.testing.internal + CloudMapServiceName: warpstream-dev + PortName: tcp_9092 + x-cloudmap: InternalZone Ingress: Myself: true ExtSources: @@ -49,6 +55,11 @@ x-route53: ZoneName: bdd-testing.compose-x.io Lookup: true +x-cloudmap: + InternalZone: + ZoneName: testing.internal + + x-acm: warp-cert: MacroParameters: From 443ba165620437f4fab7c98dc199e9f99f7846a8 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Fri, 29 Mar 2024 16:51:42 +0000 Subject: [PATCH 2/7] Working Service connect clients & servers --- ecs_composex/ecs/ecs_family/__init__.py | 1 - .../ecs/service_networking/ingress_helpers.py | 71 +++++---- .../specs/services.x-network.spec.json | 137 ++++++++++-------- use-cases/warpstream.yaml | 8 +- 4 files changed, 126 insertions(+), 91 deletions(-) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index b8b79417..6bc4391a 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -457,7 +457,6 @@ def finalize_family_settings(self, settings: ComposeXSettings): self.service_networking.set_ecs_connect(settings) if self.service_networking.ecs_connect_config and self.ecs_service: - print("ASSIGNING SERVICE CONNECT") setattr( self.ecs_service.ecs_service, "ServiceConnectConfiguration", diff --git a/ecs_composex/ecs/service_networking/ingress_helpers.py b/ecs_composex/ecs/service_networking/ingress_helpers.py index ff4207e3..2c19636f 100644 --- a/ecs_composex/ecs/service_networking/ingress_helpers.py +++ b/ecs_composex/ecs/service_networking/ingress_helpers.py @@ -314,7 +314,6 @@ def find_namespace( ): """Finds the x-cloudmap: namespace and returns the identifier to use for it""" x_resource_attribute: str = f"x-cloudmap::{namespace_id}::Arn" - print("RESOURCES?", settings.x_resources, namespace_id, x_resource_attribute) namespace, parameter = settings.get_resource_attribute(x_resource_attribute) value, params_to_add = namespace.get_resource_attribute_value(parameter, family) return value @@ -330,48 +329,61 @@ def set_ecs_connect_from_macro( Based on the MacroParameters, creates the ServiceConnectConfiguration object. Configuration is in the `macro` parameter """ - port_name = macro["PortName"] - for the_port in family.service_networking.ports: - if the_port["name"] == port_name: - break - else: - raise AttributeError( - f"No port called {port_name} in family {family.name}", - [_port["name"] for _port in family.service_networking.ports], - ) - - dns_name = set_else_none("DnsName", macro, None) - client_aliases = NoValue - if dns_name: - client_aliases = [ - ServiceConnectClientAlias(DnsName=dns_name, Port=the_port["target"]) - ] - services_props: dict = { - "DiscoveryName": set_else_none("CloudMapServiceName", macro, family.name), - "PortName": port_name, - "Timeout": set_else_none("Timeout", macro, NoValue), - "IngressPortOverride": set_else_none("IngressPortOverride", macro, NoValue), - "ClientAliases": client_aliases, - } + LOG.info(f"{family.name}.{service.name} - Setting up ecs-connect settings") + service_aliases: list[ServiceConnectService] = [] props: dict = { "Enabled": True, "Namespace": find_namespace(family, macro["x-cloudmap"], settings), - "Services": [ServiceConnectService(**services_props)], + "Services": service_aliases, } + if not keyisset("service_ports", macro): + return ServiceConnectConfiguration(**props) + + for port_name, connect_config in macro["service_ports"].items(): + for the_port in family.service_networking.ports: + if keyisset("name", the_port) and the_port["name"] == port_name: + break + else: + raise AttributeError( + f"No port called {port_name} in family {family.name}", + [_port["name"] for _port in family.service_networking.ports], + ) + + dns_name = set_else_none("DnsName", connect_config, None) + client_aliases = NoValue + if dns_name: + client_aliases = [ + ServiceConnectClientAlias(DnsName=dns_name, Port=the_port["target"]) + ] + services_props: dict = { + "DiscoveryName": set_else_none( + "CloudMapServiceName", connect_config, family.name + ), + "PortName": port_name, + "Timeout": set_else_none("Timeout", connect_config, NoValue), + "IngressPortOverride": set_else_none( + "IngressPortOverride", connect_config, NoValue + ), + "ClientAliases": client_aliases, + } + config: ServiceConnectService = ServiceConnectService(**services_props) + service_aliases.append(config) + return ServiceConnectConfiguration(**props) def process_ecs_connect_settings( family: ComposeFamily, service: ComposeService, settings: ComposeXSettings -) -> ServiceConnectConfiguration: +) -> ServiceConnectConfiguration | Ref: """Determines whether to create the ECS Service connect from the Properties or MacroParameters""" + connect_props = NoValue if keyisset("Properties", service.x_ecs_connect): props = import_record_properties( service.x_ecs_connect["Properties"], ServiceConnectConfiguration ) - return ServiceConnectConfiguration(**props) + connect_props = ServiceConnectConfiguration(**props) elif keyisset("MacroParameters", service.x_ecs_connect): - return set_ecs_connect_from_macro( + connect_props = set_ecs_connect_from_macro( family, service, service.x_ecs_connect["MacroParameters"], settings ) else: @@ -379,6 +391,7 @@ def process_ecs_connect_settings( f"{family.name} - x-network.x-ecs_connect is not set correctly. " "One of Properties or MacroParameters is required" ) + return connect_props def import_set_ecs_connect_settings( @@ -387,7 +400,7 @@ def import_set_ecs_connect_settings( if not family.service_networking.ports: LOG.warning(f"services.{family.name} - No ports defined: ignoring ECS Connect.") return - x_ecs_configs: list[Service] = [ + x_ecs_configs: list[ComposeService] = [ service for service in family.ordered_services if service.x_ecs_connect ] if not x_ecs_configs: diff --git a/ecs_composex/specs/services.x-network.spec.json b/ecs_composex/specs/services.x-network.spec.json index 54a5d993..51d464e3 100644 --- a/ecs_composex/specs/services.x-network.spec.json +++ b/ecs_composex/specs/services.x-network.spec.json @@ -32,6 +32,25 @@ } }, "x-ecs_connect": { + "$ref": "#/definitions/ecsConnect" + }, + "x-cloudmap": { + "oneOf": [ + { + "type": "string", + "description": "When you want to register the service into CloudMap. First port listed in ports[] used." + }, + { + "$ref": "#/definitions/cloudMapMappingDefinition" + } + ] + }, + "Ingress": { + "$ref": "ingress.spec.json" + } + }, + "definitions": { + "ecsConnect": { "type": "object", "oneOf": [ { @@ -51,77 +70,79 @@ "description": "Literal properties to set as in https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ecs-service-serviceconnectconfiguration.html" }, "MacroParameters": { - "type": "object", - "description": "ECS Compose-X Shorthand syntax to configure ECS Connect.", - "additionalProperties": true, - "required": [ - "PortName", - "x-cloudmap" - ], - "properties": { - "DnsName": { - "type": "string", - "description": "DNS name for the clients to find this service" - }, - "CloudMapServiceName": { - "type": "string", - "maxLength": 64, - "description": "Optional - Set the name of the service as it appears in the CloudMap namespace" - }, - "PortName": { - "type": "string", - "description": "Name of the port. Must be the same as ports[].name. If not specified, uses generated port name of the first port in the service." - }, - "x-cloudmap": { - "type": "string", - "description": "Name of the namespace defined in the x-cloudmap root level to use." - } - }, - "patternProperties": { - "x-*": {} - } + "$ref": "#/definitions/ecsConnectMacroParameters" } } }, - "x-cloudmap": { - "oneOf": [ - { + "ecsConnectMacroParameters": { + "type": "object", + "additionalProperties": false, + "description": "Mapping for connect ports, allowing to create 1 service connect server alias per port", + "properties": { + "x-cloudmap": { "type": "string", - "description": "When you want to register the service into CloudMap. First port listed in ports[] used." + "description": "Name of the namespace defined in the x-cloudmap root level to use." }, - { + "service_ports": { "type": "object", - "description": "Allows you to register the service to multiple registries, and/or use a specific port.", "additionalProperties": false, "patternProperties": { - "[a-zA-Z0-9-_.]+$": { - "type": "object", - "description": "The name of the object is the name of the x-cloudmap::.", - "additionalProperties": false, - "required": [ - "Port" - ], - "properties": { - "Port": { - "description": "The port to register in SRV record", - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "Name": { - "type": "string", - "description": "Name of the service. Do not include the cloudmap hostname. Overrides the ecs.task.family.hostname deploy label" - } - } + "^x-": {}, + "^[a-zA-Z0-9-_]+$": { + "$ref": "#/definitions/ecsConnectMacroPorts" } } } + }, + "required": [ + "x-cloudmap" ] }, - "Ingress": { - "$ref": "ingress.spec.json" + "ecsConnectMacroPorts": { + "type": "object", + "description": "ECS Compose-X Shorthand syntax to configure ECS Connect.", + "additionalProperties": true, + "properties": { + "DnsName": { + "type": "string", + "description": "DNS name for the clients to find this service" + }, + "CloudMapServiceName": { + "type": "string", + "maxLength": 64, + "description": "Optional - Set the name of the service as it appears in the CloudMap namespace" + } + }, + "patternProperties": { + "x-*": {} + } + }, + "cloudMapMappingDefinition": { + "type": "object", + "description": "Allows you to register the service to multiple registries, and/or use a specific port.", + "additionalProperties": false, + "patternProperties": { + "[a-zA-Z0-9-_.]+$": { + "type": "object", + "description": "The name of the object is the name of the x-cloudmap::.", + "additionalProperties": false, + "required": [ + "Port" + ], + "properties": { + "Port": { + "description": "The port to register in SRV record", + "type": "number", + "minimum": 0, + "maximum": 65535 + }, + "Name": { + "type": "string", + "description": "Name of the service. Do not include the cloudmap hostname. Overrides the ecs.task.family.hostname deploy label" + } + } + } + } } - }, - "definitions": { } } diff --git a/use-cases/warpstream.yaml b/use-cases/warpstream.yaml index efe8ca1e..bcbc64ac 100644 --- a/use-cases/warpstream.yaml +++ b/use-cases/warpstream.yaml @@ -1,3 +1,4 @@ +version: "3.8" secrets: CLUSTER_SECRETS: external: true @@ -22,9 +23,10 @@ services: x-network: x-ecs_connect: MacroParameters: - DnsName: wapstream.testing.internal - CloudMapServiceName: warpstream-dev - PortName: tcp_9092 + services_ports: + tcp_9092: + DnsName: wapstream.testing.internal + CloudMapServiceName: warpstream-dev x-cloudmap: InternalZone Ingress: Myself: true From b92e333babca0f60267abcd5049f5dd109fe84a1 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Fri, 29 Mar 2024 16:54:08 +0000 Subject: [PATCH 3/7] Adding YELB test --- tests/features/features/community.feature | 5 +- use-cases/yelb.yaml | 82 +++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 use-cases/yelb.yaml diff --git a/tests/features/features/community.feature b/tests/features/features/community.feature index e9ab2e8b..f39d04e1 100644 --- a/tests/features/features/community.feature +++ b/tests/features/features/community.feature @@ -1,9 +1,10 @@ Feature: community - @warpstream - Scenario Outline: WarpStream + @community + Scenario Outline: community testing Given I use as my docker-compose file Then I render the docker-compose to composex to validate Examples: | file_path | | use-cases/warpstream.yaml | + | use-cases/yelb.yaml | diff --git a/use-cases/yelb.yaml b/use-cases/yelb.yaml new file mode 100644 index 00000000..3e3abc98 --- /dev/null +++ b/use-cases/yelb.yaml @@ -0,0 +1,82 @@ +x-cloudmap: + PrivateNamespace: + Name: compose-x.internal + +networks: + public: + x-vpc: PublicSubnets +services: + yelb-ui: + image: mreferre/yelb-ui:0.10 + depends_on: + - yelb-appserver + ports: + - 80 + environment: + - UI_ENV=prod + networks: + - public + x-network: + AssignPublicIp: true + x-ecs_connect: + MacroParameters: + x-cloudmap: PrivateNamespace + Ingress: + ExtSources: + - IPv4: 86.132.105.135/32 + Name: Home + + yelb-appserver: + image: mreferre/yelb-appserver:0.7 + depends_on: + - redis-server + - yelb-db + ports: + - 4567:4567 + environment: + redishost: redis-server + yelbdbhost: yelb-db + x-network: + Ingress: + Services: + - Name: yelbui + x-ecs_connect: + MacroParameters: + service_ports: + tcp_4567: + DnsName: yelb-appserver + CloudMapServiceName: yelb-appserver + x-cloudmap: PrivateNamespace + + + redis-server: + image: redis:4.0.2 + ports: + - 6379:6379 + x-network: + x-ecs_connect: + MacroParameters: + service_ports: + tcp_6379: + DnsName: redis-server + CloudMapServiceName: redis-server + x-cloudmap: PrivateNamespace + Ingress: + Services: + - Name: yelbappserver + + yelb-db: + image: mreferre/yelb-db:0.6 + ports: + - 5432:5432 + x-network: + x-ecs_connect: + MacroParameters: + service_ports: + tcp_5432: + DnsName: yelb-db + CloudMapServiceName: yelb-db + x-cloudmap: PrivateNamespace + Ingress: + Services: + - Name: yelbappserver From 89764dc33123dcfb05baef5920e730f6ef551339 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Fri, 29 Mar 2024 17:06:32 +0000 Subject: [PATCH 4/7] Fixing up codesmells --- ecs_composex/common/settings.py | 2 +- ecs_composex/ecs/ecs_family/__init__.py | 4 ++-- ecs_composex/ecs/service_networking/ingress_helpers.py | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ecs_composex/common/settings.py b/ecs_composex/common/settings.py index f2303c45..84c950ba 100644 --- a/ecs_composex/common/settings.py +++ b/ecs_composex/common/settings.py @@ -280,7 +280,7 @@ def get_resource_attribute(self, compose_resource_arn: str) -> tuple: parts.group("return_value") ] return resource, parameter - except LookupError as error: + except LookupError: print(f"Not found {compose_resource_arn}") return None, None diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 6bc4391a..8e53cef0 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -93,7 +93,7 @@ def __init__(self, services: list[ComposeService], family_name): self.task_definition = None self.service_tags = None self.enable_execute_command = False - self.ecs_service: EcsService = None + self.ecs_service: EcsService | None = None self.runtime_cpu_arch = None self.runtime_os_family = None self.outputs = [] @@ -106,7 +106,7 @@ def __init__(self, services: list[ComposeService], family_name): self.service_scaling = None self.service_networking: ServiceNetworking | None = None self.task_compute = None - self.service_compute = ServiceCompute(self) + self.service_compute: ServiceCompute = ServiceCompute(self) self.set_enable_execute_command() set_family_hostname(self) diff --git a/ecs_composex/ecs/service_networking/ingress_helpers.py b/ecs_composex/ecs/service_networking/ingress_helpers.py index 2c19636f..12e2ac08 100644 --- a/ecs_composex/ecs/service_networking/ingress_helpers.py +++ b/ecs_composex/ecs/service_networking/ingress_helpers.py @@ -315,8 +315,7 @@ def find_namespace( """Finds the x-cloudmap: namespace and returns the identifier to use for it""" x_resource_attribute: str = f"x-cloudmap::{namespace_id}::Arn" namespace, parameter = settings.get_resource_attribute(x_resource_attribute) - value, params_to_add = namespace.get_resource_attribute_value(parameter, family) - return value + return namespace.get_resource_attribute_value(parameter, family)[0] def set_ecs_connect_from_macro( @@ -376,7 +375,6 @@ def process_ecs_connect_settings( family: ComposeFamily, service: ComposeService, settings: ComposeXSettings ) -> ServiceConnectConfiguration | Ref: """Determines whether to create the ECS Service connect from the Properties or MacroParameters""" - connect_props = NoValue if keyisset("Properties", service.x_ecs_connect): props = import_record_properties( service.x_ecs_connect["Properties"], ServiceConnectConfiguration From 7310d1faddec44c6dca358b164656fddfc257feb Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Sat, 30 Mar 2024 09:46:10 +0000 Subject: [PATCH 5/7] Update test file --- use-cases/yelb.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/use-cases/yelb.yaml b/use-cases/yelb.yaml index 3e3abc98..e581361b 100644 --- a/use-cases/yelb.yaml +++ b/use-cases/yelb.yaml @@ -39,7 +39,7 @@ services: x-network: Ingress: Services: - - Name: yelbui + - Name: yelb-ui x-ecs_connect: MacroParameters: service_ports: @@ -63,7 +63,7 @@ services: x-cloudmap: PrivateNamespace Ingress: Services: - - Name: yelbappserver + - Name: yelb-appserver yelb-db: image: mreferre/yelb-db:0.6 @@ -79,4 +79,4 @@ services: x-cloudmap: PrivateNamespace Ingress: Services: - - Name: yelbappserver + - Name: yelb-appserver From 38d3bfe59722c58db20cbab5d929e5036e1605d7 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Sun, 31 Mar 2024 15:59:32 +0100 Subject: [PATCH 6/7] Fixing up codesmell, splitting code up in smaller functions --- ecs_composex/ecs/ecs_family/__init__.py | 50 ++++++------------- .../__init__.py} | 0 .../family_helpers/compute_finalizers.py | 36 +++++++++++++ .../family_helpers/network_finalizers.py | 47 +++++++++++++++++ 4 files changed, 98 insertions(+), 35 deletions(-) rename ecs_composex/ecs/ecs_family/{family_helpers.py => family_helpers/__init__.py} (100%) create mode 100644 ecs_composex/ecs/ecs_family/family_helpers/compute_finalizers.py create mode 100644 ecs_composex/ecs/ecs_family/family_helpers/network_finalizers.py diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 8e53cef0..957fc8b9 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -453,44 +453,24 @@ def finalize_family_settings(self, settings: ComposeXSettings): Once all services have been added, we add the sidecars and deal with appropriate permissions and settings Will add xray / prometheus sidecars """ - from .family_helpers import set_service_dependency_on_all_iam_policies - - self.service_networking.set_ecs_connect(settings) - if self.service_networking.ecs_connect_config and self.ecs_service: - setattr( - self.ecs_service.ecs_service, - "ServiceConnectConfiguration", - self.service_networking.ecs_connect_config, - ) + from ecs_composex.ecs.ecs_family.family_helpers import ( + set_service_dependency_on_all_iam_policies, + ) + from ecs_composex.ecs.ecs_family.family_helpers.compute_finalizers import ( + finalize_family_compute, + finalize_scaling_settings, + ) + from ecs_composex.ecs.ecs_family.family_helpers.network_finalizers import ( + finalize_lb_settings, + finalize_network_settings, + ) - self.add_containers_images_cfn_parameters() - self.task_compute.set_task_compute_parameter() - self.task_compute.unlock_compute_for_main_container() - if self.service_compute.ecs_capacity_providers: - self.service_compute.apply_capacity_providers_to_service( - self.service_compute.ecs_capacity_providers - ) + finalize_network_settings(self, settings) + finalize_family_compute(self) set_service_dependency_on_all_iam_policies(self) - if self.service_compute.launch_type == "EXTERNAL": - if hasattr(self.service_definition, "LoadBalancers"): - setattr(self.service_definition, "LoadBalancers", NoValue) - if hasattr(self.service_definition, "ServiceRegistries"): - setattr(self.service_definition, "ServiceRegistries", NoValue) - for container in self.task_definition.ContainerDefinitions: - if hasattr(container, "LinuxParameters"): - parameters = getattr(container, "LinuxParameters") - setattr(parameters, "InitProcessEnabled", False) - if ( - self.service_definition - and self.service_definition.title in self.template.resources - ) and ( - self.service_scaling - and self.service_scaling.scalable_target - and self.service_scaling.scalable_target.title - not in self.template.resources - ): - self.template.add_resource(self.service_scaling.scalable_target) + finalize_lb_settings(self) + finalize_scaling_settings(self) self.generate_outputs() service_configs = [ [0, service] diff --git a/ecs_composex/ecs/ecs_family/family_helpers.py b/ecs_composex/ecs/ecs_family/family_helpers/__init__.py similarity index 100% rename from ecs_composex/ecs/ecs_family/family_helpers.py rename to ecs_composex/ecs/ecs_family/family_helpers/__init__.py diff --git a/ecs_composex/ecs/ecs_family/family_helpers/compute_finalizers.py b/ecs_composex/ecs/ecs_family/family_helpers/compute_finalizers.py new file mode 100644 index 00000000..3d74bdf5 --- /dev/null +++ b/ecs_composex/ecs/ecs_family/family_helpers/compute_finalizers.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright 2024 John Mille + +"""Functions to finalize the family compute & scaling settings""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ecs_composex.ecs.ecs_family import ComposeFamily + + +def finalize_family_compute(family: ComposeFamily) -> None: + """Finalizes the family compute settings""" + family.add_containers_images_cfn_parameters() + family.task_compute.set_task_compute_parameter() + family.task_compute.unlock_compute_for_main_container() + if family.service_compute.ecs_capacity_providers: + family.service_compute.apply_capacity_providers_to_service( + family.service_compute.ecs_capacity_providers + ) + + +def finalize_scaling_settings(family: ComposeFamily) -> None: + """If family has scaling target configured, ensures that the scalable target gets created.""" + if ( + family.service_definition + and family.service_definition.title in family.template.resources + ) and ( + family.service_scaling + and family.service_scaling.scalable_target + and family.service_scaling.scalable_target.title + not in family.template.resources + ): + family.template.add_resource(family.service_scaling.scalable_target) diff --git a/ecs_composex/ecs/ecs_family/family_helpers/network_finalizers.py b/ecs_composex/ecs/ecs_family/family_helpers/network_finalizers.py new file mode 100644 index 00000000..d2eb5462 --- /dev/null +++ b/ecs_composex/ecs/ecs_family/family_helpers/network_finalizers.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright 2024 John Mille + +"""Functions to finalize the family networking settings""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ecs_composex.common.settings import ComposeXSettings + from ecs_composex.ecs.ecs_family import ComposeFamily + +from troposphere import NoValue + + +def finalize_network_settings( + family: ComposeFamily, settings: ComposeXSettings +) -> None: + """ + Evaluates the ECS Connect settings to be configured by the service. + If there is a configuration to be set, ensures it's set on the ECS Service definition. + """ + family.service_networking.set_ecs_connect(settings) + if family.service_networking.ecs_connect_config and family.ecs_service: + setattr( + family.ecs_service.ecs_service, + "ServiceConnectConfiguration", + family.service_networking.ecs_connect_config, + ) + + +def finalize_lb_settings(family: ComposeFamily) -> None: + """ + Ensures that the LoadBalancers & ServiceRegistries (LB & CloudMap) are set appropriately based on + the deployment settings. Especially, resets properties if the service is deployed to ECS Anywhere. + Ensures correctness of LinuxParameters for each of the services. + """ + if family.service_compute.launch_type == "EXTERNAL": + if hasattr(family.service_definition, "LoadBalancers"): + setattr(family.service_definition, "LoadBalancers", NoValue) + if hasattr(family.service_definition, "ServiceRegistries"): + setattr(family.service_definition, "ServiceRegistries", NoValue) + for container in family.task_definition.ContainerDefinitions: + if hasattr(container, "LinuxParameters"): + parameters = getattr(container, "LinuxParameters") + setattr(parameters, "InitProcessEnabled", False) From e018602fb7ad52b8d962a5d7eea18572a1285b5d Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Sun, 31 Mar 2024 16:25:18 +0100 Subject: [PATCH 7/7] Adding docs for service connect --- docs/syntax/compose_x/ecs.details/network.rst | 101 ++++++++++++++++++ use-cases/yelb.yaml | 7 +- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/docs/syntax/compose_x/ecs.details/network.rst b/docs/syntax/compose_x/ecs.details/network.rst index e8b03e59..87e36654 100644 --- a/docs/syntax/compose_x/ecs.details/network.rst +++ b/docs/syntax/compose_x/ecs.details/network.rst @@ -14,6 +14,7 @@ services.x-network services: serviceA: x-network: + x-ecs_connect: {} AssignPublicIp: bool Ingress: {} x-cloudmap: {} @@ -41,6 +42,103 @@ This flag allows to assign an Elastic IP to the container when using ``awsvpc`` To select which subnets to place the services, see :ref:`compose_networks_syntax_reference` +x-ecs_connect (1.1.0) +====================== + +This configuration section allows you to define ECS Service Connect configuration. +It's made up of two options, `Properties` and `MacroParameters` + +`Properties` must match exactly the `ECS Service Connect properties`_ and must be all valid to work. + +.. attention:: + + No changes to input or validation will be made when set. Be sure to have everything valid. + +`MacroParameters` however, is an attempt at creating a shorthand syntax to this. + +service connect - client only +------------------------------ + +You might have applications that you want to act only as clients to other services. This will only tell ECS to make sure +to provision the Service Connect sidecar which will be there to handle the proxy-ing to server services. + +To enable the client config, you simply need to enable the feature as show below + +.. code-block:: + + x-cloudmap: + PrivateNamespace: + Name: compose-x.internal + + services: + yelb-ui: + x-network: + AssignPublicIp: true + x-ecs_connect: + MacroParameters: + x-cloudmap: PrivateNamespace + Ingress: + ExtSources: + - IPv4: 0.0.0.0/0 + Name: ANY + +service connect - server +---------------------------- + +For services that you want to act as client & server, you need to declare which ports you want to declare to Service Connect. +That's mandatory. + +For example, we have the following two services: appserver will act as both a client and a server. It will serve requests +for our yelb-ui service (the client above), and a client to the redis-server + +.. code-block:: + + x-cloudmap: + PrivateNamespace: + Name: compose-x.internal + + services: + yelb-appserver: + image: mreferre/yelb-appserver:0.7 + depends_on: + - redis-server + ports: + - 4567:4567 + environment: + redishost: redis-server + x-network: + Ingress: + Services: + - Name: yelb-ui + x-ecs_connect: + MacroParameters: + service_ports: + tcp_4567: + DnsName: yelb-appserver + CloudMapServiceName: yelb-appserver + x-cloudmap: PrivateNamespace + + + redis-server: + image: redis:4.0.2 + ports: + - 6379:6379 + x-network: + x-ecs_connect: + MacroParameters: + service_ports: + tcp_6379: + DnsName: redis-server + CloudMapServiceName: redis-server + x-cloudmap: PrivateNamespace + Ingress: + Services: + - Name: yelb-appserver + +.. hint:: + + See `the full connect example`_ uses to perform functional testing of the feature. + Ingress ====================== @@ -148,3 +246,6 @@ Definition ----------- .. literalinclude:: ../../../../ecs_composex/specs/services.x-network.spec.json + +.. _ECS Service Connect properties: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ecs-service-serviceconnectconfiguration.html +.. _the full connect example: https://github.com/compose-x/ecs_composex/tree/main/use-cases/yelb.yaml diff --git a/use-cases/yelb.yaml b/use-cases/yelb.yaml index e581361b..621628d0 100644 --- a/use-cases/yelb.yaml +++ b/use-cases/yelb.yaml @@ -1,3 +1,6 @@ +--- +# Docker compose file of the yelb application adapted for deployment via ECS Compose-X +version: "3.8" x-cloudmap: PrivateNamespace: Name: compose-x.internal @@ -23,8 +26,8 @@ services: x-cloudmap: PrivateNamespace Ingress: ExtSources: - - IPv4: 86.132.105.135/32 - Name: Home + - IPv4: 0.0.0.0/0 + Name: ANY yelb-appserver: image: mreferre/yelb-appserver:0.7