Fixing up volumes mounts when using hosts (#695)

ecs_composex/compose/compose_volumes/__init__.py

@@ -10,7 +10,7 @@
 
 from copy import deepcopy
 
-from compose_x_common.compose_x_common import keyisset
+from compose_x_common.compose_x_common import keyisset, set_else_none
 
 from ecs_composex.common.logging import LOG
 from ecs_composex.efs.efs_params import RES_KEY as EFS_KEY
@@ -53,10 +53,13 @@ def __init__(self, name, definition):
         self.use = {}
         self.lookup = {}
         self.type = "volume"
-        self.driver = "local"
+        self.driver = set_else_none("driver", definition, "local")
+        self.driver_opts = set_else_none(self.driver_opts_key, definition, {})
         self.external = False
-        self.efs_definition = evaluate_plugin_efs_properties(
-            self.definition, self.driver_opts_key
+        self.efs_definition = (
+            evaluate_plugin_efs_properties(self.definition, self.driver_opts_key)
+            if self.driver == "efs"
+            else {}
         )
         if self.efs_definition:
             LOG.info(
@@ -68,17 +71,19 @@ def __init__(self, name, definition):
             self.import_volume_from_definition()
 
     def import_volume_from_definition(self):
-        if keyisset(EFS_KEY, self.definition):
+        efs_set = set_else_none(EFS_KEY, self.definition)
+        driver = set_else_none(self.driver_key, self.definition)
+        driver_opts = set_else_none(self.driver_opts_key, self.definition)
+        if efs_set:
             self.efs_definition = self.definition[EFS_KEY]
             self.import_from_x_efs_settings()
-        elif (
-            not keyisset(EFS_KEY, self.definition)
-            and keyisset(self.driver_key, self.definition)
-            and not keyisset(self.driver_opts_key, self.definition)
-        ):
+        elif not efs_set and driver and not driver_opts:
             self.import_local_volume()
         else:
-            self.type = "volume"
+            if driver == "local":
+                self.type = set_else_none("o", driver_opts, "volume")
+            else:
+                self.type = "volume"
             self.driver = "local"
             self.is_shared = False
 
@@ -105,10 +110,7 @@ def import_local_volume(self):
             self.type = "volume"
             self.driver = "local"
             self.efs_definition = None
-        elif (
-            self.definition[self.driver_key] == "nfs"
-            or self.definition[self.driver_key] == "efs"
-        ):
+        elif self.definition[self.driver_key] in ["nfs", "efs"]:
             self.type = "bind"
             self.is_shared = True
             self.driver = "nfs"

ecs_composex/compose/compose_volumes/ecs_family_helpers.py

@@ -9,7 +9,7 @@
     from ecs_composex.ecs.ecs_family import ComposeFamily
     from ecs_composex.compose.compose_services import ComposeService
 
-from compose_x_common.compose_x_common import keyisset, keypresent
+from compose_x_common.compose_x_common import keyisset, keypresent, set_else_none
 from troposphere import If, NoValue
 from troposphere.ecs import DockerVolumeConfiguration, Host, MountPoint, Volume
 
@@ -121,13 +121,13 @@ def define_host_volumes(family):
     host_volumes = []
     for service in family.services:
         for volume in service.volumes:
+            v_volume = set_else_none("volume", volume)
+            v_source = set_else_none("source", volume)
             if (
-                (
-                    (keypresent("volume", volume) and volume["volume"] is None)
-                    or not keyisset("volume", volume)
-                )
-                and keyisset("source", volume)
-                and volume["source"].startswith(r"/")
+                not v_volume
+                and v_source
+                and v_source.startswith(r"/")
+                or v_volume.type == "bind"
             ):
                 host_volumes.append(volume)
     return host_volumes
@@ -163,14 +163,27 @@ def set_volumes(family):
         if volume.cfn_volume:
             family_definition_volumes.append(volume.cfn_volume)
     for volume_config in host_volumes:
-        cfn_volume = If(
-            USE_FARGATE_CON_T,
-            NoValue,
-            Volume(
-                Host=Host(SourcePath=volume_config["source"]),
-                DockerVolumeConfiguration=NoValue,
-                Name=NONALPHANUM.sub("", volume_config["target"]),
-            ),
-        )
-        family_definition_volumes.append(cfn_volume)
+        v_volume = set_else_none("volume", volume_config)
+        if v_volume and v_volume.driver == "local" and v_volume.type == "bind":
+            cfn_volume = If(
+                USE_FARGATE_CON_T,
+                NoValue,
+                Volume(
+                    Host=Host(SourcePath=v_volume.driver_opts["device"]),
+                    DockerVolumeConfiguration=NoValue,
+                    Name=v_volume.name,
+                ),
+            )
+        else:
+            cfn_volume = If(
+                USE_FARGATE_CON_T,
+                NoValue,
+                Volume(
+                    Host=Host(SourcePath=volume_config["source"]),
+                    DockerVolumeConfiguration=NoValue,
+                    Name=NONALPHANUM.sub("", volume_config["target"]),
+                ),
+            )
+        if cfn_volume not in family_definition_volumes:
+            family_definition_volumes.append(cfn_volume)
     set_services_mount_points(family)

ecs_composex/compose/compose_volumes/helpers.py

@@ -1,7 +1,7 @@
 #  SPDX-License-Identifier: MPL-2.0
 #  Copyright 2020-2022 John Mille <john@compose-x.io>
 
-from compose_x_common.compose_x_common import keyisset
+from compose_x_common.compose_x_common import keyisset, set_else_none
 from troposphere import NoValue
 
 
@@ -20,26 +20,25 @@ def evaluate_plugin_efs_properties(definition, driver_opts_key):
         ),
     }
     props = {}
-    if keyisset(driver_opts_key, definition) and isinstance(
-        definition[driver_opts_key], dict
-    ):
-        opts = definition[driver_opts_key]
-        if keyisset("lifecycle_policy", opts) and isinstance(
-            opts["lifecycle_policy"], str
-        ):
-            props["LifecyclePolicies"] = [{"TransitionToIA": opts["lifecycle_policy"]}]
-        if keyisset("backup_policy", opts) and isinstance(opts["backup_policy"], str):
-            props["BackupPolicy"] = {"Status": opts["backup_policy"]}
-        for name, config in efs_keys.items():
-            if not keyisset(name, opts):
-                props[config[0]] = NoValue
-            elif not isinstance(opts[name], config[1]):
-                raise TypeError(
-                    f"Property {name} is of type",
-                    type(opts[name]),
-                    "Expected",
-                    config[1],
-                )
-            else:
-                props[config[0]] = opts[name]
+    opts = set_else_none(driver_opts_key, definition, {})
+    if not opts:
+        return props
+    lifecycle_policy = set_else_none("lifecycle_policy", opts)
+    backup_policy = set_else_none("backup_policy", opts)
+    if lifecycle_policy:
+        props["LifecyclePolicies"] = [{"TransitionToIA": lifecycle_policy}]
+    if backup_policy:
+        props["BackupPolicy"] = {"Status": backup_policy}
+    for name, config in efs_keys.items():
+        if not keyisset(name, opts):
+            props[config[0]] = NoValue
+        elif not isinstance(opts[name], config[1]):
+            raise TypeError(
+                f"Property {name} is of type",
+                type(opts[name]),
+                "Expected",
+                config[1],
+            )
+        else:
+            props[config[0]] = opts[name]
     return props

ecs_composex/compose/compose_volumes/services_helpers.py

@@ -11,7 +11,7 @@
 import re
 from uuid import uuid4
 
-from compose_x_common.compose_x_common import keyisset
+from compose_x_common.compose_x_common import keyisset, set_else_none
 
 from ecs_composex.common.logging import LOG
 
@@ -36,10 +36,12 @@ def match_volumes_services_config(
         return
     else:
         for volume in volumes:
-            if not keyisset("source", vol_config) and not keyisset("volume", volume):
+            v_volume = set_else_none("volume", volume)
+            v_source = set_else_none("source", vol_config)
+            if not v_source and not v_volume:
                 LOG.error(f"volumes - Failure to process {volume}")
                 continue
-            if volume.name == vol_config["source"]:
+            if volume.name == v_source:
                 volume.services.append(service)
                 vol_config["volume"] = volume
                 service.volumes.append(vol_config)
@@ -59,7 +61,7 @@ def handle_volume_str_config(service: ComposeService, config: str, volumes: list
     """
     volume_config = {"read_only": False}
     path_finder = re.compile(
-        r"(?:(?P<source>[\S][^:]+):)?(?P<target>/[^:\n]+)(?::(?P<mode>ro|rw))?"
+        r"(?:(?P<source>[\S][^:]+):)?(?P<target>/[^:\n]+)(?::(?P<mode>ro|rw|z))?"
     )
     path_match = path_finder.match(config)
     if not path_match or (path_match and not path_match.group("target")):
@@ -147,18 +149,19 @@ def map_volumes(service: ComposeService, volumes: list = None) -> None:
     :param service: The Service to map the volumes to.
     :param list volumes:
     """
-    if keyisset(ComposeVolume.main_key, service.definition):
-        for s_volume in service.definition[ComposeVolume.main_key]:
-            if (
-                isinstance(s_volume, dict)
-                and (keyisset("type", s_volume) and s_volume["type"] == "tmpfs")
-                or keyisset("tmpfs", s_volume)
-            ):
-                handle_tmpfs(service, s_volume)
-            else:
-                if not volumes:
-                    continue
-                if isinstance(s_volume, str):
-                    handle_volume_str_config(service, s_volume, volumes)
-                elif isinstance(s_volume, dict):
-                    handle_volume_dict_config(service, s_volume, volumes)
+    if not keyisset(ComposeVolume.main_key, service.definition):
+        return
+    for s_volume in service.definition[ComposeVolume.main_key]:
+        if (
+            isinstance(s_volume, dict)
+            and (keyisset("type", s_volume) and s_volume["type"] == "tmpfs")
+            or keyisset("tmpfs", s_volume)
+        ):
+            handle_tmpfs(service, s_volume)
+        else:
+            if not volumes:
+                continue
+            if isinstance(s_volume, str):
+                handle_volume_str_config(service, s_volume, volumes)
+            elif isinstance(s_volume, dict):
+                handle_volume_dict_config(service, s_volume, volumes)