From 16e18a6b4f01da01391c792cec4bfd1d0e9a9469 Mon Sep 17 00:00:00 2001 From: Jan Pokorny Date: Wed, 2 Nov 2022 14:37:45 +0100 Subject: [PATCH] Fstab support - added support for blivet to directly modify fstab based on blivet actions - added tests and unit tests --- blivet/actionlist.py | 8 +- blivet/blivet.py | 10 +- blivet/formats/fs.py | 8 +- blivet/fstab.py | 364 ++++++++++++++++++++++++++++++ python-blivet.spec | 1 + tests/storage_tests/fstab_test.py | 75 ++++++ tests/unit_tests/fstab_test.py | 124 ++++++++++ 7 files changed, 587 insertions(+), 3 deletions(-) create mode 100644 blivet/fstab.py create mode 100644 tests/storage_tests/fstab_test.py create mode 100644 tests/unit_tests/fstab_test.py diff --git a/blivet/actionlist.py b/blivet/actionlist.py index 0a6ddb2a7..b79890962 100644 --- a/blivet/actionlist.py +++ b/blivet/actionlist.py @@ -266,12 +266,13 @@ def _post_process(self, devices=None): partition.parted_partition = pdisk.getPartitionByPath(partition.path) @with_flag("processing") - def process(self, callbacks=None, devices=None, dry_run=None): + def process(self, callbacks=None, devices=None, fstab=None, dry_run=None): """ Execute all registered actions. :param callbacks: callbacks to be invoked when actions are executed :param devices: a list of all devices current in the devicetree + :param fstab: FSTabManagerObject tied to blivet, if None fstab file will not be modified :type callbacks: :class:`~.callbacks.DoItCallbacks` """ @@ -284,6 +285,7 @@ def process(self, callbacks=None, devices=None, dry_run=None): continue with blivet_lock: + try: action.execute(callbacks) except DiskLabelCommitError: @@ -310,6 +312,10 @@ def process(self, callbacks=None, devices=None, dry_run=None): device.update_name() device.format.device = device.path + if fstab is not None: + fstab.update(devices) + fstab.write() + self._completed_actions.append(self._actions.pop(0)) _callbacks.action_executed(action=action) diff --git a/blivet/blivet.py b/blivet/blivet.py index f2bb32c63..61b0b309f 100644 --- a/blivet/blivet.py +++ b/blivet/blivet.py @@ -40,6 +40,7 @@ from .errors import StorageError, DependencyError from .size import Size from .devicetree import DeviceTree +from .fstab import FSTabManager from .formats import get_default_filesystem_type from .flags import flags from .formats import get_format @@ -71,6 +72,10 @@ def __init__(self): self.size_sets = [] self.set_default_fstype(get_default_filesystem_type()) + # fstab write location purposedly set to None. It has to be overriden + # manually when using blivet. + self.fstab = FSTabManager(src_file="/etc/fstab", dest_file=None) + self._short_product_name = 'blivet' self._next_id = 0 @@ -111,7 +116,8 @@ def do_it(self, callbacks=None): """ - self.devicetree.actions.process(callbacks=callbacks, devices=self.devices) + self.devicetree.actions.process(callbacks=callbacks, devices=self.devices, fstab=self.fstab) + self.fstab.read() @property def next_id(self): @@ -141,6 +147,8 @@ def reset(self, cleanup_only=False): self.edd_dict = get_edd_dict(self.partitioned) self.devicetree.edd_dict = self.edd_dict + self.fstab.read() + if flags.include_nodev: self.devicetree.handle_nodev_filesystems() diff --git a/blivet/formats/fs.py b/blivet/formats/fs.py index 8df881b82..fb93b2469 100644 --- a/blivet/formats/fs.py +++ b/blivet/formats/fs.py @@ -88,7 +88,8 @@ class FS(DeviceFormat): # value is already unpredictable and can change in the future... _metadata_size_factor = 1.0 - config_actions_map = {"label": "write_label"} + config_actions_map = {"label": "write_label", + "mountpoint": "change_mountpoint"} def __init__(self, **kwargs): """ @@ -627,6 +628,11 @@ def _teardown(self, **kwargs): udev.settle() + def change_mountpoint(self, dry_run=False): + # This function is intentionally left blank. Mountpoint change utilizes + # only the generic part of this code branch + pass + def read_label(self): """Read this filesystem's label. diff --git a/blivet/fstab.py b/blivet/fstab.py new file mode 100644 index 000000000..e5d9ce9c8 --- /dev/null +++ b/blivet/fstab.py @@ -0,0 +1,364 @@ +import os + +import gi + +gi.require_version("BlockDev", "2.0") +from gi.repository import BlockDev as blockdev + +from libmount import Table, Fs, MNT_ITER_FORWARD +from libmount import Error as LibmountException + +import logging +log = logging.getLogger("blivet") + + +def parser_errcb(tb, fname, line): # pylint: disable=unused-argument + print("{:s}:{:d}: parse error".format(fname, line)) + return 1 + + +class FSTabManager(): + # Read, write and modify fstab file + # This class is meant to work even without blivet. + # However some of its methods require blivet and will not function without + # it. These methods will import what they need when they are run. + + def __init__(self, src_file=None, dest_file=None): + self._table = Table() # Space for parsed fstab contents + + self.src_file = src_file + self.dest_file = dest_file + + if self.src_file is not None: + self.read(self.src_file) + + def __deepcopy__(self, memo): + clone = FSTabManager(src_file=self.src_file, dest_file=self.dest_file) + clone._table = Table() + clone._table.enable_comments(True) + + entry = self._table.next_fs() + entries = [entry] + while entry: + entries.append(entry) + entry = self._table.next_fs() + + # Libmount does not allow to simply use clone._table.add_fs(entry), so... + for entry in entries: + new_entry = Fs() + new_entry.source = entry.source + new_entry.target = entry.target + new_entry.fstype = entry.fstype + new_entry.append_options(entry.options) + new_entry.freq = entry.freq + new_entry.passno = entry.passno + if entry.comment is not None: + new_entry.comment = entry.comment + clone._table.add_fs(new_entry) + + return clone + + def _get_containing_device(self, path, devicetree): + # Return the device that a path resides on + if not os.path.exists(path): + return None + + st = os.stat(path) + major = os.major(st.st_dev) + minor = os.minor(st.st_dev) + link = "/sys/dev/block/%s:%s" % (major, minor) + if not os.path.exists(link): + return None + + try: + device_name = os.path.basename(os.readlink(link)) + except Exception: # pylint: disable=broad-except + log.error("failed to find device name for path %s", path) + return None + + if device_name.startswith("dm-"): + # have I told you lately that I love you, device-mapper? + # (code and comment copied from anaconda, kept for entertaining purposes) + device_name = blockdev.dm.name_from_node(device_name) + + return devicetree.get_device_by_name(device_name) + + def _from_device(self, device): + # Return the list of fstab options obtained from device + # *(result) of this method is meant to be used as a parameter in other methods + + spec = getattr(device, 'fstab_spec', None) + + file = None + if device.format.mountable: + file = device.format.mountpoint + if device.format.type == 'swap': + file = 'swap' + + vfstype = device.format.type + mntops = None + + return spec, file, vfstype, mntops + + def _from_entry(self, entry): + return entry.source, entry.target, entry.fstype, ','.join(entry.options) + + def read(self, src_file=''): + # Read fstab file + + # Reset table + self._table = Table() + self._table.enable_comments(True) + + # resolve which file to read + if src_file == '': + if self.src_file is None: + # No parameter given, no internal value + return + elif src_file is None: + return + else: + self.src_file = src_file + + self._table.errcb = parser_errcb + self._table.parse_fstab(self.src_file) + + def find_device_by_specs(self, blivet, spec, file_dummy=None, vfstype_dummy=None, mntops=""): # pylint: disable=unused-argument + # Parse an fstab entry for a device, return the corresponding device, + # return None if not found + # dummy arguments allow using result of _from_device/_from_entry as a parameter of this method + return blivet.devicetree.resolve_device(spec, options=mntops) + + def find_device_by_entry(self, blivet, entry): + args = self._from_entry(entry) + return self.find_device_by_specs(blivet, *args) + + def get_device_by_specs(self, blivet, spec, file, vfstype, mntops): + # Parse an fstab entry for a device, return the corresponding device, + # create new one if it does not exist + + from blivet.formats import get_format + from blivet.devices import DirectoryDevice, NFSDevice, FileDevice, NoDevice + from blivet.formats import get_device_format_class + + # no sense in doing any legwork for a noauto entry + if "noauto" in mntops.split(","): + raise ValueError("Unrecognized fstab entry value 'noauto'") + + # find device in the tree + device = blivet.devicetree.resolve_device(spec, options=mntops) + + if device: + # fall through to the bottom of this block + pass + elif ":" in spec and vfstype.startswith("nfs"): + # NFS -- preserve but otherwise ignore + device = NFSDevice(spec, + fmt=get_format(vfstype, + exists=True, + device=spec)) + elif spec.startswith("/") and vfstype == "swap": + # swap file + device = FileDevice(spec, + parents=self._get_containing_device(spec, blivet.devicetree), + fmt=get_format(vfstype, + device=spec, + exists=True), + exists=True) + elif vfstype == "bind" or "bind" in mntops: + # bind mount... set vfstype so later comparison won't + # turn up false positives + vfstype = "bind" + + # This is probably not going to do anything useful, so we'll + # make sure to try again from FSSet.mount_filesystems. The bind + # mount targets should be accessible by the time we try to do + # the bind mount from there. + parents = self._get_containing_device(spec, blivet.devicetree) + device = DirectoryDevice(spec, parents=parents, exists=True) + device.format = get_format("bind", + device=device.path, + exists=True) + elif file in ("/proc", "/sys", "/dev/shm", "/dev/pts", + "/sys/fs/selinux", "/proc/bus/usb", "/sys/firmware/efi/efivars"): + # drop these now -- we'll recreate later + return None + else: + # nodev filesystem -- preserve or drop completely? + fmt = get_format(vfstype) + fmt_class = get_device_format_class("nodev") + if spec == "none" or \ + (fmt_class and isinstance(fmt, fmt_class)): + device = NoDevice(fmt=fmt) + + if device is None: + log.error("failed to resolve %s (%s) from fstab", spec, + vfstype) + raise ValueError() + + device.setup() + fmt = get_format(vfstype, device=device.path, exists=True) + if vfstype != "auto" and None in (device.format.type, fmt.type): + log.info("Unrecognized filesystem type for %s (%s)", + device.name, vfstype) + device.teardown() + raise ValueError() + + # make sure, if we're using a device from the tree, that + # the device's format we found matches what's in the fstab + ftype = getattr(fmt, "mount_type", fmt.type) + dtype = getattr(device.format, "mount_type", device.format.type) + if hasattr(fmt, "test_mount") and vfstype != "auto" and ftype != dtype: + log.info("fstab says %s at %s is %s", dtype, file, ftype) + if fmt.test_mount(): # pylint: disable=no-member + device.format = fmt + else: + device.teardown() + log.info("There is an entry in your fstab file that contains " + "unrecognized file system type. The file says that " + "%s at %s is %s.", dtype, file, ftype) + return None + + if hasattr(device.format, "mountpoint"): + device.format.mountpoint = file + + device.format.options = mntops + + return device + + def get_device_by_entry(self, blivet, entry): + args = self._from_entry(entry) + return self.get_device_by_specs(blivet, *args) + + def add_entry_by_specs(self, spec, file, vfstype, mntops, freq=None, passno=None, comment=None): + # Add new entry into the table + + # Default mount options + if mntops is None: + mntops = 'defaults' + + # Whether the fs should be dumped by dump(8), defaults to 0 + if freq is None: + freq = 0 + + # Order of fsck run at boot time. '/' should have 1, other 2, defaults to 0 + if passno is None: + if file is None: + file = '' + if file == '/': + passno = 1 + elif file.startswith('/boot'): + passno = 2 + else: + passno = 0 + + entry = Fs() + + entry.source = spec + entry.target = file + entry.fstype = vfstype + entry.append_options(mntops) + entry.freq = freq + entry.passno = passno + + if comment is not None: + # add '# ' at the start of any comment line and newline to the end of comment + modif_comment = '# ' + comment.replace('\n', '\n# ') + '\n' + entry.comment = modif_comment + + self._table.add_fs(entry) + + def add_entry_by_device(self, device): + args = self._from_device(device) + return self.add_entry_by_specs(*args) + + def remove_entry_by_specs(self, spec, file, vfstype=None, mntops=""): + fs = self.find_entry_by_specs(spec, file, vfstype, mntops) + if fs: + self._table.remove_fs(fs) + + def remove_entry_by_device(self, device): + args = self._from_device(device) + return self.remove_entry_by_specs(*args) + + def write(self, dest_file=None): + # Commit the self._table into the file. + if dest_file is None: + dest_file = self.dest_file + if dest_file is None: + log.info("Fstab path for writing was not specified") + return + + if os.path.exists(dest_file): + self._table.replace_file(dest_file) + else: + # write will fail if underlying directories do not exist + self._table.write_file(dest_file) + + def find_entry_by_specs(self, spec, file, vfstype_dummy=None, mntops_dummy=None): # pylint: disable=unused-argument + # Return line of self._table with given spec or file + # dummy arguments allow using result of _from_device/_from_entry as a parameter of this method + + entry = None + + if spec is not None and file is not None: + try: + entry = self._table.find_pair(spec, file, MNT_ITER_FORWARD) + except LibmountException: + entry = None + + if spec is not None: + try: + entry = self._table.find_source(spec, MNT_ITER_FORWARD) + except LibmountException: + entry = None + + if file is not None: + try: + entry = self._table.find_target(file, MNT_ITER_FORWARD) + except LibmountException: + entry = None + return entry + + def find_entry_by_device(self, device): + args = self._from_device(device) + return self.find_entry_by_specs(*args) + + def update(self, devices): + # Update self._table entries based on 'devices' list + # - keep unaffected devices entries unchanged + # - remove entries no longer tied to any device + # - add new entries for devices not present in self._table + + # remove entries not tied to any device + fstab_entries = [] + entry = self._table.next_fs() + while entry: + fstab_entries.append(entry) + entry = self._table.next_fs() + + for device in devices: + args = self._from_device(device) + entry = self.find_entry_by_specs(*args) + + # remove from the list if found + try: + fstab_entries.remove(entry) + except ValueError: + pass + + # All entries left in the fstab_entries do not have their counterpart + # in devices and are to be removed from self._table + for entry in fstab_entries: + self._table.remove_fs(entry) + + # add new entries based on devices not in the table + for device in devices: + # if mountable the device should be in the fstab + if device.format.mountable: + args = self._from_device(device) + found = self.find_entry_by_specs(*args) + if found is None: + self.add_entry_by_specs(*args) + elif found.target != device.format.mountpoint: + self.add_entry_by_specs(args[0], device.format.mountpoint, args[2], args[3]) diff --git a/python-blivet.spec b/python-blivet.spec index bc6b2110f..4d97dd0ee 100644 --- a/python-blivet.spec +++ b/python-blivet.spec @@ -56,6 +56,7 @@ Requires: python3-pyudev >= %{pyudevver} Requires: parted >= %{partedver} Requires: python3-pyparted >= %{pypartedver} Requires: libselinux-python3 +Requires: python3-libmount Requires: python3-blockdev >= %{libblockdevver} Recommends: libblockdev-btrfs >= %{libblockdevver} Recommends: libblockdev-crypto >= %{libblockdevver} diff --git a/tests/storage_tests/fstab_test.py b/tests/storage_tests/fstab_test.py new file mode 100644 index 000000000..079a6ad00 --- /dev/null +++ b/tests/storage_tests/fstab_test.py @@ -0,0 +1,75 @@ +import os + +from .storagetestcase import StorageTestCase + +import blivet + +FSTAB_WRITE_FILE = "/tmp/test-blivet-fstab" + + +class FstabTestCase(StorageTestCase): + + def setUp(self): + super().setUp() + + disks = [os.path.basename(vdev) for vdev in self.vdevs] + self.storage = blivet.Blivet() + self.storage.exclusive_disks = disks + self.storage.reset() + + # make sure only the targetcli disks are in the devicetree + for disk in self.storage.disks: + self.assertTrue(disk.path in self.vdevs) + self.assertIsNone(disk.format.type) + self.assertFalse(disk.children) + + def _clean_up(self): + + self.storage.reset() + for disk in self.storage.disks: + if disk.path not in self.vdevs: + raise RuntimeError("Disk %s found in devicetree but not in disks created for tests" % disk.name) + self.storage.recursive_remove(disk) + + self.storage.do_it() + + # restore original fstab target file + self.storage.fstab.dest_file = "/etc/fstab" + + os.remove(FSTAB_WRITE_FILE) + + return super()._clean_up() + + def test_fstab(self): + disk = self.storage.devicetree.get_device_by_path(self.vdevs[0]) + self.assertIsNotNone(disk) + + # change write path of blivet.fstab + self.storage.fstab.dest_file = FSTAB_WRITE_FILE + + self.storage.initialize_disk(disk) + + pv = self.storage.new_partition(size=blivet.size.Size("100 MiB"), fmt_type="lvmpv", + parents=[disk]) + self.storage.create_device(pv) + + blivet.partitioning.do_partitioning(self.storage) + + vg = self.storage.new_vg(name="blivetTestVG", parents=[pv]) + self.storage.create_device(vg) + + lv = self.storage.new_lv(fmt_type="ext4", size=blivet.size.Size("50 MiB"), + parents=[vg], name="blivetTestLVMine") + self.storage.create_device(lv) + + # Change the mountpoint, make sure the change will make it into the fstab + ac = blivet.deviceaction.ActionConfigureFormat(device=lv, attr="mountpoint", new_value="/mnt/test2") + self.storage.devicetree.actions.add(ac) + + self.storage.do_it() + self.storage.reset() + + with open(FSTAB_WRITE_FILE, "r") as f: + contents = f.read() + self.assertTrue("blivetTestLVMine" in contents) + self.assertTrue("/mnt/test2" in contents) diff --git a/tests/unit_tests/fstab_test.py b/tests/unit_tests/fstab_test.py new file mode 100644 index 000000000..965355eb9 --- /dev/null +++ b/tests/unit_tests/fstab_test.py @@ -0,0 +1,124 @@ +import os +import unittest + +from blivet.fstab import FSTabManager +from blivet.devices import DiskDevice +from blivet.formats import get_format +from blivet import Blivet + +FSTAB_WRITE_FILE = "/tmp/test-blivet-fstab2" + + +class FSTabTestCase(unittest.TestCase): + + def setUp(self): + self.fstab = FSTabManager() + self.addCleanup(self._clean_up) + + def _clean_up(self): + try: + os.remove(FSTAB_WRITE_FILE) + except FileNotFoundError: + pass + + def test_fstab(self): + + self.fstab.src_file = None + self.fstab.dest_file = FSTAB_WRITE_FILE + + # create new entries + self.fstab.add_entry_by_specs("/dev/sda_dummy", "/mnt/mountpath", "xfs", "defaults") + self.fstab.add_entry_by_specs("/dev/sdb_dummy", "/media/newpath", "ext4", "defaults") + + # try to find nonexistent entry based on device + entry = self.fstab.find_entry_by_specs("/dev/nonexistent", None, None, None) + self.assertEqual(entry, None) + + # try to find existing entry based on device + entry = self.fstab.find_entry_by_specs("/dev/sda_dummy", None, None, None) + # check that found item is the correct one + self.assertEqual(entry.target, "/mnt/mountpath") + + self.fstab.remove_entry_by_specs(None, "/mnt/mountpath", None, None) + entry = self.fstab.find_entry_by_specs(None, "/mnt/mountpath", None, None) + self.assertEqual(entry, None) + + # write new fstab + self.fstab.write() + + # read the file and verify its contents + with open(FSTAB_WRITE_FILE, "r") as f: + contents = f.read() + self.assertTrue("/media/newpath" in contents) + + def test_from_device(self): + + device = DiskDevice("test_device", fmt=get_format("ext4", exists=True)) + device.format.mountpoint = "/media/fstab_test" + + self.assertEqual(self.fstab._from_device(device), ('/dev/test_device', '/media/fstab_test', 'ext4', None)) + + def test_update(self): + + # Reset table + self.fstab.read(None) + + # Device already present in fstab._table that should be kept there after fstab.update() + dev1 = DiskDevice("already_present_keep", fmt=get_format("ext4", exists=True)) + dev1.format.mountpoint = "/media/fstab_test1" + + # Device already present in fstab._table which should be removed by fstab.update() + dev2 = DiskDevice("already_present_remove", fmt=get_format("ext4", exists=True)) + dev2.format.mountpoint = "/media/fstab_test2" + + # Device not at fstab._table which should not be added since it is not mountable + dev3 = DiskDevice("unmountable_skip") + + # Device not at fstab._table that should be added there after fstab.update() + dev4 = DiskDevice("new", fmt=get_format("ext4", exists=True)) + dev4.format.mountpoint = "/media/fstab_test3" + + self.fstab.add_entry_by_device(dev1) + self.fstab.add_entry_by_device(dev2) + + self.fstab.update([dev1, dev3, dev4]) + + # write new fstab + self.fstab.write("/tmp/test_blivet_fstab2") + + with open("/tmp/test_blivet_fstab2", "r") as f: + contents = f.read() + + self.assertTrue("already_present_keep" in contents) + self.assertFalse("already_present_remove" in contents) + self.assertFalse("unmountable_skip" in contents) + self.assertTrue("new" in contents) + + def test_find_device(self): + # Reset table + self.fstab.read(None) + + b = Blivet() + + dev1 = DiskDevice("sda_dummy", exists=True, fmt=get_format("xfs", exists=True)) + dev1.format.mountpoint = "/mnt/mountpath" + b.devicetree._add_device(dev1) + + dev2 = self.fstab.find_device_by_specs(b, "/dev/sda_dummy", "/mnt/mountpath", "xfs", "defaults") + + self.assertEqual(dev1, dev2) + + def test_get_device(self): + + # Reset table + self.fstab.read(None) + + b = Blivet() + + dev1 = DiskDevice("sda_dummy", exists=True, fmt=get_format("xfs", exists=True)) + dev1.format.mountpoint = "/mnt/mountpath" + b.devicetree._add_device(dev1) + + dev2 = self.fstab.get_device_by_specs(b, "/dev/sda_dummy", "/mnt/mountpath", "xfs", "defaults") + + self.assertEqual(dev1, dev2)