From 129d6c347b6613276dfdb5067e17e2e2c326acae Mon Sep 17 00:00:00 2001 From: VOvchinnikov Date: Thu, 23 May 2024 13:11:20 +0200 Subject: [PATCH 1/2] Implemented filtering of "ghost" objects in `_ls` --- gcsfs/core.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gcsfs/core.py b/gcsfs/core.py index e6a11331..92281ad9 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -1001,6 +1001,15 @@ async def _ls( if versions and "generation" in entry: entry = entry.copy() entry["name"] = f"{entry['name']}#{entry['generation']}" + + if ( + entry["name"].rstrip("/") == path + and entry.get("kind", "") == "storage#object" + and (entry.get("size", "") == "0" or entry.get("size", "") == 0) + and entry.get("crc32c", "") == "AAAAAA==" + ): + # this is the "ghost" object of an empty folder, skip it + continue out.append(entry) if detail: From e9c9015bc0d31eb7c0651797739245ff4995ae29 Mon Sep 17 00:00:00 2001 From: VOvchinnikov Date: Thu, 23 May 2024 13:17:41 +0200 Subject: [PATCH 2/2] Moved filtering into deeper `_list_objects` method --- gcsfs/core.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/gcsfs/core.py b/gcsfs/core.py index 92281ad9..a788ad5f 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -580,7 +580,19 @@ async def _list_objects(self, path, prefix="", versions=False, **kwargs): return [await self._get_object(path)] else: return [] - out = pseudodirs + items + dirty_out = pseudodirs + items + + out = [] + for entry in dirty_out: + if ( + entry["name"].rstrip("/") == path + and entry.get("kind", "") == "storage#object" + and (entry.get("size", "") == "0" or entry.get("size", "") == 0) + and entry.get("crc32c", "") == "AAAAAA==" + ): + # this is the "ghost" object of an empty folder, skip it + continue + out.append(entry) use_snapshot_listing = inventory_report_info and inventory_report_info.get( "use_snapshot_listing" @@ -1001,15 +1013,6 @@ async def _ls( if versions and "generation" in entry: entry = entry.copy() entry["name"] = f"{entry['name']}#{entry['generation']}" - - if ( - entry["name"].rstrip("/") == path - and entry.get("kind", "") == "storage#object" - and (entry.get("size", "") == "0" or entry.get("size", "") == 0) - and entry.get("crc32c", "") == "AAAAAA==" - ): - # this is the "ghost" object of an empty folder, skip it - continue out.append(entry) if detail: