Implemented "ghost" files skipping

gcsfs/core.py

@@ -486,6 +486,25 @@ async def _make_bucket_requester_pays(self, path, state=True):
 
     make_bucket_requester_pays = sync_wrapper(_make_bucket_requester_pays)
 
+    @staticmethod
+    def _filter_ghost_items(items):
+        if not items:
+            items = []
+
+        filtered_items = []
+
+        for item in items:
+            if (
+                item.get("kind", "") == "storage#object"
+                and item.get("size", "") == "0"
+                and item.get("crc32c", "") == "AAAAAA=="
+            ):
+                # This is a ghost item, skip it
+                continue
+            filtered_items.append(item)
+
+        return filtered_items
+
     async def _get_object(self, path):
         """Return object information at the given path."""
         bucket, key, generation = self.split_path(path)
@@ -528,7 +547,7 @@ async def _get_object(self, path):
                 maxResults=1 if not generation else None,
                 versions="true" if generation else None,
             )
-            for item in resp.get("items", []):
+            for item in self._filter_ghost_items(resp.get("items", [])):
                 if item["name"] == key and (
                     not generation or item.get("generation") == generation
                 ):
@@ -746,7 +765,7 @@ async def _sequential_list_objects_helper(
         )
 
         prefixes.extend(page.get("prefixes", []))
-        items.extend(page.get("items", []))
+        items.extend(self._filter_ghost_items(page.get("items", [])))
         next_page_token = page.get("nextPageToken", None)
 
         while next_page_token is not None:
@@ -766,7 +785,7 @@ async def _sequential_list_objects_helper(
 
             assert page["kind"] == "storage#objects"
             prefixes.extend(page.get("prefixes", []))
-            items.extend(page.get("items", []))
+            items.extend(self._filter_ghost_items(page.get("items", [])))
             next_page_token = page.get("nextPageToken", None)
 
         items = [self._process_object(bucket, i) for i in items]
@@ -780,7 +799,7 @@ async def _list_buckets(self):
             page = await self._call("GET", "b", project=self.project, json_out=True)
 
             assert page["kind"] == "storage#buckets"
-            items.extend(page.get("items", []))
+            items.extend(self._filter_ghost_items(page.get("items", [])))
             next_page_token = page.get("nextPageToken", None)
 
             while next_page_token is not None:
@@ -793,7 +812,7 @@ async def _list_buckets(self):
                 )
 
                 assert page["kind"] == "storage#buckets"
-                items.extend(page.get("items", []))
+                items.extend(self._filter_ghost_items(page.get("items", [])))
                 next_page_token = page.get("nextPageToken", None)
 
             buckets = [