Skip to content

Commit

Permalink
Removed filtering from low level object listing
Browse files Browse the repository at this point in the history
  • Loading branch information
VOvchinnikov committed May 22, 2024
1 parent 59d1710 commit 7229b84
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions gcsfs/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,7 @@ def _filter_ghost_items(items):
for item in items:
if (
item.get("kind", "") == "storage#object"
and item.get("size", "") == "0"
and (item.get("size", "") == "0" or item.get("size", "") == 0)
and item.get("crc32c", "") == "AAAAAA=="
):
# This is a ghost item, skip it
Expand Down Expand Up @@ -599,7 +599,7 @@ async def _list_objects(self, path, prefix="", versions=False, **kwargs):
return [await self._get_object(path)]
else:
return []
out = pseudodirs + items
out = pseudodirs + self._filter_ghost_items(items)

use_snapshot_listing = inventory_report_info and inventory_report_info.get(
"use_snapshot_listing"
Expand Down Expand Up @@ -765,7 +765,7 @@ async def _sequential_list_objects_helper(
)

prefixes.extend(page.get("prefixes", []))
items.extend(self._filter_ghost_items(page.get("items", [])))
items.extend(page.get("items", []))
next_page_token = page.get("nextPageToken", None)

while next_page_token is not None:
Expand All @@ -785,7 +785,7 @@ async def _sequential_list_objects_helper(

assert page["kind"] == "storage#objects"
prefixes.extend(page.get("prefixes", []))
items.extend(self._filter_ghost_items(page.get("items", [])))
items.extend(page.get("items", []))
next_page_token = page.get("nextPageToken", None)

items = [self._process_object(bucket, i) for i in items]
Expand Down Expand Up @@ -1421,6 +1421,10 @@ async def _find(
bucket, delimiter="", prefix=_prefix, versions=versions
)

# Now filter the objects to remove the "ghost" ones. This cannot be done at "_do_list_objects" level,
# since there's some logic, reliant on presence of the "ghost" objects
objects = self._filter_ghost_items(objects)

dirs = {}
cache_entries = {}

Expand Down

0 comments on commit 7229b84

Please sign in to comment.