Skip to content

Commit

Permalink
clean up iterator cache after run
Browse files Browse the repository at this point in the history
  • Loading branch information
pudo committed Jun 4, 2024
1 parent 3cf8a79 commit cf32207
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions nomenklatura/store/versioned.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,12 +307,12 @@ def get_inverted(self, id: str) -> Generator[Tuple[Property, CE], None, None]:
ident = Identifier.get(id)
for ent_id in self.store.linker.connected(ident):
keys.extend([f"inv:{d}:{v}:{ent_id}" for d, v in self.vers])
entities: Set[str] = set()
refs = (
{bv(v) for v in self.store.db.sunion(keys)}
if len(keys) > 0
else self.store.db.smembers(keys[0])
)
entities: Set[str] = set()
for v in refs:
entity_id = v.decode("utf-8")
entities.add(self.store.linker.get_canonical(entity_id))
Expand Down Expand Up @@ -348,7 +348,7 @@ def entities(self) -> Generator[CE, None, None]:
if len(self.vers) == 1:
scope_name = b(f"ents:{self.vers[0][0]}:{self.vers[0][1]}")
if len(self.vers) > 1:
version = max((v for _, v in self.vers), default="latest")
version = Version.new().id + ":iter"
scope_name = b(f"ents:{self.scope.name}:{version}")
parts = [b(f"ents:{d}:{v}") for d, v in self.vers]
self.store.db.sunionstore(scope_name, parts)
Expand All @@ -357,6 +357,7 @@ def entities(self) -> Generator[CE, None, None]:
# de-duplicated entity multiple times. This intrinsically leaks
# memory, so we're being careful to only record entity IDs
# that are part of a cluster with more than one ID.
try:
seen: Set[str] = set()
for id in self.store.db.sscan_iter(scope_name):
entity_id = id.decode("utf-8")
Expand All @@ -370,3 +371,6 @@ def entities(self) -> Generator[CE, None, None]:
entity = self.get_entity(entity_id)
if entity is not None:
yield entity
finally:
if len(self.vers) > 1:
self.store.db.delete(scope_name)

0 comments on commit cf32207

Please sign in to comment.