Use tantivy index for dedupe

nomenklatura/__init__.py

@@ -3,13 +3,15 @@
 from nomenklatura.resolver import Resolver
 from nomenklatura.store import Store, View
 from nomenklatura.index import Index
+from nomenklatura.index import TantivyIndex
 
 __version__ = "3.11.4"
 __all__ = [
     "Dataset",
     "CompositeEntity",
     "Resolver",
     "Index",
+    "TantivyIndex",
     "Store",
     "View",
 ]

nomenklatura/index/__init__.py

@@ -1,3 +1,4 @@
 from nomenklatura.index.index import Index
+from nomenklatura.index.tantivy_index import TantivyIndex
 
-__all__ = ["Index"]
+__all__ = ["Index", "TantivyIndex"]

nomenklatura/xref.py

@@ -1,13 +1,16 @@
 import logging
 from typing import List, Optional, Type
 from followthemoney.schema import Schema
+from tempfile import mkdtemp
+from pathlib import Path
+import shutil
 
 from nomenklatura.dataset import DS
 from nomenklatura.entity import CE
 from nomenklatura.store import Store
 from nomenklatura.judgement import Judgement
 from nomenklatura.resolver import Resolver
-from nomenklatura.index import Index
+from nomenklatura.index import TantivyIndex
 from nomenklatura.matching import DefaultAlgorithm, ScoringAlgorithm
 
 log = logging.getLogger(__name__)
@@ -39,7 +42,8 @@ def xref(
 ) -> None:
     log.info("Begin xref: %r, resolver: %s", store, resolver)
     view = store.default_view(external=external)
-    index = Index(view)
+    working_dir = Path(mkdtemp())
+    index = TantivyIndex(view, working_dir)
     index.build()
     try:
         scores: List[float] = []
@@ -94,3 +98,5 @@ def xref(
 
     except KeyboardInterrupt:
         log.info("User cancelled, xref will end gracefully.")
+    finally:
+        shutil.rmtree(working_dir, ignore_errors=True)

tests/conftest.py

@@ -6,7 +6,7 @@
 from tempfile import mkdtemp
 
 from nomenklatura import settings
-from nomenklatura.index.tantivy_index import TantivyIndex
+from nomenklatura.index import TantivyIndex
 from nomenklatura.store import load_entity_file_store, SimpleMemoryStore
 from nomenklatura.kv import get_redis
 from nomenklatura.db import get_engine, get_metadata