Pydantic 2

contrib/test_bench.py

@@ -22,7 +22,7 @@
 
 async def test_example():
     ds = await get_dataset("default")
-    example = EntityExample.parse_obj(EXAMPLE)
+    example = EntityExample.model_validate(EXAMPLE)
     entity = Entity.from_example(example)
     query = entity_query(ds, entity)
     pprint(query)

kubernetes.example.yml

@@ -1,19 +1,4 @@
 ---
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-prod
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: info@opensanctions.org
-    privateKeySecretRef:
-      name: letsencrypt-prod
-    solvers:
-      - http01:
-          ingress:
-            class: nginx
----
 apiVersion: v1
 kind: Service
 metadata:
@@ -28,37 +13,6 @@ spec:
       targetPort: 8000
       name: http
 ---
-# Supposes you have an ingress, and ideally cert-manager installed on your
-# cluster. You should also consider running the service internally to the
-# cluster without exposing it on an ingress.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: yente-ingress
-  annotations:
-    # you need to configure letsencrypt for your cluster:
-    # cert-manager.io/cluster-issuer: letsencrypt-prod
-    acme.cert-manager.io/http01-edit-in-place: "true"
-  labels:
-    app: opensanctions
-spec:
-  ingressClassName: nginx
-  tls:
-    - hosts:
-        - api.opensanctions.org
-      secretName: tls-api.opensanctions.org
-  rules:
-    - host: api.opensanctions.org
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: yente
-                port:
-                  number: 8000
----
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -67,7 +21,7 @@ data:
   manifest.yml: |
     catalogs:
       - url: "https://data.opensanctions.org/datasets/latest/index.json"
-        scope: all
+        scope: default
         resource_name: entities.ftm.json
       # - url: "https://data.opensanctions.org/graph/catalog.json"
       #   resource_name: entities.ftm.json
@@ -107,10 +61,10 @@ spec:
               name: http
           resources:
             requests:
-              memory: 300Mi
+              memory: 600Mi
               cpu: 200m
             limits:
-              memory: 300Mi
+              memory: 600Mi
               cpu: 200m
           securityContext:
             readOnlyRootFilesystem: true
@@ -122,6 +76,8 @@ spec:
             - mountPath: /tmp
               name: tmp-volume
           env:
+            - name: YENTE_PORT
+              value: 8000
             - name: YENTE_TITLE
               value: "OpenSanctions API"
             - name: YENTE_LOG_JSON
@@ -205,10 +161,10 @@ spec:
                 - reindex
               resources:
                 requests:
-                  memory: 300M
+                  memory: 600M
                   cpu: 400m
                 limits:
-                  memory: 300M
+                  memory: 600M
                   cpu: 400m
               securityContext:
                 readOnlyRootFilesystem: true

setup.py

@@ -17,22 +17,22 @@
     namespace_packages=[],
     install_requires=[
         "followthemoney==3.4.3",
-        "nomenklatura==3.2.0",
+        "nomenklatura==3.2.1",
         "asyncstdlib==3.10.8",
         "aiocron==1.8",
         "aiocsv==1.2.4",
         "aiofiles==23.1.0",
         "types-aiofiles==23.1.0.4",
         "aiohttp[speedups]==3.8.4",
         "elasticsearch[async]==8.8.0",
-        "fastapi==0.99.1",
+        "fastapi==0.100.0",
         "uvicorn[standard]==0.22.0",
         "python-multipart==0.0.6",
         "email-validator==2.0.0.post2",
         "structlog==23.1.0",
         "pyicu==2.11",
         "jellyfish==1.0.0",
-        "orjson==3.9.1",
+        "orjson==3.9.2",
         "text-unidecode==1.3",
         "click==8.0.4",
         "normality==2.4.0",

yente/data/common.py

@@ -10,38 +10,27 @@
 
 
 class ErrorResponse(BaseModel):
-    detail: str = Field(..., example="Detailed error message")
+    detail: str = Field(..., examples=["Detailed error message"])
 
 
 class EntityResponse(BaseModel):
-    id: str = Field(..., example="NK-A7z....")
-    caption: str = Field(..., example="John Doe")
-    schema_: str = Field(..., example="LegalEntity", alias="schema")
-    properties: EntityProperties = Field(..., example={"name": ["John Doe"]})
-    datasets: List[str] = Field([], example=["us_ofac_sdn"])
-    referents: List[str] = Field([], example=["ofac-1234"])
+    id: str = Field(..., examples=["NK-A7z...."])
+    caption: str = Field(..., examples=["John Doe"])
+    schema_: str = Field(..., examples=["LegalEntity"], alias="schema")
+    properties: EntityProperties = Field(..., examples=[{"name": ["John Doe"]}])
+    datasets: List[str] = Field([], examples=[["us_ofac_sdn"]])
+    referents: List[str] = Field([], examples=[["ofac-1234"]])
     target: bool = Field(False)
-    first_seen: Optional[datetime] = Field(..., example=datetime.utcnow())
-    last_seen: Optional[datetime] = Field(..., example=datetime.utcnow())
-    last_change: Optional[datetime] = Field(..., example=datetime.utcnow())
+    first_seen: Optional[datetime] = Field(..., examples=[datetime.utcnow()])
+    last_seen: Optional[datetime] = Field(..., examples=[datetime.utcnow()])
+    last_change: Optional[datetime] = Field(..., examples=[datetime.utcnow()])
 
     @classmethod
     def from_entity(cls, entity: Entity) -> "EntityResponse":
-        return cls.construct(
-            id=entity.id,
-            caption=entity._caption,
-            schema=entity.schema.name,
-            properties=dict(entity.properties),
-            datasets=list(entity.datasets),
-            referents=list(entity.referents),
-            target=entity.target,
-            first_seen=entity.first_seen,
-            last_seen=entity.last_seen,
-            last_change=entity.last_change,
-        )
+        return cls.model_validate(entity.to_dict())
 
 
-EntityResponse.update_forward_refs()
+EntityResponse.model_rebuild()
 
 
 class ScoredEntityResponse(EntityResponse):
@@ -53,46 +42,35 @@ class ScoredEntityResponse(EntityResponse):
     def from_entity_result(
         cls, entity: Entity, result: MatchingResult, threshold: float
     ) -> "ScoredEntityResponse":
-        return cls.construct(
-            id=entity.id,
-            caption=entity._caption,
-            schema=entity.schema.name,
-            properties=entity.properties,
-            datasets=list(entity.datasets),
-            referents=list(entity.referents),
-            target=entity.target,
-            first_seen=entity.first_seen,
-            last_seen=entity.last_seen,
-            last_change=entity.last_change,
-            score=result["score"],
-            match=result["score"] >= threshold,
-            features=result["features"],
-        )
+        data = entity.to_dict()
+        data.update(result)
+        data["match"] = result["score"] >= threshold
+        return cls.model_validate(data)
 
 
 class StatusResponse(BaseModel):
     status: str = "ok"
 
 
 class SearchFacetItem(BaseModel):
-    name: str = Field(..., example="ru")
-    label: str = Field(..., example="Russia")
-    count: int = Field(1, example=42)
+    name: str = Field(..., examples=["ru"])
+    label: str = Field(..., examples=["Russia"])
+    count: int = Field(1, examples=[42])
 
 
 class SearchFacet(BaseModel):
-    label: str = Field(..., example="Countries")
+    label: str = Field(..., examples=["Countries"])
     values: List[SearchFacetItem]
 
 
 class TotalSpec(BaseModel):
-    value: int = Field(..., example=42)
-    relation: str = Field("eq", example="eq")
+    value: int = Field(..., examples=[42])
+    relation: str = Field("eq", examples=["eq"])
 
 
 class ResultsResponse(BaseModel):
-    limit: int = Field(..., example=20)
-    offset: int = Field(0, example=0)
+    limit: int = Field(..., examples=[20])
+    offset: int = Field(0, examples=[0])
     total: TotalSpec
 
 
@@ -102,10 +80,10 @@ class SearchResponse(ResultsResponse):
 
 
 class EntityExample(BaseModel):
-    id: Optional[str] = Field(None, example="my-entity-id")
-    schema_: str = Field(..., example=settings.BASE_SCHEMA, alias="schema")
+    id: Optional[str] = Field(None, examples=["my-entity-id"])
+    schema_: str = Field(..., examples=[settings.BASE_SCHEMA], alias="schema")
     properties: Dict[str, Union[str, List[str]]] = Field(
-        ..., example={"name": ["John Doe"]}
+        ..., examples=[{"name": ["John Doe"]}]
     )
 
 
@@ -114,7 +92,7 @@ class EntityMatchQuery(BaseModel):
 
 
 class EntityMatches(BaseModel):
-    status: int = Field(200, example=200)
+    status: int = Field(200, examples=[200])
     results: List[ScoredEntityResponse]
     total: TotalSpec
     query: EntityExample
@@ -123,16 +101,16 @@ class EntityMatches(BaseModel):
 class EntityMatchResponse(BaseModel):
     responses: Dict[str, EntityMatches]
     matcher: FeatureDocs
-    limit: int = Field(..., example=5)
+    limit: int = Field(..., examples=[5])
 
 
 class DatasetModel(BaseModel):
     name: str
     title: str
-    summary: Optional[str]
-    url: Optional[str]
+    summary: Optional[str] = None
+    url: Optional[str] = None
     load: bool
-    entities_url: Optional[str]
+    entities_url: Optional[str] = None
     version: str
     children: List[str]
 
@@ -143,7 +121,7 @@ class DataCatalogModel(BaseModel):
 
 class Algorithm(BaseModel):
     name: str
-    description: Optional[str]
+    description: Optional[str] = None
     features: FeatureDocs
 
 

yente/data/freebase.py

@@ -11,9 +11,9 @@
 
 
 class FreebaseType(BaseModel):
-    id: str = Field(..., example="Person")
-    name: str = Field(..., example="People")
-    description: Optional[str] = Field(None, example="...")
+    id: str = Field(..., examples=["Person"])
+    name: str = Field(..., examples=["People"])
+    description: Optional[str] = None
 
     @classmethod
     def from_schema(cls, schema: Schema) -> "FreebaseType":
@@ -22,19 +22,19 @@ def from_schema(cls, schema: Schema) -> "FreebaseType":
 
 
 class FreebaseProperty(BaseModel):
-    id: str = Field(..., example="birthDate")
-    name: str = Field(..., example="Date of birth")
-    description: Optional[str] = Field(None, example="...")
+    id: str = Field(..., examples=["birthDate"])
+    name: str = Field(..., examples=["Date of birth"])
+    description: Optional[str] = None
 
     @classmethod
     def from_prop(cls, prop: Property) -> "FreebaseProperty":
         return cls(id=prop.qname, name=prop.label, description=prop.description)
 
 
 class FreebaseEntity(BaseModel):
-    id: str = Field(..., example="NK-A7z....")
-    name: str = Field(..., example="John Doe")
-    description: Optional[str] = Field(None, example="...")
+    id: str = Field(..., examples=["NK-A7z...."])
+    name: str = Field(..., examples=["John Doe"])
+    description: Optional[str] = None
     type: List[FreebaseType]
 
     @classmethod
@@ -49,8 +49,8 @@ def from_proxy(cls, proxy: EntityProxy) -> "FreebaseEntity":
 
 
 class FreebaseScoredEntity(FreebaseEntity):
-    score: Optional[float] = Field(..., example=0.99)
-    match: Optional[bool] = Field(..., example=False)
+    score: Optional[float] = Field(..., examples=[0.99])
+    match: Optional[bool] = Field(..., examples=[False])
 
     @classmethod
     def from_scored(cls, data: ScoredEntityResponse) -> "FreebaseScoredEntity":
@@ -89,11 +89,11 @@ class FreebasePropertySuggestResponse(FreebaseSuggestResponse):
 
 
 class FreebaseManifestView(BaseModel):
-    url: str
+    url: AnyHttpUrl
 
 
 class FreebaseManifestPreview(BaseModel):
-    url: str
+    url: AnyHttpUrl
     width: int
     height: int
 
@@ -110,8 +110,8 @@ class FreebaseManifestSuggest(BaseModel):
 
 
 class FreebaseManifest(BaseModel):
-    versions: List[str] = Field(..., example=["0.2"])
-    name: str = Field(..., example=settings.TITLE)
+    versions: List[str] = Field(..., examples=[["0.2"]])
+    name: str = Field(..., examples=[settings.TITLE])
     identifierSpace: AnyHttpUrl
     schemaSpace: AnyHttpUrl
     view: FreebaseManifestView

yente/data/manifest.py

@@ -42,7 +42,7 @@ class Manifest(BaseModel):
     @classmethod
     async def load(cls) -> "Manifest":
         data = await load_yaml_url(settings.MANIFEST)
-        manifest = cls.parse_obj(data)
+        manifest = cls.model_validate(data)
         for catalog in manifest.catalogs:
             await catalog.fetch(manifest)
         # TODO: load remote metadata from a `metadata_url` on each dataset?

yente/routers/admin.py

@@ -89,7 +89,7 @@ async def catalog() -> DataCatalogModel:
     data sources are included, and how often they should be loaded.
     """
     catalog = await get_catalog()
-    return DataCatalogModel.parse_obj(catalog.to_dict())
+    return DataCatalogModel.model_validate(catalog.to_dict())
 
 
 @router.get(