diff --git a/src/maggma/stores/open_data.py b/src/maggma/stores/open_data.py index 8fc02548a..5a0d1f492 100644 --- a/src/maggma/stores/open_data.py +++ b/src/maggma/stores/open_data.py @@ -4,6 +4,7 @@ from typing import Dict, Generator, List, Optional, Tuple, Union import jsonlines +import numpy as np import pandas as pd from boto3 import client as boto_client from botocore import UNSIGNED @@ -625,6 +626,8 @@ def _write_doc_and_update_index(self, items: pd.DataFrame, index: pd.DataFrame) self.index.update(index) def _write_doc_to_s3(self, doc: pd.DataFrame, index: pd.DataFrame) -> None: + doc = doc.replace({pd.NaT: None}).replace({"NaT": None}).replace({np.NaN: None}) + string_io = StringIO() with jsonlines.Writer(string_io, dumps=json_util.dumps) as writer: for _, row in doc.iterrows():