diff --git a/README.md b/README.md index 45b2dca9..169c7ace 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ with safe_open("model.safetensors", framework="pt", device="cpu") as f: - 8 bytes: `N`, a u64 int, containing the size of the header - N bytes: a JSON utf-8 string representing the header. - - The header is a dict like `{"TENSOR_NAME": {"dtype": "F16", "shape": [1, 16, 256], "offsets": [BEGIN, END]}, "NEXT_TENSOR_NAME": {...}, ...}`, where offsets point to the tensor data relative to the beginning of the byte buffer, with `BEGIN` as the starting offset and `END` as the one-past offset (so total tensor byte size = `END - BEGIN`). + - The header is a dict like `{"TENSOR_NAME": {"dtype": "F16", "shape": [1, 16, 256], "data_offsets": [BEGIN, END]}, "NEXT_TENSOR_NAME": {...}, ...}`, where offsets point to the tensor data relative to the beginning of the byte buffer, with `BEGIN` as the starting offset and `END` as the one-past offset (so total tensor byte size = `END - BEGIN`). - A special key `__metadata__` is allowed to contain free form text-to-text map. - Rest of the file: byte-buffer. diff --git a/safetensors/README.md b/safetensors/README.md index 45b2dca9..169c7ace 100644 --- a/safetensors/README.md +++ b/safetensors/README.md @@ -77,7 +77,7 @@ with safe_open("model.safetensors", framework="pt", device="cpu") as f: - 8 bytes: `N`, a u64 int, containing the size of the header - N bytes: a JSON utf-8 string representing the header. - - The header is a dict like `{"TENSOR_NAME": {"dtype": "F16", "shape": [1, 16, 256], "offsets": [BEGIN, END]}, "NEXT_TENSOR_NAME": {...}, ...}`, where offsets point to the tensor data relative to the beginning of the byte buffer, with `BEGIN` as the starting offset and `END` as the one-past offset (so total tensor byte size = `END - BEGIN`). + - The header is a dict like `{"TENSOR_NAME": {"dtype": "F16", "shape": [1, 16, 256], "data_offsets": [BEGIN, END]}, "NEXT_TENSOR_NAME": {...}, ...}`, where offsets point to the tensor data relative to the beginning of the byte buffer, with `BEGIN` as the starting offset and `END` as the one-past offset (so total tensor byte size = `END - BEGIN`). - A special key `__metadata__` is allowed to contain free form text-to-text map. - Rest of the file: byte-buffer. diff --git a/safetensors/src/lib.rs b/safetensors/src/lib.rs index 8e201af1..f3dcf867 100644 --- a/safetensors/src/lib.rs +++ b/safetensors/src/lib.rs @@ -54,7 +54,7 @@ //! //! - 8 bytes: `N`, a u64 int, containing the size of the header //! - N bytes: a JSON utf-8 string representing the header. -//! - The header is a dict like `{"TENSOR_NAME": {"dtype": "F16", "shape": [1, 16, 256], "offsets": [BEGIN, END]}, "NEXT_TENSOR_NAME": {...}, ...}`, where offsets point to the tensor data relative to the beginning of the byte buffer, with `BEGIN` as the starting offset and `END` as the one-past offset (so total tensor byte size = `END - BEGIN`). +//! - The header is a dict like `{"TENSOR_NAME": {"dtype": "F16", "shape": [1, 16, 256], "data_offsets": [BEGIN, END]}, "NEXT_TENSOR_NAME": {...}, ...}`, where offsets point to the tensor data relative to the beginning of the byte buffer, with `BEGIN` as the starting offset and `END` as the one-past offset (so total tensor byte size = `END - BEGIN`). //! - A special key `__metadata__` is allowed to contain free form text-to-text map. //! - Rest of the file: byte-buffer. //!