From 9f1f761a6933193ac7fc5263b2c453198957bbaf Mon Sep 17 00:00:00 2001 From: Roman Shkarin Date: Tue, 3 Sep 2024 18:06:42 +0200 Subject: [PATCH 1/6] Reset lock on fork --- gcsfs/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gcsfs/core.py b/gcsfs/core.py index 786f9295..b855ba6e 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -27,6 +27,10 @@ from .inventory_report import InventoryReport from .retry import errs, retry_request, validate_response +os.register_at_fork( + after_in_child=asyn.reset_lock, +) + logger = logging.getLogger("gcsfs") From 38968eb59f83a96c149ea63b4228a2f382d5d3d1 Mon Sep 17 00:00:00 2001 From: Metamess Date: Thu, 23 May 2024 17:05:17 +0200 Subject: [PATCH 2/6] Add seek(0) to request data to prevent issues on retries (#624) --- gcsfs/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gcsfs/core.py b/gcsfs/core.py index b855ba6e..b7a4c898 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -425,6 +425,8 @@ async def _request( self, method, path, *args, headers=None, json=None, data=None, **kwargs ): await self._set_session() + if hasattr(data, "seek"): + data.seek(0) async with self.session.request( method=method, url=self._format_path(path, args), From 6a639b3b927648f8b836e2bc3a2c658167ca7960 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Mon, 3 Jun 2024 20:53:43 -0400 Subject: [PATCH 3/6] changelog (#626) --- docs/source/changelog.rst | 5 +++++ requirements.txt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 928a0f79..ab8a0c6d 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -4,6 +4,11 @@ Changelog Note: in some releases, there are no changes, because we always guarantee relasing in step with fsspec. +2024.6.0 +-------- + +* Add seek(0) to request data to prevent issues on retries (#624) + 2024.5.0 -------- diff --git a/requirements.txt b/requirements.txt index ddf22697..878178bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiohttp!=4.0.0a0, !=4.0.0a1 decorator>4.1.2 -fsspec==2024.5.0 +fsspec==2024.6.0 google-auth>=1.2 google-auth-oauthlib google-cloud-storage From 005ceb679d96cc2105824d398fe66e766160aef5 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 27 Jun 2024 10:54:36 -0400 Subject: [PATCH 4/6] changelog (#629) --- docs/source/changelog.rst | 5 +++++ requirements.txt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index ab8a0c6d..1bf64c29 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -4,6 +4,11 @@ Changelog Note: in some releases, there are no changes, because we always guarantee relasing in step with fsspec. +2024.6.1 +-------- + +no changes + 2024.6.0 -------- diff --git a/requirements.txt b/requirements.txt index 878178bd..1d290de1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiohttp!=4.0.0a0, !=4.0.0a1 decorator>4.1.2 -fsspec==2024.6.0 +fsspec==2024.6.1 google-auth>=1.2 google-auth-oauthlib google-cloud-storage From fce76ab543803796831e58cece370c86c048f251 Mon Sep 17 00:00:00 2001 From: Keunhong Park Date: Thu, 18 Jul 2024 06:54:44 -0700 Subject: [PATCH 5/6] Handle case when error message is a string. (#631) Fixes https://github.com/fsspec/gcsfs/issues/630. --- gcsfs/retry.py | 8 ++++++-- gcsfs/tests/test_retry.py | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gcsfs/retry.py b/gcsfs/retry.py index 1c4012b3..0d787a4e 100644 --- a/gcsfs/retry.py +++ b/gcsfs/retry.py @@ -99,7 +99,11 @@ def validate_response(status, content, path, args=None): content = content.decode() try: error = json.loads(content)["error"] - msg = error["message"] + # Sometimes the error message is a string. + if isinstance(error, str): + msg = error + else: + msg = error["message"] except json.decoder.JSONDecodeError: msg = content @@ -109,7 +113,7 @@ def validate_response(status, content, path, args=None): raise requests.exceptions.ProxyError() elif "invalid" in str(msg): raise ValueError(f"Bad Request: {path}\n{msg}") - elif error: + elif error and not isinstance(error, str): raise HttpError(error) elif status: raise HttpError({"code": status, "message": msg}) # text-like diff --git a/gcsfs/tests/test_retry.py b/gcsfs/tests/test_retry.py index 79058f52..e5d4ed41 100644 --- a/gcsfs/tests/test_retry.py +++ b/gcsfs/tests/test_retry.py @@ -103,6 +103,15 @@ def test_validate_response(): validate_response(502, b"", "/path") +def test_validate_response_error_is_string(): + # HttpError with JSON body + j = '{"error": "Too Many Requests"}' + with pytest.raises(HttpError) as e: + validate_response(429, j, "/path") + assert e.value.code == 429 + assert e.value.message == "Too Many Requests, 429" + + @pytest.mark.parametrize( ["file_path", "validate_get_error", "validate_list_error", "expected_error"], [ From d8d6512c58edce55fc8520981bcdd0be912999db Mon Sep 17 00:00:00 2001 From: VOvchinnikov Date: Wed, 4 Sep 2024 15:23:42 +0200 Subject: [PATCH 6/6] Limited `google-auth` to version until `2.33` In `2.33` Google added some changes that make tests with anything Google being very slow. When `fsspec`/`gcsfs` people fix this, the limitation could be lifted --- environment_gcsfs.yaml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_gcsfs.yaml b/environment_gcsfs.yaml index 98da14c0..b2cc4c16 100644 --- a/environment_gcsfs.yaml +++ b/environment_gcsfs.yaml @@ -9,7 +9,7 @@ dependencies: - fusepy<3 - google-api-core - google-api-python-client - - google-auth + - google-auth>=1.2, <2.33 - google-auth-oauthlib - google-cloud-core - google-cloud-storage diff --git a/requirements.txt b/requirements.txt index 1d290de1..94902d37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiohttp!=4.0.0a0, !=4.0.0a1 decorator>4.1.2 fsspec==2024.6.1 -google-auth>=1.2 +google-auth>=1.2, <2.33 google-auth-oauthlib google-cloud-storage requests