Merge branch 'fsspec:main' into master

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout source
@@ -49,8 +49,6 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.1.0
+      - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-      - uses: pre-commit/action@v3.0.0
+      - uses: pre-commit/action@v2.0.0

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+known_third_party = aiohttp,click,decorator,fsspec,fuse,google,google_auth_oauthlib,pytest,requests,setuptools

.pre-commit-config.yaml

@@ -18,15 +18,11 @@ repos:
     rev: 6.0.0
     hooks:
       - id: flake8
-        files: gcsfs/
-  - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v2.2.0
     hooks:
-      - id: isort
-        args: ["--profile", "black", "--filter-files"]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.2.3
+      - id: seed-isort-config
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.7.0
     hooks:
-      - id: pyupgrade
-        args:
-          - --py37-plus
+      - id: isort

.readthedocs.yaml

@@ -0,0 +1,18 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: miniconda3-4.7
+
+conda:
+  environment: docs/environment.yml
+
+python:
+  install:
+    - method: pip
+      path: .
+
+sphinx:
+  configuration: docs/source/conf.py
+  fail_on_warning: true

docs/environment.yml

@@ -0,0 +1,8 @@
+name: s3fs
+channels:
+  - defaults
+dependencies:
+  - python= 3.9
+  - docutils<0.17
+  - sphinx
+  - sphinx_rtd_theme

docs/source/_static/custom.css

@@ -0,0 +1,5 @@
+.classifier:before {
+    font-style: normal;
+    margin: 0.5em;
+    content: ":";
+}

docs/source/api.rst

@@ -38,8 +38,10 @@ API
 
 .. autoclass:: GCSFileSystem
    :members:
+   :inherited-members:
 
 .. autoclass:: GCSFile
    :members:
+   :inherited-members:
 
 .. currentmodule:: gcsfs.mapping

docs/source/changelog.rst

@@ -1,6 +1,56 @@
 Changelog
 =========
 
+2023.9.0
+--------
+
+* bulk operations edge cases (#576, 572)
+* inventory report based file listing (#573)
+* pickle HttpError (#571)
+* avoid warnings (#569)
+* maxdepth in find() (#566)
+* invalidate dircache (#564)
+* standard metadata field names (#563)
+* performance of building cache in find() (#561)
+
+
+2023.6.0
+--------
+
+* allow raw/session token for auth (#554)
+* fix listings_expiry_time kwargs (#551)
+* allow setting fixed metadata on put/pipe (#550)
+
+2023.5.0
+--------
+
+* Allow emulator host without protocol (#548)
+* Prevent upload retry from closing the file being sent (#540)
+
+2023.4.0
+--------
+
+No changes
+
+2023.3.0
+--------
+
+* Don't let find() mess up dircache (#531)
+* Drop py3.7 (#529)
+* Update docs (#528)
+* Make times UTC (#527)
+* Use BytesIO for large bodies (#525)
+* Fix: Don't append generation when it is absent (#523)
+* get/put/cp consistency tests (#521)
+
+2023.1.0
+--------
+
+* Support create time (#516, 518)
+* defer async session creation (#513, 514)
+* support listing of file versions (#509)
+* fix ``sign`` following versioned split protocol (#513)
+
 2022.11.0
 ---------
 

docs/source/conf.py

@@ -12,9 +12,6 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import os
-import sys
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -35,7 +32,7 @@
     "sphinx.ext.viewcode",
     "sphinx.ext.autosummary",
     "sphinx.ext.extlinks",
-    "numpydoc",
+    "sphinx.ext.napoleon",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -68,13 +65,6 @@
 # The full version, including alpha/beta/rc tags.
 release = version
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
 # today = ''
@@ -115,15 +105,7 @@
 
 # -- Options for HTML output ----------------------------------------------
 
-# Taken from docs.readthedocs.io:
-# on_rtd is whether we are on readthedocs.io
-on_rtd = os.getenv("READTHEDOCS", None) == "True"
-
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-
-    html_theme = "sphinx_rtd_theme"
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -154,6 +136,10 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
+# Custom CSS file to override read the docs default CSS.
+# Contains workaround for RTD not rendering colon between argument name and type
+html_css_files = ["custom.css"]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
@@ -297,4 +283,4 @@
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 # texinfo_no_detailmenu = False
 
-extlinks = {"pr": ("https://github.com/fsspec/gcsfs/pull/%s", "PR #")}
+extlinks = {"pr": ("https://github.com/fsspec/gcsfs/pull/%s", "PR #%s")}

docs/source/fuse.rst

@@ -1,7 +1,7 @@
 GCSFS and FUSE
 ==============
 
-Warning, this functionality is **experimental**
+Warning, this functionality is **experimental**.
 
 FUSE_ is a mechanism to mount user-level filesystems in unix-like
 systems (linux, osx, etc.). GCSFS is able to use FUSE to present remote

docs/source/index.rst

@@ -10,7 +10,7 @@ Please file issues and requests on github_ and we welcome pull requests.
 .. _github: https://github.com/fsspec/gcsfs/issues
 
 
-This package depends on fsspec_ , and inherits many useful behaviours from there,
+This package depends on fsspec_, and inherits many useful behaviours from there,
 including integration with Dask, and the facility for key-value dict-like
 objects of the type used by zarr.
 
@@ -19,12 +19,16 @@ objects of the type used by zarr.
 Installation
 ------------
 
-The GCSFS library can be installed using ``conda`` or ``pip``:
+The GCSFS library can be installed using ``conda``:
 
 .. code-block:: bash
 
    conda install -c conda-forge gcsfs
-   or
+
+or ``pip``:
+
+.. code-block:: bash
+
    pip install gcsfs
 
 or by cloning the repository:
@@ -50,7 +54,7 @@ Locate and read a file:
    ...     print(f.read())
    b'Hello, world'
 
-(see also ``walk`` and ``glob``)
+(see also :meth:`~gcsfs.core.GCSFileSystem.walk` and :meth:`~gcsfs.core.GCSFileSystem.glob`)
 
 Read with delimited blocks:
 
@@ -128,7 +132,7 @@ to ``GCSFileSystem``, for example:
                       storage_options={"token": "anon"})
 
 This gives the chance to pass any credentials or other necessary
-arguments needed to s3fs.
+arguments needed to gcsfs.
 
 
 Async
@@ -146,14 +150,11 @@ await the client creation before making any GCS call.
 
 .. code-block:: python
 
-    loop = ...  # however you create your loop
-
-    async def run_program(loop):
-        gcs = GCSFileSystem(..., asynchronous=True, loop=loop)
-        await gcs.set_session()
-        ...  # perform work
+    async def run_program():
+        gcs = GCSFileSystem(asynchronous=True)
+        print(await gcs._ls(""))
 
-    asyncio.run(run_program(loop))  # or call from your async code
+    asyncio.run(run_program())  # or call from your async code
 
 Concurrent async operations are also used internally for bulk operations
 such as ``pipe/cat``, ``get/put``, ``cp/mv/rm``. The async calls are
@@ -162,6 +163,10 @@ from normal code. If you are *not*
 using async-style programming, you do not need to know about how this
 works, but you might find the implementation interesting.
 
+For every synchronous function there is asynchronous one prefixed by ``_``, but
+the ``open`` operation does not support async operation. If you need it to open
+some file in async manner, it's better to asynchronously download it to
+temporary location and working with it from there.
 
 Proxy
 -----
@@ -176,7 +181,7 @@ proxy settings from the environment provide ``session_kwargs`` as follows:
 
 For further reference check `aiohttp proxy support`_.
 
-.. _aiohttp proxy support: https://docs.aiohttp.org/en/stable/client_advanced.html?highlight=proxy#proxy-support
+.. _aiohttp proxy support: https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support
 
 
 Contents

gcsfs/__init__.py

@@ -6,3 +6,7 @@
 from .mapping import GCSMap
 
 __all__ = ["GCSFileSystem", "GCSMap"]
+
+from . import _version
+
+__version__ = _version.get_versions()["version"]