Merge branch 'main' into Issue753

pyproject.toml

@@ -17,7 +17,8 @@ select = [
     "LOG",
     "PERF",
     "PLE",
-    "UP006"
+    "UP006",
+    "F401",
 ]
 extend-safe-fixes = [
     "UP006",

scripts/compression/plot.py

@@ -6,7 +6,7 @@
 from argparse import ArgumentParser, Namespace
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Dict, List, Optional
+from typing import Optional
 
 import numpy as np
 from matplotlib import pyplot as plt

scripts/hashing/plot.py

@@ -5,7 +5,7 @@
 
 from argparse import ArgumentParser, Namespace
 from collections import defaultdict
-from typing import Dict, List, Optional, Tuple
+from typing import Optional
 
 import numpy as np
 from matplotlib import pyplot as plt

scripts/partition/diff.py

@@ -5,7 +5,6 @@
 
 from argparse import ArgumentParser, Namespace
 from time import time
-from typing import List
 
 import numpy as np
 from tqdm import tqdm

scripts/partition/plot.py

@@ -4,7 +4,6 @@
 """Plot results of comparing streaming dataset partitioning algorithms."""
 
 from argparse import ArgumentParser, Namespace
-from typing import Dict, List, Tuple
 
 from matplotlib import pyplot as plt
 

scripts/samples/bench_and_plot.py

@@ -9,7 +9,7 @@
 from contextlib import contextmanager
 from shutil import rmtree
 from time import time, time_ns
-from typing import Any, Callable, Dict, Iterator, List, Tuple
+from typing import Any, Callable, Iterator
 
 import numpy as np
 from matplotlib import pyplot as plt

scripts/serialization/compare.py

@@ -25,7 +25,7 @@
 from glob import glob
 from shutil import rmtree
 from time import time
-from typing import Any, Callable, Dict, Iterator, List
+from typing import Any, Callable, Iterator
 
 import numpy as np
 import pandas as pd

scripts/shuffle/plot.py

@@ -4,7 +4,6 @@
 """Plot results of comparing streaming dataset shuffling algorithms."""
 
 from argparse import ArgumentParser, Namespace
-from typing import Dict, List, Tuple
 
 from matplotlib import pyplot as plt
 

simulation/core/main.py

@@ -5,7 +5,7 @@
 
 import logging
 import time
-from typing import Generator, Tuple, Union
+from typing import Generator, Union
 
 import numpy as np
 from core.node_tracker import NodeTracker

simulation/core/node_tracker.py

@@ -3,7 +3,7 @@
 
 """Class for tracking node information during simulation."""
 
-from typing import Optional, Tuple
+from typing import Optional
 
 import numpy as np
 from core.last_used_ordered_set import LastUsedOrderedSet

simulation/core/shard_downloads.py

@@ -3,7 +3,7 @@
 
 """Functions for simulating shard downloads and calculating needed cache limit for downloads."""
 
-from typing import Optional, Tuple
+from typing import Optional
 
 import numpy as np
 from core.node_tracker import NodeTracker

simulation/core/shuffle_quality.py

@@ -4,7 +4,6 @@
 """Determine shuffle quality of a run over a fixed number of samples."""
 
 import logging
-from typing import Tuple
 
 import numpy as np
 from core.utils import remove_padded_samples

simulation/core/sim_spanner.py

@@ -3,8 +3,6 @@
 
 """Mapping of global sample index to shard index for simulation purposes."""
 
-from typing import Tuple
-
 from streaming.base.spanner import Spanner
 
 

simulation/core/utils.py

@@ -3,8 +3,6 @@
 
 """Peripheral functions for simulation functionality."""
 
-from typing import Tuple
-
 import numpy as np
 from core.sim_dataset import SimulationDataset
 from core.sim_time import Time, TimeUnit

simulation/core/yaml_processing.py

@@ -3,7 +3,7 @@
 
 """Ingest yaml and create SimulationDataset."""
 
-from typing import Optional, Tuple
+from typing import Optional
 
 from core.sim_dataset import SimulationDataset
 from core.sim_time import Time, TimeUnit, ensure_time

streaming/__init__.py

@@ -6,7 +6,7 @@
 import streaming.multimodal as multimodal
 import streaming.text as text
 import streaming.vision as vision
-from streaming._version import __version__
+from streaming._version import __version__  # noqa: F401
 from streaming.base import (CSVWriter, JSONWriter, LocalDataset, MDSWriter, Stream,
                             StreamingDataLoader, StreamingDataset, TSVWriter, XSVWriter)
 

tests/conftest.py

@@ -9,9 +9,9 @@
 import pytest
 from moto import mock_aws
 
-from tests.common.utils import compressed_local_remote_dir  # pyright: ignore
-from tests.common.utils import get_free_tcp_port  # pyright: ignore
-from tests.common.utils import local_remote_dir  # pyright: ignore
+from tests.common.utils import compressed_local_remote_dir  # pyright: ignore # noqa: F401
+from tests.common.utils import get_free_tcp_port  # pyright: ignore # noqa: F401
+from tests.common.utils import local_remote_dir  # pyright: ignore # noqa: F401
 
 MY_BUCKET = 'streaming-test-bucket'
 MY_PREFIX = 'train'