Skip to content

Commit

Permalink
remove v0.7.0 warning
Browse files Browse the repository at this point in the history
  • Loading branch information
ez2rok committed Sep 23, 2024
1 parent 0d59dcf commit fc3df60
Showing 1 changed file with 4 additions and 11 deletions.
15 changes: 4 additions & 11 deletions streaming/base/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,9 +415,7 @@ def __init__(self,
f'predownload to at-least batch_size.')
elif self.predownload is None:
logger.warning(f'Because `predownload` was not specified, it will default to ' +
f'8*batch_size if batch_size is not None, otherwise 64. Prior to ' +
f'Streaming v0.7.0, `predownload` defaulted to ' +
f'max(batch_size, 256 * batch_size // num_canonical_nodes).')
f'8*batch_size if batch_size is not None, otherwise 64.')
self.predownload = 8 * self.batch_size if self.batch_size is not None else 64

# Convert epoch size from string to int, if needed. Cannot be negative.
Expand Down Expand Up @@ -671,8 +669,7 @@ def _set_shuffle_block_size(self, world: World):
if not world.worker_of_rank:
logger.warning(f'Because `shuffle_block_size` was not specified, it will ' +
f'default to max(4_000_000 // num_canonical_nodes, 1 << 18) if ' +
f'num_canonical_nodes is not None, otherwise 262144. Prior to ' +
f'Streaming v0.7.0, `shuffle_block_size` defaulted to 262144.')
f'num_canonical_nodes is not None, otherwise 262144.')
self.shuffle_block_size = max(4_000_000 // self.num_canonical_nodes, 1 << 18) \
if self.num_canonical_nodes is not None else 1 << 18

Expand Down Expand Up @@ -700,9 +697,7 @@ def _resume(self, world: World, epoch: int) -> tuple[int, int]:
logger.warning(
f'Because `num_canonical_nodes` was not specified, and ' +
f'`shuffle_algo` is {self.shuffle_algo}, it will default to ' +
f'be equal to physical nodes. Prior to Streaming ' +
f'v0.7.0, `num_canonical_nodes` defaulted to 64 * physical ' +
f'nodes.')
f'be equal to physical nodes.')
self.num_canonical_nodes = world.num_nodes
self._set_shuffle_block_size(world)
return epoch, 0
Expand All @@ -723,9 +718,7 @@ def _resume(self, world: World, epoch: int) -> tuple[int, int]:
logger.warning(
f'Because `num_canonical_nodes` was not specified, and ' +
f'`shuffle_algo` is {self.shuffle_algo}, it will default to ' +
f'be equal to physical nodes. Prior to Streaming ' +
f'v0.7.0, `num_canonical_nodes` defaulted to 64 * physical ' +
f'nodes.')
f'be equal to physical nodes.')
self.num_canonical_nodes = world.num_nodes
self._set_shuffle_block_size(world)
return epoch, 0
Expand Down

0 comments on commit fc3df60

Please sign in to comment.