Add shuffle_after_epoch_seed argument to file-based readers.

* Adds an optional `shuffle_after_epoch_seed` (int32) argument to all
  readers that support `shuffle_after_epoch`: readers.file, readers.numpy,
  readers.fits, readers.coco, and readers.nemo_asr.
* Previously the per-epoch shuffle was always seeded with a fixed
  constant, making the dataset order identical across training runs.
  The new argument lets users supply a custom base seed so that
  different training runs produce statistically independent orderings,
  while still guaranteeing a consistent global permutation across shards.
* When omitted, the old fixed seed is used, preserving backward compatibility.
* Tests added in test_shuffling.py and test_numpy.py.

Signed-off-by: Janusz Lisiecki <jlisiecki@nvidia.com>

Author: JanuszL

dali/operators/reader/coco_reader_op.cc

@@ -93,6 +93,22 @@ This readers produces the following outputs::
   .AddOptionalArg("shuffle_after_epoch",
       R"code(If set to True, the reader shuffles the entire  dataset after each epoch.)code",
       false)
+  .AddOptionalArg<int32_t>("shuffle_after_epoch_seed",
+      R"code(Random seed for the dataset shuffling performed after each epoch.
+
+If not provided, a fixed default seed is used, which results in the same shuffling
+pattern across different training runs. Providing a custom seed allows for different
+shuffle patterns across training runs, which may be desirable for better statistical
+properties.
+
+.. note::
+    When using multiple DALI pipelines (e.g., for multi-GPU training), all pipeline
+    instances should use the same ``shuffle_after_epoch_seed`` to ensure a consistent
+    global shuffle across all shards.
+
+.. note::
+    This argument has no effect unless ``shuffle_after_epoch`` is set to ``True``.)code",
+      nullptr, false)
   .AddOptionalArg<string>("file_root",
       R"code(Path to a directory that contains the data files.
 

dali/operators/reader/file_reader_op.cc

@@ -98,6 +98,22 @@ This argument is mutually exclusive with `files`.)", nullptr)
 
 `stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)",
       false)
+  .AddOptionalArg<int32_t>("shuffle_after_epoch_seed",
+      R"(Random seed for the dataset shuffling performed after each epoch.
+
+If not provided, a fixed default seed is used, which results in the same shuffling
+pattern across different training runs. Providing a custom seed allows for different
+shuffle patterns across training runs, which may be desirable for better statistical
+properties.
+
+.. note::
+    When using multiple DALI pipelines (e.g., for multi-GPU training), all pipeline
+    instances should use the same ``shuffle_after_epoch_seed`` to ensure a consistent
+    global shuffle across all shards.
+
+.. note::
+    This argument has no effect unless ``shuffle_after_epoch`` is set to ``True``.)",
+      nullptr, false)
   .AddOptionalArg<vector<string>>("files", R"(A list of file paths to read the data from.
 
 If `file_root` is provided, the paths are treated as being relative to it.

dali/operators/reader/fits_reader_op.cc

@@ -71,6 +71,22 @@ This argument is mutually exclusive with `files`.)",
 
 `stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)",
                     false)
+    .AddOptionalArg<int32_t>("shuffle_after_epoch_seed",
+                    R"(Random seed for the dataset shuffling performed after each epoch.
+
+If not provided, a fixed default seed is used, which results in the same shuffling
+pattern across different training runs. Providing a custom seed allows for different
+shuffle patterns across training runs, which may be desirable for better statistical
+properties.
+
+.. note::
+    When using multiple DALI pipelines (e.g., for multi-GPU training), all pipeline
+    instances should use the same ``shuffle_after_epoch_seed`` to ensure a consistent
+    global shuffle across all shards.
+
+.. note::
+    This argument has no effect unless ``shuffle_after_epoch`` is set to ``True``.)",
+                    nullptr, false)
     .AddOptionalArg<vector<string>>("files", R"(A list of file paths to read the data from.
 
 If `file_root` is provided, the paths are treated as being relative to it.

dali/operators/reader/loader/file_label_loader.h

@@ -58,6 +58,9 @@ class DLL_PUBLIC FileLabelLoaderBase : public Loader<CPUBackend, ImageLabelWrapp
       shuffle_after_epoch_(shuffle_after_epoch),
       current_index_(0),
       current_epoch_(0) {
+    int32_t seed_arg = kDaliDataloaderSeed;
+    spec.TryGetArgument(seed_arg, "shuffle_after_epoch_seed");
+    shuffle_after_epoch_seed_ = seed_arg;
 
     vector<string> files;
     vector<int> labels;
@@ -227,7 +230,7 @@ class DLL_PUBLIC FileLabelLoaderBase : public Loader<CPUBackend, ImageLabelWrapp
         // the random distribution.
         file_label_entries_ = backup_file_label_entries_;
       }
-      std::mt19937 g(kDaliDataloaderSeed + current_epoch_);
+      std::mt19937 g(static_cast<uint32_t>(shuffle_after_epoch_seed_ + current_epoch_));
       std::shuffle(file_label_entries_.begin(), file_label_entries_.end(), g);
     }
   }
@@ -261,6 +264,7 @@ class DLL_PUBLIC FileLabelLoaderBase : public Loader<CPUBackend, ImageLabelWrapp
   bool has_file_root_arg_ = false;
 
   bool shuffle_after_epoch_;
+  int32_t shuffle_after_epoch_seed_;
   Index current_index_;
   int current_epoch_;
   FileStream::MappingReserver mmap_reserver_;

dali/operators/reader/loader/file_loader.h

@@ -50,6 +50,9 @@ class FileLoader : public Loader<Backend, Target, true> {
         shuffle_after_epoch_(shuffle_after_epoch),
         current_index_(0),
         current_epoch_(0) {
+    int32_t seed_arg = kDaliDataloaderSeed;
+    spec.TryGetArgument(seed_arg, "shuffle_after_epoch_seed");
+    shuffle_after_epoch_seed_ = seed_arg;
     vector<string> files;
 
     file_discovery_opts_.label_from_subdir = false;
@@ -161,7 +164,7 @@ class FileLoader : public Loader<Backend, Target, true> {
         // reduce the randomness.
         file_entries_ = backup_file_entries_;
       }
-      std::mt19937 g(kDaliDataloaderSeed + current_epoch_);
+      std::mt19937 g(static_cast<uint32_t>(shuffle_after_epoch_seed_ + current_epoch_));
       std::shuffle(file_entries_.begin(), file_entries_.end(), g);
     }
   }
@@ -195,6 +198,7 @@ class FileLoader : public Loader<Backend, Target, true> {
   bool has_file_root_arg_ = false;
 
   bool shuffle_after_epoch_;
+  int32_t shuffle_after_epoch_seed_;
   Index current_index_;
   int current_epoch_;
   typename InputStream::MappingReserver mmap_reserver_;

dali/operators/reader/loader/nemo_asr_loader.cc

@@ -104,7 +104,7 @@ void NemoAsrLoader::Reset(bool wrap_to_shard) {
       // reduce the randomness.
       std::iota(shuffled_indices_.begin(), shuffled_indices_.end(), 0);
     }
-    std::mt19937 g(kDaliDataloaderSeed + current_epoch_);
+    std::mt19937 g(static_cast<uint32_t>(shuffle_after_epoch_seed_ + current_epoch_));
     std::shuffle(shuffled_indices_.begin(), shuffled_indices_.end(), g);
   }
 }

dali/operators/reader/loader/nemo_asr_loader.h

@@ -113,6 +113,11 @@ class DLL_PUBLIC NemoAsrLoader : public Loader<CPUBackend, AsrSample, true> {
       : Loader<CPUBackend, AsrSample, true>(spec),
         manifest_filepaths_(spec.GetRepeatedArgument<std::string>("manifest_filepaths")),
         shuffle_after_epoch_(spec.GetArgument<bool>("shuffle_after_epoch")),
+        shuffle_after_epoch_seed_([&spec]() {
+          int32_t seed = kDaliDataloaderSeed;
+          spec.TryGetArgument(seed, "shuffle_after_epoch_seed");
+          return seed;
+        }()),
         sample_rate_(spec.GetArgument<float>("sample_rate")),
         quality_(spec.GetArgument<float>("quality")),
         downmix_(spec.GetArgument<bool>("downmix")),
@@ -172,6 +177,7 @@ class DLL_PUBLIC NemoAsrLoader : public Loader<CPUBackend, AsrSample, true> {
   std::vector<size_t> shuffled_indices_;
 
   bool shuffle_after_epoch_;
+  int32_t shuffle_after_epoch_seed_;
   Index current_index_ = 0;
   int current_epoch_ = 0;
 

dali/operators/reader/nemo_asr_reader_op.cc

@@ -86,6 +86,22 @@ This reader produces between 1 and 3 outputs:
   .AddOptionalArg("shuffle_after_epoch",
     "If true, reader shuffles whole dataset after each epoch",
     false)
+  .AddOptionalArg<int32_t>("shuffle_after_epoch_seed",
+    R"code(Random seed for the dataset shuffling performed after each epoch.
+
+If not provided, a fixed default seed is used, which results in the same shuffling
+pattern across different training runs. Providing a custom seed allows for different
+shuffle patterns across training runs, which may be desirable for better statistical
+properties.
+
+.. note::
+    When using multiple DALI pipelines (e.g., for multi-GPU training), all pipeline
+    instances should use the same ``shuffle_after_epoch_seed`` to ensure a consistent
+    global shuffle across all shards.
+
+.. note::
+    This argument has no effect unless ``shuffle_after_epoch`` is set to ``True``.)code",
+    nullptr, false)
   .AddOptionalArg("sample_rate",
     "If specified, the target sample rate, in Hz, to which the audio is resampled.",
     -1.0f)

dali/operators/reader/numpy_reader_op.cc

@@ -137,6 +137,22 @@ This argument is mutually exclusive with `files`.)", nullptr)
 
 `stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)",
       false)
+  .AddOptionalArg<int32_t>("shuffle_after_epoch_seed",
+      R"(Random seed for the dataset shuffling performed after each epoch.
+
+If not provided, a fixed default seed is used, which results in the same shuffling
+pattern across different training runs. Providing a custom seed allows for different
+shuffle patterns across training runs, which may be desirable for better statistical
+properties.
+
+.. note::
+    When using multiple DALI pipelines (e.g., for multi-GPU training), all pipeline
+    instances should use the same ``shuffle_after_epoch_seed`` to ensure a consistent
+    global shuffle across all shards.
+
+.. note::
+    This argument has no effect unless ``shuffle_after_epoch`` is set to ``True``.)",
+      nullptr, false)
   .AddOptionalArg<vector<string>>("files", R"(A list of file paths to read the data from.
 
 If `file_root` is provided, the paths are treated as being relative to it.

dali/test/python/reader/test_numpy.py

@@ -1132,3 +1132,92 @@ def my_pipeline(files):
         p = my_pipeline(files=[fname])
         # shouldn't throw
         assert_array_equal(p.run()[0][0], data)
+
+
+def _collect_numpy_epoch_order(filenames, seed=None):
+    """Run a numpy reader pipeline for one epoch and return the list of sample values (int).
+
+    Each file stores a unique integer value, so the returned list represents the read order.
+    """
+    num_samples = len(filenames)
+
+    @pipeline_def(batch_size=1, num_threads=1, device_id=0)
+    def make_pipe():
+        data = fn.readers.numpy(
+            device="cpu",
+            files=filenames,
+            shuffle_after_epoch=True,
+            shard_id=0,
+            num_shards=1,
+            shuffle_after_epoch_seed=seed,
+        )
+        return data
+
+    pipe = make_pipe()
+    pipe.build()
+
+    order = []
+    for _ in range(num_samples):
+        out = pipe.run()
+        order.append(int(to_array(out[0])[0][0]))
+
+    return order
+
+
+def test_shuffle_after_epoch_seed_numpy_reproducible():
+    """Same shuffle_after_epoch_seed should produce the same order across runs."""
+    with tempfile.TemporaryDirectory() as test_data_root:
+        num_samples = 20
+        filenames = []
+        for i in range(num_samples):
+            fname = os.path.join(test_data_root, "sample_{:03d}.npy".format(i))
+            np.save(fname, np.array([i], dtype=np.float32))
+            filenames.append(fname)
+
+        seed = 42
+        order1 = _collect_numpy_epoch_order(filenames, seed=seed)
+        order2 = _collect_numpy_epoch_order(filenames, seed=seed)
+
+        assert (
+            order1 == order2
+        ), "Same shuffle_after_epoch_seed should produce the same reading order"
+
+
+def test_shuffle_after_epoch_seed_numpy_different_seeds():
+    """Different shuffle_after_epoch_seed values should produce different orders."""
+    with tempfile.TemporaryDirectory() as test_data_root:
+        num_samples = 20
+        filenames = []
+        for i in range(num_samples):
+            fname = os.path.join(test_data_root, "sample_{:03d}.npy".format(i))
+            np.save(fname, np.array([i], dtype=np.float32))
+            filenames.append(fname)
+
+        order_seed1 = _collect_numpy_epoch_order(filenames, seed=11111)
+        order_seed2 = _collect_numpy_epoch_order(filenames, seed=99999)
+
+        assert (
+            order_seed1 != order_seed2
+        ), "Different shuffle_after_epoch_seed values should produce different orders"
+        # Both orderings should contain all samples
+        assert (
+            sorted(order_seed1) == sorted(order_seed2) == list(range(num_samples))
+        ), "All samples should be present regardless of seed"
+
+
+def test_shuffle_after_epoch_seed_numpy_default_reproducible():
+    """Without explicit seed, two pipelines should produce the same default order."""
+    with tempfile.TemporaryDirectory() as test_data_root:
+        num_samples = 20
+        filenames = []
+        for i in range(num_samples):
+            fname = os.path.join(test_data_root, "sample_{:03d}.npy".format(i))
+            np.save(fname, np.array([i], dtype=np.float32))
+            filenames.append(fname)
+
+        order1 = _collect_numpy_epoch_order(filenames, seed=None)
+        order2 = _collect_numpy_epoch_order(filenames, seed=None)
+
+        assert (
+            order1 == order2
+        ), "Without explicit seed, default behavior should be reproducible (backward compat)"