fix(config): drain dist-archive substreams before zipping

.changeset/dist-archive-no-merge-stream.md

@@ -0,0 +1,5 @@
+---
+"@jspsych/config": patch
+---
+
+Fix `createCoreDistArchive` dropping plugins from `dist.zip`. Recent release archives were missing the alphabetical tail of `dist/*.js` (e.g. `plugin-survey-multi-*`, `plugin-video-*`, `plugin-virtual-chinrest`, `plugin-visual-search-circle`, the `plugin-webgazer-*` plugins) along with some `examples/*.html` files. Under gulp 5's streamx-based Vinyl, `merge-stream` would race with `gulp-zip` and finalize the archive before slow upstreams (notably the 3 MB `plugin-survey/dist/index.browser.js`) had finished flushing. The task now drains every substream first and feeds the collected files into `gulp-zip` from a single ordered stream.

packages/config/gulp.js

@@ -1,16 +1,28 @@
 import { readFileSync } from "fs";
 import { sep as pathSeparator } from "path";
+import { Readable } from "stream";
 
 import glob from "glob";
 import gulp from "gulp";
 import file from "gulp-file";
 import rename from "gulp-rename";
 import replace from "gulp-replace";
 import zip from "gulp-zip";
-import merge from "merge-stream";
 
 const { dest, src } = gulp;
 
+// Drains a Vinyl object stream and resolves with the emitted files. We collect
+// every substream up front instead of feeding `gulp-zip` through `merge-stream`
+// because under gulp 5 (streamx-based Vinyl) the zip sink can finalize before
+// slow upstreams flush, silently dropping files from `dist.zip`.
+const collectFiles = (stream) =>
+  new Promise((resolve, reject) => {
+    const files = [];
+    stream.on("data", (vinylFile) => files.push(vinylFile));
+    stream.on("end", () => resolve(files));
+    stream.on("error", reject);
+  });
+
 const readJsonFile = (filename) => JSON.parse(readFileSync(filename, "utf8"));
 
 const getAllPackages = () =>
@@ -63,60 +75,75 @@ const getVersionFileContents = () =>
     "",
   ].join("\n");
 
-export const createCoreDistArchive = () =>
-  merge(
+export const createCoreDistArchive = async () => {
+  const fileGroups = await Promise.all([
     // index.browser.js files
-    src("packages/*/dist/index.browser.js", { root: "packages/" })
-      // Rename dist files
-      .pipe(
-        rename((path) => {
-          const packageName = path.dirname.split(pathSeparator)[0];
-
-          path.dirname = "/dist";
-          path.basename = packageName;
-        })
-      )
-      // Remove sourceMappingURL comments
-      .pipe(replace(/\/\/# sourceMappingURL=.*\n/g, "")),
+    collectFiles(
+      src("packages/*/dist/index.browser.js", { root: "packages/" })
+        // Rename dist files
+        .pipe(
+          rename((path) => {
+            const packageName = path.dirname.split(pathSeparator)[0];
+
+            path.dirname = "/dist";
+            path.basename = packageName;
+          })
+        )
+        // Remove sourceMappingURL comments
+        .pipe(replace(/\/\/# sourceMappingURL=.*\n/g, ""))
+    ),
 
     // jspsych.css
-    src("packages/jspsych/css/jspsych.css").pipe(rename("/dist/jspsych.css")),
+    collectFiles(src("packages/jspsych/css/jspsych.css").pipe(rename("/dist/jspsych.css"))),
 
     // survey.css
-    src("packages/plugin-survey/css/survey.css").pipe(rename("/dist/survey.css")),
-    src("packages/plugin-survey/css/survey.min.css").pipe(rename("/dist/survey.min.css")),
+    collectFiles(src("packages/plugin-survey/css/survey.css").pipe(rename("/dist/survey.css"))),
+    collectFiles(
+      src("packages/plugin-survey/css/survey.min.css").pipe(rename("/dist/survey.min.css"))
+    ),
 
     // Examples HTML files
-    src(["examples/**/*.html"], { base: "." })
-      // Rewrite script source paths
-      .pipe(
-        replace(
-          /<script src="(.*)\/packages\/(.*)\/dist\/index\.browser\.js"/g,
-          '<script src="$1/dist/$2.js"'
+    collectFiles(
+      src(["examples/**/*.html"], { base: "." })
+        // Rewrite script source paths
+        .pipe(
+          replace(
+            /<script src="(.*)\/packages\/(.*)\/dist\/index\.browser\.js"/g,
+            '<script src="$1/dist/$2.js"'
+          )
         )
-      )
-      // Rewrite jspsych css source paths
-      .pipe(
-        replace(
-          /<link rel="stylesheet" href="(.*)\/packages\/jspsych\/css\/(.*)"/g,
-          '<link rel="stylesheet" href="$1/dist/$2"'
+        // Rewrite jspsych css source paths
+        .pipe(
+          replace(
+            /<link rel="stylesheet" href="(.*)\/packages\/jspsych\/css\/(.*)"/g,
+            '<link rel="stylesheet" href="$1/dist/$2"'
+          )
         )
-      ),
+    ),
 
     // Examples files other than HTML (e.g. media files)
     // Note: `encoding: false` means that the files contents are treated as binary.
     // This prevents Gulp from corrupting binary files such as images when it reads them.
     // Needed since Gulp v5.
-    src(["examples/**/*", "!examples/**/*.html"], { base: ".", encoding: false }),
+    collectFiles(src(["examples/**/*", "!examples/**/*.html"], { base: ".", encoding: false })),
 
     // VERSION.md
-    file("VERSION.md", getVersionFileContents(), { src: true }),
+    collectFiles(file("VERSION.md", getVersionFileContents(), { src: true })),
 
     // Other files
-    src(["*.md", "license.txt"])
-  )
-    .pipe(zip("dist.zip"))
-    .pipe(dest("."));
+    collectFiles(src(["*.md", "license.txt"])),
+  ]);
+
+  // Await the destination's `finish` so gulp doesn't mark this task complete
+  // before `dist.zip` is fully written (async-done resolves a returned Promise
+  // by its resolved value, not by re-handling a stream the Promise resolves to).
+  const writeStream = Readable.from(fileGroups.flat()).pipe(zip("dist.zip")).pipe(dest("."));
+  await new Promise((resolve, reject) => {
+    writeStream.on("finish", resolve);
+    writeStream.on("end", resolve);
+    writeStream.on("error", reject);
+  });
+};
 
 /**
  * Updates each unpkg link with a precise version number to the corresponding package's current

packages/config/package.json

@@ -63,7 +63,6 @@
     "jest": "29.7.0",
     "jest-canvas-mock": "2.5.0",
     "jest-environment-jsdom": "29.7.0",
-    "merge-stream": "2.0.0",
     "rollup": "^4.22.4",
     "rollup-plugin-dts": "6.1.1",
     "rollup-plugin-esbuild": "6.1.1",