chore(testing): bump bust for uncached CI run

Forces e2e/nx (and other) tasks to re-run without cache hits so the
spread.test.ts stress loops actually exercise the daemon path under
the new fixes.

e2e/nx/src/spread.test.ts

@@ -3,10 +3,13 @@ import {
   newProject,
   readJson,
   runCLI,
+  tmpProjPath,
   uniq,
   updateFile,
   updateJson,
 } from '@nx/e2e-utils';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
 
 describe('Spread Token Merging', () => {
   let proj: string;
@@ -22,6 +25,32 @@ describe('Spread Token Merging', () => {
     existingNxJson = readJson('nx.json');
   });
   afterEach(() => {
+    // Print the daemon log into the test's stdout BEFORE reset (which
+    // stops the daemon and may rotate the file). CI captures stdout
+    // per test so the [watcher] lines end up alongside the failure
+    // assertion in the build output.
+    try {
+      const daemonLog = join(
+        tmpProjPath(),
+        '.nx',
+        'workspace-data',
+        'd',
+        'daemon.log'
+      );
+      if (existsSync(daemonLog)) {
+        const contents = readFileSync(daemonLog, 'utf-8');
+        console.log(
+          `\n========== daemon.log for "${
+            expect.getState().currentTestName ?? 'unknown'
+          }" ==========\n${contents}\n========== end daemon.log ==========\n`
+        );
+      } else {
+        console.log(`[spread-debug] no daemon log at ${daemonLog}`);
+      }
+    } catch (e) {
+      console.log(`[spread-debug] failed to read daemon log: ${e}`);
+    }
+
     updateFile('nx.json', JSON.stringify(existingNxJson, null, 2));
     // Reset daemon cache so the next test does not see stale plugin-inferred
     // project graph data.  The PR enabling NX_DAEMON=true in runCLI means the
@@ -925,4 +954,193 @@ describe('Spread Token Merging', () => {
       expect(project.targets.echo.dependsOn).toEqual(['prebuild', '^build']);
     });
   });
+
+  /**
+   * Race-condition stress.
+   *
+   * These tests deliberately mutate nx.json (and tools/*) in tight loops
+   * with no settle time between the write and the `show project` query,
+   * and without a `reset` in between — so the long-lived daemon must pick
+   * up every change through its file watcher before it serves the graph.
+   *
+   * If the daemon answers from a stale cached graph (the watcher event
+   * for the latest nx.json write has not landed yet), the resolved
+   * build.inputs will not match the expectation and the test fails.
+   * Each loop iteration is a fresh chance to hit that window, so a flake
+   * that shows up ~1-in-20 in a single-shot test shows up far more often
+   * here. The restored daemon.log dump (afterEach) captures the
+   * [watcher]/recompute lines so a failure is diagnosable on CI.
+   */
+  describe('rapid reconfiguration (race-condition stress)', () => {
+    it('reflects the latest specified plugin after rapid nx.json swaps', () => {
+      const lib = uniq('lib');
+      runCLI(`generate @nx/js:lib libs/${lib}`);
+      updateJson(`libs/${lib}/project.json`, (c) => {
+        c.targets = {};
+        return c;
+      });
+
+      // Pre-create a pool of plugins, each stamping a distinct input.
+      const pluginCount = 6;
+      for (let i = 0; i < pluginCount; i++) {
+        createPlugin(
+          `race-plugin-${i}`,
+          `{
+          build: {
+            executor: 'nx:run-commands',
+            options: { command: 'echo build' },
+            inputs: ['from-plugin-${i}'],
+          }
+        }`
+        );
+      }
+
+      // Swap the active plugin and query immediately, no settle time.
+      // A stale daemon graph surfaces as the previous iteration's input.
+      for (let i = 0; i < pluginCount; i++) {
+        updateJson('nx.json', (json) => {
+          json.plugins = [`./tools/race-plugin-${i}`];
+          return json;
+        });
+        const project = getResolvedProject(lib);
+        expect(project.targets.build.inputs).toEqual([`from-plugin-${i}`]);
+      }
+    });
+
+    it('reflects plugin-list growth and shrink across rapid nx.json edits', () => {
+      const lib = uniq('lib');
+      runCLI(`generate @nx/js:lib libs/${lib}`);
+      updateJson(`libs/${lib}/project.json`, (c) => {
+        c.targets = {};
+        return c;
+      });
+
+      createPlugin(
+        'race-base',
+        `{
+        build: {
+          executor: 'nx:run-commands',
+          options: { command: 'echo build' },
+          inputs: ['base'],
+        }
+      }`
+      );
+      createPlugin(
+        'race-spread',
+        `{
+        build: {
+          executor: 'nx:run-commands',
+          options: { command: 'echo build' },
+          inputs: ['spread', '...'],
+        }
+      }`
+      );
+
+      // Alternate between one plugin and two (with spread). The resolved
+      // inputs differ each step, so a stale graph is caught immediately.
+      const steps: { plugins: string[]; inputs: string[] }[] = [
+        { plugins: ['./tools/race-base'], inputs: ['base'] },
+        {
+          plugins: ['./tools/race-base', './tools/race-spread'],
+          inputs: ['spread', 'base'],
+        },
+        { plugins: ['./tools/race-spread'], inputs: ['spread'] },
+        {
+          plugins: ['./tools/race-base', './tools/race-spread'],
+          inputs: ['spread', 'base'],
+        },
+        { plugins: ['./tools/race-base'], inputs: ['base'] },
+      ];
+      for (const { plugins, inputs } of steps) {
+        updateJson('nx.json', (json) => {
+          json.plugins = plugins;
+          return json;
+        });
+        const project = getResolvedProject(lib);
+        expect(project.targets.build.inputs).toEqual(inputs);
+      }
+    });
+
+    it('reflects rapid project.json edits against a stable plugin base', () => {
+      const lib = uniq('lib');
+      runCLI(`generate @nx/js:lib libs/${lib}`);
+
+      createPlugin(
+        'race-infer',
+        `{
+        build: {
+          executor: 'nx:run-commands',
+          options: { command: 'echo build' },
+          inputs: ['inferred'],
+        }
+      }`
+      );
+      updateJson('nx.json', (json) => {
+        json.plugins = ['./tools/race-infer'];
+        return json;
+      });
+
+      // Plugin set is fixed; only project.json changes each round. The
+      // daemon must observe that file change before answering — a stale
+      // graph surfaces as a previous iteration's project input.
+      for (let i = 0; i < 6; i++) {
+        updateJson(`libs/${lib}/project.json`, (c) => {
+          c.targets = {
+            build: {
+              inputs: [`project-${i}`, '...'],
+            },
+          };
+          return c;
+        });
+        const project = getResolvedProject(lib);
+        expect(project.targets.build.inputs).toEqual([
+          `project-${i}`,
+          'inferred',
+        ]);
+      }
+    });
+
+    /**
+     * Mirrors the shape of the flaky single-shot test
+     * ("...with target defaults overriding"): per iteration a plugin
+     * file, nx.json (plugins + targetDefaults) and project.json all
+     * change together, then a single `show project` query. A stale
+     * daemon graph here surfaces as build being undefined entirely —
+     * the plugin set the graph was built against never ran.
+     */
+    it('reflects a plugin + nx.json + project.json change applied together, repeatedly', () => {
+      for (let i = 0; i < 4; i++) {
+        const lib = uniq('lib');
+        runCLI(`generate @nx/js:lib libs/${lib}`);
+
+        createPlugin(
+          `combo-infer-${i}`,
+          `{
+          build: {
+            executor: 'nx:run-commands',
+            options: { command: 'echo build' },
+            inputs: ['inferred-${i}'],
+          }
+        }`
+        );
+        updateJson('nx.json', (json) => {
+          json.plugins = [`./tools/combo-infer-${i}`];
+          json.targetDefaults = { build: { inputs: [`defaults-${i}`] } };
+          return json;
+        });
+        updateJson(`libs/${lib}/project.json`, (c) => {
+          c.targets = { build: { inputs: [`project-${i}`, '...'] } };
+          return c;
+        });
+
+        const project = getResolvedProject(lib);
+        // Target defaults (no spread) replace the inferred inputs, then
+        // project.json spreads against the resolved defaults.
+        expect(project.targets.build.inputs).toEqual([
+          `project-${i}`,
+          `defaults-${i}`,
+        ]);
+      }
+    });
+  });
 });

nx.json

@@ -408,7 +408,7 @@
   "nxCloudId": "62d013ea0852fe0a2df74438",
   "nxCloudUrl": "https://staging.nx.app",
   "parallel": 1,
-  "bust": 3235,
+  "bust": 3244,
   "defaultBase": "master",
   "sync": {
     "applyChanges": true

packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts

@@ -107,35 +107,46 @@ let cacheHasBeenPersisted = false;
 function kickOffRecompute() {
   let myPromise: Promise<SerializedProjectGraph>;
   myPromise = (async () => {
-    // Single read shared with getPluginsSeparated below. This collapses
-    // what would otherwise be two independent nx.json reads (our snap +
-    // the plugin loader's) into one, so the snap hash and the plugin
-    // set the compute uses always reflect the same disk state.
-    const nxJson = readNxJson(workspaceRoot);
-    const myPluginsHash = hashObject(nxJson.plugins ?? []);
+    // The whole body must resolve, never reject: scheduleProjectGraphRecomputation
+    // calls kickOffRecompute() fire-and-forget, so a rejected myPromise has no
+    // awaiter and crashes the daemon with an unhandled rejection. The prologue
+    // (readNxJson / getPluginsSeparated) can throw — e.g. a plugin fails to
+    // load — so a failure here is turned into an errorResult the next requester
+    // surfaces, same as processFilesAndCreateAndSerializeProjectGraph already does.
+    try {
+      // Single read shared with getPluginsSeparated below. This collapses
+      // what would otherwise be two independent nx.json reads (our snap +
+      // the plugin loader's) into one, so the snap hash and the plugin
+      // set the compute uses always reflect the same disk state.
+      const nxJson = readNxJson(workspaceRoot);
+      const myPluginsHash = hashObject(nxJson.plugins ?? []);
 
-    const plugins = await getPluginsSeparated(nxJson, workspaceRoot);
+      const plugins = await getPluginsSeparated(nxJson, workspaceRoot);
 
-    // Plugin set we just loaded may already be stale vs disk.
-    if (isStale(myPluginsHash)) return chainToSuccessor(myPromise);
+      // Plugin set we just loaded may already be stale vs disk.
+      if (isStale(myPluginsHash)) return chainToSuccessor(myPromise);
 
-    const result = await processFilesAndCreateAndSerializeProjectGraph(plugins);
+      const result =
+        await processFilesAndCreateAndSerializeProjectGraph(plugins);
 
-    // Compute may have run against plugins that are now stale.
-    if (isStale(myPluginsHash)) return chainToSuccessor(myPromise);
+      // Compute may have run against plugins that are now stale.
+      if (isStale(myPluginsHash)) return chainToSuccessor(myPromise);
 
-    if (
-      cachedSerializedProjectGraphPromise === myPromise &&
-      result.projectGraph
-    ) {
-      notifyProjectGraphRecomputationListeners(
-        result.projectGraph,
-        result.sourceMaps,
-        result.error
-      );
-      persistProjectGraphToDisk(result);
+      if (
+        cachedSerializedProjectGraphPromise === myPromise &&
+        result.projectGraph
+      ) {
+        notifyProjectGraphRecomputationListeners(
+          result.projectGraph,
+          result.sourceMaps,
+          result.error
+        );
+        persistProjectGraphToDisk(result);
+      }
+      return result;
+    } catch (e) {
+      return errorResult(e);
     }
-    return result;
   })();
   cachedSerializedProjectGraphPromise = myPromise;
 }

packages/nx/src/project-graph/plugins/get-plugins.spec.ts

@@ -1,6 +1,111 @@
 import { createSerializableError } from '../../utils/serializable-error';
 import { reasonToError } from './get-plugins';
 
+// Covers the daemon race where two recomputes call getPluginsSeparated
+// back-to-back with the same NEW plugin config (after nx.json change).
+// The earlier code path updated `currentPluginsConfigurationHash` to the
+// new hash before populating `cachedSeparatedPlugins`, so a concurrent
+// caller arriving in that window passed the cache check and received the
+// STALE SeparatedPlugins from the previous load. In the daemon this
+// surfaced as the project graph being built without the freshly-added
+// specified plugins (see spread.test.ts middle-plugin flake).
+describe('getPluginsSeparated — concurrent load of same new config', () => {
+  it('does not serve stale cached plugins to a concurrent caller mid-load', async () => {
+    await jest.isolateModulesAsync(async () => {
+      jest.doMock('./isolation/enabled', () => ({
+        __esModule: true,
+        isIsolationEnabled: () => false,
+      }));
+
+      let firstLoadGate: () => void;
+      const firstLoadParked = new Promise<void>((resolve) => {
+        firstLoadGate = resolve;
+      });
+
+      let specifiedLoadCount = 0;
+      const makeFakePlugin = (name: string) =>
+        ({ name }) as unknown as import('./loaded-nx-plugin').LoadedNxPlugin;
+
+      // Only specified plugins (paths starting with "./") are gated;
+      // default plugin loads (absolute paths from getDefaultPlugins)
+      // run to completion so Promise.allSettled below progresses.
+      jest.doMock('./in-process-loader', () => ({
+        __esModule: true,
+        loadNxPlugin: (config: unknown) => {
+          const name =
+            typeof config === 'string'
+              ? config
+              : ((config as { plugin: string }).plugin ?? 'unknown');
+          const isSpecifiedPlugin = name.startsWith('./');
+          let parkThis = false;
+          if (isSpecifiedPlugin) {
+            specifiedLoadCount++;
+            parkThis = specifiedLoadCount === 1;
+          }
+          const promise = (async () => {
+            if (parkThis) await firstLoadParked;
+            return makeFakePlugin(`loaded:${name}`);
+          })();
+          return [promise, () => {}] as const;
+        },
+      }));
+
+      const { getPluginsSeparated } = require('./get-plugins');
+
+      // Seed the cache with an EMPTY specified-plugin set so the race
+      // window has a meaningfully stale value to return. This matches
+      // the daemon's startup state right before the test mutates
+      // nx.json to add specified plugins.
+      const seed = await getPluginsSeparated({ plugins: [] }, '/tmp/fake-root');
+      expect(seed.specifiedPlugins).toEqual([]);
+
+      // First call after the nx.json change: falls through cache (hash
+      // differs from seed), bumps currentPluginsConfigurationHash to the
+      // NEW hash, and parks loading './tools/plugin-a'.
+      const callA = getPluginsSeparated(
+        { plugins: ['./tools/plugin-a'] },
+        '/tmp/fake-root'
+      );
+
+      // Yield through the microtask + setImmediate queues so call A has
+      // (synchronously) stamped `currentPluginsConfigurationHash` to the
+      // NEW hash and is suspended on `Promise.allSettled` awaiting the
+      // parked load.
+      await new Promise((r) => setImmediate(r));
+      await new Promise((r) => setImmediate(r));
+
+      // Concurrent call with the SAME new config arriving in that race
+      // window. Without the fix it hits the cache check
+      // (`cachedSeparatedPlugins && newHash === currentHash`) and gets
+      // the seed's empty `specifiedPlugins` — the daemon would then
+      // build a project graph with no specified plugins loaded.
+      const callB = getPluginsSeparated(
+        { plugins: ['./tools/plugin-a'] },
+        '/tmp/fake-root'
+      );
+
+      // Unblock the parked load and let both calls settle.
+      firstLoadGate!();
+      const [resultA, resultB] = await Promise.all([callA, callB]);
+
+      expect(resultA.specifiedPlugins).toHaveLength(1);
+      expect(resultA.specifiedPlugins[0]).toMatchObject({
+        name: 'loaded:./tools/plugin-a',
+      });
+      // The bug: B used to return `specifiedPlugins: []` because the
+      // cache check passed with the seed's stale data still attached
+      // to the freshly-updated hash.
+      expect(resultB.specifiedPlugins).toHaveLength(1);
+      expect(resultB.specifiedPlugins[0]).toMatchObject({
+        name: 'loaded:./tools/plugin-a',
+      });
+      expect(resultA.specifiedPlugins.map((p: any) => p.name)).toEqual(
+        resultB.specifiedPlugins.map((p: any) => p.name)
+      );
+    });
+  });
+});
+
 describe('reasonToError', () => {
   it('should return the same Error instance when given a real Error', () => {
     const error = new Error('real error');

packages/nx/src/project-graph/plugins/get-plugins.ts

@@ -26,6 +26,17 @@ let loadedPlugins: LoadedNxPlugin[];
 let cachedSeparatedPlugins: SeparatedPlugins;
 let pendingPluginsPromise: Promise<LoadedNxPlugin[]> | undefined;
 let cleanupSpecifiedPlugins: () => void | undefined;
+// In-flight `getPluginsSeparated` call paired with the plugin-config hash
+// it's loading for. Lets a concurrent caller arriving with the same new
+// config dedupe onto the existing load instead of either (a) hitting the
+// cache check after `currentPluginsConfigurationHash` was bumped but
+// before `cachedSeparatedPlugins` was replaced — that race served the
+// previous load's stale result and surfaced as the spread.test.ts
+// middle-plugin flake — or (b) racing a parallel reload that thrashes
+// workers.
+let pendingSeparatedPluginsLoad:
+  | { hash: string; promise: Promise<SeparatedPlugins> }
+  | undefined;
 
 export interface SeparatedPlugins {
   specifiedPlugins: LoadedNxPlugin[];
@@ -81,6 +92,21 @@ export async function getPluginsSeparated(
     return cachedSeparatedPlugins;
   }
 
+  // A load for this exact plugin set is already in flight (e.g. the
+  // watcher kicked off a recompute and a `nx show` request landed
+  // before it finished). Dedupe onto it instead of starting a parallel
+  // reload that thrashes workers — and, more importantly, instead of
+  // falling through to the cache check after a concurrent caller bumps
+  // `currentPluginsConfigurationHash` to the new hash but hasn't yet
+  // replaced `cachedSeparatedPlugins` (which was the spread.test.ts
+  // middle-plugin flake).
+  if (
+    pendingSeparatedPluginsLoad &&
+    pendingSeparatedPluginsLoad.hash === pluginsConfigurationHash
+  ) {
+    return pendingSeparatedPluginsLoad.promise;
+  }
+
   // Plugins config changed (e.g. `nx add @nx/maven` updated nx.json). The
   // cached SeparatedPlugins is invalidated by the early-return above, but
   // pendingPluginsPromise — the in-flight load — would otherwise be reused
@@ -88,36 +114,63 @@ export async function getPluginsSeparated(
   // down the old workers and force a fresh load.
   cleanupSpecifiedPlugins?.();
   pendingPluginsPromise = undefined;
-  currentPluginsConfigurationHash = pluginsConfigurationHash;
-  const results = await Promise.allSettled([
-    getOnlyDefaultPlugins(root),
-    (pendingPluginsPromise ??= loadSpecifiedNxPlugins(
-      pluginsConfiguration,
-      root
-    )),
-  ]);
 
-  const errors: Error[] = [];
-  const defaultPlugins: LoadedNxPlugin[] = [];
-  const specifiedPlugins: LoadedNxPlugin[] = [];
+  const myHash = pluginsConfigurationHash;
+  const myPromise = (async (): Promise<SeparatedPlugins> => {
+    try {
+      const results = await Promise.allSettled([
+        getOnlyDefaultPlugins(root),
+        (pendingPluginsPromise ??= loadSpecifiedNxPlugins(
+          pluginsConfiguration,
+          root
+        )),
+      ]);
 
-  for (let i = 0; i < results.length; i++) {
-    const result = results[i];
-    if (result.status === 'fulfilled') {
-      (i === 0 ? defaultPlugins : specifiedPlugins).push(...result.value);
-    } else {
-      errors.push(reasonToError(result.reason));
+      const errors: Error[] = [];
+      const defaultPlugins: LoadedNxPlugin[] = [];
+      const specifiedPlugins: LoadedNxPlugin[] = [];
+
+      for (let i = 0; i < results.length; i++) {
+        const result = results[i];
+        if (result.status === 'fulfilled') {
+          (i === 0 ? defaultPlugins : specifiedPlugins).push(...result.value);
+        } else {
+          errors.push(reasonToError(result.reason));
+        }
+      }
+
+      if (errors.length > 0) {
+        throw new AggregateError(
+          errors,
+          errors.map((e) => e.message).join('\n')
+        );
+      }
+
+      const newCache: SeparatedPlugins = { specifiedPlugins, defaultPlugins };
+
+      // Only commit the cache + hash if we're still the most recent
+      // in-flight load. A newer config arrived during our load → that
+      // newer load will commit its own (correct) result; we must not
+      // overwrite it with our older one.
+      if (pendingSeparatedPluginsLoad?.promise === myPromise) {
+        cachedSeparatedPlugins = newCache;
+        currentPluginsConfigurationHash = myHash;
+        loadedPlugins = specifiedPlugins.concat(defaultPlugins);
+      }
+
+      return newCache;
+    } finally {
+      // Always drop the in-flight marker for our promise — on success
+      // the cache is committed above, on error we want the next caller
+      // to fall through and retry rather than be handed our rejection.
+      if (pendingSeparatedPluginsLoad?.promise === myPromise) {
+        pendingSeparatedPluginsLoad = undefined;
+      }
     }
-  }
+  })();
 
-  if (errors.length > 0) {
-    throw new AggregateError(errors, errors.map((e) => e.message).join('\n'));
-  }
-
-  cachedSeparatedPlugins = { specifiedPlugins, defaultPlugins };
-  loadedPlugins = specifiedPlugins.concat(defaultPlugins);
-
-  return cachedSeparatedPlugins;
+  pendingSeparatedPluginsLoad = { hash: myHash, promise: myPromise };
+  return myPromise;
 }
 
 /**