ci: run the plugins test suite in parallel processes (#6051)

* test(plugins): make the suite safe to run in parallel processes

Two things broke when the suite ran across several Ginkgo processes.

buildTestPlugins ran in every process, so N copies of make raced in the same
directory. The packaging rule made that worse by staging every plugin through
one shared plugin.wasm, so concurrent targets clobbered each other and left
orphaned temp files behind. That also ruled out make -j.

Stage each package under its own per-target directory, and move the build into
SynchronizedBeforeSuite so process 1 does it once while the others wait.

* ci: run the plugins suite in parallel processes

With the compilation cache warm the suite is bound by spec execution, which
splits cleanly across processes. Run it as its own step with the ginkgo CLI,
already declared as a tool in go.mod, and drop the package from the main go
test invocation so it is not run twice.

Locally, with -race: 69s to 19s warm, and 256s to 96s cold.

* ci: give the plugins suite its own job so it runs concurrently

Running it as a second step in the go job serialised it against the other 90
packages, which cancelled out the parallel win: the job went from 5m41s to
only 5m29s even though the suite itself dropped from ~175s to 82s.

Move it to its own job so the two run at the same time. The WASM compilation
cache moves with it, since the go job no longer runs the suite.

.github/workflows/pipeline.yml

@@ -131,24 +131,14 @@ jobs:
         uses: actions/checkout@v7
 
       - uses: actions/setup-go@v6
-        id: setup-go
         with:
           go-version-file: go.mod
 
       - name: Download dependencies
         run: go mod download
 
-      # Without this, the plugins suite recompiles every test plugin WASM module,
-      # which dominates the job runtime under -race.
-      - name: Cache the plugins test suite WASM compilation cache
-        uses: actions/cache@v6
-        with:
-          path: plugins/testdata/.wazero-cache
-          key: wazero-${{ runner.os }}-go${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('plugins/testdata/*/*.go', 'plugins/testdata/*/go.*', 'plugins/pdk/go/**/*.go', 'plugins/pdk/go/go.*') }}
-          restore-keys: wazero-${{ runner.os }}-
-
       - name: Test
-        run: go test -shuffle=on -tags netgo,sqlite_fts5 -race ./... -v
+        run: go test -shuffle=on -tags netgo,sqlite_fts5 -race -v $(go list ./... | grep -v '/plugins$')
 
       - name: Test ndpgen
         run: |
@@ -157,6 +147,30 @@ jobs:
           go build -o ndpgen .
           ./ndpgen --help
 
+  go-plugins:
+    name: Test Go plugins
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v7
+
+      - uses: actions/setup-go@v6
+        id: setup-go
+        with:
+          go-version-file: go.mod
+
+      # Without this, the suite recompiles every test plugin WASM module,
+      # which dominates its runtime under -race.
+      - name: Cache the WASM compilation cache
+        uses: actions/cache@v6
+        with:
+          path: plugins/testdata/.wazero-cache
+          key: wazero-${{ runner.os }}-go${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('plugins/testdata/*/*.go', 'plugins/testdata/*/go.*', 'plugins/pdk/go/**/*.go', 'plugins/pdk/go/go.*') }}
+          restore-keys: wazero-${{ runner.os }}-
+
+      - name: Test plugins
+        run: go tool ginkgo -p -race -tags netgo,sqlite_fts5 ./plugins/
+
   go-windows:
     name: Test Go code (Windows)
     runs-on: windows-2022
@@ -294,7 +308,7 @@ jobs:
 
   build:
     name: Build
-    needs: [js, go, go-windows, go-lint, i18n-lint, git-version, check-push-enabled, validate-migrations]
+    needs: [js, go, go-plugins, go-windows, go-lint, i18n-lint, git-version, check-push-enabled, validate-migrations]
     strategy:
       matrix:
         platform: [ linux/amd64, linux/arm64, linux/arm/v5, linux/arm/v6, linux/arm/v7, linux/386, linux/riscv64, darwin/amd64, darwin/arm64, windows/amd64, windows/386 ]

.gitignore

@@ -47,3 +47,4 @@ zz_*_test.go
 
 # wazero compilation cache for the plugins test suite
 /plugins/testdata/.wazero-cache/
+/plugins/testdata/*.stage/

plugins/plugins_suite_test.go

@@ -7,6 +7,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 	"os/exec"
@@ -39,7 +40,6 @@ var (
 
 func TestPlugins(t *testing.T) {
 	tests.Init(t, false)
-	buildTestPlugins(t, testDataDir)
 
 	// Set globally so tests using configtest.SetupConfig inherit it. The cache
 	// persists between runs; entries are content-addressed, so a stale one only misses.
@@ -51,16 +51,11 @@ func TestPlugins(t *testing.T) {
 	RunSpecs(t, "Plugins Suite")
 }
 
-func buildTestPlugins(t *testing.T, path string) {
-	t.Helper()
+func buildTestPlugins(path string) {
 	start := time.Now()
-	t.Logf("[BeforeSuite] Current working directory: %s", path)
-	cmd := exec.Command("make", "-C", path)
-	out, err := cmd.CombinedOutput()
-	t.Logf("[BeforeSuite] Make output: %s elapsed: %s", string(out), time.Since(start))
-	if err != nil {
-		t.Fatalf("Failed to build test plugins: %v", err)
-	}
+	out, err := exec.Command("make", "-C", path).CombinedOutput()
+	fmt.Fprintf(GinkgoWriter, "[BeforeSuite] built test plugins in %s:\n%s", time.Since(start), out)
+	Expect(err).ToNot(HaveOccurred(), "failed to build test plugins")
 }
 
 // createTestManager creates a new plugin Manager with the given plugin config.
@@ -146,7 +141,10 @@ func createTestManagerWithPluginsAndMetrics(pluginConfig map[string]map[string]s
 	return manager, tmpDir
 }
 
-var _ = BeforeSuite(func() {
+var _ = SynchronizedBeforeSuite(func() {
+	// Build once: the testdata Makefile is not safe to run concurrently.
+	buildTestPlugins(testDataDir)
+}, func() {
 	// Get testdata directory (where test plugin .ndp packages live)
 	_, currentFile, _, ok := runtime.Caller(0)
 	Expect(ok).To(BeTrue())

plugins/testdata/Makefile

@@ -10,17 +10,19 @@ all: $(PLUGINS:%=%.ndp)
 
 clean:
 	rm -f $(PLUGINS:%=%.ndp) $(PLUGINS:%=%.wasm)
-	rm -rf .wazero-cache
+	rm -rf .wazero-cache $(PLUGINS:%=%.stage)
 
 # PDK source files that trigger rebuild when changed (recursive)
 PDK_SOURCES := $(shell find ../pdk/go -name '*.go' 2>/dev/null)
 
 # Build the .ndp package (zip containing manifest.json + plugin.wasm)
+# Stage under a per-target name: a shared plugin.wasm breaks concurrent builds.
 %.ndp: %.wasm %/manifest.json
 	@rm -f $@
-	@cp $< plugin.wasm
-	zip -j $@ $*/manifest.json plugin.wasm
-	@rm -f plugin.wasm
+	@rm -rf $*.stage && mkdir -p $*.stage
+	@cp $< $*.stage/plugin.wasm
+	zip -j $@ $*/manifest.json $*.stage/plugin.wasm
+	@rm -rf $*.stage
 	@mv $< $<.tmp && mv $<.tmp $<  # Touch wasm to ensure it's older than ndp
 
 # Build the wasm binary. -buildvcs=false keeps the bytes stable across commits, so