test: run real external-update and notify path in multi-repo e2e

e2e/harness/multi_repo.go

@@ -31,6 +31,11 @@ type MultiRepoHarness struct {
 	act         *ActRunner
 	repos       map[string]*RepoContext // name -> repo context
 	primaryRepo string                  // name of primary repo
+	// base is the single-repo Harness created during SetupInfra. Its act/gitea
+	// bound helpers (GenerateWorkflows, ensureCLIBinary, localizeWorkflows,
+	// waitForBranchHead, ...) are reused per repo by re-pointing base.repo. We
+	// keep one instance so the sync.Once CLI build happens exactly once.
+	base *Harness
 }
 
 // NewMultiRepoHarness creates a harness for multi-repo testing
@@ -53,6 +58,11 @@ func (h *MultiRepoHarness) SetupInfra(ctx context.Context) error {
 	h.act = base.act
 	h.networkName = base.networkName
 	h.network = base.network
+	// Keep the base Harness so its act/gitea-bound single-repo helpers can be
+	// reused for real per-repo workflow generation and runs. base.gitea/base.act
+	// already equal h.gitea/h.act, so re-pointing base.repo is all that is needed
+	// to target a given repo.
+	h.base = base
 
 	return nil
 }
@@ -97,47 +107,45 @@ func (h *MultiRepoHarness) CreateRepo(ctx context.Context, setup MultiRepoSetup)
 
 	// Add CICD config file
 	if setup.Config != nil {
-		// Wrap in ci: structure for manifest
-		manifest := map[string]interface{}{
-			"ci": map[string]interface{}{
-				"config": setup.Config,
-			},
+		// Build the manifest the CLI expects: top-level `ci:` wrapping both the
+		// pipeline `config:` and the initial `state:`. The single-repo harness
+		// only wraps `ci.config`, but the `cascade external update` verb reads
+		// `ci.state.<env>` and fails if it is absent, so the multi-repo manifest
+		// must carry the scenario's state block under `ci.state` as well.
+		ci := map[string]interface{}{
+			"config": setup.Config,
 		}
-
-		// Merge in any provided manifest state
-		if setup.Manifest != nil {
-			if ci, ok := manifest["ci"].(map[string]interface{}); ok {
-				for k, v := range setup.Manifest {
-					if k == "ci" {
-						if ciState, ok := v.(map[string]interface{}); ok {
-							for ck, cv := range ciState {
-								ci[ck] = cv
-							}
-						}
-					} else {
-						ci[k] = v
-					}
-				}
-			}
+		if state := extractManifestState(setup.Manifest); state != nil {
+			ci["state"] = state
 		}
+		manifest := map[string]interface{}{"ci": ci}
 
 		configYAML, err := yaml.Marshal(manifest)
 		if err != nil {
 			return nil, fmt.Errorf("failed to marshal config: %w", err)
 		}
-		initialFiles[".github/cicd.yaml"] = string(configYAML)
-
-		// Create stub workflow files for builds and deploys, tagged with the
-		// repo name so multi-repo scenarios running in parallel don't collide
-		// on shared act container names.
+		// Write to .github/manifest.yaml (the CLI's auto-detected default). The
+		// previous .github/cicd.yaml path was never read by generate-workflow or
+		// the external update verb.
+		initialFiles[".github/manifest.yaml"] = string(configYAML)
+
+		// Create stub workflow files for local builds and deploys, tagged with
+		// the repo name so multi-repo scenarios running in parallel don't collide
+		// on shared act container names. generate-workflow reads each local
+		// reusable workflow at its literal path (e.g. .github/workflows/deploy.yaml)
+		// to discover inputs/outputs, so a stub must exist there; the previous
+		// guard skipped any path containing a slash, which dropped every
+		// .github/workflows/*.yaml stub and broke generation. External (cross-repo)
+		// deploys live under config.External and are intentionally not stubbed
+		// here - they are referenced, not run, in the primary's repo.
 		repoTag := scenarioTagFromTestName(setup.Name)
 		for _, build := range setup.Config.Builds {
-			if build.Workflow != "" && !strings.Contains(build.Workflow, "/") {
+			if isLocalWorkflowPath(build.Workflow) {
 				initialFiles[build.Workflow] = generateStubWorkflow(build.Name, repoTag)
 			}
 		}
 		for _, deploy := range setup.Config.Deploys {
-			if deploy.Workflow != "" && !strings.Contains(deploy.Workflow, "/") {
+			if isLocalWorkflowPath(deploy.Workflow) {
 				initialFiles[deploy.Workflow] = generateStubWorkflow(deploy.Name, repoTag)
 			}
 		}
@@ -179,6 +187,24 @@ func (h *MultiRepoHarness) CreateRepo(ctx context.Context, setup MultiRepoSetup)
 		repoCtx.ExecCtx.RecordTag(tag, true)
 	}
 
+	// Generate real workflows for this repo via `cascade generate-workflow`,
+	// reusing the single-repo machinery bound to the shared act/gitea. This
+	// produces orchestrate.yaml (carrying the Notify step for satellites) and,
+	// for a primary with external repos, external-update.yaml. Generation clones
+	// into the shared /tmp/repo and is destructive, so it must run while this
+	// repo is the active target; later steps re-clone via
+	// prepareRepoInActContainer before running anything.
+	if setup.Config != nil && h.base != nil && h.act != nil {
+		h.base.repo = repoCtx.Repo
+		if err := h.base.GenerateWorkflows(ctx); err != nil {
+			return nil, fmt.Errorf("failed to generate workflows for %s: %w", setup.Name, err)
+		}
+		// Refresh HEAD: GenerateWorkflows pushed a workflows commit.
+		if head, err := h.gitea.getHeadSHA(ctx, repoCtx.Repo); err == nil && head != "" {
+			repoCtx.HeadSHA = head
+		}
+	}
+
 	// Store in harness
 	h.repos[setup.Name] = repoCtx
 	if setup.IsPrimary {
@@ -188,6 +214,91 @@ func (h *MultiRepoHarness) CreateRepo(ctx context.Context, setup MultiRepoSetup)
 	return repoCtx, nil
 }
 
+// isLocalWorkflowPath reports whether a build/deploy workflow reference is a
+// local reusable workflow in this repo (under .github/workflows/) that needs a
+// stub file, as opposed to an empty value or a cross-repo `owner/repo/...`
+// reference.
+func isLocalWorkflowPath(workflow string) bool {
+	return workflow != "" && strings.HasPrefix(workflow, ".github/")
+}
+
+// extractManifestState pulls the `state` submap out of a scenario's Manifest
+// block so it can be placed under `ci.state` in the generated manifest. The
+// scenario YAML shapes the block as either `{state: {...}}` (the common case)
+// or `{ci: {state: {...}}}`; both are handled. It returns nil when no state is
+// present.
+func extractManifestState(manifest map[string]interface{}) interface{} {
+	if manifest == nil {
+		return nil
+	}
+	if state, ok := manifest["state"]; ok {
+		return state
+	}
+	if ci, ok := manifest["ci"].(map[string]interface{}); ok {
+		if state, ok := ci["state"]; ok {
+			return state
+		}
+	}
+	return nil
+}
+
+// prepareRepoInActContainer re-clones the given repo fresh into the shared
+// /tmp/repo and copies the cascade CLI binary into /tmp/repo/.github/bin. This
+// mirrors the clone+copy that GenerateWorkflows performs (minus the generate),
+// and is required before running any workflow under act because /tmp/repo holds
+// whichever repo was generated or run last in the single shared act container.
+func (h *MultiRepoHarness) prepareRepoInActContainer(ctx context.Context, repoCtx *RepoContext) error {
+	if h.act == nil {
+		return fmt.Errorf("act runner not initialized")
+	}
+	if h.base == nil {
+		return fmt.Errorf("base harness not initialized")
+	}
+
+	externalURL := h.act.GiteaURL()
+	externalHost := strings.TrimPrefix(strings.TrimPrefix(externalURL, "http://"), "https://")
+	cloneURL := fmt.Sprintf("http://%s:%s@%s/%s/%s.git",
+		AdminUsername, AdminPassword,
+		externalHost,
+		AdminUsername, repoCtx.Repo.Name)
+
+	cloneCmd := []string{
+		"bash", "-c",
+		fmt.Sprintf("rm -rf /tmp/repo && git clone %s /tmp/repo", cloneURL),
+	}
+	exitCode, _, err := h.act.Container().Exec(ctx, cloneCmd)
+	if err != nil || exitCode != 0 {
+		return fmt.Errorf("failed to clone %s into act container (exit %d): %w", repoCtx.Name, exitCode, err)
+	}
+
+	// Copy the (already-built) CLI binary into the repo so the mock setup-cli
+	// action can install it onto PATH inside job containers.
+	binaryPath, err := h.base.ensureCLIBinary(ctx)
+	if err != nil {
+		return err
+	}
+	if err := h.act.Container().CopyFileToContainer(ctx, binaryPath, "/usr/local/bin/cascade", 0755); err != nil {
+		return fmt.Errorf("failed to copy CLI to container: %w", err)
+	}
+	copyToRepoCmd := []string{
+		"bash", "-c",
+		"mkdir -p /tmp/repo/.github/bin && cp /usr/local/bin/cascade /tmp/repo/.github/bin/cascade",
+	}
+	if _, _, err := h.act.Container().Exec(ctx, copyToRepoCmd); err != nil {
+		return fmt.Errorf("failed to stage CLI in repo: %w", err)
+	}
+
+	// Recreate the master branch alias the generated workflows reference via
+	// `@master` action refs, matching GenerateWorkflows.
+	masterCmd := []string{
+		"bash", "-c",
+		"cd /tmp/repo && git branch master 2>/dev/null || git branch -f master HEAD",
+	}
+	_, _, _ = h.act.Container().Exec(ctx, masterCmd)
+
+	return nil
+}
+
 // SetupPrimarySatellite creates primary + satellite repos with proper configuration
 func (h *MultiRepoHarness) SetupPrimarySatellite(ctx context.Context, primary MultiRepoSetup, satellites ...MultiRepoSetup) error {
 	// Ensure primary is marked correctly
@@ -334,9 +445,23 @@ func (h *MultiRepoHarness) RunWorkflowInRepo(ctx context.Context, repoName strin
 	return h.act.RunWorkflow(ctx, opts)
 }
 
-// SimulateCrossRepoDispatch simulates workflow_dispatch from one repo to another
-// This is used to test satellite -> primary notification flows
-func (h *MultiRepoHarness) SimulateCrossRepoDispatch(ctx context.Context, sourceRepo, targetRepo, workflow string, inputs map[string]string) error {
+// RealCrossRepoDispatch performs the REAL primary-side half of a satellite ->
+// primary notification. The satellite's generated Notify step would dispatch
+// the target repo's external-update.yaml with these inputs; gitea does not
+// implement the GitHub Actions workflow_dispatch API, so that live network hop
+// is a no-op under act (see the network-hop residue note below). The harness
+// bridges it by running the target's external-update.yaml directly under act
+// with the same inputs. That workflow checks out the target repo from gitea,
+// runs `cascade external update`, and commits+pushes ci.state.<env>.external.<name>
+// back to gitea. Assertions then read that real manifest.
+//
+// Network-hop residue: the satellite's generated Notify step calls
+// github.rest.actions.createWorkflowDispatch against GITHUB_API_URL, which in
+// e2e points at gitea. Gitea does not implement that API, so the live cross-repo
+// dispatch is a no-op under act. We assert the generated Notify step CONTENT
+// (RunSatelliteOrchestrateAndAssertNotify) plus the real verb effect here; the
+// live dispatch network hop is real-GitHub-only residue.
+func (h *MultiRepoHarness) RealCrossRepoDispatch(ctx context.Context, sourceRepo, targetRepo, workflow string, inputs map[string]string) error {
 	source := h.repos[sourceRepo]
 	if source == nil {
 		return fmt.Errorf("source repo %s not found", sourceRepo)
@@ -347,52 +472,88 @@ func (h *MultiRepoHarness) SimulateCrossRepoDispatch(ctx context.Context, source
 		return fmt.Errorf("target repo %s not found", targetRepo)
 	}
 
-	// Validate the dispatch makes sense
 	if target.Config != nil && !target.Config.IsPrimary() {
-		// Allow dispatch to non-primary for flexibility, but log warning
 		h.t.Logf("Warning: dispatching to non-primary repo %s", targetRepo)
 	}
 
-	// Run the external-update workflow (or specified workflow) in target repo
+	// Re-clone the target (primary) into /tmp/repo so the external-update run
+	// operates on the right repo regardless of what was generated/run last.
+	if err := h.prepareRepoInActContainer(ctx, target); err != nil {
+		return fmt.Errorf("prepare primary %s for external-update: %w", targetRepo, err)
+	}
+	h.base.repo = target.Repo
+
 	opts := RunOpts{
 		WorkflowPath: workflow,
 		Event:        "workflow_dispatch",
 		Inputs:       inputs,
 	}
-
-	_, err := h.RunWorkflowInRepo(ctx, targetRepo, opts)
+	result, err := h.act.RunWorkflowFromRepo(ctx, opts)
 	if err != nil {
-		return fmt.Errorf("failed to run workflow in target repo: %w", err)
+		return fmt.Errorf("failed to run external-update workflow in %s: %w", targetRepo, err)
+	}
+	if result.Conclusion != "success" {
+		return fmt.Errorf("external-update workflow in %s concluded %q: %s\n%s",
+			targetRepo, result.Conclusion, result.Error, result.Logs)
 	}
 
-	// If this is an external-update dispatch, update the target's external state
-	if strings.Contains(workflow, "external-update") {
-		if err := h.updateExternalState(ctx, targetRepo, inputs); err != nil {
-			return fmt.Errorf("failed to update external state: %w", err)
-		}
+	// The verb committed+pushed the manifest back to gitea. Refresh the target's
+	// recorded HEAD so subsequent reads/operations see the new tip.
+	if head, err := h.gitea.getHeadSHA(ctx, target.Repo); err == nil && head != "" {
+		target.HeadSHA = head
 	}
 
 	return nil
 }
 
-// updateExternalState updates the external state in the target repo's manifest
-func (h *MultiRepoHarness) updateExternalState(ctx context.Context, repoName string, inputs map[string]string) error {
-	repo := h.repos[repoName]
-	if repo == nil {
-		return fmt.Errorf("repo %s not found", repoName)
+// RunSatelliteOrchestrateAndAssertNotify asserts that the satellite's generated
+// orchestrate.yaml carries the real Notify step that dispatches the primary's
+// external-update workflow. We assert the generated CONTENT rather than running
+// the satellite orchestrate to green: the Notify step's createWorkflowDispatch
+// call targets the GitHub Actions API (GITHUB_API_URL), which points at gitea in
+// e2e and is not implemented there, so the live dispatch is a documented no-op
+// (the network hop is bridged by RealCrossRepoDispatch). Asserting the generated
+// step is the maximal observable subset under gitea/act and is far cheaper than
+// a full orchestrate run.
+func (h *MultiRepoHarness) RunSatelliteOrchestrateAndAssertNotify(ctx context.Context, satelliteName string) error {
+	sat := h.repos[satelliteName]
+	if sat == nil {
+		return fmt.Errorf("satellite repo %s not found", satelliteName)
+	}
+	if sat.Config == nil || sat.Config.Notify == nil {
+		return nil // Not a notifying satellite; nothing to assert.
+	}
+
+	content, err := h.gitea.GetFileContent(ctx, sat.Repo, ".github/workflows/orchestrate.yaml")
+	if err != nil {
+		return fmt.Errorf("read generated orchestrate.yaml for %s: %w", satelliteName, err)
 	}
 
-	deployName := inputs["deploy_name"]
-	env := inputs["environment"]
-	sha := inputs["sha"]
-	version := inputs["version"]
+	// The generated Notify step targets the primary repo named in notify.repo.
+	repoParts := strings.SplitN(sat.Config.Notify.Repo, "/", 2)
+	wantOwner, wantName := "", ""
+	if len(repoParts) == 2 {
+		wantOwner, wantName = repoParts[0], repoParts[1]
+	}
+	wantWorkflow := sat.Config.Notify.GetWorkflow()
 
-	if deployName == "" || env == "" {
-		return nil // Not enough info to update state
+	required := []string{
+		"Notify Primary Repo",
+		"createWorkflowDispatch",
+		fmt.Sprintf("workflow_id: '%s'", wantWorkflow),
+	}
+	if wantOwner != "" {
+		required = append(required, fmt.Sprintf("owner: '%s'", wantOwner))
+	}
+	if wantName != "" {
+		required = append(required, fmt.Sprintf("repo: '%s'", wantName))
 	}
 
-	// Record in execution context
-	repo.ExecCtx.RecordExternalDeployState(env, deployName, sha, version)
+	for _, want := range required {
+		if !strings.Contains(content, want) {
+			return fmt.Errorf("satellite %s orchestrate.yaml missing notify content %q", satelliteName, want)
+		}
+	}
 
 	return nil
 }