+ diff --git a/.gitmodules b/.gitmodules index cfa2b86..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "examples/bmeg-dictionary"] - path = examples/bmeg-dictionary - url = https://github.com/bmeg/bmeg-dictionary.git diff --git a/README.md b/README.md index 9e79793..8074cfe 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,9 @@ name: census_2010 params: census: - type: File + type: file default: ../data/census_2010_byzip.json date: - type: string default: "2010-01-01" schema: type: path @@ -37,6 +36,9 @@ outputs: json: path: census_data.ndjson +outputs: + + pipelines: transform: - from: censusData diff --git a/cmd/root.go b/cmd/root.go index d23161c..25e0bb2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,6 +5,7 @@ import ( "github.com/bmeg/sifter/cmd/inspect" "github.com/bmeg/sifter/cmd/run" + "github.com/bmeg/sifter/cmd/web" "github.com/spf13/cobra" ) @@ -18,6 +19,7 @@ var RootCmd = &cobra.Command{ func init() { RootCmd.AddCommand(run.Cmd) RootCmd.AddCommand(inspect.Cmd) + RootCmd.AddCommand(web.Cmd) } var genBashCompletionCmd = &cobra.Command{ diff --git a/cmd/run/main.go b/cmd/run/main.go index 8b3f016..cbab677 100644 --- a/cmd/run/main.go +++ b/cmd/run/main.go @@ -14,6 +14,8 @@ var outDir string = "" var paramsFile string = "" var verbose bool = false var cmdParams map[string]string +var captureDir string = "" +var captureLimit int = 10 // Cmd is the declaration of the command line var Cmd = &cobra.Command{ @@ -46,11 +48,11 @@ var Cmd = &cobra.Command{ } pb := playbook.Playbook{} playbook.ParseBytes(yaml, "./playbook.yaml", &pb) - if err := Execute(pb, "./", "./", outDir, params); err != nil { + if err := Execute(pb, "./", "./", outDir, params, captureDir, captureLimit); err != nil { return err } } else { - if err := ExecuteFile(playFile, "./", outDir, params); err != nil { + if err := ExecuteFile(playFile, "./", outDir, params, captureDir, captureLimit); err != nil { return err } } @@ -65,4 +67,7 @@ func init() { flags.BoolVarP(&verbose, "verbose", "v", verbose, "Verbose logging") flags.StringToStringVarP(&cmdParams, "param", "p", cmdParams, "Parameter variable") flags.StringVarP(¶msFile, "params-file", "f", paramsFile, "Parameter file") + flags.StringVarP(&captureDir, "capture-dir", "d", "", "Directory for capture files (default: None)") + flags.IntVarP(&captureLimit, "capture-limit", "l", 10, "Max records to capture per step (0 = unlimited)") + flags.StringVarP(&outDir, "output", "o", outDir, "Output directory for playbook results (default: current directory or value specified in playbook)") } diff --git a/cmd/run/run.go b/cmd/run/run.go index 42651b2..0c491e0 100644 --- a/cmd/run/run.go +++ b/cmd/run/run.go @@ -1,6 +1,7 @@ package run import ( + "fmt" "os" "path/filepath" @@ -9,7 +10,7 @@ import ( "github.com/bmeg/sifter/task" ) -func ExecuteFile(playFile string, workDir string, outDir string, inputs map[string]string) error { +func ExecuteFile(playFile string, workDir string, outDir string, inputs map[string]string, debugDir string, debugLimit int) error { logger.Info("Starting", "playFile", playFile) pb := playbook.Playbook{} if err := playbook.ParseFile(playFile, &pb); err != nil { @@ -19,10 +20,10 @@ func ExecuteFile(playFile string, workDir string, outDir string, inputs map[stri a, _ := filepath.Abs(playFile) baseDir := filepath.Dir(a) logger.Debug("parsed file", "baseDir", baseDir, "playbook", pb) - return Execute(pb, baseDir, workDir, outDir, inputs) + return Execute(pb, baseDir, workDir, outDir, inputs, debugDir, debugLimit) } -func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string, params map[string]string) error { +func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string, params map[string]string, debugDir string, debugLimit int) error { if outDir == "" { outDir = pb.GetDefaultOutDir() @@ -32,6 +33,32 @@ func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string os.MkdirAll(outDir, 0777) } + // Setup debug capture directory if enabled + // Enable if: user explicitly set dir, OR user changed limit from default + enableDebug := debugDir != "" || (debugLimit != 10) + if enableDebug { + if debugDir == "" { + debugDir = filepath.Join(workDir, "debug-capture") + } else if !filepath.IsAbs(debugDir) { + debugDir = filepath.Join(workDir, debugDir) + } + if info, err := os.Stat(debugDir); err != nil { + if os.IsNotExist(err) { + if mkErr := os.MkdirAll(debugDir, 0777); mkErr != nil { + logger.Error("Failed to create debug directory", "error", mkErr) + return mkErr + } + } else { + logger.Error("Failed to access debug directory", "path", debugDir, "error", err) + return err + } + } else if !info.IsDir() { + logger.Error("Debug path exists but is not a directory", "path", debugDir) + return fmt.Errorf("debug path %s exists but is not a directory", debugDir) + } + logger.Info("Debug capture enabled", "dir", debugDir, "limit", debugLimit) + } + nInputs, err := pb.PrepConfig(params, workDir) if err != nil { return err @@ -39,6 +66,6 @@ func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string logger.Debug("Running", "outDir", outDir) t := task.NewTask(pb.Name, baseDir, workDir, outDir, nInputs) - err = pb.Execute(t) + err = pb.ExecuteWithCapture(t, debugDir, debugLimit) return err } diff --git a/cmd/web/main.go b/cmd/web/main.go new file mode 100644 index 0000000..4a4e9b9 --- /dev/null +++ b/cmd/web/main.go @@ -0,0 +1,183 @@ +package web + +import ( + "embed" + "encoding/json" + "fmt" + "io/fs" + "log" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/spf13/cobra" +) + +//go:embed static/* +var staticFS embed.FS + +var playbookDir string +var siteDir string +var port string = "8081" + +// Cmd is the declaration of the command line +var Cmd = &cobra.Command{ + Use: "web + + + +
+
+