feat: add basic mem store boot-time initialization

.github/workflows/lint.yaml

@@ -22,4 +22,4 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v8
         with:
-          version: v2.1
+          version: v2.11

cmd/cloud-init-server/main.go

@@ -55,6 +55,7 @@ var (
 	wireGuardMiddleware  func(http.Handler) http.Handler
 	storageBackend       = "mem"           // Default to memstore
 	dbPath               = "cloud-init.db" // Default database path for quackstore
+	memPath              string
 	store                cistore.Store
 )
 
@@ -99,6 +100,7 @@ func setupFlags(flags *pflag.FlagSet) {
 	flags.BoolVar(&debug, "debug", parseBool(getEnv("DEBUG", "false")), "Enable debug logging")
 	flags.StringVar(&storageBackend, "storage-backend", getEnv("STORAGE_BACKEND", "mem"), "Storage backend to use (mem or quack)")
 	flags.StringVar(&dbPath, "db-path", getEnv("DB_PATH", "cloud-init.db"), "Path to the database file for quackstore backend")
+	flags.StringVar(&memPath, "mem-path", getEnv("MEM_PATH", ""), "Path to initial in-memory store configuration")
 }
 
 // bindViperToFlags binds each flag to Viper so environment variables work seamlessly.
@@ -124,6 +126,7 @@ func bindViperToFlags() {
 	_ = viper.BindEnv("debug")
 	_ = viper.BindEnv("storage_backend")
 	_ = viper.BindEnv("db_path")
+	_ = viper.BindEnv("mem_path")
 }
 
 // startServer is where we run our main program logic
@@ -149,6 +152,7 @@ func startServer() error {
 			Bool("debug", debug).
 			Str("storage-backend", storageBackend).
 			Str("db-path", dbPath).
+			Str("mem-path", memPath).
 			Msg("Resolved configuration")
 	}
 
@@ -161,7 +165,14 @@ func startServer() error {
 	var err error
 	switch storageBackend {
 	case "mem":
-		store = memstore.NewMemStore()
+		if memPath == "" {
+			store = memstore.NewMemStore()
+		} else {
+			store, err = memstore.NewMemStoreFromPath(memPath)
+			if err != nil {
+				return fmt.Errorf("failed to initialize in-memory store from path: %w", err)
+			}
+		}
 	case "quack":
 		store, err = quackstore.NewQuackStore(dbPath)
 		if err != nil {

go.mod

@@ -1,6 +1,6 @@
 module github.com/OpenCHAMI/cloud-init
 
-go 1.24.0
+go 1.26.0
 
 require (
 	github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18
@@ -17,6 +17,7 @@ require (
 	github.com/spf13/viper v1.20.1
 	github.com/stretchr/testify v1.11.1
 	github.com/swaggo/swag v1.16.6
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (

go.sum

@@ -230,3 +230,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

internal/memstore/ciMemStore.go

@@ -3,11 +3,15 @@ package memstore
 import (
 	"crypto/rand"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 
-	"github.com/OpenCHAMI/cloud-init/pkg/cistore"
 	"github.com/rs/zerolog/log"
+	"sigs.k8s.io/yaml"
+
+	"github.com/OpenCHAMI/cloud-init/pkg/cistore"
 )
 
 type MemStore struct {
@@ -30,6 +34,49 @@ func NewMemStore() *MemStore {
 	}
 }
 
+const (
+	groupsFile    = "groups.yaml"
+	instancesFile = "instances.yaml"
+	defaultsFile  = "clusterdefaults.yaml"
+)
+
+func NewMemStoreFromPath(path string) (*MemStore, error) {
+	store := NewMemStore()
+
+	groupsPath := filepath.Join(path, groupsFile)
+	instancesPath := filepath.Join(path, instancesFile)
+	defaultsPath := filepath.Join(path, defaultsFile)
+
+	groups, err := os.ReadFile(groupsPath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening %q: %w", groupsPath, err)
+	}
+	err = yaml.Unmarshal(groups, &store.Groups)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshaling %q: %w", groupsPath, err)
+	}
+
+	instances, err := os.ReadFile(instancesPath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening %q: %w", instancesPath, err)
+	}
+	err = yaml.Unmarshal(instances, &store.Instances)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshaling %q: %w", instancesPath, err)
+	}
+
+	defaults, err := os.ReadFile(defaultsPath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening %q: %w", defaultsPath, err)
+	}
+	err = yaml.Unmarshal(defaults, &store.ClusterDefaults)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshaling %q: %w", defaultsPath, err)
+	}
+
+	return store, err
+}
+
 func (m *MemStore) GetGroups() map[string]cistore.GroupData {
 	m.GroupsMutex.RLock()
 	defer m.GroupsMutex.RUnlock()

internal/memstore/ciMemStore_test.go

@@ -1,8 +1,13 @@
 package memstore
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	storetesting "github.com/OpenCHAMI/cloud-init/pkg/cistore/testing"
 )
 
@@ -18,3 +23,76 @@ func TestMemStore(t *testing.T) {
 	// Run the standard test suite
 	storetesting.RunStoreTests(t, store, cleanup)
 }
+
+func TestNewMemStoreFromPath(t *testing.T) {
+	testDir, err := os.MkdirTemp("", "cimemstore")
+	require.NoError(t, err)
+	// not really worth trying to wrap the deferred removes, worst case it's (small) leaked tempfiles
+	defer os.RemoveAll(testDir) // nolint:errcheck
+
+	invalidDir, err := os.MkdirTemp("", "cimemstore")
+	require.NoError(t, err)
+	defer os.RemoveAll(invalidDir) // nolint:errcheck
+
+	_, err = NewMemStoreFromPath(testDir)
+	require.Error(t, err)
+	require.ErrorContains(t, err, fmt.Sprintf("error opening %q", filepath.Join(testDir, groupsFile)))
+
+	for _, file := range []string{groupsFile, instancesFile, defaultsFile} {
+		err := os.WriteFile(filepath.Join(invalidDir, file), []byte(testInvalidFile), 0666)
+		require.NoError(t, err)
+	}
+
+	_, err = NewMemStoreFromPath(invalidDir)
+	require.Error(t, err)
+	require.ErrorContains(t, err, fmt.Sprintf("error unmarshaling %q", filepath.Join(invalidDir, groupsFile)))
+
+	err = os.WriteFile(filepath.Join(testDir, groupsFile), []byte(testGroupsFile), 0666)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(testDir, instancesFile), []byte(testInstancesFile), 0666)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(testDir, defaultsFile), []byte(testDefaultsFile), 0666)
+	require.NoError(t, err)
+
+	store, err := NewMemStoreFromPath(testDir)
+	require.NoError(t, err)
+	require.Len(t, store.Groups, 3)
+	require.Len(t, store.Instances, 0)
+	require.Equal(t, "http://test.example/", store.ClusterDefaults.BaseUrl)
+	require.Equal(t, "Login nodes", store.Groups["login"].Description)
+
+}
+
+// Instances and groups follow similar map[string]Type structs. The files must be present, but may be empty if no
+// such resources are loaded, so instances is empty to confirm this works.
+
+const (
+	testGroupsFile = `allnodes:
+  name: allnodes
+  description: All nodes
+  file:
+    content: I2Nsb3VkLWNvbmZpZwpzc2hfYXV0aG9yaXplZF9rZXlzOgotICJzc2gtcnNhIEFBQUFCM056YUMxeWMyRUFBQUFEQVFBQkFBQUJnUUNHejVpSjFGRjVBWFA1eGVWbE9EdldlRGpiblllL25KdDkwS21ySlhwL3FveFd4RTc5WVRwWlhlWlVPeTVDdXZWME9ObG5nK3crS0lheDZrTkFsMWVkUTQyQ2hZMUFBdXdDNWlFb0Y3VmZuS25ndWhJTS9YakxidWtwbGp4NU5SeUg0L1VoMmJ6RzhXNVNiM3ptQWRUNitYMlVkcTBqMFF0RWtWaEFVbnUycXdLZGdZdWJnS0JEWENZWDdqWXdEaWxNM0pvdE1IcFJ4WS8wZEt2QW81VE45VUo0ZGJaWGIxMldaWlBTWWgxeVJDNXB1SnJLMURFQ2lZMzRmKysxWkhyYXB5TlZnODBmN09KSWJxRVNrMkNNMk5jeXNLdU03dkRxMVdLam1QM0p2WTFvdXppbUllcVFadjg1Qis3UWlpZkMxS2JJMHM1dGZNWlh6akxBUWhYT2FzaTczT3oxQzlsN21SRFVtSnoraVFSWm83WXp5NnNXaUUrdVlhK0hCa2docnpSeTNKaEZjUTdaVWhGVllQVE4xNFZEbDhpY1R5RWY5c0lBUUJmb3VLY3orSEloS2RLMkVaR0ppOGlBaUFOTnNwUmNOQWRMajExaEpvSDFOM2kyeER2L0JvK25lWjhlT05pelZBZlF5U245eGVGWC9FSG1uR2lGak1EOVVncz0gZm9vQGV4YW1wbGUiCi0gInNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQmdRQzAvd0tDSHVqZXQyYmU2dWFLSEZ3MXk5TVVBUWZSYnZ3eUQyN3Bab3VQSmxkcThEYVlROExTMkdkbEhmTDYxRVp0Y0p0Mno3ZWZPWkV6YXVqWFlKTk9VZ1Q2YU9vdFZpZ0tZMnhPVmM3RmxVYXdyd2RtTlR0RGsrMXBXT0dadHZJU3g0cXU0NExrNzlXMzZTeGF3aTdheXovNGpOQy9TSFQyTmRqSEF6L3YzY0ZiN3k3R0pmNjQzL2pic0hCOVRWcllsaXY0S0VnRnBHNkdQcUdtanJCY3kxWXJYN1JZem53V2lYaVFrZlpSVUpLbUl1a2pnenAyZlllT28wVWNJT0lZcGs3RGI1TnlSQXNMWkxtWU5sdy9ZWC9xWnN6dkNvYkEzeUtlaUNBYWlFUmtxcFVnNE5Cd2xSMzBCY1RtandUMWNwY256am4zTHN4MUx4akc2RlYreHJTYkxhd3djcFlWeG5iMkVuWkpYbFFOZzdqSmZSc3ZoNEp1ZjlUUWZONS9IWlBvV0huS0pjZFVLaXoyTmtXckZjUE9sVHAvVCs4VzExakp6MjYwY3UxQURucW5EbWNUaVV2SXF2WjBJMGU1amhaay9oMnJ6UDNpSWlUQzhkdEgzMmY0OXZIcGFBbXhTamZZNzV4YnpueDM3NGtaYkY4N2krdkFsNWRFV1JNPSBiYXJAZXhhbXBsZSIK
+    encoding: base64
+login:
+  name: login
+  description: Login nodes
+  file:
+    content: I2Nsb3VkLWNvbmZpZwpwYWNrYWdlczoKLSBmb3J0dW5lLW1vZAp3cml0ZV9maWxlczoKLSBlbmNvZGluZzogYjY0CiAgY29udGVudDogImFHVnNiRzhnYkc5bmFXNEsiCiAgb3duZXI6IG11bmdlOm11bmdlCiAgcGF0aDogL2V0Yy9oZWxsbwogIHBlcm1pc3Npb25zOiAnMDYwMCcK
+    encoding: base64
+compute:
+  name: compute
+  description: Compute nodes
+  file:
+    content: I2Nsb3VkLWNvbmZpZwpwYWNrYWdlczoKLSBjb3dzYXkKd3JpdGVfZmlsZXM6Ci0gZW5jb2Rpbmc6IGI2NAogIGNvbnRlbnQ6ICJhR1ZzYkc4Z1kyOXRjSFYwWlFvPSIKICBvd25lcjogcm9vdDpyb290CiAgcGF0aDogL2V0Yy9jb21wdXRlX2hlbGxvCiAgcGVybWlzc2lvbnM6ICcwNjAwJwo=
+    encoding: base64
+`
+	testDefaultsFile = `cloud-provider: openchami
+region: us-west-2
+availability-zone: us-west-2a
+cluster-name: venado
+base-url: http://test.example/
+`
+	testInstancesFile = ``
+
+	testInvalidFile = `this is not yaml`
+)

pkg/cistore/models.go

@@ -8,11 +8,11 @@ import (
 )
 
 type GroupData struct {
-	Name        string                 `json:"name" example:"compute" description:"Group name"`
-	Description string                 `json:"description,omitempty" example:"The compute group" description:"A short description of the group"`
-	Data        map[string]interface{} `json:"meta-data,omitempty" description:"json map of a string (key) to a struct (value) representing group meta-data"`
-	File        CloudConfigFile        `json:"file,omitempty" description:"Cloud-Init configuration for group"`
-	Versions    map[string]string      `json:"versions,omitempty" description:"Map of group versions"`
+	Name        string                 `json:"name" yaml:"name" example:"compute" description:"Group name"`
+	Description string                 `json:"description,omitempty" yaml:"description,omitempty" example:"The compute group" description:"A short description of the group"`
+	Data        map[string]interface{} `json:"meta-data,omitempty" yaml:"meta-data,omitempty" description:"json map of a string (key) to a struct (value) representing group meta-data"`
+	File        CloudConfigFile        `json:"file,omitempty" yaml:"file,omitempty" description:"Cloud-Init configuration for group"`
+	Versions    map[string]string      `json:"versions,omitempty" yaml:"versions,omitempty" description:"Map of group versions"`
 }
 
 func (g *GroupData) ParseFromJSON(body []byte) error {
@@ -66,9 +66,9 @@ type ClusterDefaults struct {
 }
 
 type CloudConfigFile struct {
-	Content  []byte `json:"content" swaggertype:"string" example:"IyMgdGVtcGxhdGU6IGppbmphCiNjbG91ZC1jb25maWcKbWVyZ2VfaG93OgotIG5hbWU6IGxpc3QKICBzZXR0aW5nczogW2FwcGVuZF0KLSBuYW1lOiBkaWN0CiAgc2V0dGluZ3M6IFtub19yZXBsYWNlLCByZWN1cnNlX2xpc3RdCnVzZXJzOgogIC0gbmFtZTogcm9vdAogICAgc3NoX2F1dGhvcml6ZWRfa2V5czoge3sgZHMubWV0YV9kYXRhLmluc3RhbmNlX2RhdGEudjEucHVibGljX2tleXMgfX0KZGlzYWJsZV9yb290OiBmYWxzZQo=" description:"Cloud-Init configuration content whose encoding depends on the value of 'encoding'"`
-	Name     string `json:"filename"`
-	Encoding string `json:"encoding,omitempty" enums:"base64,plain"`
+	Content  []byte `json:"content" yaml:"content" swaggertype:"string" example:"IyMgdGVtcGxhdGU6IGppbmphCiNjbG91ZC1jb25maWcKbWVyZ2VfaG93OgotIG5hbWU6IGxpc3QKICBzZXR0aW5nczogW2FwcGVuZF0KLSBuYW1lOiBkaWN0CiAgc2V0dGluZ3M6IFtub19yZXBsYWNlLCByZWN1cnNlX2xpc3RdCnVzZXJzOgogIC0gbmFtZTogcm9vdAogICAgc3NoX2F1dGhvcml6ZWRfa2V5czoge3sgZHMubWV0YV9kYXRhLmluc3RhbmNlX2RhdGEudjEucHVibGljX2tleXMgfX0KZGlzYWJsZV9yb290OiBmYWxzZQo=" description:"Cloud-Init configuration content whose encoding depends on the value of 'encoding'"`
+	Name     string `json:"filename" yaml:"filename"`
+	Encoding string `json:"encoding,omitempty" yaml:"encoding,omitempty" enums:"base64,plain"`
 }
 
 // UnmarshalJSON implements json.Unmarshaler