diff --git a/.golangci.yaml b/.golangci.yaml index 53395e546..65d93820b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -30,6 +30,7 @@ linters: - unused - usestdlibvars - whitespace + - wsl_v5 settings: depguard: rules: diff --git a/cmd/addon-operator/main.go b/cmd/addon-operator/main.go index 4290cbe49..6576d701a 100644 --- a/cmd/addon-operator/main.go +++ b/cmd/addon-operator/main.go @@ -45,6 +45,7 @@ func main() { kpApp.Action(func(_ *kingpin.ParseContext) error { klogtolog.InitAdapter(shapp.DebugKubernetesAPI, logger.Named("klog")) stdliblogtolog.InitAdapter(logger) + return nil }) @@ -85,6 +86,7 @@ func start(logger *log.Logger) func(_ *kingpin.ParseContext) error { if os.Getenv("ADDON_OPERATOR_HA") == "true" { operator.Logger.Info("Addon-operator is starting in HA mode") runHAMode(ctx, operator) + return nil } @@ -179,6 +181,7 @@ func runHAMode(ctx context.Context, operator *addon_operator.AddonOperator) { go func() { <-ctx.Done() log.Info("Context canceled received") + if err := syscall.Kill(1, syscall.SIGUSR2); err != nil { operator.Logger.Fatal("Couldn't shutdown addon-operator", log.Err(err)) } diff --git a/cmd/post-renderer/main.go b/cmd/post-renderer/main.go index 011e8ad23..480f3fee3 100644 --- a/cmd/post-renderer/main.go +++ b/cmd/post-renderer/main.go @@ -15,6 +15,7 @@ func main() { fmt.Fprintf(os.Stderr, "couldn't read input from stdin: %s", err) os.Exit(1) } + buf := bytes.NewBuffer(inputBytes) renderer := post_renderer.NewPostRenderer(map[string]string{ diff --git a/examples/700-go-hook/modules/001-module-go-hooks/hooks/go_hooks.go b/examples/700-go-hook/modules/001-module-go-hooks/hooks/go_hooks.go index 1daf87030..1876d8392 100644 --- a/examples/700-go-hook/modules/001-module-go-hooks/hooks/go_hooks.go +++ b/examples/700-go-hook/modules/001-module-go-hooks/hooks/go_hooks.go @@ -44,6 +44,7 @@ type podSpecFilteredObj v1.PodSpec func ObjFilter(obj *unstructured.Unstructured) (gohook.FilterResult, error) { pod := &v1.Pod{} + err := sdk.FromUnstructured(obj, pod) if err != nil { return nil, err diff --git a/pkg/addon-operator/admission_http_server.go b/pkg/addon-operator/admission_http_server.go index 44d53b78b..b175457fc 100644 --- a/pkg/addon-operator/admission_http_server.go +++ b/pkg/addon-operator/admission_http_server.go @@ -58,6 +58,7 @@ func (as *AdmissionServer) start(ctx context.Context) { go func() { cert := path.Join(as.certsDir, "tls.crt") + key := path.Join(as.certsDir, "tls.key") if err := srv.ListenAndServeTLS(cert, key); err != nil { if errors.Is(err, http.ErrServerClosed) { @@ -72,10 +73,12 @@ func (as *AdmissionServer) start(ctx context.Context) { <-ctx.Done() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer func() { // extra handling here cancel() }() + if err := srv.Shutdown(ctx); err != nil { log.Fatal("Server Shutdown Failed", log.Err(err)) } diff --git a/pkg/addon-operator/bootstrap.go b/pkg/addon-operator/bootstrap.go index a23a72ad4..53a106fe4 100644 --- a/pkg/addon-operator/bootstrap.go +++ b/pkg/addon-operator/bootstrap.go @@ -35,6 +35,7 @@ func (op *AddonOperator) bootstrap() error { // Initialize the debug server for troubleshooting and monitoring // TODO: rewrite shapp global variables to the addon-operator one var err error + op.DebugServer, err = shell_operator.RunDefaultDebugServer(shapp.DebugUnixSocket, shapp.DebugHttpServerAddr, op.Logger.Named("debug-server")) if err != nil { log.Error("Fatal: start Debug server", log.Err(err)) diff --git a/pkg/addon-operator/converge/converge.go b/pkg/addon-operator/converge/converge.go index d6e4a27b9..8aa373c12 100644 --- a/pkg/addon-operator/converge/converge.go +++ b/pkg/addon-operator/converge/converge.go @@ -61,6 +61,7 @@ func (cs *ConvergeState) SetOnConvergeFinish(callback func()) { func (cs *ConvergeState) SetFirstRunPhase(ph FirstConvergePhase) { cs.phaseMu.Lock() defer cs.phaseMu.Unlock() + cs.firstRunPhase = ph if ph == FirstDone { close(cs.FirstRunDoneC) @@ -70,12 +71,14 @@ func (cs *ConvergeState) SetFirstRunPhase(ph FirstConvergePhase) { func (cs *ConvergeState) GetFirstRunPhase() FirstConvergePhase { cs.phaseMu.RLock() defer cs.phaseMu.RUnlock() + return cs.firstRunPhase } func (cs *ConvergeState) SetPhase(ph ConvergePhase) { cs.phaseMu.Lock() defer cs.phaseMu.Unlock() + cs.phase = ph if ph == RunBeforeAll && cs.onConvergeStart != nil { @@ -90,6 +93,7 @@ func (cs *ConvergeState) SetPhase(ph ConvergePhase) { func (cs *ConvergeState) GetPhase() ConvergePhase { cs.phaseMu.RLock() defer cs.phaseMu.RUnlock() + return cs.phase } @@ -131,6 +135,7 @@ func IsConvergeTask(t sh_task.Task) bool { return true } } + return false } @@ -140,6 +145,7 @@ func IsFirstConvergeTask(t sh_task.Task) bool { case task.ModulePurge, task.DiscoverHelmReleases, task.GlobalHookEnableKubernetesBindings, task.GlobalHookEnableScheduleBindings: return true } + return false } @@ -152,6 +158,7 @@ func NewConvergeModulesTask(description string, convergeEvent ConvergeEvent, log }). WithQueuedAt(time.Now()) convergeTask.SetProp(ConvergeEventProp, convergeEvent) + return convergeTask } @@ -164,5 +171,6 @@ func NewApplyKubeConfigValuesTask(description string, logLabels map[string]strin GlobalValuesChanged: globalValuesChanged, }). WithQueuedAt(time.Now()) + return convergeTask } diff --git a/pkg/addon-operator/debug_server.go b/pkg/addon-operator/debug_server.go index 6d3af23f0..aea5f267b 100644 --- a/pkg/addon-operator/debug_server.go +++ b/pkg/addon-operator/debug_server.go @@ -51,6 +51,7 @@ func (op *AddonOperator) RegisterDebugGlobalRoutes(dbgSrv *debug.Server) { func(_ *http.Request) (any, error) { kubeHookNames := op.ModuleManager.GetGlobalHooksInOrder(types.OnKubernetesEvent) snapshots := make(map[string]any) + for _, hName := range kubeHookNames { h := op.ModuleManager.GetGlobalHook(hName) snapshots[hName] = h.GetHookController().SnapshotsDump() @@ -68,12 +69,14 @@ func (op *AddonOperator) RegisterDebugGraphRoutes(dbgSrv *debug.Server) { if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(err.Error())) + return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusOK) _, _ = w.Write(dotDesc) + return } @@ -81,6 +84,7 @@ func (op *AddonOperator) RegisterDebugGraphRoutes(dbgSrv *debug.Server) { if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = fmt.Fprintf(w, "couldn't get graph's image: %s", err) + return } @@ -88,6 +92,7 @@ func (op *AddonOperator) RegisterDebugGraphRoutes(dbgSrv *debug.Server) { if err = png.Encode(buf, image); err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(fmt.Errorf("couldn't encode png graph's image").Error())) + return } @@ -101,6 +106,7 @@ func (op *AddonOperator) RegisterDebugModuleRoutes(dbgSrv *debug.Server) { dbgSrv.RegisterHandler(http.MethodGet, "/module/list.{format:(json|yaml|text)}", func(_ *http.Request) (any, error) { mods := op.ModuleManager.GetEnabledModuleNames() sort.Strings(mods) + return map[string][]string{"enabledModules": mods}, nil }) @@ -110,6 +116,7 @@ func (op *AddonOperator) RegisterDebugModuleRoutes(dbgSrv *debug.Server) { withGlobal := false withGlobalStr := r.URL.Query().Get("global") + v, err := strconv.ParseBool(withGlobalStr) if err == nil { withGlobal = v @@ -149,11 +156,13 @@ func (op *AddonOperator) RegisterDebugModuleRoutes(dbgSrv *debug.Server) { dbgSrv.RegisterHandler(http.MethodGet, "/module/{name}/render", func(r *http.Request) (any, error) { modName := chi.URLParam(r, "name") + debugMode, err := strconv.ParseBool(r.URL.Query().Get("debug")) if err != nil { // if empty or unparsable - set false debugMode = false } + diffMode, err := strconv.ParseBool(r.URL.Query().Get("diff")) if err != nil { // if empty or unparsable - set false @@ -228,6 +237,7 @@ func (op *AddonOperator) RegisterDebugModuleRoutes(dbgSrv *debug.Server) { defer differ.TearDown() const maxRetries = 4 + buffer := new(bytes.Buffer) printer := diff.Printer{} diffProgram := &diff.DiffProgram{ @@ -250,6 +260,7 @@ func (op *AddonOperator) RegisterDebugModuleRoutes(dbgSrv *debug.Server) { if !apierrors.IsNotFound(err) { return err } + info.Object = nil } @@ -325,6 +336,7 @@ func (op *AddonOperator) RegisterDebugModuleRoutes(dbgSrv *debug.Server) { } mHooks := m.GetHooks() + snapshots := make(map[string]any) for _, h := range mHooks { snapshots[h.GetName()] = h.GetHookController().SnapshotsDump() @@ -347,6 +359,7 @@ func (op *AddonOperator) RegisterDiscoveryRoute(dbgSrv *debug.Server) { _, _ = fmt.Fprintf(buf, "%s %s\n", method, route) return nil } + return nil } diff --git a/pkg/addon-operator/diff/diff.go b/pkg/addon-operator/diff/diff.go index be89638cc..94ee76776 100644 --- a/pkg/addon-operator/diff/diff.go +++ b/pkg/addon-operator/diff/diff.go @@ -27,6 +27,7 @@ func NewDiffer(from, to string) (*Differ, error) { if err != nil { return nil, err } + return &Differ{Differ: d}, nil } @@ -60,6 +61,7 @@ func (ow *objectWrapper) Merged() (runtime.Object, error) { if err != nil { return nil, err } + return ow.preprocessObject(obj), nil } @@ -97,7 +99,9 @@ func (p *Printer) Print(obj runtime.Object, w io.Writer) error { if err != nil { return err } + _, err = w.Write(data) + return err } @@ -106,9 +110,11 @@ func toUnstructured(obj runtime.Object) (*unstructured.Unstructured, error) { if unstr, ok := obj.(*unstructured.Unstructured); ok { return unstr, nil } + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) if err != nil { return nil, err } + return &unstructured.Unstructured{Object: data}, nil } diff --git a/pkg/addon-operator/ensure_crds.go b/pkg/addon-operator/ensure_crds.go index 747d968c5..5667a7eb2 100644 --- a/pkg/addon-operator/ensure_crds.go +++ b/pkg/addon-operator/ensure_crds.go @@ -18,6 +18,7 @@ func (op *AddonOperator) EnsureCRDs(module *modules.BasicModule) ([]string, erro if cp == nil { return nil, nil } + if err := cp.Run(context.TODO()); err != nil { return nil, err } diff --git a/pkg/addon-operator/handler_module_manager.go b/pkg/addon-operator/handler_module_manager.go index b3a1a9811..ebab0c2d9 100644 --- a/pkg/addon-operator/handler_module_manager.go +++ b/pkg/addon-operator/handler_module_manager.go @@ -21,6 +21,7 @@ import ( func (op *AddonOperator) StartModuleManagerEventHandler() { go func() { logEntry := op.Logger.With(pkg.LogKeyOperatorComponent, "handleManagerEvents") + for { select { case schedulerEvent := <-op.ModuleManager.SchedulerEventCh(): @@ -70,6 +71,7 @@ func (op *AddonOperator) StartModuleManagerEventHandler() { pkg.LogKeyEventSource: "KubeConfigExtenderChanged", } eventLogEntry := utils.EnrichLoggerWithLabels(logEntry, logLabels) + switch event.Type { case config.KubeConfigInvalid: op.ModuleManager.SetKubeConfigValid(false) @@ -81,6 +83,7 @@ func (op *AddonOperator) StartModuleManagerEventHandler() { slog.Any(pkg.LogKeyModuleValuesChanged, event.ModuleValuesChanged), slog.Any(pkg.LogKeyModuleEnabledStateChanged, event.ModuleEnabledStateChanged), slog.Any(pkg.LogKeyModuleMaintenanceChanged, event.ModuleMaintenanceChanged)) + if !op.ModuleManager.GetKubeConfigValid() { eventLogEntry.Info("KubeConfig become valid") } @@ -113,7 +116,9 @@ func (op *AddonOperator) StartModuleManagerEventHandler() { op.engine.TaskQueues.GetMain().CancelTaskDelay() op.logTaskAdd(eventLogEntry, "KubeConfigExtender is updated, put first", kubeConfigTask) } + eventLogEntry.Info("Kube config modification detected, ignore until starting first converge") + break } @@ -146,6 +151,7 @@ func (op *AddonOperator) StartModuleManagerEventHandler() { if kubeConfigTask != nil { op.engine.TaskQueues.GetMain().AddFirst(kubeConfigTask) } + logEntry.Info("ConvergeModules: kube config modification detected, rerun all modules required") op.engine.TaskQueues.GetMain().AddLast(convergeTask) } @@ -169,10 +175,12 @@ func (op *AddonOperator) StartModuleManagerEventHandler() { if kubeConfigTask != nil { reloadTasks := op.CreateReloadModulesTasks(modulesToRerun, kubeConfigTask.GetLogLabels(), "KubeConfig-Changed-Modules") op.engine.TaskQueues.GetMain().AddFirst(kubeConfigTask) + if len(reloadTasks) > 0 { for i := len(reloadTasks) - 1; i >= 0; i-- { op.engine.TaskQueues.GetMain().AddAfter(kubeConfigTask.GetId(), reloadTasks[i]) } + logEntry.Info("ConvergeModules: kube config modification detected, append tasks to rerun modules", slog.Int(pkg.LogKeyCount, len(reloadTasks)), slog.Any(pkg.LogKeyModules, modulesToRerun)) @@ -199,6 +207,7 @@ func (op *AddonOperator) StartModuleManagerEventHandler() { // helm reslease in unexpected state event if HelmReleaseStatusEvent.UnexpectedStatus { op.engine.MetricStorage.CounterAdd(metrics.ModulesHelmReleaseRedeployedTotal, 1.0, map[string]string{pkg.MetricKeyModule: HelmReleaseStatusEvent.ModuleName}) + eventDescription = "HelmReleaseUnexpectedStatus" additionalDescription = "unexpected helm release status" } else { diff --git a/pkg/addon-operator/http_server.go b/pkg/addon-operator/http_server.go index d73818af5..c748e3850 100644 --- a/pkg/addon-operator/http_server.go +++ b/pkg/addon-operator/http_server.go @@ -53,6 +53,7 @@ func (op *AddonOperator) checkLeaderReadiness(w http.ResponseWriter) { if leader == "" { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("HA mode is enabled but no leader is elected\n")) + return } @@ -66,6 +67,7 @@ func (op *AddonOperator) checkLeaderReadiness(w http.ResponseWriter) { if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("HA mode is enabled but couldn't craft a request to the leader\n")) + return } @@ -73,6 +75,7 @@ func (op *AddonOperator) checkLeaderReadiness(w http.ResponseWriter) { if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("HA mode is enabled but couldn't send a request to the leader\n")) + return } defer resp.Body.Close() @@ -80,6 +83,7 @@ func (op *AddonOperator) checkLeaderReadiness(w http.ResponseWriter) { if resp.StatusCode != http.StatusOK { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("HA mode is enabled but the leader's status response code isn't OK\n")) + return } @@ -129,8 +133,10 @@ func (op *AddonOperator) handleConvergeStatus(writer http.ResponseWriter, reques if request.URL.Query().Get("output") == "json" { writer.Header().Set("Content-Type", "application/json") + response := generateConvergeJSON(op.ConvergeState.GetFirstRunPhase(), convergeTasks) _ = json.NewEncoder(writer).Encode(response) + return } diff --git a/pkg/addon-operator/kube_client.go b/pkg/addon-operator/kube_client.go index 3499bfe2b..8ac590ee5 100644 --- a/pkg/addon-operator/kube_client.go +++ b/pkg/addon-operator/kube_client.go @@ -26,19 +26,24 @@ func defaultKubeClient(metricStorage metricsstorage.Storage, metricLabels map[st client.WithConfigPath(shapp.KubeConfig) client.WithMetricStorage(metric.NewMetricsAdapter(metricStorage, logger.Named("kube-client-metrics-adapter"))) client.WithMetricLabels(metricLabels) + return client } func InitDefaultHelmResourcesManager(ctx context.Context, namespace string, metricStorage metricsstorage.Storage, logger *log.Logger) (helm_resources_manager.HelmResourcesManager, error) { kubeClient := defaultKubeClient(metricStorage, DefaultHelmMonitorKubeClientMetricLabels, logger.Named("helm-monitor-kube-client")) kubeClient.WithRateLimiterSettings(app.HelmMonitorKubeClientQps, app.HelmMonitorKubeClientBurst) + if err := kubeClient.Init(); err != nil { return nil, fmt.Errorf("initialize Kubernetes client for Helm resources manager: %s\n", err) } + mgr, err := helm_resources_manager.NewHelmResourcesManager(ctx, kubeClient, logger.Named("helm-resource-manager")) if err != nil { return nil, fmt.Errorf("initialize Helm resources manager: %s\n", err) } + mgr.WithDefaultNamespace(namespace) + return mgr, nil } diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index c6a2d6a7d..6f944fa28 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -170,6 +170,7 @@ func NewAddonOperator(ctx context.Context, metricsStorage, hookMetricStorage met if err != nil { panic(err) } + crdExtraLabels := labelSelector.MatchLabels // use `heritage=addon-operator` by default if not set @@ -189,6 +190,7 @@ func NewAddonOperator(ctx context.Context, metricsStorage, hookMetricStorage met func (op *AddonOperator) WithLeaderElector(config *leaderelection.LeaderElectionConfig) error { var err error + op.LeaderElector, err = leaderelection.NewLeaderElector(*config) if err != nil { return err @@ -233,6 +235,7 @@ func (op *AddonOperator) Setup() error { if err != nil { return fmt.Errorf("global hooks directory: %s", err) } + log.Info("global hooks directory", slog.String(pkg.LogKeyDir, globalHooksDir)) @@ -254,10 +257,12 @@ func ensureTempDirectory(inDir string) (string, error) { // No path to temporary dir, use default temporary dir. if inDir == "" { tmpPath := app.AppName + "-*" + dir, err := os.MkdirTemp("", tmpPath) if err != nil { return "", fmt.Errorf("create tmp dir in '%s': %s", tmpPath, err) } + return dir, nil } @@ -273,6 +278,7 @@ func ensureTempDirectory(inDir string) (string, error) { return "", fmt.Errorf("create tmp dir '%s': %s", dir, err) } } + return dir, nil } @@ -364,6 +370,7 @@ func (op *AddonOperator) InitModuleManager() error { op.KubeConfigManager.SafeReadConfig(func(config *config.KubeConfig) { err = op.ModuleManager.ApplyNewKubeConfigValues(config, true) }) + if err != nil { return fmt.Errorf("init module manager: load initial config for KubeConfigManager: %s", err) } @@ -448,6 +455,7 @@ func (op *AddonOperator) BootstrapMainQueue(tqs *queue.TaskQueueSet) { tasks := op.CreateBootstrapTasks(logLabels) op.logTaskAdd(logEntry, "append", tasks...) + for _, tsk := range tasks { op.engine.TaskQueues.GetMain().AddLast(tsk) } @@ -473,6 +481,7 @@ func (op *AddonOperator) BootstrapMainQueue(tqs *queue.TaskQueueSet) { func (op *AddonOperator) CreateBootstrapTasks(logLabels map[string]string) []sh_task.Task { const eventDescription = "Operator-Startup" + queuedAt := time.Now() // 'OnStartup' global hooks. @@ -619,6 +628,7 @@ func (op *AddonOperator) CreateAndStartQueuesForGlobalHooks() { slog.String(pkg.LogKeyHook, hookName)) } } + for _, hookBinding := range h.GetHookConfig().OnKubernetesEvents { if !op.IsQueueExists(hookBinding.Queue) { op.CreateAndStartQueue(hookBinding.Queue) @@ -709,6 +719,7 @@ func (op *AddonOperator) CreateReloadModulesTasks(moduleNames []string, logLabel }) newTasks = append(newTasks, newTask.WithQueuedAt(queuedAt)) } + return newTasks } @@ -759,6 +770,7 @@ func formatTaskDetails(tsk sh_task.Task, hm task.HookMetadata, phase string) str if hm.DoModuleStartup { details += " with doModuleStartup" } + return details case task.ParallelModuleRun: @@ -829,6 +841,7 @@ func formatConvergeTaskDetails(tsk sh_task.Task, phase string) string { if taskEvent, ok := tsk.GetProp(converge.ConvergeEventProp).(converge.ConvergeEvent); ok { return fmt.Sprintf(" for %s in phase '%s'", string(taskEvent), phase) } + return "" } @@ -846,6 +859,8 @@ func (op *AddonOperator) getConvergeQueues() []*queue.TaskQueue { for i := 0; i < app.NumberOfParallelQueues; i++ { convergeQueues = append(convergeQueues, op.engine.TaskQueues.GetByName(fmt.Sprintf(app.ParallelQueueNamePattern, i))) } + convergeQueues = append(convergeQueues, op.engine.TaskQueues.GetMain()) + return convergeQueues } diff --git a/pkg/addon-operator/queue.go b/pkg/addon-operator/queue.go index fd876e3ca..dbd88d40b 100644 --- a/pkg/addon-operator/queue.go +++ b/pkg/addon-operator/queue.go @@ -19,8 +19,10 @@ func QueueHasPendingModuleRunTask(q *queue.TaskQueue, moduleName string) bool { if q == nil { return false } + modules := ModulesWithPendingModuleRun(q) _, has := modules[moduleName] + return has } @@ -64,6 +66,7 @@ func ConvergeTasksInQueue(q *queue.TaskQueue) int { } convergeTasks := 0 + q.IterateSnapshot(func(t sh_task.Task) { if converge.IsConvergeTask(t) || converge.IsFirstConvergeTask(t) { convergeTasks++ @@ -79,6 +82,7 @@ func ConvergeModulesInQueue(q *queue.TaskQueue) int { } tasks := 0 + q.IterateSnapshot(func(t sh_task.Task) { taskType := t.GetType() if converge.IsConvergeTask(t) && (taskType == task.ModuleRun || taskType == task.ModuleDelete) { @@ -128,16 +132,20 @@ func RemoveCurrentConvergeTasks(convergeQueues []*queue.TaskQueue, logLabels map hm.ParallelRunMetadata.CancelF() } } + logEntry.Debug("Drained converge task", slog.String(pkg.LogKeyType, string(t.GetType())), slog.String(pkg.LogKeyModule, hm.ModuleName), slog.String(pkg.LogKeyDescription, hm.EventDescription), slog.String(pkg.LogKeyQueue, queue.Name)) + return false } + return true }) } + return convergeDrained } @@ -154,6 +162,7 @@ func RemoveCurrentConvergeTasksFromId(q *queue.TaskQueue, afterId string, logLab IDFound := false convergeDrained := false stop := false + q.DeleteFunc(func(t sh_task.Task) bool { if stop { return true @@ -175,13 +184,16 @@ func RemoveCurrentConvergeTasksFromId(q *queue.TaskQueue, afterId string, logLab if t.GetType() == task.ConvergeModules { stop = true } + hm := task.HookMetadataAccessor(t) logEntry.Debug("Drained converge task", slog.String(pkg.LogKeyType, string(t.GetType())), slog.String(pkg.LogKeyModule, hm.ModuleName), slog.String(pkg.LogKeyDescription, hm.EventDescription)) + return false } + return true }) @@ -199,14 +211,17 @@ func RemoveAdjacentConvergeModules(q *queue.TaskQueue, afterId string, logLabels IDFound := false stop := false + q.DeleteFunc(func(t sh_task.Task) bool { if stop { return true } + if !IDFound { if t.GetId() == afterId { IDFound = true } + return true } @@ -216,10 +231,12 @@ func RemoveAdjacentConvergeModules(q *queue.TaskQueue, afterId string, logLabels logEntry.Debug("Drained adjacent ConvergeModules task", slog.String(pkg.LogKeyType, string(t.GetType())), slog.String(pkg.LogKeyDescription, hm.EventDescription)) + return false } stop = true + return true }) } @@ -228,13 +245,16 @@ func ModuleEnsureCRDsTasksInQueueAfterId(q *queue.TaskQueue, afterId string) boo if q == nil { return false } + IDFound := false taskFound := false stop := false + q.DeleteFunc(func(t sh_task.Task) bool { if stop { return true } + if !IDFound { if t.GetId() == afterId { IDFound = true diff --git a/pkg/app/app.go b/pkg/app/app.go index 2a7a41dae..980f26212 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -176,5 +176,6 @@ func DefineStartCommandFlags(kpApp *kingpin.Application, cmd *kingpin.CmdClause) shapp.DefineLoggingFlags(cmd) shapp.DebugUnixSocket = DefaultDebugUnixSocket + shapp.DefineDebugFlags(kpApp, cmd) } diff --git a/pkg/app/debug-cmd.go b/pkg/app/debug-cmd.go index 1333ffcf0..354c8a716 100644 --- a/pkg/app/debug-cmd.go +++ b/pkg/app/debug-cmd.go @@ -25,11 +25,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := globalRequest(client).List(outputFormat) if err != nil { return err } + fmt.Println(string(dump)) + return nil }) @@ -39,11 +42,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := globalRequest(client).Values(outputFormat) if err != nil { return err } + fmt.Println(string(dump)) + return nil }) @@ -53,11 +59,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := globalRequest(client).Config(outputFormat) if err != nil { return err } + fmt.Println(string(dump)) + return nil }) @@ -67,11 +76,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := globalRequest(client).Patches() if err != nil { return err } + fmt.Println(string(dump)) + return nil }) @@ -81,11 +93,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + out, err := globalRequest(client).Snapshots(outputFormat) if err != nil { return err } + fmt.Println(string(out)) + return nil }) @@ -101,11 +116,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + modules, err := moduleRequest(client).List(outputFormat) if err != nil { return err } + fmt.Println(string(modules)) + return nil }) @@ -120,11 +138,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := moduleRequest(client).Name(moduleName).Values(outputFormat, showGlobal) if err != nil { return err } + fmt.Println(string(dump)) + return nil }) moduleValuesCmd.Arg("module_name", "").Required().StringVar(&moduleName) @@ -141,11 +162,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := moduleRequest(client).Name(moduleName).Render(debug, diff) if err != nil { return err } + fmt.Println(string(dump)) + return nil }) moduleRenderCmd.Arg("module_name", "").Required().StringVar(&moduleName) @@ -158,11 +182,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := moduleRequest(client).Name(moduleName).Config(outputFormat, showGlobal) if err != nil { return err } + fmt.Println(string(dump)) + return nil }) moduleConfigCmd.Arg("module_name", "").Required().StringVar(&moduleName) @@ -174,11 +201,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + dump, err := moduleRequest(client).Name(moduleName).Patches() if err != nil { return err } + fmt.Println(string(dump)) + return nil }) modulePatchesCmd.Arg("module_name", "").Required().StringVar(&moduleName) @@ -189,11 +219,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + out, err := moduleRequest(client).Name(moduleName).ResourceMonitor(outputFormat) if err != nil { return err } + fmt.Println(string(out)) + return nil }) moduleResourceMonitorCmd.Arg("module_name", "").StringVar(&moduleName) @@ -204,11 +237,14 @@ func DefineDebugCommands(kpApp *kingpin.Application) { if err != nil { return err } + out, err := moduleRequest(client).Name(moduleName).Snapshots(outputFormat) if err != nil { return err } + fmt.Println(string(out)) + return nil }) moduleSnapshotsCmd.Arg("module_name", "").Required().StringVar(&moduleName) diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index b5f072ef9..153045395 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -43,6 +43,7 @@ func (f *ClientFactory) NewClient(logger *log.Logger, options ...ClientOption) c return c } + return nil } @@ -67,6 +68,7 @@ func InitHelmClientFactory(helmopts *Options, labels map[string]string) (*Client switch clientType { case Helm3Lib: factory.NewClientFn = helm3lib.NewClient + err = helm3lib.Init(&helm3lib.Options{ Namespace: helmopts.Namespace, HistoryMax: helmopts.HistoryMax, @@ -87,6 +89,7 @@ func InitHelmClientFactory(helmopts *Options, labels map[string]string) (*Client if helmopts.Namespace != "" { opts.Namespace = &helmopts.Namespace } + return nelm.NewNelmClient(opts, logger.Named("nelm"), labels) } } diff --git a/pkg/helm/helm3lib/helm3lib.go b/pkg/helm/helm3lib/helm3lib.go index a7e1eb11b..ae1f5f032 100644 --- a/pkg/helm/helm3lib/helm3lib.go +++ b/pkg/helm/helm3lib/helm3lib.go @@ -121,6 +121,7 @@ func buildConfigFlagsFromEnv(ns *string, env *cli.EnvSettings) *genericclioption config.Burst = env.BurstLimit return config } + return flags } @@ -154,6 +155,7 @@ func initAndVersion(logger *log.Logger) error { } logger.Info("Helm 3 version", slog.String(pkg.LogKeyVersion, chartutil.DefaultCapabilities.HelmVersion.Version)) + return nil } @@ -169,6 +171,7 @@ func (h *LibClient) LastReleaseStatus(releaseName string) (string /*revision*/, if errors.Is(err, driver.ErrReleaseNotFound) || strings.HasPrefix(err.Error(), "no revision for release") { return "0", "", fmt.Errorf("release '%s' not found\n", releaseName) } + return "", "", err } @@ -183,9 +186,12 @@ func (h *LibClient) UpgradeRelease(releaseName, modulePath string, valuesPaths [ if err := actionConfigInit(h.Logger); err != nil { return err } + return h.upgradeRelease(releaseName, modulePath, valuesPaths, setValues, labels, namespace) } + h.Logger.Debug("helm release upgraded", slog.String(pkg.LogKeyVersion, releaseName)) + return nil } @@ -198,6 +204,7 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ if namespace != "" { upg.Namespace = namespace } + if h.hasLabelsToApply() { upg.PostRenderer = post_renderer.NewPostRenderer(h.labels) } @@ -221,12 +228,14 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ if len(setValues) > 0 { m := make(map[string]interface{}) + for _, sv := range setValues { arr := strings.Split(sv, "=") if len(arr) == 2 { m[arr[0]] = arr[1] } } + resultValues = chartutil.CoalesceTables(resultValues, m) } @@ -239,6 +248,7 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ slog.String(pkg.LogKeyRelease, releaseName), slog.String(pkg.LogKeyChart, modulePath), slog.String(pkg.LogKeyNamespace, namespace)) + histClient := action.NewHistory(actionConfig) // Max is not working!!! Sort the final of releases by your own // histClient.Max = 1 @@ -248,6 +258,7 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ if namespace != "" { instClient.Namespace = namespace } + if h.hasLabelsToApply() { instClient.PostRenderer = post_renderer.NewPostRenderer(h.labels) } @@ -259,9 +270,12 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ instClient.Labels = labels _, err = instClient.Run(loaded, resultValues) + return err } + h.Logger.Debug("old releases found", slog.Int(pkg.LogKeyCount, len(releases))) + if len(releases) > 0 { // https://github.com/fluxcd/helm-controller/issues/149 // looking through this issue you can find the common error: another operation (install/upgrade/rollback) is in progress @@ -273,8 +287,10 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ slog.String(pkg.LogKeyRelease, nsReleaseName), slog.Int(pkg.LogKeyVersion, latestRelease.Version), slog.String(pkg.LogKeyStatus, string(latestRelease.Info.Status))) + if latestRelease.Info.Status.IsPending() { objectName := fmt.Sprintf("%s.%s.v%d", storage.HelmStorageType, latestRelease.Name, latestRelease.Version) + kubeClient, err := actionConfig.KubernetesClientSet() if err != nil { return fmt.Errorf("couldn't get kubernetes client set: %w", err) @@ -287,10 +303,12 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ slog.String(pkg.LogKeyRelease, nsReleaseName), slog.String(pkg.LogKeyStatus, string(latestRelease.Info.Status)), slog.String(pkg.LogKeyDriver, driver.ConfigMapsDriverName)) + err := kubeClient.CoreV1().ConfigMaps(latestRelease.Namespace).Delete(context.TODO(), objectName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("couldn't delete configmap %s of release %s: %w", objectName, nsReleaseName, err) } + h.Logger.Debug("ConfigMap was deleted", slog.String(pkg.LogKeyName, objectName)) case driver.SecretsDriverName: @@ -299,10 +317,12 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ slog.String(pkg.LogKeyRelease, nsReleaseName), slog.String(pkg.LogKeyStatus, string(latestRelease.Info.Status)), slog.String(pkg.LogKeyDriver, driver.ConfigMapsDriverName)) + err := kubeClient.CoreV1().Secrets(latestRelease.Namespace).Delete(context.TODO(), objectName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("couldn't delete secret %s of release %s: %w", objectName, nsReleaseName, err) } + h.Logger.Debug("Secret was deleted", slog.String(pkg.LogKeyName, objectName)) default: @@ -321,6 +341,7 @@ func (h *LibClient) upgradeRelease(releaseName, modulePath string, valuesPaths [ if err != nil { return fmt.Errorf("helm upgrade failed: %s\n", err) } + h.Logger.Info("Helm upgrade successful", slog.String(pkg.LogKeyRelease, releaseName), slog.String(pkg.LogKeyChart, modulePath), @@ -338,29 +359,35 @@ func (h *LibClient) rollbackLatestRelease(releases []*release.Release) { if latestRelease.Version == 1 || options.HistoryMax == 1 || len(releases) == 1 { rb := action.NewUninstall(actionConfig) rb.KeepHistory = false + _, err := rb.Run(latestRelease.Name) if err != nil { h.Logger.Warn("Failed to uninstall pending release", slog.String(pkg.LogKeyRelease, nsReleaseName), log.Err(err)) + return } } else { previousVersion := latestRelease.Version - 1 + for i := 1; i < len(releases); i++ { if !releases[i].Info.Status.IsPending() { previousVersion = releases[i].Version break } } + rb := action.NewRollback(actionConfig) rb.Version = previousVersion rb.CleanupOnFail = true + err := rb.Run(latestRelease.Name) if err != nil { h.Logger.Warn("Failed to rollback pending release", slog.String(pkg.LogKeyRelease, nsReleaseName), log.Err(err)) + return } } @@ -377,6 +404,7 @@ var ErrLabelIsNotFound = errors.New("label is not found") func (h *LibClient) GetReleaseLabels(releaseName, labelName string) (string, error) { gv := action.NewGet(actionConfig) + rel, err := gv.Run(releaseName) if err != nil { return "", fmt.Errorf("helm get failed: %w", err) @@ -392,10 +420,12 @@ func (h *LibClient) GetReleaseLabels(releaseName, labelName string) (string, err // Deprecated: use GetReleaseLabels instead func (h *LibClient) GetReleaseChecksum(releaseName string) (string, error) { gv := action.NewGet(actionConfig) + rel, err := gv.Run(releaseName) if err != nil { return "", fmt.Errorf("helm get failed: %s", err) } + if checksum, ok := rel.Labels["moduleChecksum"]; ok { return checksum, nil } @@ -405,6 +435,7 @@ func (h *LibClient) GetReleaseChecksum(releaseName string) (string, error) { if err != nil { return "", fmt.Errorf("helm get failed: %s", err) } + if recordedChecksum, hasKey := releaseValues["_addonOperatorModuleChecksum"]; hasKey { if recordedChecksumStr, ok := recordedChecksum.(string); ok { return recordedChecksumStr, nil @@ -418,12 +449,14 @@ func (h *LibClient) DeleteRelease(releaseName string) error { h.Logger.Debug("helm release: execute helm uninstall", slog.String(pkg.LogKeyRelease, releaseName)) un := action.NewUninstall(actionConfig) + _, err := un.Run(releaseName) if err != nil { return fmt.Errorf("helm uninstall %s invocation error: %v\n", releaseName, err) } h.Logger.Debug("helm release deleted", slog.String(pkg.LogKeyRelease, releaseName)) + return nil } @@ -432,9 +465,11 @@ func (h *LibClient) IsReleaseExists(releaseName string) (bool, error) { if err == nil { return true, nil } + if revision == "0" { return false, nil } + return false, err } @@ -443,6 +478,7 @@ func (h *LibClient) ListReleasesNames() ([]string, error) { l := action.NewList(actionConfig) // list all releases regardless of their state l.StateMask = action.ListAll + list, err := l.Run() if err != nil { return nil, fmt.Errorf("helm list failed: %s", err) @@ -459,6 +495,7 @@ func (h *LibClient) ListReleasesNames() ([]string, error) { } sort.Strings(releases) + return releases, nil } @@ -476,12 +513,14 @@ func (h *LibClient) Render(releaseName, modulePath string, valuesPaths, setValue if len(setValues) > 0 { m := make(map[string]interface{}) + for _, sv := range setValues { arr := strings.Split(sv, "=") if len(arr) == 2 { m[arr[0]] = arr[1] } } + resultValues = chartutil.CoalesceTables(resultValues, m) } @@ -510,6 +549,7 @@ func (h *LibClient) Render(releaseName, modulePath string, valuesPaths, setValue if !debug { return "", fmt.Errorf("%w\n\nUse --debug flag to render out invalid YAML", err) } + if rs == nil { return "", err } @@ -533,6 +573,7 @@ func (h *LibClient) newDryRunInstAction(namespace, releaseName string) *action.I if h.hasLabelsToApply() { inst.PostRenderer = post_renderer.NewPostRenderer(h.labels) } + inst.ReleaseName = releaseName inst.UseReleaseName = true inst.Replace = true // Skip the name check @@ -547,6 +588,7 @@ func (h *LibClient) ListReleases() ([]*release.Release, error) { l := action.NewList(actionConfig) // list all releases regardless of their state l.StateMask = action.ListAll + list, err := l.Run() if err != nil { return nil, fmt.Errorf("helm list failed: %w", err) diff --git a/pkg/helm/nelm/nelm.go b/pkg/helm/nelm/nelm.go index a74972118..f7bc9851a 100644 --- a/pkg/helm/nelm/nelm.go +++ b/pkg/helm/nelm/nelm.go @@ -94,6 +94,7 @@ func (s *SafeNelmActions) ReleaseGet(ctx context.Context, name, namespace string err = fmt.Errorf("panic in ReleaseGet: %v", r) } }() + return s.wrapped.ReleaseGet(ctx, name, namespace, opts) } @@ -115,6 +116,7 @@ func (s *SafeNelmActions) ReleaseInstall(ctx context.Context, name, namespace st err = fmt.Errorf("panic in ReleaseInstall: %v", r) } }() + return s.wrapped.ReleaseInstall(ctx, name, namespace, opts) } @@ -132,6 +134,7 @@ func (s *SafeNelmActions) ReleaseUninstall(ctx context.Context, name, namespace err = fmt.Errorf("panic in ReleaseUninstall: %v", r) } }() + return s.wrapped.ReleaseUninstall(ctx, name, namespace, opts) } @@ -147,6 +150,7 @@ func (s *SafeNelmActions) ReleaseList(ctx context.Context, opts action.ReleaseLi err = fmt.Errorf("panic in ReleaseList: %v", r) } }() + return s.wrapped.ReleaseList(ctx, opts) } @@ -166,6 +170,7 @@ func (s *SafeNelmActions) ChartRender(ctx context.Context, opts action.ChartRend err = fmt.Errorf("panic in ChartRender: %v", r) } }() + return s.wrapped.ChartRender(ctx, opts) } @@ -260,6 +265,7 @@ func (c *NelmClient) WithExtraLabels(labels map[string]string) { if c.labels == nil { c.labels = make(map[string]string) } + maps.Copy(c.labels, labels) } } @@ -269,6 +275,7 @@ func (c *NelmClient) WithExtraAnnotations(annotations map[string]string) { if c.annotations == nil { c.annotations = make(map[string]string) } + maps.Copy(c.annotations, annotations) } } @@ -389,6 +396,7 @@ func (c *NelmClient) GetReleaseValues(releaseName string) (utils.Values, error) if err != nil { return nil, fmt.Errorf("get nelm release %q: %w", releaseName, err) } + if releaseGetResult == nil || releaseGetResult.Values == nil { return nil, fmt.Errorf("no values found for release %q", releaseName) } @@ -469,9 +477,11 @@ func (c *NelmClient) IsReleaseExists(releaseName string) (bool, error) { if err == nil { return true, nil } + if revision == "0" { return false, nil } + return false, err } @@ -489,11 +499,13 @@ func (c *NelmClient) ListReleasesNames() ([]string, error) { } releaseNames := make([]string, 0) + for _, release := range releaseListResult.Releases { chartName := "unknown" if release.Chart != nil { chartName = release.Chart.Name } + if release.Name == "" { c.logger.Warn("release name is empty, skipped", slog.String(pkg.LogKeyChart, chartName)) continue @@ -549,20 +561,24 @@ func (c *NelmClient) Render(releaseName, modulePath string, valuesPaths, setValu if !debug { return "", fmt.Errorf("render nelm chart %q: %w\n\nUse --debug flag to render out invalid YAML", modulePath, err) } + return "", fmt.Errorf("render nelm chart %q: %w", modulePath, err) } c.logger.Info("Render nelm templates for chart was successful", slog.String(pkg.LogKeyChart, modulePath)) var result strings.Builder + for _, resource := range chartRenderResult.Resources { b, err := yaml.Marshal(resource.Unstruct) if err != nil { return "", fmt.Errorf("marshal resource: %w", err) } + if result.Len() > 0 { result.WriteString("---\n") } + result.Write(b) } diff --git a/pkg/helm/test/mock/mock.go b/pkg/helm/test/mock/mock.go index 10d150a0a..68cc7b858 100644 --- a/pkg/helm/test/mock/mock.go +++ b/pkg/helm/test/mock/mock.go @@ -29,6 +29,7 @@ func (c *Client) ListReleasesNames() ([]string, error) { if c.ReleaseNames != nil { return c.ReleaseNames, nil } + return []string{}, nil } diff --git a/pkg/helm_resources_manager/helm_resources_manager.go b/pkg/helm_resources_manager/helm_resources_manager.go index 53d846fbe..7523e8b23 100644 --- a/pkg/helm_resources_manager/helm_resources_manager.go +++ b/pkg/helm_resources_manager/helm_resources_manager.go @@ -63,11 +63,13 @@ func NewHelmResourcesManager(ctx context.Context, kclient *klient.Client, logger } cfg := kclient.RestConfig() + defaultLabelSelector, err := labels.Parse(app.ExtraLabels) if err != nil { cancel() return nil, err } + cache, err := cr_cache.New(cfg, cr_cache.Options{ DefaultLabelSelector: defaultLabelSelector, }) @@ -81,10 +83,12 @@ func NewHelmResourcesManager(ctx context.Context, kclient *klient.Client, logger }() log.Debug("Helm resource manager: cache's been started") + if synced := cache.WaitForCacheSync(cctx); !synced { cancel() return nil, fmt.Errorf("Couldn't sync helm resource informer cache") } + log.Debug("Helm resourcer manager: cache has been synced") return &helmResourcesManager{ @@ -140,12 +144,14 @@ func (hm *helmResourcesManager) StartMonitor(moduleName string, manifests []mani func (hm *helmResourcesManager) absentResourcesCallback(moduleName string, unexpectedStatus bool, absent []manifest.Manifest, defaultNs string) { log.Debug("Detect absent resources for module", slog.String(pkg.LogKeyModule, moduleName)) + for _, m := range absent { log.Debug("absent module", slog.String(pkg.LogKeyNamespace, m.Namespace(defaultNs)), slog.String(pkg.LogKeyKind, m.Kind()), slog.String(pkg.LogKeyModule, m.Name())) } + hm.eventCh <- ReleaseStatusEvent{ ModuleName: moduleName, Absent: absent, @@ -164,17 +170,21 @@ func (hm *helmResourcesManager) StopMonitors() { func (hm *helmResourcesManager) PauseMonitors() { hm.l.RLock() + for _, monitor := range hm.monitors { monitor.Pause() } + hm.l.RUnlock() } func (hm *helmResourcesManager) ResumeMonitors() { hm.l.RLock() + for _, monitor := range hm.monitors { monitor.Resume() } + hm.l.RUnlock() } @@ -189,17 +199,21 @@ func (hm *helmResourcesManager) StopMonitor(moduleName string) { func (hm *helmResourcesManager) PauseMonitor(moduleName string) { hm.l.RLock() + if monitor, ok := hm.monitors[moduleName]; ok { monitor.Pause() } + hm.l.RUnlock() } func (hm *helmResourcesManager) ResumeMonitor(moduleName string) { hm.l.RLock() + if monitor, ok := hm.monitors[moduleName]; ok { monitor.Resume() } + hm.l.RUnlock() } @@ -207,21 +221,25 @@ func (hm *helmResourcesManager) HasMonitor(moduleName string) bool { hm.l.RLock() _, ok := hm.monitors[moduleName] hm.l.RUnlock() + return ok } func (hm *helmResourcesManager) AbsentResources(moduleName string) ([]manifest.Manifest, error) { hm.l.RLock() defer hm.l.RUnlock() + if monitor, ok := hm.monitors[moduleName]; ok { return monitor.AbsentResources() } + return nil, nil } func (hm *helmResourcesManager) GetMonitor(moduleName string) *ResourcesMonitor { hm.l.RLock() defer hm.l.RUnlock() + return hm.monitors[moduleName] } diff --git a/pkg/helm_resources_manager/resources_monitor.go b/pkg/helm_resources_manager/resources_monitor.go index 696913ba3..3d85488ce 100644 --- a/pkg/helm_resources_manager/resources_monitor.go +++ b/pkg/helm_resources_manager/resources_monitor.go @@ -110,6 +110,7 @@ func (r *ResourcesMonitor) Start() { r.logger.Debug("Helm release is in unexpected status", slog.String(pkg.LogKeyModule, r.moduleName), slog.String(pkg.LogKeyStatus, status)) + if r.absentCb != nil { r.absentCb(r.moduleName, true, []manifest.Manifest{}, r.defaultNamespace) } @@ -123,6 +124,7 @@ func (r *ResourcesMonitor) Start() { if len(absent) > 0 { r.logger.Debug("Absent resources detected") + if r.absentCb != nil { r.absentCb(r.moduleName, false, absent, r.defaultNamespace) } @@ -144,10 +146,12 @@ func (r *ResourcesMonitor) GetHelmReleaseStatus(moduleName string) (string, erro if err != nil { return "", err } + r.logger.Debug("Helm release", slog.String(pkg.LogKeyModule, moduleName), slog.String(pkg.LogKeyRevision, revision), slog.String(pkg.LogKeyStatus, status)) + return status, nil } @@ -181,8 +185,10 @@ func (r *ResourcesMonitor) AbsentResources() ([]manifest.Manifest, error) { for nsgvk, manifests := range gvkMap { wg.Add(1) + go r.checkGVKManifests(ctx, &wg, nsgvk, manifests, resC, concurrency) } + go func() { wg.Wait() close(resC) @@ -226,10 +232,12 @@ func (r *ResourcesMonitor) buildGVKMap() (map[namespacedGVK][]manifest.Manifest, Version: apiRes.Version, Kind: apiRes.Kind, } + ns := m.Namespace(r.defaultNamespace) if !apiRes.Namespaced { ns = "" } + nsgvk := namespacedGVK{ Namespace: ns, GVK: gvk, @@ -251,6 +259,7 @@ func (r *ResourcesMonitor) checkGVKManifests(ctx context.Context, wg *sync.WaitG defer wg.Done() concurrency <- struct{}{} + defer func() { <-concurrency }() @@ -265,6 +274,7 @@ func (r *ResourcesMonitor) checkGVKManifests(ctx context.Context, wg *sync.WaitG resC <- manifestResult{ err: err, } + return } @@ -274,6 +284,7 @@ func (r *ResourcesMonitor) checkGVKManifests(ctx context.Context, wg *sync.WaitG hasAbsent: true, manifest: mf, } + return } } @@ -289,6 +300,7 @@ func (r *ResourcesMonitor) listResources(ctx context.Context, nsgvk namespacedGV }) log.Debug("List objects from cache", slog.String(pkg.LogKeyNsgvk, fmt.Sprintf("%v", nsgvk))) + err := r.cache.List(ctx, objList, cr_client.InNamespace(nsgvk.Namespace)) if err != nil { return nil, fmt.Errorf("couldn't list objects from cache: %v", err) diff --git a/pkg/kube_config_manager/backend/configmap/configmap.go b/pkg/kube_config_manager/backend/configmap/configmap.go index 60504db14..9396e2a0e 100644 --- a/pkg/kube_config_manager/backend/configmap/configmap.go +++ b/pkg/kube_config_manager/backend/configmap/configmap.go @@ -60,6 +60,7 @@ func (b Backend) LoadConfig(ctx context.Context, _ ...string) (*config.KubeConfi if obj == nil { b.logger.Info("Initial config from ConfigMap: resource is not found", slog.String(pkg.LogKeyName, b.name)) + return nil, nil } @@ -130,6 +131,7 @@ func (b Backend) getConfigMap(ctx context.Context) (*v1.ConfigMap, error) { if errors.IsNotFound(err) { return nil, nil } + if err != nil { return nil, err } @@ -139,6 +141,7 @@ func (b Backend) getConfigMap(ctx context.Context) (*v1.ConfigMap, error) { func parseConfigMapData(data map[string]string) (*config.KubeConfig, error) { var err error + cfg := config.NewConfig() // Parse values in global section. @@ -201,6 +204,7 @@ func getModulesNamesFromConfigData(configData map[string]string) (map[string]boo if utils.ModuleNameToValuesKey(modName) != key { return nil, fmt.Errorf("bad module name '%s': should be camelCased", key) } + res[modName] = true } @@ -300,6 +304,7 @@ func (b Backend) mergeValues(ctx context.Context, moduleName string, values util } obj.Data[moduleName] = string(data) + return nil }) } @@ -313,6 +318,7 @@ func (b Backend) updateConfigMap(ctx context.Context, transformFn func(*v1.Confi } isUpdate := true + if obj == nil { obj = &v1.ConfigMap{} obj.Name = b.name @@ -333,6 +339,7 @@ func (b Backend) updateConfigMap(ctx context.Context, transformFn func(*v1.Confi } else { _, err = b.client.CoreV1().ConfigMaps(b.namespace).Create(ctx, obj, metav1.CreateOptions{}) } + return err } @@ -352,6 +359,7 @@ func (b Backend) StartInformer(ctx context.Context, eventC chan config.Event) { _, _ = cmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { b.logConfigMapEvent(ctx, obj, "add") + err := b.handleConfigMapEvent(obj.(*v1.ConfigMap), eventC) if err != nil { b.logger.Error("Handle ConfigMap 'add' error", @@ -361,6 +369,7 @@ func (b Backend) StartInformer(ctx context.Context, eventC chan config.Event) { }, UpdateFunc: func(_ interface{}, obj interface{}) { b.logConfigMapEvent(ctx, obj, "update") + err := b.handleConfigMapEvent(obj.(*v1.ConfigMap), eventC) if err != nil { b.logger.Error("Handle ConfigMap 'update' error", @@ -390,8 +399,10 @@ func (b Backend) logConfigMapEvent(ctx context.Context, obj interface{}, eventNa slog.String(pkg.LogKeyConfigMapName, b.name), slog.String(pkg.LogKeyEventName, eventName), log.Err(err)) + return } + b.logger.Info("Dump ConfigMap", slog.String(pkg.LogKeyConfigMapName, b.name), slog.String(pkg.LogKeyEventName, eventName), @@ -412,6 +423,7 @@ func (b Backend) handleConfigMapEvent(obj *v1.ConfigMap, eventC chan config.Even b.logger.Error("ConfigMap invalid", slog.String(pkg.LogKeyConfigMapName, b.name), log.Err(err)) + return err } diff --git a/pkg/kube_config_manager/checksums.go b/pkg/kube_config_manager/checksums.go index de0974f78..ae2880357 100644 --- a/pkg/kube_config_manager/checksums.go +++ b/pkg/kube_config_manager/checksums.go @@ -15,6 +15,7 @@ func (c *Checksums) Add(name string, checksum string) { if !c.HasChecksum(name) { c.RemoveAll(name) } + c.sums[name][checksum] = struct{}{} } @@ -52,6 +53,7 @@ func (c *Checksums) HasEqualChecksum(name string, checksum string) bool { return true } } + return false } @@ -60,6 +62,7 @@ func (c *Checksums) Names() map[string]struct{} { for name := range c.sums { names[name] = struct{}{} } + return names } @@ -67,5 +70,6 @@ func (c *Checksums) Dump(moduleName string) map[string]struct{} { if checksums, has := c.sums[moduleName]; has { return checksums } + return nil } diff --git a/pkg/kube_config_manager/context/context.go b/pkg/kube_config_manager/context/context.go index 5106b3b30..81a0111bf 100644 --- a/pkg/kube_config_manager/context/context.go +++ b/pkg/kube_config_manager/context/context.go @@ -18,6 +18,8 @@ func IsKubeConfigManagerDebug(ctx context.Context) bool { if val == nil { return false } + debug, ok := val.(bool) + return ok && debug } diff --git a/pkg/kube_config_manager/kube_config_manager.go b/pkg/kube_config_manager/kube_config_manager.go index f03de7e03..6797dd852 100644 --- a/pkg/kube_config_manager/kube_config_manager.go +++ b/pkg/kube_config_manager/kube_config_manager.go @@ -156,6 +156,7 @@ func (kcm *KubeConfigManager) loadConfig() error { } kcm.currentConfig = newConfig + return nil } @@ -165,6 +166,7 @@ func (kcm *KubeConfigManager) currentModuleNames() map[string]struct{} { for name := range kcm.currentConfig.Modules { names[name] = struct{}{} } + return names } @@ -177,7 +179,9 @@ func (kcm *KubeConfigManager) isGlobalChanged(newConfig *config.KubeConfig) bool kcm.logger.Info("Global section deleted") return true } + kcm.logger.Debug("Global section is empty") + return false } @@ -187,6 +191,7 @@ func (kcm *KubeConfigManager) isGlobalChanged(newConfig *config.KubeConfig) bool // Remove known checksum, do not fire event on self-update. kcm.knownChecksums.Remove(utils.GlobalValuesKey, newChecksum) kcm.logger.Debug("Global section self-update") + return false } @@ -212,9 +217,11 @@ func (kcm *KubeConfigManager) handleConfigEvent(obj config.Event) { kcm.configEventCh <- config.KubeConfigEvent{ Type: config.KubeConfigInvalid, } + kcm.logger.Error("Config invalid", slog.String(pkg.LogKeyName, obj.Key), log.Err(obj.Err)) + return } @@ -224,18 +231,19 @@ func (kcm *KubeConfigManager) handleConfigEvent(obj config.Event) { kcm.m.Lock() kcm.currentConfig = config.NewConfig() kcm.m.Unlock() + kcm.configEventCh <- config.KubeConfigEvent{ Type: config.KubeConfigChanged, } case utils.GlobalValuesKey: // global values - kcm.m.Lock() globalChanged := kcm.isGlobalChanged(obj.Config) // Update state after successful parsing. kcm.currentConfig.Global = obj.Config.Global kcm.m.Unlock() + if globalChanged { kcm.configEventCh <- config.KubeConfigEvent{ Type: config.KubeConfigChanged, @@ -267,6 +275,7 @@ func (kcm *KubeConfigManager) handleDeleteEvent(moduleName string, cfg *config.M moduleMaintenanceChanged := make(map[string]utils.Maintenance) kcm.logger.Info("module section deleted", slog.String(pkg.LogKeyName, moduleName)) + modulesChanged = append(modulesChanged, moduleName) if kcm.currentConfig.Modules[moduleName].GetEnabled() != "" && kcm.currentConfig.Modules[moduleName].GetEnabled() != "n/d" { modulesStateChanged = append(modulesStateChanged, moduleName) @@ -289,8 +298,11 @@ func (kcm *KubeConfigManager) handleDeleteEvent(moduleName string, cfg *config.M } func (kcm *KubeConfigManager) handleUpdateEvent(moduleName string, cfg *config.ModuleKubeConfig) { - var modulesChanged []string - var modulesStateChanged []string + var ( + modulesChanged []string + modulesStateChanged []string + ) + moduleMaintenanceChanged := make(map[string]utils.Maintenance) var changed bool @@ -298,11 +310,13 @@ func (kcm *KubeConfigManager) handleUpdateEvent(moduleName string, cfg *config.M if currentCfg, has := kcm.currentConfig.Modules[moduleName]; has { if currentCfg.Checksum != cfg.Checksum { changed = true + modulesChanged = append(modulesChanged, moduleName) } if currentCfg.GetEnabled() != cfg.GetEnabled() { changed = true + modulesStateChanged = append(modulesStateChanged, moduleName) } @@ -340,7 +354,9 @@ func (kcm *KubeConfigManager) handleBatchConfigEvent(obj config.Event) { kcm.configEventCh <- config.KubeConfigEvent{ Type: config.KubeConfigInvalid, } + kcm.logger.Error("Batch Config invalid", log.Err(obj.Err)) + return } @@ -349,6 +365,7 @@ func (kcm *KubeConfigManager) handleBatchConfigEvent(obj config.Event) { kcm.m.Lock() kcm.currentConfig = config.NewConfig() kcm.m.Unlock() + kcm.configEventCh <- config.KubeConfigEvent{ Type: config.KubeConfigChanged, } @@ -421,6 +438,7 @@ func (kcm *KubeConfigManager) handleBatchConfigEvent(obj config.Event) { modulesStateChanged = append(modulesStateChanged, moduleName) } } + kcm.logger.Info("Module sections deleted", slog.String(pkg.LogKeyModules, fmt.Sprintf("%+v", currentModuleNames))) } @@ -479,6 +497,7 @@ func (kcm *KubeConfigManager) SafeReadConfig(handler func(config *config.KubeCon if handler == nil { return } + kcm.withLock(func() { handler(kcm.currentConfig) }) @@ -488,6 +507,7 @@ func (kcm *KubeConfigManager) withLock(fn func()) { if fn == nil { return } + kcm.m.Lock() fn() kcm.m.Unlock() diff --git a/pkg/module_manager/environment_manager/evironment_manager.go b/pkg/module_manager/environment_manager/evironment_manager.go index 2e758454c..e0e075141 100644 --- a/pkg/module_manager/environment_manager/evironment_manager.go +++ b/pkg/module_manager/environment_manager/evironment_manager.go @@ -201,6 +201,7 @@ func (m *Manager) AssembleEnvironmentForModule(moduleName, modulePath string, ta if errors.Is(err, os.ErrExist) { continue } + return fmt.Errorf("create null file: %w", err) } diff --git a/pkg/module_manager/go_hook/go_hook.go b/pkg/module_manager/go_hook/go_hook.go index bae349655..e04405e0a 100644 --- a/pkg/module_manager/go_hook/go_hook.go +++ b/pkg/module_manager/go_hook/go_hook.go @@ -141,5 +141,6 @@ func BoolDeref(ptr *bool, def bool) bool { if ptr != nil { return *ptr } + return def } diff --git a/pkg/module_manager/go_hook/metrics/collector.go b/pkg/module_manager/go_hook/metrics/collector.go index bc295e899..76c4a4610 100644 --- a/pkg/module_manager/go_hook/metrics/collector.go +++ b/pkg/module_manager/go_hook/metrics/collector.go @@ -65,6 +65,7 @@ func (dms *MemoryMetricsCollector) Expire(group string) { if group == "" { group = dms.defaultGroup } + dms.metrics = append(dms.metrics, operation.MetricOperation{ Group: group, Action: operation.ActionExpireMetrics, diff --git a/pkg/module_manager/loader/fs/fs.go b/pkg/module_manager/loader/fs/fs.go index e04cd9be4..816121530 100644 --- a/pkg/module_manager/loader/fs/fs.go +++ b/pkg/module_manager/loader/fs/fs.go @@ -129,6 +129,7 @@ func (fl *FileSystemLoader) LoadModules() ([]*modules.BasicModule, error) { if err != nil { return nil, err } + result = append(result, bm) } } @@ -148,6 +149,7 @@ func readDir(modulesDir string) ([]os.DirEntry, error) { if err != nil && os.IsNotExist(err) { return nil, fmt.Errorf("path '%s' does not exist", modulesDir) } + if err != nil { return nil, fmt.Errorf("listing modules directory '%s': %s", modulesDir, err) } @@ -162,6 +164,7 @@ func (fl *FileSystemLoader) findModulesInDir(modulesDir string) ([]moduleDefinit } mods := make([]moduleDefinition, 0) + for _, dirEntry := range dirEntries { name, absPath, err := resolveDirEntry(modulesDir, dirEntry) if err != nil { @@ -198,6 +201,7 @@ func resolveDirEntry(dirPath string, entry os.DirEntry) (string, string, error) if e.Err.Error() == "no such file or directory" { log.Warn("Symlink target does not exist. Ignoring module", slog.String(pkg.LogKeyDir, dirPath)) + return "", "", nil } } @@ -213,6 +217,7 @@ func resolveDirEntry(dirPath string, entry os.DirEntry) (string, string, error) log.Warn("Ignore dir while searching for modules", slog.String(pkg.LogKeyDir, absPath)) } + return "", "", nil } @@ -221,6 +226,7 @@ func resolveSymlinkToDir(dirPath string, entry os.DirEntry) (string, error) { if err != nil { return "", err } + targetDirPath, isTargetDir, err := utils.SymlinkInfo(filepath.Join(dirPath, info.Name()), info) if err != nil { return "", err @@ -273,6 +279,7 @@ func parseUintOrDefault(num string, defaultValue uint32) uint32 { if err != nil { return defaultValue } + return uint32(val) } diff --git a/pkg/module_manager/models/hooks/global_hook.go b/pkg/module_manager/models/hooks/global_hook.go index f55cb3127..06f1e2a3a 100644 --- a/pkg/module_manager/models/hooks/global_hook.go +++ b/pkg/module_manager/models/hooks/global_hook.go @@ -59,10 +59,13 @@ func (h *GlobalHook) GetConfigDescription() string { if h.config.BeforeAll != nil { msgs = append(msgs, fmt.Sprintf("beforeAll:%d", int64(h.config.BeforeAll.Order))) } + if h.config.AfterAll != nil { msgs = append(msgs, fmt.Sprintf("afterAll:%d", int64(h.config.AfterAll.Order))) } + msgs = append(msgs, h.executableHook.GetHookConfigDescription()) + return strings.Join(msgs, ", ") } @@ -78,6 +81,7 @@ func (h *GlobalHook) Order(binding shell_op_types.BindingType) float64 { return h.config.OnStartup.Order } } + return 0.0 } @@ -93,6 +97,7 @@ func (h *GlobalHook) SynchronizationNeeded() bool { return true } } + return false } @@ -103,5 +108,6 @@ func (h *GlobalHook) GetGoHookInputSettings() *gohook.HookConfigSettings { } gohook := h.executableHook.(*kind.GoHook) + return gohook.GetConfig().Settings } diff --git a/pkg/module_manager/models/hooks/global_hook_config.go b/pkg/module_manager/models/hooks/global_hook_config.go index c08a2442c..979479df9 100644 --- a/pkg/module_manager/models/hooks/global_hook_config.go +++ b/pkg/module_manager/models/hooks/global_hook_config.go @@ -53,12 +53,14 @@ func (c *GlobalHookConfig) HasBinding(binding BindingType) bool { if c.HookConfig.HasBinding(binding) { return true } + switch binding { case BeforeAll: return c.BeforeAll != nil case AfterAll: return c.AfterAll != nil } + return false } @@ -74,9 +76,11 @@ func (c *GlobalHookConfig) BindingsCount() int { if c.HasBinding(Schedule) { res += len(c.Schedules) } + if c.HasBinding(OnKubernetesEvent) { res += len(c.OnKubernetesEvents) } + return res } diff --git a/pkg/module_manager/models/hooks/kind/batch_hook.go b/pkg/module_manager/models/hooks/kind/batch_hook.go index f80561cc8..f8979170c 100644 --- a/pkg/module_manager/models/hooks/kind/batch_hook.go +++ b/pkg/module_manager/models/hooks/kind/batch_hook.go @@ -123,6 +123,7 @@ func (h *BatchHook) Execute(ctx context.Context, configVersion string, bContext } versionedContextList := bindingcontext.ConvertBindingContextList(configVersion, bContext) + bindingContextBytes, err := versionedContextList.Json() if err != nil { return nil, err @@ -138,6 +139,7 @@ func (h *BatchHook) Execute(ctx context.Context, configVersion string, bContext if shapp.DebugKeepTmpFiles { return } + for _, f := range tmpFiles { err := os.Remove(f) if err != nil { @@ -148,6 +150,7 @@ func (h *BatchHook) Execute(ctx context.Context, configVersion string, bContext } } }() + configValuesPatchPath := tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"] valuesPatchPath := tmpFiles["VALUES_JSON_PATCH_PATH"] metricsPath := tmpFiles["METRICS_PATH"] @@ -184,6 +187,7 @@ func (h *BatchHook) Execute(ctx context.Context, configVersion string, bContext WithChroot(utils.GetModuleChrootPath(h.moduleName)) usage, err := cmd.RunAndLogLines(ctx, logLabels) + result.Usage = usage if err != nil { return result, fmt.Errorf("run and log lines: %w", err) @@ -250,6 +254,7 @@ func GetBatchHookConfig(moduleName, hookPath string) (*BatchHookConfig, error) { hooks := make([]sdkhook.HookConfig, 0) buf := bytes.NewReader(o) + err = json.NewDecoder(buf).Decode(&hooks) if err != nil { return nil, fmt.Errorf("decode: %w", err) @@ -268,6 +273,7 @@ func GetBatchHookConfig(moduleName, hookPath string) (*BatchHookConfig, error) { cfgs := &sdkhook.BatchHookConfig{} buf := bytes.NewReader(o) + err = json.NewDecoder(buf).Decode(&cfgs) if err != nil { return nil, fmt.Errorf("decode: %w", err) @@ -278,6 +284,7 @@ func GetBatchHookConfig(moduleName, hookPath string) (*BatchHookConfig, error) { outputLog := &BatchHookLog{} buf := bytes.NewReader(o) + err = json.NewDecoder(buf).Decode(&cfgs) if err != nil { return nil, fmt.Errorf("decode: %w", err) @@ -402,6 +409,7 @@ func (h *BatchHook) GetAfterDeleteHelm() *float64 { // PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths func (h *BatchHook) prepareTmpFilesForHookRun(bindingContext []byte, moduleSafeName string, configValues, values utils.Values) (map[string]string, error) { var err error + tmpFiles := make(map[string]string) tmpFiles["CONFIG_VALUES_PATH"], err = h.prepareConfigValuesJsonFile(moduleSafeName, configValues) @@ -448,12 +456,14 @@ func (h *BatchHook) prepareMetricsFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } // BINDING_CONTEXT_PATH func (h *BatchHook) prepareBindingContextJsonFile(moduleSafeName string, bindingContext []byte) (string, error) { path := filepath.Join(h.TmpDir, fmt.Sprintf("%s.module-hook-%s-binding-context-%s.json", moduleSafeName, h.SafeName(), uuid.Must(uuid.NewV4()).String())) + err := utils.DumpData(path, bindingContext) if err != nil { return "", err @@ -468,6 +478,7 @@ func (h *BatchHook) prepareConfigValuesJsonPatchFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -477,6 +488,7 @@ func (h *BatchHook) prepareValuesJsonPatchFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -486,6 +498,7 @@ func (h *BatchHook) prepareKubernetesPatchFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -497,6 +510,7 @@ func (h *BatchHook) prepareConfigValuesJsonFile(moduleSafeName string, configVal } path := filepath.Join(h.TmpDir, fmt.Sprintf("%s.module-config-values-%s.json", moduleSafeName, uuid.Must(uuid.NewV4()).String())) + err = utils.DumpData(path, data) if err != nil { return "", err @@ -516,6 +530,7 @@ func (h *BatchHook) prepareValuesJsonFile(moduleSafeName string, values utils.Va } path := filepath.Join(h.TmpDir, fmt.Sprintf("%s.module-values-%s.json", moduleSafeName, uuid.Must(uuid.NewV4()).String())) + err = utils.DumpData(path, data) if err != nil { return "", err diff --git a/pkg/module_manager/models/hooks/kind/gohook.go b/pkg/module_manager/models/hooks/kind/gohook.go index 36f1843c5..c7a0e6ea7 100644 --- a/pkg/module_manager/models/hooks/kind/gohook.go +++ b/pkg/module_manager/models/hooks/kind/gohook.go @@ -289,10 +289,13 @@ func newHookConfigFromGoConfig(input *gohook.HookConfig) (config.HookConfig, err monitor.WithFieldSelector(kubeCfg.FieldSelector) monitor.WithNamespaceSelector(kubeCfg.NamespaceSelector) monitor.WithLabelSelector(kubeCfg.LabelSelector) + if kubeCfg.FilterFunc == nil { return config.HookConfig{}, errors.New(`"FilterFunc" in KubernetesConfig cannot be nil`) } + filterFunc := kubeCfg.FilterFunc + monitor.FilterFunc = func(obj *unstructured.Unstructured) (interface{}, error) { return filterFunc(obj) } @@ -305,15 +308,18 @@ func newHookConfigFromGoConfig(input *gohook.HookConfig) (config.HookConfig, err kubeConfig := htypes.OnKubernetesEventConfig{} kubeConfig.Monitor = monitor kubeConfig.AllowFailure = input.AllowFailure + if kubeCfg.Name == "" { return c, spew.Errorf(`"name" is a required field in binding: %v`, kubeCfg) } + kubeConfig.BindingName = kubeCfg.Name if input.Queue == "" { kubeConfig.Queue = defaultHookQueueName } else { kubeConfig.Queue = input.Queue } + kubeConfig.Group = defaultHookGroupName kubeConfig.ExecuteHookOnSynchronization = gohook.BoolDeref(kubeCfg.ExecuteHookOnSynchronization, true) @@ -328,6 +334,7 @@ func newHookConfigFromGoConfig(input *gohook.HookConfig) (config.HookConfig, err // schedule bindings with includeSnapshotsFrom // are depend on kubernetes bindings. c.Schedules = []htypes.ScheduleConfig{} + for _, inSch := range input.Schedule { res := htypes.ScheduleConfig{} @@ -353,6 +360,7 @@ func newHookConfigFromGoConfig(input *gohook.HookConfig) (config.HookConfig, err } else { res.Queue = input.Queue } + res.Group = "main" c.Schedules = append(c.Schedules, res) @@ -361,30 +369,40 @@ func newHookConfigFromGoConfig(input *gohook.HookConfig) (config.HookConfig, err // Update IncludeSnapshotsFrom for every binding with a group. // Merge binding's IncludeSnapshotsFrom with snapshots list calculated for group. groupSnapshots := make(map[string][]string) + for _, kubeCfg := range c.OnKubernetesEvents { if kubeCfg.Group == "" { continue } + if _, ok := groupSnapshots[kubeCfg.Group]; !ok { groupSnapshots[kubeCfg.Group] = make([]string, 0) } + groupSnapshots[kubeCfg.Group] = append(groupSnapshots[kubeCfg.Group], kubeCfg.BindingName) } + newKubeEvents := make([]htypes.OnKubernetesEventConfig, 0) + for _, cfg := range c.OnKubernetesEvents { if snapshots, ok := groupSnapshots[cfg.Group]; ok { cfg.IncludeSnapshotsFrom = config.MergeArrays(cfg.IncludeSnapshotsFrom, snapshots) } + newKubeEvents = append(newKubeEvents, cfg) } + c.OnKubernetesEvents = newKubeEvents newSchedules := make([]htypes.ScheduleConfig, 0) + for _, cfg := range c.Schedules { if snapshots, ok := groupSnapshots[cfg.Group]; ok { cfg.IncludeSnapshotsFrom = config.MergeArrays(cfg.IncludeSnapshotsFrom, snapshots) } + newSchedules = append(newSchedules, cfg) } + c.Schedules = newSchedules /*** END Copy Paste ***/ diff --git a/pkg/module_manager/models/hooks/kind/shellhook.go b/pkg/module_manager/models/hooks/kind/shellhook.go index 6a16e3dd6..fa02f2629 100644 --- a/pkg/module_manager/models/hooks/kind/shellhook.go +++ b/pkg/module_manager/models/hooks/kind/shellhook.go @@ -127,6 +127,7 @@ func (sh *ShellHook) Execute(ctx context.Context, configVersion string, bContext } versionedContextList := bindingcontext.ConvertBindingContextList(configVersion, bContext) + bindingContextBytes, err := versionedContextList.Json() if err != nil { return nil, err @@ -141,6 +142,7 @@ func (sh *ShellHook) Execute(ctx context.Context, configVersion string, bContext if shapp.DebugKeepTmpFiles { return } + for _, f := range tmpFiles { err := os.Remove(f) if err != nil { @@ -151,6 +153,7 @@ func (sh *ShellHook) Execute(ctx context.Context, configVersion string, bContext } } }() + configValuesPatchPath := tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"] valuesPatchPath := tmpFiles["VALUES_JSON_PATCH_PATH"] metricsPath := tmpFiles["METRICS_PATH"] @@ -160,6 +163,7 @@ func (sh *ShellHook) Execute(ctx context.Context, configVersion string, bContext for envName, filePath := range tmpFiles { envs = append(envs, fmt.Sprintf("%s=%s", envName, filePath)) } + command := sh.GetPath() args := make([]string, 0) @@ -178,6 +182,7 @@ func (sh *ShellHook) Execute(ctx context.Context, configVersion string, bContext WithChroot(utils.GetModuleChrootPath(sh.moduleName)) usage, err := cmd.RunAndLogLines(ctx, logLabels) + result.Usage = usage if err != nil { return result, err @@ -265,6 +270,7 @@ func (sh *ShellHook) getConfig() ([]byte, error) { slog.String(pkg.LogKeyHook, sh.Name), log.Err(err), slog.String(pkg.LogKeyOutput, string(output))) + return nil, err } @@ -288,12 +294,14 @@ func (sh *ShellHook) GetConfigForModule(moduleKind string) (*config.HookConfig, } vu := config.NewDefaultVersionedUntyped() + err = vu.Load(cfgData) if err != nil { return nil, err } var validationFunc func(version string) *spec.Schema + switch moduleKind { case "global": { @@ -320,6 +328,7 @@ func (sh *ShellHook) GetConfigForModule(moduleKind string) (*config.HookConfig, sh.Config = cfg sh.ScheduleConfig = &HookScheduleConfig{} + err = yaml.Unmarshal(cfgData, sh.ScheduleConfig) if err != nil { return nil, fmt.Errorf("unmarshal schedule yaml hook config: %s", err) @@ -421,6 +430,7 @@ func getGlobalHookConfigSchema(version string) *spec.Schema { example: 10 ` } + config.Schemas[globalHookVersion] = schema } @@ -459,6 +469,7 @@ func getModuleHookConfigSchema(version string) *spec.Schema { example: 10 ` } + config.Schemas[globalHookVersion] = schema } @@ -468,6 +479,7 @@ func getModuleHookConfigSchema(version string) *spec.Schema { // PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths func (sh *ShellHook) prepareTmpFilesForHookRun(bindingContext []byte, moduleSafeName string, configValues, values utils.Values) (map[string]string, error) { var err error + tmpFiles := make(map[string]string) tmpFiles["CONFIG_VALUES_PATH"], err = sh.prepareConfigValuesJsonFile(moduleSafeName, configValues) @@ -514,12 +526,14 @@ func (sh *ShellHook) prepareMetricsFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } // BINDING_CONTEXT_PATH func (sh *ShellHook) prepareBindingContextJsonFile(moduleSafeName string, bindingContext []byte) (string, error) { path := filepath.Join(sh.TmpDir, fmt.Sprintf("%s.module-hook-%s-binding-context-%s.json", moduleSafeName, sh.SafeName(), uuid.Must(uuid.NewV4()).String())) + err := utils.DumpData(path, bindingContext) if err != nil { return "", err @@ -534,6 +548,7 @@ func (sh *ShellHook) prepareConfigValuesJsonPatchFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -543,6 +558,7 @@ func (sh *ShellHook) prepareValuesJsonPatchFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -552,6 +568,7 @@ func (sh *ShellHook) prepareKubernetesPatchFile() (string, error) { if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -563,6 +580,7 @@ func (sh *ShellHook) prepareConfigValuesJsonFile(moduleSafeName string, configVa } path := filepath.Join(sh.TmpDir, fmt.Sprintf("%s.module-config-values-%s.json", moduleSafeName, uuid.Must(uuid.NewV4()).String())) + err = utils.DumpData(path, data) if err != nil { return "", err @@ -582,6 +600,7 @@ func (sh *ShellHook) prepareValuesJsonFile(moduleSafeName string, values utils.V } path := filepath.Join(sh.TmpDir, fmt.Sprintf("%s.module-values-%s.json", moduleSafeName, uuid.Must(uuid.NewV4()).String())) + err = utils.DumpData(path, data) if err != nil { return "", err diff --git a/pkg/module_manager/models/hooks/module_hook.go b/pkg/module_manager/models/hooks/module_hook.go index b5de31908..fe2c51c09 100644 --- a/pkg/module_manager/models/hooks/module_hook.go +++ b/pkg/module_manager/models/hooks/module_hook.go @@ -55,6 +55,7 @@ func (mh *ModuleHook) Order(binding shell_op_types.BindingType) float64 { return mh.config.AfterDeleteHelm.Order } } + return 0.0 } @@ -81,6 +82,7 @@ func (mh *ModuleHook) SynchronizationNeeded() bool { return true } } + return false } @@ -98,16 +100,19 @@ func (mh *ModuleHook) WithTmpDir(tmpDir string) { func (mh *ModuleHook) ApplyBindingActions(bindingActions []gohook.BindingAction) error { for _, action := range bindingActions { bindingIdx := -1 + for i, binding := range mh.config.OnKubernetesEvents { if binding.BindingName == action.Name { bindingIdx = i } } + if bindingIdx == -1 { continue } monitorCfg := mh.config.OnKubernetesEvents[bindingIdx].Monitor + switch strings.ToLower(action.Action) { case "disable": // Empty kind - "null" monitor. @@ -128,6 +133,7 @@ func (mh *ModuleHook) ApplyBindingActions(bindingActions []gohook.BindingAction) return err } } + return nil } @@ -138,12 +144,15 @@ func (mh *ModuleHook) GetConfigDescription() string { if mh.config.BeforeHelm != nil { bd = append(bd, fmt.Sprintf("beforeHelm:%d", int64(mh.config.BeforeHelm.Order))) } + if mh.config.AfterHelm != nil { bd = append(bd, fmt.Sprintf("afterHelm:%d", int64(mh.config.AfterHelm.Order))) } + if mh.config.AfterDeleteHelm != nil { bd = append(bd, fmt.Sprintf("afterDeleteHelm:%d", int64(mh.config.AfterDeleteHelm.Order))) } + bd = append(bd, mh.executableHook.GetHookConfigDescription()) return strings.Join(bd, ", ") @@ -156,5 +165,6 @@ func (mh *ModuleHook) GetGoHookInputSettings() *gohook.HookConfigSettings { } gohook := mh.executableHook.(*kind.GoHook) + return gohook.GetConfig().Settings } diff --git a/pkg/module_manager/models/hooks/module_hook_config.go b/pkg/module_manager/models/hooks/module_hook_config.go index bca7ddaf2..6d7f3f98a 100644 --- a/pkg/module_manager/models/hooks/module_hook_config.go +++ b/pkg/module_manager/models/hooks/module_hook_config.go @@ -85,9 +85,11 @@ func (c *ModuleHookConfig) BindingsCount() int { if c.HasBinding(Schedule) { res += len(c.Schedules) } + if c.HasBinding(OnKubernetesEvent) { res += len(c.OnKubernetesEvents) } + return res } diff --git a/pkg/module_manager/models/modules/basic.go b/pkg/module_manager/models/modules/basic.go index 8b99900e0..4064ff7d4 100644 --- a/pkg/module_manager/models/modules/basic.go +++ b/pkg/module_manager/models/modules/basic.go @@ -125,6 +125,7 @@ func (bm *BasicModule) SetCritical(value bool) { // if file name do not start with `_` or `doc-` prefix func getCRDsFromPath(path string, crdsFilters string) []string { var crdFilesPaths []string + err := filepath.Walk( filepath.Join(path, "crds"), func(path string, _ os.FileInfo, err error) error { @@ -161,7 +162,9 @@ func normalizeHookPath(modulePath, hookPath string) (string, error) { if hooksIdx == -1 { return filepath.Rel(modulePath, hookPath) } + relPath := hookPath[hooksIdx+1:] + return relPath, nil } @@ -218,6 +221,7 @@ func (bm *BasicModule) SetHooksControllersReady() { // ResetState drops the module state func (bm *BasicModule) ResetState() { bm.l.Lock() + var maintenanceState MaintenanceState if bm.state.maintenanceState == Unmanaged { @@ -252,6 +256,7 @@ func (bm *BasicModule) RegisterHooks(logger *log.Logger) ([]*hooks.ModuleHook, e } logger.Debug("Found hooks", slog.Int(pkg.LogKeyCount, len(searchModuleHooksResult.Hooks))) + if logger.GetLevel() == log.LevelDebug { for _, h := range searchModuleHooksResult.Hooks { logger.Debug("ModuleHook", @@ -358,8 +363,10 @@ func (bm *BasicModule) searchModuleShellHooks() ([]*kind.ShellHook, error) { } } }) + options = append(options, kind.WithPythonVenv(discoveredPythonVenvPath)) } + hookName, err := normalizeHookPath(filepath.Dir(bm.Path), hookPath) if err != nil { return nil, fmt.Errorf("could not get hook name: %w", err) @@ -446,7 +453,9 @@ func (bm *BasicModule) searchModuleBatchHooks() (*searchModuleBatchHooksResult, func RecursiveGetBatchHookExecutablePaths(moduleName, dir string, logger *log.Logger, excludedDirs ...string) ([]string, error) { paths := make([]string, 0) + excludedDirs = append(excludedDirs, "lib") + err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { if err != nil { return err @@ -581,6 +590,7 @@ func (bm *BasicModule) registerHooks(hks []*hooks.ModuleHook, logger *log.Logger func (bm *BasicModule) GetPhase() ModuleRunPhase { bm.l.RLock() defer bm.l.RUnlock() + return bm.state.Phase } @@ -646,6 +656,7 @@ func (bm *BasicModule) GetMaintenanceState() MaintenanceState { // sets KubernetesSnapshots and runs the hook. func (bm *BasicModule) RunHooksByBinding(ctx context.Context, binding sh_op_types.BindingType, logLabels map[string]string) error { var err error + moduleHooks := bm.GetHooks(binding) for _, moduleHook := range moduleHooks { @@ -666,6 +677,7 @@ func (bm *BasicModule) RunHooksByBinding(ctx context.Context, binding sh_op_type bc.Snapshots = moduleHook.GetHookController().KubernetesSnapshots() bc.Metadata.IncludeAllSnapshots = true } + bc.Metadata.BindingType = binding metricLabels := map[string]string{ @@ -680,8 +692,10 @@ func (bm *BasicModule) RunHooksByBinding(ctx context.Context, binding sh_op_type defer measure.Duration(func(d time.Duration) { bm.dc.MetricStorage.HistogramObserve(metrics.ModuleHookRunSeconds, d.Seconds(), metricLabels, nil) })() + err = bm.executeHook(ctx, moduleHook, binding, []bindingcontext.BindingContext{bc}, logLabels, metricLabels) }() + if err != nil { return err } @@ -729,17 +743,21 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec logEntry := utils.EnrichLoggerWithLabels(bm.logger, logLabels) enabledScriptPath := filepath.Join(bm.Path, "enabled") + configValuesPath, err := bm.prepareConfigValuesJsonFile(tmpDir) if err != nil { logEntry.Error("Prepare CONFIG_VALUES_PATH file", slog.String(pkg.LogKeyPath, enabledScriptPath), log.Err(err)) + return false, err } + defer func() { if bm.keepTemporaryHookFiles { return } + err := os.Remove(configValuesPath) if err != nil { bm.logger.With(pkg.LogKeyModule, bm.GetName()). @@ -754,12 +772,15 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec logEntry.Error("Prepare VALUES_PATH file", slog.String(pkg.LogKeyPath, enabledScriptPath), log.Err(err)) + return false, err } + defer func() { if bm.keepTemporaryHookFiles { return } + err := os.Remove(valuesPath) if err != nil { bm.logger.With(pkg.LogKeyModule, bm.GetName()). @@ -774,12 +795,15 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec logEntry.Error("Prepare MODULE_ENABLED_RESULT file", slog.String(pkg.LogKeyPath, enabledScriptPath), log.Err(err)) + return false, err } + defer func() { if bm.keepTemporaryHookFiles { return } + err := os.Remove(enabledResultFilePath) if err != nil { bm.logger.With(pkg.LogKeyModule, bm.GetName()). @@ -794,12 +818,15 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec logEntry.Error("Prepare MODULE_ENABLED_REASON file", slog.String(pkg.LogKeyPath, reasonFilePath), log.Err(err)) + return false, err } + defer func() { if bm.keepTemporaryHookFiles { return } + err := os.Remove(reasonFilePath) if err != nil { bm.logger.With(pkg.LogKeyModule, bm.GetName()). @@ -850,10 +877,12 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec bm.dc.MetricStorage.HistogramObserve(metrics.ModuleHookRunUserCPUSeconds, usage.User.Seconds(), metricLabels, nil) bm.dc.MetricStorage.GaugeSet(metrics.ModuleHookRunMaxRSSBytes, float64(usage.MaxRss)*1024, metricLabels) } + if err != nil { logEntry.Error("Fail to run enabled script", slog.String(pkg.LogKeyPath, enabledScriptPath), log.Err(err)) + return false, err } @@ -862,6 +891,7 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec logEntry.Error("Read enabled result", slog.String(pkg.LogKeyPath, enabledScriptPath), log.Err(err)) + return false, fmt.Errorf("bad enabled result") } @@ -869,9 +899,11 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec if moduleEnabled { result = "Enabled" } + logEntry.Info("Enabled script run successful", slog.Bool(pkg.LogKeyResult, moduleEnabled), slog.String(pkg.LogKeyStatus, result)) + var reason string if !moduleEnabled { reason, err = bm.readModuleEnabledReason(reasonFilePath) @@ -879,15 +911,19 @@ func (bm *BasicModule) RunEnabledScript(ctx context.Context, tmpDir string, prec logEntry.Error("Read enabled result", slog.String(pkg.LogKeyPath, enabledScriptPath), log.Err(err)) + return false, fmt.Errorf("bad enabled result") } } + bm.l.Lock() + bm.state.enabledScriptResult = &moduleEnabled if reason != "" { bm.state.enabledScriptReason = &reason } bm.l.Unlock() + return moduleEnabled, nil } @@ -901,6 +937,7 @@ func (bm *BasicModule) prepareModuleEnabledResultFile(tmpdir string) (string, er if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -909,6 +946,7 @@ func (bm *BasicModule) prepareModuleEnabledReasonFile(tmpdir string) (string, er if err := utils.CreateEmptyWritableFile(path); err != nil { return "", err } + return path, nil } @@ -935,6 +973,7 @@ func (bm *BasicModule) readModuleEnabledReason(path string) (string, error) { if err != nil { return "", fmt.Errorf("cannot read %s: %w", path, err) } + return strings.TrimSpace(string(data)), nil } @@ -946,6 +985,7 @@ func (bm *BasicModule) prepareValuesJsonFileWith(tmpdir string, values utils.Val } path := filepath.Join(tmpdir, fmt.Sprintf("%s.module-values-%s.json", bm.safeName(), uuid.Must(uuid.NewV4()).String())) + err = utils.DumpData(path, data) if err != nil { return "", err @@ -973,6 +1013,7 @@ func (bm *BasicModule) valuesForEnabledScript(precedingEnabledModules []string) }, }, ) + return res } @@ -993,6 +1034,7 @@ func (bm *BasicModule) prepareConfigValuesJsonFile(tmpDir string) (string, error } path := filepath.Join(tmpDir, fmt.Sprintf("%s.module-config-values-%s.json", bm.safeName(), uuid.Must(uuid.NewV4()).String())) + err = utils.DumpData(path, data) if err != nil { return "", err @@ -1031,6 +1073,7 @@ func (bm *BasicModule) executeHook(ctx context.Context, h *hooks.ModuleHook, bin if bindingType == sh_op_types.OnKubernetesEvent || bindingType == sh_op_types.Schedule { logStartLevel = log.LevelDebug } + logEntry.Log(ctx, logStartLevel.Level(), "Module hook start", slog.String(bm.GetName(), h.GetName())) for _, info := range h.GetHookController().SnapshotsInfo() { @@ -1125,6 +1168,7 @@ func (bm *BasicModule) executeHook(ctx context.Context, h *hooks.ModuleHook, bin logEntry.Debug("Module hook kube module config values stay unchanged", slog.String(pkg.LogKeyModule, h.GetName()), slog.String(pkg.LogKeyValues, bm.valuesStorage.GetConfigValues(false).DebugString())) + return fmt.Errorf("module hook '%s': set kube module config failed: %s", h.GetName(), err) } @@ -1144,6 +1188,7 @@ func (bm *BasicModule) executeHook(ctx context.Context, h *hooks.ModuleHook, bin if err != nil { return fmt.Errorf("module hook '%s': dynamic module values update error: %s", h.GetName(), err) } + if valuesPatchResult.ValuesChanged { logEntry.Debug("Module hook: validate module values before update", slog.String(pkg.LogKeyModule, h.GetName())) @@ -1159,6 +1204,7 @@ func (bm *BasicModule) executeHook(ctx context.Context, h *hooks.ModuleHook, bin // Save patch set if everything is ok. bm.valuesStorage.appendValuesPatch(valuesPatchResult.ValuesPatch) + err = bm.valuesStorage.CommitValues() if err != nil { return fmt.Errorf("error on commit values: %w", err) @@ -1251,6 +1297,7 @@ func (bm *BasicModule) InjectRegistryValue(registry *Registry) { func (bm *BasicModule) Synchronization() *SynchronizationState { bm.l.RLock() defer bm.l.RUnlock() + return bm.state.synchronizationState } @@ -1263,6 +1310,7 @@ func (bm *BasicModule) SynchronizationNeeded() bool { return true } } + return false } @@ -1298,6 +1346,7 @@ func (bm *BasicModule) GetHookErrorsSummary() string { } sort.Strings(hooksState) + return strings.Join(hooksState, "\n") } @@ -1305,6 +1354,7 @@ func (bm *BasicModule) GetHookErrorsSummary() string { func (bm *BasicModule) GetEnabledScriptResult() *bool { bm.l.RLock() defer bm.l.RUnlock() + return bm.state.enabledScriptResult } @@ -1312,6 +1362,7 @@ func (bm *BasicModule) GetEnabledScriptResult() *bool { func (bm *BasicModule) GetEnabledScriptReason() *string { bm.l.RLock() defer bm.l.RUnlock() + return bm.state.enabledScriptReason } @@ -1332,6 +1383,7 @@ func (bm *BasicModule) GetLastHookError() error { func (bm *BasicModule) GetModuleError() error { bm.l.RLock() defer bm.l.RUnlock() + return bm.state.lastModuleErr } diff --git a/pkg/module_manager/models/modules/global.go b/pkg/module_manager/models/modules/global.go index fb652bc42..a0805be07 100644 --- a/pkg/module_manager/models/modules/global.go +++ b/pkg/module_manager/models/modules/global.go @@ -108,10 +108,12 @@ func (gm *GlobalModule) GetHookByName(name string) *hooks.GlobalHook { func (gm *GlobalModule) GetHooks(bt ...sh_op_types.BindingType) []*hooks.GlobalHook { if len(bt) > 0 { t := bt[0] + res, ok := gm.byBinding[t] if !ok { return []*hooks.GlobalHook{} } + sort.Slice(res, func(i, j int) bool { return res[i].Order(t) < res[j].Order(t) }) @@ -255,6 +257,7 @@ func (gm *GlobalModule) executeHook(ctx context.Context, h *hooks.GlobalHook, bi logEntry.Debug("Global hook kube config global values stay unchanged", slog.String(pkg.LogKeyHook, h.GetName()), slog.String(pkg.LogKeyValue, gm.valuesStorage.GetConfigValues(false).DebugString())) + return fmt.Errorf("global hook '%s': set kube config failed: %s", h.GetName(), err) } @@ -285,12 +288,14 @@ func (gm *GlobalModule) executeHook(ctx context.Context, h *hooks.GlobalHook, bi if valuesPatchResult != nil && valuesPatchResult.ValuesChanged { logEntry.Debug("Global hook: validate global values before update", slog.String(pkg.LogKeyHook, h.GetName())) + validationErr := gm.valuesStorage.validateValues(valuesPatchResult.Values) if validationErr != nil { return fmt.Errorf("cannot apply values patch for global values: %w", validationErr) } gm.valuesStorage.appendValuesPatch(valuesPatchResult.ValuesPatch) + err = gm.valuesStorage.CommitValues() if err != nil { return fmt.Errorf("error on commit values: %w", err) @@ -450,6 +455,7 @@ func (gm *GlobalModule) searchAndRegisterHooks() ([]*hooks.GlobalHook, error) { } gm.logger.Debug("Found global hooks", slog.Int(pkg.LogKeyCount, len(hks))) + if gm.logger.GetLevel() == log.LevelDebug { for _, h := range hks { gm.logger.Debug("GlobalHook", @@ -546,6 +552,7 @@ func (gm *GlobalModule) searchGlobalShellHooks(hooksDir string) ([]*kind.ShellHo if _, err := os.Stat(hooksSubDir); !os.IsNotExist(err) { hooksDir = hooksSubDir } + hooksRelativePaths, err := utils_file.RecursiveGetExecutablePaths(hooksDir) if err != nil { return nil, err @@ -580,6 +587,7 @@ func (gm *GlobalModule) searchGlobalShellHooks(hooksDir string) ([]*kind.ShellHo if len(hks) > 0 { count = strconv.Itoa(len(hks)) } + gm.logger.Info("Found global shell hooks in dir", slog.String(pkg.LogKeyCount, count), slog.String(pkg.LogKeyDir, hooksDir)) diff --git a/pkg/module_manager/models/modules/helm.go b/pkg/module_manager/models/modules/helm.go index 5bcd0ac09..3257ab94b 100644 --- a/pkg/module_manager/models/modules/helm.go +++ b/pkg/module_manager/models/modules/helm.go @@ -110,6 +110,7 @@ func NewHelmModule(bm *BasicModule, namespace string, tmpDir string, deps *HelmM if hm.logger == nil { hm.logger = log.NewLogger().Named("helm-module") } + hm.logger = hm.logger.With(slog.String(pkg.LogKeyModule, hm.name)) isHelm, err := hm.isHelmChart() @@ -120,6 +121,7 @@ func NewHelmModule(bm *BasicModule, namespace string, tmpDir string, deps *HelmM if !isHelm { hm.logger.Info("module has neither Chart.yaml nor templates/ dir, is't not a helm chart", slog.String(pkg.LogKeyName, bm.GetName())) + return nil, ErrModuleIsNotHelm } @@ -147,6 +149,7 @@ func (hm *HelmModule) isHelmChart() (bool, error) { if err == nil { return true, nil } + if os.IsNotExist(err) { // if templates not exists - it's not a helm module return false, nil @@ -213,6 +216,7 @@ func (hm *HelmModule) RunHelmInstall(ctx context.Context, logLabels map[string]s logEntry.Warn("get release label failed", log.Err(err), slog.String(pkg.LogKeyRelease, helmReleaseName)) return fmt.Errorf("get release label failed: %w", err) } + logEntry.Debug("release not found when checking unmanaged state", slog.String(pkg.LogKeyRelease, helmReleaseName)) } @@ -236,6 +240,7 @@ func (hm *HelmModule) RunHelmInstall(ctx context.Context, logLabels map[string]s // Render templates to prevent excess helm runs. var renderedManifests string + func() { metricLabels := map[string]string{ pkg.MetricKeyModule: hm.name, @@ -257,6 +262,7 @@ func (hm *HelmModule) RunHelmInstall(ctx context.Context, logLabels map[string]s false, ) }() + if err != nil { return err } @@ -276,18 +282,21 @@ func (hm *HelmModule) RunHelmInstall(ctx context.Context, logLabels map[string]s span.AddEvent("ModuleRun-HelmPhase-helm-check-upgrade") // Skip upgrades if nothing is changed var runUpgradeRelease bool + func() { metricLabels := map[string]string{ pkg.MetricKeyModule: hm.name, pkg.MetricKeyActivation: logLabels[pkg.LogKeyEventType], pkg.MetricKeyOperation: "check-upgrade", } + defer measure.Duration(func(d time.Duration) { hm.dependencies.MetricsStorage.HistogramObserve(metrics.HelmOperationSeconds, d.Seconds(), metricLabels, nil) })() runUpgradeRelease, err = hm.shouldRunHelmUpgrade(helmClient, helmReleaseName, checksum, manifests, logLabels) }() + if err != nil { return err } @@ -354,6 +363,7 @@ func (hm *HelmModule) shouldRunHelmUpgrade(helmClient client.HelmClient, release logEntry.Debug("helm release: should run upgrade", slog.String(pkg.LogKeyRelease, releaseName), slog.String(pkg.LogKeyStatus, strings.ToLower(status))) + return true, nil } @@ -363,6 +373,7 @@ func (hm *HelmModule) shouldRunHelmUpgrade(helmClient client.HelmClient, release logEntry.Debug("helm release get values error, no upgrade", slog.String(pkg.LogKeyRelease, releaseName), log.Err(err)) + return false, err } @@ -373,6 +384,7 @@ func (hm *HelmModule) shouldRunHelmUpgrade(helmClient client.HelmClient, release slog.String(pkg.LogKeyRelease, releaseName), slog.String(pkg.LogKeyChecksum, recordedChecksum), slog.String(pkg.LogKeyNewChecksum, checksum)) + return true, nil } @@ -388,11 +400,13 @@ func (hm *HelmModule) shouldRunHelmUpgrade(helmClient client.HelmClient, release slog.String(pkg.LogKeyRelease, releaseName), slog.Int(pkg.LogKeyCount, len(absent)), ) + return true, nil } logEntry.Debug("helm release is unchanged: skip release upgrade", slog.String(pkg.LogKeyRelease, releaseName)) + return false, nil } @@ -403,6 +417,7 @@ func (hm *HelmModule) PrepareValuesYamlFile() (string, error) { } path := filepath.Join(hm.tmpDir, fmt.Sprintf("%s.module-values.yaml-%s", hm.safeName(), uuid.Must(uuid.NewV4()).String())) + err = utils.DumpData(path, data) if err != nil { return "", err diff --git a/pkg/module_manager/models/modules/hook_storage.go b/pkg/module_manager/models/modules/hook_storage.go index 735e232ef..8728d4aaf 100644 --- a/pkg/module_manager/models/modules/hook_storage.go +++ b/pkg/module_manager/models/modules/hook_storage.go @@ -30,6 +30,7 @@ func (hs *HooksStorage) AddHook(hk *hooks.ModuleHook) { defer hs.lock.Unlock() hName := hk.GetName() + hs.byName[hName] = hk for _, binding := range hk.GetHookConfig().Bindings() { hs.byBinding[binding] = append(hs.byBinding[binding], hk) @@ -42,10 +43,12 @@ func (hs *HooksStorage) getHooks(bt ...sh_op_types.BindingType) []*hooks.ModuleH if len(bt) > 0 { t := bt[0] + res, ok := hs.byBinding[t] if !ok { return []*hooks.ModuleHook{} } + sort.Slice(res, func(i, j int) bool { oi, oj := res[i].Order(t), res[j].Order(t) if oi != oj { diff --git a/pkg/module_manager/models/modules/synchronization_state.go b/pkg/module_manager/models/modules/synchronization_state.go index 4732f7156..746d0aa75 100644 --- a/pkg/module_manager/models/modules/synchronization_state.go +++ b/pkg/module_manager/models/modules/synchronization_state.go @@ -46,13 +46,16 @@ func NewSynchronizationState() *SynchronizationState { func (s *SynchronizationState) HasQueued() bool { s.m.RLock() defer s.m.RUnlock() + queued := false + for _, state := range s.state { if state.Queued { queued = true break } } + return queued } @@ -60,16 +63,21 @@ func (s *SynchronizationState) HasQueued() bool { func (s *SynchronizationState) IsCompleted() bool { s.m.RLock() defer s.m.RUnlock() + done := true + for _, state := range s.state { if !state.Done { done = false + log.Debug("Synchronization isn't done", slog.String(pkg.LogKeyHook, state.HookName), slog.String(pkg.LogKeyBinding, state.BindingName)) + break } } + return done } @@ -77,7 +85,9 @@ func (s *SynchronizationState) IsCompleted() bool { func (s *SynchronizationState) QueuedForBinding(metadata TaskMetadata) { s.m.Lock() defer s.m.Unlock() + var state *kubernetesBindingSynchronizationState + state, ok := s.state[metadata.GetKubernetesBindingID()] if !ok { state = &kubernetesBindingSynchronizationState{ @@ -86,18 +96,22 @@ func (s *SynchronizationState) QueuedForBinding(metadata TaskMetadata) { } s.state[metadata.GetKubernetesBindingID()] = state } + state.Queued = true } func (s *SynchronizationState) DoneForBinding(id string) { s.m.Lock() defer s.m.Unlock() + var state *kubernetesBindingSynchronizationState + state, ok := s.state[id] if !ok { state = &kubernetesBindingSynchronizationState{} s.state[id] = state } + log.Debug("Synchronization done", slog.String(pkg.LogKeyHook, state.HookName), slog.String(pkg.LogKeyBinding, state.BindingName)) @@ -107,6 +121,7 @@ func (s *SynchronizationState) DoneForBinding(id string) { func (s *SynchronizationState) DebugDumpState(logEntry *log.Logger) { s.m.RLock() defer s.m.RUnlock() + for id, state := range s.state { logEntry.Debug(fmt.Sprintf("%s/%s: queued=%v done=%v id=%s", state.HookName, state.BindingName, state.Queued, state.Done, id)) } diff --git a/pkg/module_manager/models/modules/values_storage.go b/pkg/module_manager/models/modules/values_storage.go index 0219dc2fa..93c7cefba 100644 --- a/pkg/module_manager/models/modules/values_storage.go +++ b/pkg/module_manager/models/modules/values_storage.go @@ -66,6 +66,7 @@ func NewValuesStorage(moduleName string, staticValues utils.Values, configBytes, schemaStorage: schemaStorage, moduleName: moduleName, } + err = vs.calculateResultValues() if err != nil { return nil, fmt.Errorf("critical error occurred with calculating values for %q: %w", moduleName, err) @@ -217,6 +218,7 @@ func (vs *ValuesStorage) SaveConfigValues(configV utils.Values) { defer vs.lock.Unlock() vs.configValues = configV + err := vs.calculateResultValues() if err != nil { panic(err) diff --git a/pkg/module_manager/models/moduleset/moduleset.go b/pkg/module_manager/models/moduleset/moduleset.go index a8bba5d14..4c87d1992 100644 --- a/pkg/module_manager/models/moduleset/moduleset.go +++ b/pkg/module_manager/models/moduleset/moduleset.go @@ -20,12 +20,14 @@ type ModulesSet struct { func (s *ModulesSet) SetInited() { s.lck.Lock() defer s.lck.Unlock() + s.inited = true } func (s *ModulesSet) IsInited() bool { s.lck.RLock() defer s.lck.RUnlock() + return s.inited } @@ -47,6 +49,7 @@ func (s *ModulesSet) Add(mods ...*modules.BasicModule) { if _, ok := s.modules[module.GetName()]; !ok { s.orderedNames = nil } + s.modules[module.GetName()] = module } } @@ -54,16 +57,19 @@ func (s *ModulesSet) Add(mods ...*modules.BasicModule) { func (s *ModulesSet) Get(name string) *modules.BasicModule { s.lck.RLock() defer s.lck.RUnlock() + return s.modules[name] } func (s *ModulesSet) List() []*modules.BasicModule { s.lck.Lock() defer s.lck.Unlock() + list := make([]*modules.BasicModule, 0, len(s.modules)) for _, name := range s.namesInOrder() { list = append(list, s.modules[name]) } + return list } @@ -77,6 +83,7 @@ func (s *ModulesSet) Len() int { func (s *ModulesSet) NamesInOrder() []string { s.lck.Lock() defer s.lck.Unlock() + return s.namesInOrder() } @@ -84,13 +91,16 @@ func (s *ModulesSet) namesInOrder() []string { if s.orderedNames == nil { s.orderedNames = s.sortModuleNames() } + return s.orderedNames } func (s *ModulesSet) Has(name string) bool { s.lck.RLock() defer s.lck.RUnlock() + _, ok := s.modules[name] + return ok } @@ -105,6 +115,7 @@ func (s *ModulesSet) sortModuleNames() []string { if mods[i].GetOrder() != mods[j].GetOrder() { return mods[i].GetOrder() < mods[j].GetOrder() } + return mods[i].GetName() < mods[j].GetName() }) // return names array. @@ -112,5 +123,6 @@ func (s *ModulesSet) sortModuleNames() []string { for _, mod := range mods { names = append(names, mod.GetName()) } + return names } diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 33f4b4d84..752c29ce0 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -245,6 +245,7 @@ func (mm *ModuleManager) ApplyNewKubeConfigValues(kubeConfig *config.KubeConfig, slog.String(pkg.LogKeyValues, fmt.Sprintf("%v", newGlobalValues))) mm.global.SaveConfigValues(newGlobalValues) } + delete(valuesMap, mm.global.GetName()) } @@ -278,6 +279,7 @@ func (mm *ModuleManager) validateNewKubeConfig(kubeConfig *config.KubeConfig, al if validationErr != nil { _ = multierror.Append(validationErrors, validationErr) } + valuesMap[mm.global.GetName()] = newValues } @@ -315,6 +317,7 @@ func (mm *ModuleManager) validateNewKubeConfig(kubeConfig *config.KubeConfig, al if validationErr != nil { _ = multierror.Append(validationErrors, validationErr) } + valuesMap[mod.GetName()] = newValues } } @@ -330,11 +333,13 @@ func (mm *ModuleManager) warnAboutUnknownModules(kubeConfig *config.KubeConfig) } unknownNames := make([]string, 0) + for moduleName := range kubeConfig.Modules { if !mm.modules.Has(moduleName) { unknownNames = append(unknownNames, moduleName) } } + if len(unknownNames) > 0 { mm.logger.Warn("KubeConfigManager has values for unknown modules", slog.Any(pkg.LogKeyModules, unknownNames)) @@ -361,6 +366,7 @@ func (mm *ModuleManager) Init(logger *log.Logger) error { if err != nil { return fmt.Errorf("couldn't create static extender: %w", err) } + if err := mm.moduleScheduler.AddExtender(staticExtender); err != nil { return fmt.Errorf("couldn't add static extender: %w", err) } @@ -417,10 +423,13 @@ func (mm *ModuleManager) checkConfig() { if mm.ctx.Err() != nil { return } + mm.kubeConfigLock.RLock() + if !mm.kubeConfigValid || !mm.kubeConfigValuesValid { mm.dependencies.MetricStorage.CounterAdd(metrics.ConfigValuesErrorsTotal, 1.0, map[string]string{}) } + mm.kubeConfigLock.RUnlock() time.Sleep(5 * time.Second) } @@ -499,11 +508,13 @@ func (mm *ModuleManager) SetGlobalDiscoveryAPIVersions(apiVersions []string) { // UpdateModulesMetrics updates modules' states metrics func (mm *ModuleManager) UpdateModulesMetrics() { mm.dependencies.MetricStorage.Grouped().ExpireGroupMetricByName(moduleInfoMetricGroup, moduleInfoMetricName) + for _, module := range mm.GetModuleNames() { enabled := "false" if mm.IsModuleEnabled(module) { enabled = "true" } + mm.dependencies.MetricStorage.Grouped().GaugeSet(moduleInfoMetricGroup, moduleInfoMetricName, 1, map[string]string{pkg.MetricKeyModule: module, "enabled": enabled}) } } @@ -514,6 +525,7 @@ func (mm *ModuleManager) SetModuleMaintenanceState(moduleName string, state util mm.logger.Info("set module management state", slog.String(pkg.LogKeyModule, moduleName), slog.String(pkg.LogKeyState, state.String())) + if state == utils.NoResourceReconciliation { mm.dependencies.MetricStorage.Grouped().GaugeSet(moduleMaintenanceMetricGroup, moduleMaintenanceMetricName, 1, map[string]string{pkg.MetricKeyModule: moduleName, "state": utils.NoResourceReconciliation.String()}) } else { @@ -567,13 +579,16 @@ func (mm *ModuleManager) RefreshEnabledState(logLabels map[string]string) (*Modu case *fs.FileSystemLoader: default: logEntry.Debug("non-default module loader detected - applying enabledModules patch") + enabledModulesAndFakeCRDmodules := make([]string, 0, len(enabledModules)) for _, moduleName := range enabledModules { if mm.ModuleHasCRDs(moduleName) { enabledModulesAndFakeCRDmodules = append(enabledModulesAndFakeCRDmodules, fmt.Sprintf("%s-crd", moduleName)) } + enabledModulesAndFakeCRDmodules = append(enabledModulesAndFakeCRDmodules, moduleName) } + mm.global.SetEnabledModules(enabledModulesAndFakeCRDmodules) } @@ -754,7 +769,9 @@ func (mm *ModuleManager) RunModule(ctx context.Context, moduleName string, logLa treg := trace.StartRegion(ctx, "ModuleRun-HelmPhase-beforeHelm") err = bm.RunHooksByBinding(ctx, BeforeHelm, logLabels) + treg.End() + if err != nil { return false, fmt.Errorf("run hooks by binding: %w", err) } @@ -784,6 +801,7 @@ func (mm *ModuleManager) RunModule(ctx context.Context, moduleName string, logLa } treg.End() + if err != nil && !errors.Is(err, modules.ErrModuleIsNotHelm) { return false, fmt.Errorf("run helm install: %w", err) } @@ -792,7 +810,9 @@ func (mm *ModuleManager) RunModule(ctx context.Context, moduleName string, logLa oldValuesChecksum := oldValues.Checksum() treg = trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-afterHelm") err = bm.RunHooksByBinding(ctx, AfterHelm, logLabels) + treg.End() + if err != nil { return false, fmt.Errorf("run hooks by binding: %w", err) } @@ -907,6 +927,7 @@ func (mm *ModuleManager) DisableModuleScheduleBindings(moduleName string) { if !ml.HooksControllersReady() { return } + schHooks := ml.GetHooks(Schedule) for _, mh := range schHooks { mh.GetHookController().DisableScheduleBindings() @@ -1043,10 +1064,12 @@ func (mm *ModuleManager) applyEnabledPatch(enabledPatch utils.ValuesPatch, exten modName := strings.TrimSuffix(op.Path, utils.EnabledSuffix) modName = strings.TrimPrefix(modName, "/") modName = utils.ModuleNameFromValuesKey(modName) + v, err := utils.ModuleEnabledValue(op.Value) if err != nil { return fmt.Errorf("apply enabled patch operation '%s' for %s: %w", op.Op, op.Path, err) } + switch op.Op { case "add": mm.logger.Debug("apply dynamic enable", @@ -1056,6 +1079,7 @@ func (mm *ModuleManager) applyEnabledPatch(enabledPatch utils.ValuesPatch, exten mm.logger.Debug("apply dynamic enable: module removed from dynamic enable", slog.String(pkg.LogKeyModule, modName)) } + extender.UpdateStatus(modName, op.Op, *v) mm.logger.Info("dynamically enabled module status change", slog.String(pkg.LogKeyModule, modName), @@ -1099,16 +1123,19 @@ func (mm *ModuleManager) GlobalSynchronizationState() *modules.SynchronizationSt func (mm *ModuleManager) ApplyBindingActions(moduleHook *hooks.ModuleHook, bindingActions []gohook.BindingAction) error { for _, action := range bindingActions { bindingIdx := -1 + for i, binding := range moduleHook.GetHookConfig().OnKubernetesEvents { if binding.BindingName == action.Name { bindingIdx = i } } + if bindingIdx == -1 { continue } monitorCfg := moduleHook.GetHookConfig().OnKubernetesEvents[bindingIdx].Monitor + switch strings.ToLower(action.Action) { case "disable": // Empty kind - "null" monitor. @@ -1129,6 +1156,7 @@ func (mm *ModuleManager) ApplyBindingActions(moduleHook *hooks.ModuleHook, bindi return fmt.Errorf("update monitor: %w", err) } } + return nil } @@ -1189,6 +1217,7 @@ func (mm *ModuleManager) PushRunModuleTask(moduleName string, doModuleStartup bo func (mm *ModuleManager) AreModulesInited() bool { mm.l.Lock() defer mm.l.Unlock() + return mm.modules.IsInited() } @@ -1465,6 +1494,7 @@ func (mm *ModuleManager) registerModules(scriptEnabledExtender *script_extender. mm.logger.Warn("module is not registered, because it has a duplicate", slog.String(pkg.LogKeyModule, mod.GetName()), slog.String(pkg.LogKeyPath, mod.GetPath())) + continue } @@ -1531,8 +1561,10 @@ func queueHasPendingModuleRunTaskWithStartup(q *queue.TaskQueue, moduleName stri if q == nil { return false } + modules := modulesWithPendingTasks(q, task.ModuleRun) meta, has := modules[moduleName] + return has && meta.doStartup } diff --git a/pkg/module_manager/module_manager_hooks.go b/pkg/module_manager/module_manager_hooks.go index c3aabcde7..0cd2efda9 100644 --- a/pkg/module_manager/module_manager_hooks.go +++ b/pkg/module_manager/module_manager_hooks.go @@ -44,6 +44,7 @@ func (mm *ModuleManager) loadGlobalValues() (*globalValues, error) { if key == utils.GlobalValuesKey { section := commonStaticValues.GetKeySection(utils.GlobalValuesKey) resultGlobalValues = utils.MergeValues(resultGlobalValues, section) + continue } diff --git a/pkg/module_manager/scheduler/extenders/dynamically_enabled/dynamic.go b/pkg/module_manager/scheduler/extenders/dynamically_enabled/dynamic.go index a24a7c435..de08c2ca0 100644 --- a/pkg/module_manager/scheduler/extenders/dynamically_enabled/dynamic.go +++ b/pkg/module_manager/scheduler/extenders/dynamically_enabled/dynamic.go @@ -27,6 +27,7 @@ func NewExtender() *Extender { e := &Extender{ modulesStatus: make(map[string]bool), } + return e } diff --git a/pkg/module_manager/scheduler/extenders/error/permanent.go b/pkg/module_manager/scheduler/extenders/error/permanent.go index 2f5a00684..7baef336a 100644 --- a/pkg/module_manager/scheduler/extenders/error/permanent.go +++ b/pkg/module_manager/scheduler/extenders/error/permanent.go @@ -18,6 +18,7 @@ func Permanent(err error) *PermanentError { if err == nil { return nil } + return &PermanentError{ Err: err, } diff --git a/pkg/module_manager/scheduler/extenders/kube_config/kube_config.go b/pkg/module_manager/scheduler/extenders/kube_config/kube_config.go index 6a99e12a8..c7d57e3df 100644 --- a/pkg/module_manager/scheduler/extenders/kube_config/kube_config.go +++ b/pkg/module_manager/scheduler/extenders/kube_config/kube_config.go @@ -52,6 +52,7 @@ func (e *Extender) sendNotify(kubeConfigEvent config.KubeConfigEvent) { func (e *Extender) SetNotifyChannel(ctx context.Context, ch chan extenders.ExtenderEvent) { e.notifyCh = ch + go func() { for { select { diff --git a/pkg/module_manager/scheduler/extenders/script_enabled/script.go b/pkg/module_manager/scheduler/extenders/script_enabled/script.go index 1c2da4ced..c4874d6c7 100644 --- a/pkg/module_manager/scheduler/extenders/script_enabled/script.go +++ b/pkg/module_manager/scheduler/extenders/script_enabled/script.go @@ -65,10 +65,12 @@ func (e *Extender) AddBasicModule(module node.ModuleInterface) { } enabledScriptPath := filepath.Join(module.GetPath(), "enabled") + f, err := os.Stat(enabledScriptPath) if err != nil { if os.IsNotExist(err) { moduleD.scriptState = noEnabledScript + log.Debug("MODULE is ENABLED. Enabled script doesn't exist!", slog.String(pkg.LogKeyModule, module.GetName())) } else { @@ -79,6 +81,7 @@ func (e *Extender) AddBasicModule(module node.ModuleInterface) { } else { if utils_file.CheckExecutablePermissions(f) != nil { moduleD.scriptState = nonExecutableScript + log.Warn("Found non-executable enabled script for module - assuming enabled state", slog.String(pkg.LogKeyModule, module.GetName())) } @@ -93,23 +96,29 @@ func (e *Extender) Name() extenders.ExtenderName { func (e *Extender) Filter(moduleName string, logLabels map[string]string) (*bool, error) { if moduleDescriptor, found := e.basicModuleDescriptors[moduleName]; found { - var err error - var enabled *bool + var ( + err error + enabled *bool + ) switch moduleDescriptor.scriptState { case "": var isEnabled bool + refreshLogLabels := utils.MergeLabels(logLabels, map[string]string{ pkg.LogKeyExtender: "ScriptEnabled", }) + isEnabled, err = moduleDescriptor.module.RunEnabledScript(context.Background(), e.tmpDir, e.GetEnabledModules(), refreshLogLabels) if err != nil { err = fmt.Errorf("failed to execute '%s' module's enabled script: %v", moduleDescriptor.module.GetName(), err) } + enabled = &isEnabled case statError: log.Error(moduleDescriptor.stateDescription) + enabled = pointer.To(false) err = errors.New(moduleDescriptor.stateDescription) diff --git a/pkg/module_manager/scheduler/extenders/static/static.go b/pkg/module_manager/scheduler/extenders/static/static.go index 097577446..cd682117f 100644 --- a/pkg/module_manager/scheduler/extenders/static/static.go +++ b/pkg/module_manager/scheduler/extenders/static/static.go @@ -25,19 +25,23 @@ type Extender struct { func NewExtender(staticValuesFilePaths string) (*Extender, error) { result := make(map[string]bool) + dirs := utils.SplitToPaths(staticValuesFilePaths) for _, dir := range dirs { valuesFile := filepath.Join(dir, "values.yaml") + fileInfo, err := os.Stat(valuesFile) if err != nil { log.Warn("Couldn't stat file", slog.String(pkg.LogKeyFile, valuesFile)) + continue } if fileInfo.IsDir() { log.Error("File is a directory", slog.String(pkg.LogKeyFile, valuesFile)) + continue } @@ -46,8 +50,10 @@ func NewExtender(staticValuesFilePaths string) (*Extender, error) { if os.IsNotExist(err) { log.Debug("File doesn't exist", slog.String(pkg.LogKeyFile, valuesFile)) + continue } + return nil, err } defer f.Close() diff --git a/pkg/module_manager/scheduler/node/mock/node_mock.go b/pkg/module_manager/scheduler/node/mock/node_mock.go index 1f5c1ade1..620af1922 100644 --- a/pkg/module_manager/scheduler/node/mock/node_mock.go +++ b/pkg/module_manager/scheduler/node/mock/node_mock.go @@ -38,6 +38,7 @@ func (m MockModule) RunEnabledScript(_ context.Context, _ string, _ []string, _ } depsEnabled := true + if len(m.ListOfRequiredModules) > 0 && m.EnabledModules != nil { for _, requiredModule := range m.ListOfRequiredModules { if !slices.Contains(*m.EnabledModules, requiredModule) { diff --git a/pkg/module_manager/scheduler/scheduler.go b/pkg/module_manager/scheduler/scheduler.go index 75b65abac..6108bb577 100644 --- a/pkg/module_manager/scheduler/scheduler.go +++ b/pkg/module_manager/scheduler/scheduler.go @@ -99,6 +99,7 @@ func NewScheduler(ctx context.Context, logger *log.Logger) *Scheduler { nodeHash := func(n *node.Node) string { return n.GetName() } + return &Scheduler{ ctx: ctx, extenders: make([]extenderContainer, 0), @@ -122,11 +123,13 @@ func (s *Scheduler) GetGraphDOTDescription() ([]byte, error) { func (s *Scheduler) getGraphDOTDescription() ([]byte, error) { var b bytes.Buffer + writer := bufio.NewWriter(&b) if err := draw.DOT(s.dag, writer, draw.GraphAttribute("label", "Module Scheduler's Graph")); err != nil { return nil, fmt.Errorf("couldn't write graph file: %w", err) } + writer.Flush() return b.Bytes(), nil @@ -198,6 +201,7 @@ func (s *Scheduler) Initialize() error { // add edges for _, module := range s.modules { var connected bool + for _, parents := range s.getEdgesFromExtenders(module.GetName()) { for _, hint := range parents { parent, err := s.dag.Vertex(hint.Name) @@ -281,6 +285,7 @@ func (s *Scheduler) addWeightVertex(vertex *node.Node) error { child *node.Node dfsErr error ) + err := graph.DFS(s.dag, s.root.GetName(), func(name string) bool { currVertex, props, err := s.dag.VertexWithProperties(name) if err != nil { @@ -342,6 +347,7 @@ func (s *Scheduler) addWeightVertex(vertex *node.Node) error { // ApplyExtenders excludes and reorders attached extenders according to the APPLIED_MODULE_EXTENDERS env variable func (s *Scheduler) ApplyExtenders(extendersEnv string) error { appliedExtenders := []extenders.ExtenderName{} + if len(extendersEnv) == 0 { log.Warn("ADDON_OPERATOR_APPLIED_MODULE_EXTENDERS variable isn't set - default list will be applied", slog.Any(pkg.LogKeyValues, defaultAppliedExtenders)) @@ -373,6 +379,7 @@ func (s *Scheduler) ApplyExtenders(extendersEnv string) error { } newExtenders := []extenderContainer{} + for _, appliedExt := range appliedExtenders { for _, e := range s.extenders { if e.ext.Name() == appliedExt { @@ -380,11 +387,13 @@ func (s *Scheduler) ApplyExtenders(extendersEnv string) error { if ne, ok := e.ext.(extenders.NotificationExtender); ok { ne.SetNotifyChannel(s.ctx, s.extCh) } + if se, ok := e.ext.(extenders.StatefulExtender); ok { se.SetModulesStateHelper(func() []string { return s.vertexStateBuffer.enabledModules }) } + break } } @@ -402,6 +411,7 @@ func (s *Scheduler) ApplyExtenders(extendersEnv string) error { log.Info("The list of applied module extenders", slog.Any(pkg.LogKeyFinalList, finalList)) + return nil } @@ -419,6 +429,7 @@ func (s *Scheduler) setExtendersMeta() { // getEdgesFromExtenders cycles through topological extenders to build a map of additional edges for the module func (s *Scheduler) getEdgesFromExtenders(moduleName string) map[extenders.ExtenderName][]extenders.Hint { result := make(map[extenders.ExtenderName][]extenders.Hint, 0) + for _, e := range s.extenders { if tope, ok := e.ext.(extenders.TopologicalExtender); ok { if hints := tope.GetTopologicalHints(moduleName); len(hints) > 0 { @@ -439,6 +450,7 @@ func (s *Scheduler) AddExtender(ext extenders.Extender) error { } s.extenders = append(s.extenders, extenderContainer{ext: ext}) + return nil } @@ -457,6 +469,7 @@ func (s *Scheduler) getModuleVerticesByOrder(logLabels map[string]string) ([]*no if err != nil { return nil, fmt.Errorf("could not get %s vertex from the graph: %w", vertexName, err) } + nodes = append(nodes, vertex) } } @@ -509,6 +522,7 @@ func (s *Scheduler) customBFS(root string, visit func(node, prevNode string, pre } slices.Sort(sliceOfAdjacencies) + for _, adj := range sliceOfAdjacencies { queue = append(queue, vertex{name: adj, prevName: currentHash, prevDepth: depth}) } @@ -542,10 +556,12 @@ func (s *Scheduler) printSummary() (map[string]bool, error) { func (s *Scheduler) GetUpdatedByExtender(moduleName string) (string, error) { s.l.Lock() defer s.l.Unlock() + vertex, err := s.dag.Vertex(moduleName) if err != nil { return "", err } + return vertex.GetUpdatedBy(), err } @@ -557,6 +573,7 @@ func (s *Scheduler) GetFunctionalDependencies() map[string][]extenders.Hint { if tope, ok := ext.ext.(extenders.TopologicalExtender); ok { if parents := tope.GetTopologicalHints(module.GetName()); len(parents) > 0 { var deps []extenders.Hint + for _, parent := range parents { if !s.IsModuleCritical(parent.Name) { deps = append(deps, parent) @@ -565,6 +582,7 @@ func (s *Scheduler) GetFunctionalDependencies() map[string][]extenders.Hint { result[module.GetName()] = deps } + break } } @@ -588,6 +606,7 @@ func (s *Scheduler) IsModuleCritical(moduleName string) bool { func (s *Scheduler) IsModuleEnabled(moduleName string) bool { s.l.Lock() defer s.l.Unlock() + vertex, err := s.dag.Vertex(moduleName) if err != nil { return false @@ -601,6 +620,7 @@ func (s *Scheduler) IsModuleEnabled(moduleName string) bool { func (s *Scheduler) GetEnabledModuleNames() []string { s.l.Lock() defer s.l.Unlock() + return s.getEnabledModuleNames() } @@ -621,6 +641,7 @@ func (s *Scheduler) getModuleNamesByOrder(onlyEnabled bool, logLabels map[string } var dagOrder int + logEntry := utils.EnrichLoggerWithLabels(s.logger, logLabels) if s.enabledModules != nil { @@ -644,6 +665,7 @@ func (s *Scheduler) getModuleNamesByOrder(onlyEnabled bool, logLabels map[string slog.String(pkg.LogKeyVertex, name)) var depth int + vertex, props, err := s.dag.VertexWithProperties(name) if err != nil { bfsErr = fmt.Errorf("couldn not get %s vertex from the graph: %w", name, err) @@ -716,6 +738,7 @@ func (s *Scheduler) getModuleNamesByOrder(onlyEnabled bool, logLabels map[string // if s.errList isn't empty, we try to recalculate the graph in case there were some minor errors last time. func (s *Scheduler) GetGraphState(logLabels map[string]string) ( /*enabled modules*/ []string /*enabled modules sorted by order*/, [][]string /*modules diff*/, map[string]bool, error) { var recalculateGraph bool + logEntry := utils.EnrichLoggerWithLabels(s.logger, logLabels) s.l.Lock() defer s.l.Unlock() @@ -723,12 +746,14 @@ func (s *Scheduler) GetGraphState(logLabels map[string]string) ( /*enabled modul // graph hasn't been initialized yet if s.enabledModules == nil { logEntry.Info("Module Scheduler: graph hasn't been calculated yet") + recalculateGraph = true } if s.err != nil { logEntry.Warn("Module Scheduler: graph in a faulty state and will be recalculated", slog.String(pkg.LogKeyError, s.err.Error())) + recalculateGraph = true } @@ -757,9 +782,11 @@ func (s *Scheduler) RecalculateGraph(logLabels map[string]string) (bool, []strin func (s *Scheduler) Filter(extName extenders.ExtenderName, moduleName string, logLabels map[string]string) (*bool, error) { s.l.Lock() defer s.l.Unlock() + if s.root == nil { return nil, fmt.Errorf("graph is empty") } + if s.err != nil { return nil, fmt.Errorf("graph is in a faulty state: %w", s.err) } @@ -780,7 +807,9 @@ func (s *Scheduler) Filter(extName extenders.ExtenderName, moduleName string, lo // - true if some other parameters, apart from state, of a vertex has changed. func (s *Scheduler) recalculateGraphState(logLabels map[string]string) ( /* Graph's state has changed */ bool /* list of the vertices, which updateBy statuses have changed */, []string) { stateDiff, metaDiff := make(map[string]bool), make(map[string]struct{}, 0) + var graphErr error + logEntry := utils.EnrichLoggerWithLabels(s.logger, logLabels) vertices, err := s.getModuleVerticesByOrder(logLabels) @@ -799,6 +828,7 @@ outerCycle: filterErr error moduleStatus *bool ) + moduleName := vertex.GetName() s.vertexStateBuffer.state[moduleName] = &vertexState{} @@ -827,11 +857,13 @@ outerCycle: s.vertexStateBuffer.state[moduleName].enabled = *moduleStatus s.vertexStateBuffer.state[moduleName].updatedBy = string(e.ext.Name()) } + break } // continue checking extenders continue } + s.vertexStateBuffer.state[moduleName].enabled = *moduleStatus s.vertexStateBuffer.state[moduleName].updatedBy = string(e.ext.Name()) } @@ -859,6 +891,7 @@ outerCycle: if graphErr != nil { s.err = graphErr logEntry.Warn("Module Scheduler: Graph converge failed", log.Err(s.err)) + return true, nil } @@ -881,8 +914,10 @@ outerCycle: if err != nil { graphErr = multierror.Append(graphErr, fmt.Errorf("couldn't get %s vertex from the graph: %v", vertexName, err)) s.err = graphErr + return true, nil } + vertex.SetState(state.enabled) vertex.SetUpdatedBy(state.updatedBy) } @@ -899,6 +934,7 @@ outerCycle: for moduleName := range metaDiff { metaDiffSlice = append(metaDiffSlice, moduleName) } + slices.Sort(metaDiffSlice) return len(stateDiff) > 0, metaDiffSlice diff --git a/pkg/task/functional/scheduler.go b/pkg/task/functional/scheduler.go index 8357d0372..54840fa5c 100644 --- a/pkg/task/functional/scheduler.go +++ b/pkg/task/functional/scheduler.go @@ -91,6 +91,7 @@ func (s *Scheduler) runScheduleLoop(ctx context.Context) { // runProcessLoop waits for requests to be processed func (s *Scheduler) runProcessLoop(ctx context.Context) { var idx int + for { select { case <-ctx.Done(): @@ -163,6 +164,7 @@ func (s *Scheduler) reschedule(done string) { // check if all dependencies done ready := true + for _, dep := range req.Dependencies { // - if optional and not present in requests -> ignore // - if optional and present but not done -> not ready @@ -171,6 +173,7 @@ func (s *Scheduler) reschedule(done string) { continue } } + if _, ok := s.done[dep.Name]; !ok { ready = false break @@ -180,6 +183,7 @@ func (s *Scheduler) reschedule(done string) { // schedule module if ready if ready { s.logger.Debug("trigger scheduling", slog.String(pkg.LogKeyScheduled, req.Name), slog.Any(pkg.LogKeyDone, done)) + s.scheduled[req.Name] = struct{}{} s.processCh <- req } diff --git a/pkg/task/helpers/helpers.go b/pkg/task/helpers/helpers.go index a68d4e143..dea09a7b2 100644 --- a/pkg/task/helpers/helpers.go +++ b/pkg/task/helpers/helpers.go @@ -69,6 +69,7 @@ func formatTaskDetails(tsk sh_task.Task, hm task.HookMetadata, phase string) str if hm.DoModuleStartup { details += " with doModuleStartup" } + return details case task.ParallelModuleRun: @@ -139,5 +140,6 @@ func formatConvergeTaskDetails(tsk sh_task.Task, phase string) string { if taskEvent, ok := tsk.GetProp(converge.ConvergeEventProp).(converge.ConvergeEvent); ok { return fmt.Sprintf(" for %s in phase '%s'", string(taskEvent), phase) } + return "" } diff --git a/pkg/task/hook_metadata.go b/pkg/task/hook_metadata.go index 4314daf51..4fe6342a6 100644 --- a/pkg/task/hook_metadata.go +++ b/pkg/task/hook_metadata.go @@ -65,6 +65,7 @@ type ParallelRunModuleMetadata struct { func (pm *ParallelRunMetadata) GetModulesMetadata() map[string]ParallelRunModuleMetadata { pm.l.Lock() defer pm.l.Unlock() + return pm.modules } @@ -72,6 +73,7 @@ func (pm *ParallelRunMetadata) SetModuleMetadata(moduleName string, metadata Par if pm.modules == nil { pm.modules = make(map[string]ParallelRunModuleMetadata) } + pm.l.Lock() pm.modules[moduleName] = metadata pm.l.Unlock() @@ -81,6 +83,7 @@ func (pm *ParallelRunMetadata) DeleteModuleMetadata(moduleName string) { if pm.modules == nil { return } + pm.l.Lock() delete(pm.modules, moduleName) pm.l.Unlock() @@ -89,10 +92,12 @@ func (pm *ParallelRunMetadata) DeleteModuleMetadata(moduleName string) { func (pm *ParallelRunMetadata) ListModules() []string { pm.l.Lock() defer pm.l.Unlock() + result := make([]string, 0, len(pm.modules)) for module := range pm.modules { result = append(result, module) } + return result } @@ -118,6 +123,7 @@ func HookMetadataAccessor(t task.Task) HookMetadata { log.Error("Possible Bug! task metadata is not of type ModuleHookMetadata", slog.String(pkg.LogKeyType, string(t.GetType())), slog.String(pkg.LogKeyGot, fmt.Sprintf("%T", meta))) + return HookMetadata{} } @@ -136,11 +142,14 @@ func (hm HookMetadata) GetDescription() string { for bindingName := range bindingsMap { bindings = append(bindings, bindingName) } + sort.Strings(bindings) + bindingNames := "" if len(bindings) > 0 { bindingNames = ":" + strings.Join(bindings, ",") } + if len(hm.BindingContext) > 1 { bindingNames = fmt.Sprintf("%s in %d contexts", bindingNames, len(hm.BindingContext)) } @@ -156,6 +165,7 @@ func (hm HookMetadata) GetDescription() string { if hm.DoModuleStartup { osh = ":doStartup" } + return fmt.Sprintf("%s%s%s:%s", hm.ModuleName, osh, bindingNames, hm.EventDescription) } diff --git a/pkg/task/parallel/parallel.go b/pkg/task/parallel/parallel.go index f7b4d3c70..c5db20959 100644 --- a/pkg/task/parallel/parallel.go +++ b/pkg/task/parallel/parallel.go @@ -74,7 +74,9 @@ func (pq *TaskChannels) Set(id string, c TaskChannel) { func (pq *TaskChannels) Get(id string) (TaskChannel, bool) { pq.l.Lock() defer pq.l.Unlock() + c, ok := pq.channels[id] + return c, ok } diff --git a/pkg/task/queue/queue.go b/pkg/task/queue/queue.go index 8f79712f8..13f7f3663 100644 --- a/pkg/task/queue/queue.go +++ b/pkg/task/queue/queue.go @@ -120,6 +120,7 @@ func convergeTasksInQueue(q *queue.TaskQueue) int { } convergeTasks := 0 + q.IterateSnapshot(func(t sh_task.Task) { if converge.IsConvergeTask(t) || converge.IsFirstConvergeTask(t) { convergeTasks++ @@ -245,10 +246,12 @@ func (s *Service) ModuleEnsureCRDsTasksInMainQueueAfterId(afterId string) bool { IDFound := false taskFound := false stop := false + q.DeleteFunc(func(t sh_task.Task) bool { if stop { return true } + if !IDFound { if t.GetId() == afterId { IDFound = true diff --git a/pkg/task/service/converge.go b/pkg/task/service/converge.go index e018768f0..040a157c6 100644 --- a/pkg/task/service/converge.go +++ b/pkg/task/service/converge.go @@ -119,6 +119,7 @@ func ConvergeTasksInQueue(q *queue.TaskQueue) int { } convergeTasks := 0 + q.IterateSnapshot(func(t sh_task.Task) { if converge.IsConvergeTask(t) || converge.IsFirstConvergeTask(t) { convergeTasks++ @@ -134,6 +135,7 @@ func ConvergeModulesInQueue(q *queue.TaskQueue) int { } tasks := 0 + q.IterateSnapshot(func(t sh_task.Task) { taskType := t.GetType() if converge.IsConvergeTask(t) && (taskType == task.ModuleRun || taskType == task.ModuleDelete) { diff --git a/pkg/task/service/logs.go b/pkg/task/service/logs.go index 4cfb7c3de..35f671a3c 100644 --- a/pkg/task/service/logs.go +++ b/pkg/task/service/logs.go @@ -60,7 +60,9 @@ func (s *TaskHandlerService) taskPhase(tsk sh_task.Task) string { case task.ModuleRun: hm := task.HookMetadataAccessor(tsk) mod := s.moduleManager.GetModule(hm.ModuleName) + return string(mod.GetPhase()) } + return "" } diff --git a/pkg/task/service/metric.go b/pkg/task/service/metric.go index cd129f67a..8e2213f90 100644 --- a/pkg/task/service/metric.go +++ b/pkg/task/service/metric.go @@ -43,6 +43,7 @@ func (s *TaskHandlerService) UpdateWaitInQueueMetric(t sh_task.Task) { // set binding name instead of type metricLabels[pkg.MetricKeyBinding] = hm.Binding } + if t.GetType() == task.ModuleHookRun { // set binding name instead of type metricLabels[pkg.MetricKeyHook] = hm.HookName diff --git a/pkg/task/service/service.go b/pkg/task/service/service.go index ef753fff8..8958902b2 100644 --- a/pkg/task/service/service.go +++ b/pkg/task/service/service.go @@ -140,6 +140,7 @@ func (s *TaskHandlerService) Handle(ctx context.Context, t sh_task.Task) queue.T res.AfterHandle = func() { s.CheckConvergeStatus(t) + if origAfterHandle != nil { origAfterHandle() } diff --git a/pkg/task/tasks/converge-modules/task.go b/pkg/task/tasks/converge-modules/task.go index 69ca1132a..3298d1422 100644 --- a/pkg/task/tasks/converge-modules/task.go +++ b/pkg/task/tasks/converge-modules/task.go @@ -162,6 +162,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { } s.logger.Debug("ConvergeModules: send module disabled events") + go func() { for _, moduleName := range s.moduleManager.GetModuleNames() { if _, enabled := enabledModules[moduleName]; !enabled { @@ -173,12 +174,15 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { } }() } + tasks := s.CreateConvergeModulesTasks(state, s.shellTask.GetLogLabels(), string(taskEvent)) s.convergeState.SetPhase(converge.WaitDeleteAndRunModules) + if len(tasks) > 0 { res.AddHeadTasks(tasks...) res.Status = queue.Keep + s.logTaskAdd("head", tasks...) return res @@ -193,14 +197,17 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { // wait until functional converge done if !s.functionalScheduler.Finished() { s.logger.Warn("ConvergeModules: functional scheduler not finished") + res.Status = queue.Keep res.DelayBeforeNextTask = repeatInterval if s.queueService.GetQueueLength(queue.MainQueueName) > 1 { s.logger.Debug("ConvergeModules: main queue has pending tasks, pass them") + res.DelayBeforeNextTask = 0 res.AddTailTasks(s.shellTask.DeepCopyWithNewUUID()) res.Status = queue.Success + s.logTaskAdd("tail", s.shellTask) } @@ -212,10 +219,13 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { tasks, handleErr := s.CreateAfterAllTasks(s.shellTask.GetLogLabels(), hm.EventDescription) if handleErr == nil { s.convergeState.SetPhase(converge.WaitAfterAll) + if len(tasks) > 0 { res.AddHeadTasks(tasks...) res.Status = queue.Keep + s.logTaskAdd("head", tasks...) + return res } } @@ -234,6 +244,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { if handleErr != nil { res.Status = queue.Fail + s.logger.Error("ConvergeModules failed, requeue task to retry after delay.", slog.String(pkg.LogKeyPhase, string(s.convergeState.GetPhase())), slog.Int(pkg.LogKeyCount, s.shellTask.GetFailureCount()+1), @@ -241,10 +252,12 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { s.metricStorage.CounterAdd(metrics.ModulesDiscoverErrorsTotal, 1.0, map[string]string{}) s.shellTask.UpdateFailureMessage(handleErr.Error()) s.shellTask.WithQueuedAt(time.Now()) + return res } s.logger.Debug("ConvergeModules success") + res.Status = queue.Success return res @@ -355,12 +368,14 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo // Add ModuleDelete tasks to delete helm releases of disabled modules. log.Debug("The following modules are going to be disabled", slog.Any(pkg.LogKeyModules, state.ModulesToDisable)) + for _, moduleName := range state.ModulesToDisable { ev := events.ModuleEvent{ ModuleName: moduleName, EventType: events.ModuleDisabled, } s.moduleManager.SendModuleEvent(ev) + newLogLabels := utils.MergeLabels(logLabels) newLogLabels[pkg.LogKeyModule] = moduleName delete(newLogLabels, pkg.LogKeyTaskID) @@ -384,6 +399,7 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo slog.String(pkg.LogKeyModules, fmt.Sprintf("%v", state.AllEnabledModulesByOrder))) var functionalModules []string + for _, modules := range state.AllEnabledModulesByOrder { if len(modules) == 0 { continue @@ -397,10 +413,12 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo newLogLabels := utils.MergeLabels(logLabels) delete(newLogLabels, pkg.LogKeyTaskID) + switch { // create parallel moduleRun task case len(modules) > 1: parallelRunMetadata := task.ParallelRunMetadata{} + newLogLabels[pkg.LogKeyModules] = strings.Join(modules, ",") for _, moduleName := range modules { ev := events.ModuleEvent{ @@ -408,7 +426,9 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo EventType: events.ModuleEnabled, } s.moduleManager.SendModuleEvent(ev) + doModuleStartup := false + if _, has := newlyEnabled[moduleName]; has { // add EnsureCRDs task if module is about to be enabled if s.moduleManager.ModuleHasCRDs(moduleName) { @@ -421,12 +441,15 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo IsReloadAll: true, }).WithQueuedAt(queuedAt)) } + doModuleStartup = true } + parallelRunMetadata.SetModuleMetadata(moduleName, task.ParallelRunModuleMetadata{ DoModuleStartup: doModuleStartup, }) } + parallelRunMetadata.Context, parallelRunMetadata.CancelF = context.WithCancel(context.Background()) newTask := sh_task.NewTask(task.ParallelModuleRun). WithLogLabels(newLogLabels). @@ -447,8 +470,10 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo EventType: events.ModuleEnabled, } s.moduleManager.SendModuleEvent(ev) + newLogLabels[pkg.LogKeyModule] = modules[0] doModuleStartup := false + if _, has := newlyEnabled[modules[0]]; has { // add EnsureCRDs task if module is about to be enabled if s.moduleManager.ModuleHasCRDs(modules[0]) { @@ -461,8 +486,10 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo IsReloadAll: true, }).WithQueuedAt(queuedAt)) } + doModuleStartup = true } + newTask := sh_task.NewTask(task.ModuleRun). WithLogLabels(newLogLabels). WithQueueName("main"). @@ -510,6 +537,7 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo IsReloadAll: true, }).WithQueuedAt(queuedAt)) } + doModuleStartup = true } @@ -532,6 +560,7 @@ func (s *Task) CreateConvergeModulesTasks(state *module_manager.ModulesState, lo // ConvregeState.CRDsEnsured if there are new ensureCRDsTasks to execute if s.convergeState.CRDsEnsured && len(resultingTasks) > 0 { log.Debug("CheckCRDsEnsured: set to false") + s.convergeState.CRDsEnsured = false } diff --git a/pkg/task/tasks/global-hook-enable-kubernetes-bindings/task.go b/pkg/task/tasks/global-hook-enable-kubernetes-bindings/task.go index 5416f9755..91f783b99 100644 --- a/pkg/task/tasks/global-hook-enable-kubernetes-bindings/task.go +++ b/pkg/task/tasks/global-hook-enable-kubernetes-bindings/task.go @@ -81,6 +81,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { defer span.End() var res queue.TaskResult + s.logger.Debug("Global hook enable kubernetes bindings") hm := task.HookMetadataAccessor(s.shellTask) @@ -104,6 +105,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { if len(info.BindingContext) > 0 { taskLogLabels[pkg.LogKeyBindingName] = info.BindingContext[0].Binding } + delete(taskLogLabels, pkg.LogKeyTaskID) kubernetesBindingID := uuid.Must(uuid.NewV4()).String() @@ -152,7 +154,9 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { log.Err(err)) s.shellTask.UpdateFailureMessage(err.Error()) s.shellTask.WithQueuedAt(queuedAt) + res.Status = queue.Fail + return res } // Substitute current task with Synchronization tasks for the main queue. diff --git a/pkg/task/tasks/global-hook-run/task.go b/pkg/task/tasks/global-hook-run/task.go index 54ca26728..9c24d2542 100644 --- a/pkg/task/tasks/global-hook-run/task.go +++ b/pkg/task/tasks/global-hook-run/task.go @@ -108,16 +108,19 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { isSynchronization := hm.IsSynchronization() shouldRunHook := true + if isSynchronization { // Synchronization is not a part of v0 contract, skip hook execution. if taskHook.GetHookConfig().Version == "v0" { s.logger.Info("Execute on Synchronization ignored for v0 hooks") + shouldRunHook = false res.Status = queue.Success } // Check for "executeOnSynchronization: false". if !hm.ExecuteOnSynchronization { s.logger.Info("Execute on Synchronization disabled in hook config: ExecuteOnSynchronization=false") + shouldRunHook = false res.Status = queue.Success } @@ -310,6 +313,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { // Unlock Kubernetes events for all monitors when Synchronization task is done. s.logger.Debug("Synchronization done, unlock Kubernetes events") + for _, monitorID := range hm.MonitorIDs { taskHook.GetHookController().UnlockKubernetesEventsFor(monitorID) } diff --git a/pkg/task/tasks/module-hook-run/task.go b/pkg/task/tasks/module-hook-run/task.go index 24a997ae3..53db6438a 100644 --- a/pkg/task/tasks/module-hook-run/task.go +++ b/pkg/task/tasks/module-hook-run/task.go @@ -156,6 +156,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { if hm.WaitForSynchronization != thm.WaitForSynchronization { return true } + if hm.ExecuteOnSynchronization != thm.ExecuteOnSynchronization { return true } @@ -169,6 +170,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { baseModule.Synchronization().DoneForBinding(thm.KubernetesBindingId) } + return false // do not stop combine process on this task }) @@ -202,6 +204,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { if err != nil { if hm.AllowFailure { allowed = 1.0 + s.logger.Info("Module hook failed, but allowed to fail.", log.Err(err)) res.Status = queue.Success @@ -227,11 +230,13 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { s.logger.Debug("Module hook success", slog.String(pkg.LogKeyName, hm.HookName)) res.Status = queue.Success + s.moduleManager.UpdateModuleHookStatusAndNotify(baseModule, hm.HookName, nil) // Handle module values change. reloadModule := false eventDescription := "" + switch hm.BindingType { case htypes.Schedule: if beforeChecksum != afterChecksum { @@ -253,6 +258,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { } } } + if reloadModule { // relabel logLabels := s.shellTask.GetLogLabels() diff --git a/pkg/task/tasks/module-run/task.go b/pkg/task/tasks/module-run/task.go index 46d67727d..01ade299a 100644 --- a/pkg/task/tasks/module-run/task.go +++ b/pkg/task/tasks/module-run/task.go @@ -129,10 +129,12 @@ func (s *Task) Handle(ctx context.Context) (res queue.TaskResult) { //nolint:non })() var moduleRunErr error + valuesChanged := false defer func(res *queue.TaskResult, valuesChanged *bool) { s.moduleManager.UpdateModuleLastErrorAndNotify(baseModule, moduleRunErr) + if moduleRunErr != nil { res.Status = queue.Fail @@ -198,6 +200,7 @@ func (s *Task) Handle(ctx context.Context) (res queue.TaskResult) { //nolint:non if moduleRunErr == nil { s.moduleManager.SetModulePhaseAndNotify(baseModule, modules.OnStartupDone) } + treg.End() } else { s.moduleManager.SetModulePhaseAndNotify(baseModule, modules.OnStartupDone) @@ -213,6 +216,7 @@ func (s *Task) Handle(ctx context.Context) (res queue.TaskResult) { //nolint:non span.AddEvent("module on startup done") s.logger.Debug("ModuleRun phase", slog.String(pkg.LogKeyPhase, string(baseModule.GetPhase()))) + if baseModule.HasKubernetesHooks() { s.moduleManager.SetModulePhaseAndNotify(baseModule, modules.QueueSynchronizationTasks) } else { @@ -261,6 +265,7 @@ func (s *Task) Handle(ctx context.Context) (res queue.TaskResult) { //nolint:non delete(taskLogLabels, pkg.LogKeyTaskID) kubernetesBindingID := uuid.Must(uuid.NewV4()).String() + parallelRunMetadata := &task.ParallelRunMetadata{} if hm.ParallelRunMetadata != nil && len(hm.ParallelRunMetadata.ChannelId) != 0 { parallelRunMetadata.ChannelId = hm.ParallelRunMetadata.ChannelId @@ -344,7 +349,9 @@ func (s *Task) Handle(ctx context.Context) (res queue.TaskResult) { //nolint:non if len(mainSyncTasks) > 0 { res.AddHeadTasks(mainSyncTasks...) res.Status = queue.Keep + s.logTaskAdd("head", mainSyncTasks...) + return res } } @@ -372,6 +379,7 @@ func (s *Task) Handle(ctx context.Context) (res queue.TaskResult) { //nolint:non slog.Bool(pkg.LogKeySyncDone, baseModule.Synchronization().IsCompleted())) baseModule.Synchronization().DebugDumpState(s.logger) } + s.logger.Debug("Synchronization not completed, keep ModuleRun task in repeat mode") s.shellTask.WithQueuedAt(time.Now()) } diff --git a/pkg/task/tasks/parallel-module-run/task.go b/pkg/task/tasks/parallel-module-run/task.go index f8e73fc1a..174f8b0b0 100644 --- a/pkg/task/tasks/parallel-module-run/task.go +++ b/pkg/task/tasks/parallel-module-run/task.go @@ -86,7 +86,9 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { if hm.ParallelRunMetadata == nil { s.logger.Error("Possible bug! Couldn't get task ParallelRunMetadata for a parallel task.", slog.String(pkg.LogKeyDescription, hm.EventDescription)) + res.Status = queue.Fail + return res } @@ -126,6 +128,7 @@ func (s *Task) Handle(ctx context.Context) queue.TaskResult { // map to hold modules' errors tasksErrors := make(map[string]string) + L: for { select { @@ -143,6 +146,7 @@ L: continue L } + if parallelEvent.Succeeded() { hm.ParallelRunMetadata.DeleteModuleMetadata(parallelEvent.ModuleName()) @@ -171,6 +175,7 @@ L: s.parallelTaskChannels.Delete(s.shellTask.GetId()) t := time.NewTimer(time.Second * 3) + for { select { // wait for several seconds if any ModuleRun task wants to send an event diff --git a/pkg/utils/fschecksum.go b/pkg/utils/fschecksum.go index 27755955c..e2b149435 100644 --- a/pkg/utils/fschecksum.go +++ b/pkg/utils/fschecksum.go @@ -11,10 +11,13 @@ import ( func CalculateStringsChecksum(stringArr ...string) string { hasher := md5.New() + sort.Strings(stringArr) + for _, value := range stringArr { _, _ = hasher.Write([]byte(value)) } + return hex.EncodeToString(hasher.Sum(nil)) } @@ -23,6 +26,7 @@ func CalculateChecksumOfFile(path string) (string, error) { if err != nil { return "", err } + return CalculateStringsChecksum(string(content)), nil } @@ -30,14 +34,17 @@ func CalculateChecksumOfDirectory(dir string) (string, error) { res := "" var checkErr error + files, err := FilesFromRoot(dir, func(dir string, name string, _ os.FileInfo) bool { fPath := path.Join(dir, name) + checksum, err := CalculateChecksumOfFile(fPath) if err != nil { // return only bad files for logging checkErr = err return true } + res = CalculateStringsChecksum(res, checksum) // good files are skipped return false @@ -45,6 +52,7 @@ func CalculateChecksumOfDirectory(dir string) (string, error) { if err != nil { return "", err } + if checkErr != nil { return "", fmt.Errorf("calculate checksum of %+v: %v", files, err) } @@ -71,6 +79,7 @@ func CalculateChecksumOfPaths(paths ...string) (string, error) { if err != nil { return "", err } + res = CalculateStringsChecksum(res, checksum) } diff --git a/pkg/utils/fswalk.go b/pkg/utils/fswalk.go index 205d77eec..7d5dae14d 100644 --- a/pkg/utils/fswalk.go +++ b/pkg/utils/fswalk.go @@ -41,6 +41,7 @@ func FilesFromRoot(root string, filterFn func(dir string, name string, info os.F if err != nil { return nil, err } + if len(symlinkedDirs) == 0 { return files, nil } @@ -63,6 +64,7 @@ func FilesFromRoot(root string, filterFn func(dir string, name string, info os.F if err != nil { return nil, err } + for k, v := range symlinked { newSymlinkedDirs[k] = v } @@ -126,6 +128,7 @@ func WalkSymlinks(target string, linkName string, files map[string]map[string]st if err != nil { return err } + if target != "" && isDir { // symlink to directory -> save it for future listing symlinkedDirectories[resPath] = target @@ -135,11 +138,13 @@ func WalkSymlinks(target string, linkName string, files map[string]map[string]st // Walk found a file or symlink to file, just store it. // FIXME symlink can have +x, but target file is not, so filterFn is not working properly fDir := path.Dir(resPath) + fName := path.Base(resPath) if filterFn == nil || filterFn(fDir, fName, info) { if _, has := files[fDir]; !has { files[fDir] = map[string]string{} } + files[fDir][fName] = "" } @@ -162,7 +167,9 @@ func FindExecutableFilesInPath(dir string) ([]string, []string, error) { if info.Mode()&0o111 != 0 { return true } + nonExecutables = append(nonExecutables, path.Join(dir, name)) + return false }) if err != nil { diff --git a/pkg/utils/helpers.go b/pkg/utils/helpers.go index 2fa4c8421..8ce3e5b54 100644 --- a/pkg/utils/helpers.go +++ b/pkg/utils/helpers.go @@ -7,6 +7,7 @@ func DumpData(filePath string, data []byte) error { if err != nil { return err } + return nil } @@ -17,5 +18,6 @@ func CreateEmptyWritableFile(filePath string) error { } _ = file.Close() + return nil } diff --git a/pkg/utils/jsonpatch.go b/pkg/utils/jsonpatch.go index 407a624b5..005758dae 100644 --- a/pkg/utils/jsonpatch.go +++ b/pkg/utils/jsonpatch.go @@ -386,6 +386,7 @@ func (p Patch) test(doc *container, op Operation) error { if op.value().IsRawEmpty() { return nil } + return errors.Wrapf(ErrTestFailed, "testing value %s failed", path) } else if op.value() == nil { return errors.Wrapf(ErrTestFailed, "testing value %s failed", path) diff --git a/pkg/utils/loader.go b/pkg/utils/loader.go index d7f5c718f..a90fa2df4 100644 --- a/pkg/utils/loader.go +++ b/pkg/utils/loader.go @@ -21,26 +21,32 @@ const ( // SplitToPaths split concatenated dirs to the array func SplitToPaths(dir string) []string { res := make([]string, 0) + paths := strings.Split(dir, PathsSeparator) for _, path := range paths { if path == "" { continue } + res = append(res, path) } + return res } // LoadValuesFileFromDir finds and parses values.yaml files in the specified directory. func LoadValuesFileFromDir(dir string, strictModeEnabled bool) (Values, error) { valuesFilePath := filepath.Join(dir, ValuesFileName) + valuesYaml, err := os.ReadFile(valuesFilePath) if err != nil && os.IsNotExist(err) && !strictModeEnabled { log.Debug("No static values file", slog.String(pkg.LogKeyPath, valuesFilePath), log.Err(err)) + return nil, nil } + if err != nil { return nil, fmt.Errorf("load values file '%s': %s", valuesFilePath, err) } @@ -67,11 +73,13 @@ func ReadOpenAPIFiles(openApiDir string) ([]byte /*configValuesBytes*/, []byte / if openApiDir == "" { return nil, nil, nil } + if _, err := os.Stat(openApiDir); os.IsNotExist(err) { return nil, nil, nil } configValuesBytes := make([]byte, 0) + configPath := filepath.Join(openApiDir, ConfigValuesFileName) if _, err := os.Stat(configPath); !os.IsNotExist(err) { configValuesBytes, err = os.ReadFile(configPath) @@ -81,6 +89,7 @@ func ReadOpenAPIFiles(openApiDir string) ([]byte /*configValuesBytes*/, []byte / } valuesBytes := make([]byte, 0) + valuesPath := filepath.Join(openApiDir, ValuesFileName) if _, err := os.Stat(valuesPath); !os.IsNotExist(err) { valuesBytes, err = os.ReadFile(valuesPath) diff --git a/pkg/utils/merge_labels.go b/pkg/utils/merge_labels.go index d1226dc3f..9b52e9239 100644 --- a/pkg/utils/merge_labels.go +++ b/pkg/utils/merge_labels.go @@ -11,11 +11,13 @@ import ( // Can be used to copy a map if just one argument is used. func MergeLabels(labelsMaps ...map[string]string) map[string]string { labels := make(map[string]string) + for _, labelsMap := range labelsMaps { for k, v := range labelsMap { labels[k] = v } } + return labels } diff --git a/pkg/utils/mergemap.go b/pkg/utils/mergemap.go index 31fc8e89d..00b5ea2d1 100644 --- a/pkg/utils/mergemap.go +++ b/pkg/utils/mergemap.go @@ -33,6 +33,7 @@ func merge(dst, src map[string]interface{}, depth int) map[string]interface{} { dst[key] = srcVal } } + return dst } @@ -50,7 +51,9 @@ func mapify(i interface{}) (map[string]interface{}, bool) { for _, k := range value.MapKeys() { m[k.String()] = value.MapIndex(k).Interface() } + return m, true } + return map[string]interface{}{}, false } diff --git a/pkg/utils/module_config.go b/pkg/utils/module_config.go index e0cd5a88c..daff8c642 100644 --- a/pkg/utils/module_config.go +++ b/pkg/utils/module_config.go @@ -62,6 +62,7 @@ func (mc *ModuleConfig) GetEnabled() string { if mc == nil { return "" } + switch { case mc.IsEnabled == nil: return "n/d" @@ -84,6 +85,7 @@ func NewModuleConfig(moduleName string, values Values) *ModuleConfig { if values == nil { values = make(Values) } + return &ModuleConfig{ ModuleName: moduleName, IsEnabled: nil, @@ -143,6 +145,7 @@ func (mc *ModuleConfig) LoadFromValues(values Values) (*ModuleConfig, error) { if err != nil { return nil, err } + mc.values = values default: return nil, fmt.Errorf("load '%s' values: module config should be array or map. Got: %s", mc.ModuleName, spew.Sdump(moduleValuesData)) @@ -215,7 +218,9 @@ func ModuleEnabledValue(i interface{}) (*bool, error) { if v { return &ModuleEnabled, nil } + return &ModuleDisabled, nil } + return nil, fmt.Errorf("unsupported module enabled value: %v", i) } diff --git a/pkg/utils/module_list.go b/pkg/utils/module_list.go index af9781c23..5ff627d19 100644 --- a/pkg/utils/module_list.go +++ b/pkg/utils/module_list.go @@ -13,6 +13,7 @@ func SortReverseByReference(in []string, ref []string) []string { for _, v := range in { inValues[v] = true } + for _, v := range ref { if _, hasKey := inValues[v]; hasKey { // prepend @@ -40,6 +41,7 @@ func SortByReference(in []string, ref []string) []string { for _, v := range in { inValues[v] = true } + for _, v := range ref { if _, hasKey := inValues[v]; hasKey { res = append(res, v) @@ -67,6 +69,7 @@ func KeysSortedByReference(m map[string]struct{}, ref []string) []string { // not present in 'ignored' arrays. func ListSubtract(src []string, ignored ...[]string) []string { ignoredMap := make(map[string]bool) + for _, arr := range ignored { for _, v := range arr { ignoredMap[v] = true @@ -91,6 +94,7 @@ func ListIntersection(arrs ...[]string) []string { // Counts each item in arrays. m := make(map[string]int) + for _, a := range arrs { for _, v := range a { m[v]++ diff --git a/pkg/utils/values.go b/pkg/utils/values.go index 60ab21902..aea806943 100644 --- a/pkg/utils/values.go +++ b/pkg/utils/values.go @@ -88,6 +88,7 @@ func (v Values) DebugString() string { if err != nil { return "bad values: " + err.Error() } + return string(b) } @@ -107,6 +108,7 @@ func (v Values) GetKeySection(key string) Values { if !has { return Values{} } + switch sec := section.(type) { case map[string]interface{}: return sec @@ -127,12 +129,15 @@ func (v Values) Global() Values { globalValues, has := v[GlobalValuesKey] if has { data := map[string]interface{}{GlobalValuesKey: globalValues} + newV, err := NewValues(data) if err != nil { log.Error("get global Values", log.Err(err)) } + return newV } + return make(Values) } @@ -141,14 +146,17 @@ func (v Values) SectionByKey(key string) Values { sectionValues, has := v[key] if has { data := map[string]interface{}{key: sectionValues} + newV, err := NewValues(data) if err != nil { log.Error("get section Values", slog.String(pkg.LogKeyKey, key), log.Err(err)) } + return newV } + return make(Values) } @@ -199,6 +207,7 @@ func deepCopyMap(originalMap map[string]interface{}) map[string]interface{} { for key, value := range originalMap { copiedMap[key] = valueDeepCopy(value) } + return copiedMap } @@ -214,6 +223,7 @@ func valueDeepCopy(item interface{}) interface{} { case reflect.Ptr: newVal := reflect.New(typ.Elem()) newVal.Elem().Set(reflect.ValueOf(valueDeepCopy(val.Elem().Interface()))) + return newVal.Interface() case reflect.Map: @@ -221,6 +231,7 @@ func valueDeepCopy(item interface{}) interface{} { for _, k := range val.MapKeys() { newMap.SetMapIndex(k, reflect.ValueOf(valueDeepCopy(val.MapIndex(k).Interface()))) } + return newMap.Interface() case reflect.Slice: @@ -228,6 +239,7 @@ func valueDeepCopy(item interface{}) interface{} { for i := 0; i < val.Len(); i++ { newSlice.Index(i).Set(reflect.ValueOf(valueDeepCopy(val.Index(i).Interface()))) } + return newSlice.Interface() } diff --git a/pkg/utils/values_patch.go b/pkg/utils/values_patch.go index 2027fc005..df23a2d50 100644 --- a/pkg/utils/values_patch.go +++ b/pkg/utils/values_patch.go @@ -37,10 +37,12 @@ func (p *ValuesPatch) ToJSONPatch() (patch.Patch, error) { if err != nil { return nil, err } + patch, err := sdkutils.DecodePatch(data) if err != nil { return nil, err } + return patch, nil } @@ -53,6 +55,7 @@ func (p *ValuesPatch) ApplyStrict(doc []byte) ([]byte, error) { if err != nil { return nil, err } + return patch.Apply(doc) } @@ -70,16 +73,19 @@ func (p *ValuesPatch) ApplyIgnoreNonExistentPaths(doc []byte) ([]byte, error) { if err != nil { return nil, err } + pd, err = patch.ApplyContainer(pd) // Ignore errors for remove operation. if op.Op == "remove" && IsNonExistentPathError(err) { continue } + if err != nil { return nil, err } } + return json.Marshal(pd) } @@ -87,6 +93,7 @@ func (p *ValuesPatch) MergeOperations(src *ValuesPatch) { if src == nil { return } + p.Operations = append(p.Operations, src.Operations...) } @@ -94,6 +101,7 @@ func JsonPatchFromReader(r io.Reader) (Patch, error) { operations := make([]Operation, 0) dec := json.NewDecoder(r) + for { var jsonStreamItem interface{} if err := dec.Decode(&jsonStreamItem); err == io.EOF { @@ -109,6 +117,7 @@ func JsonPatchFromReader(r io.Reader) (Patch, error) { if err != nil { return nil, err } + operations = append(operations, operation) } case map[string]interface{}: @@ -116,6 +125,7 @@ func JsonPatchFromReader(r io.Reader) (Patch, error) { if err != nil { return nil, err } + operations = append(operations, operation) } } @@ -138,10 +148,12 @@ func DecodeJsonPatchOperation(v interface{}) (Operation, error) { } var res Operation + err = json.Unmarshal(data, &res) if err != nil { return nil, fmt.Errorf("unmarshal operation from bytes: %s", err) } + return res, nil } @@ -207,6 +219,7 @@ func CompactPatches(existedOperations []*sdkutils.ValuesPatchOperation, newOpera if _, ok := patchesTree[op.Path]; !ok { patchesTree[op.Path] = make([]*sdkutils.ValuesPatchOperation, 0) } + patchesTree[op.Path] = append(patchesTree[op.Path], op) } @@ -252,6 +265,7 @@ func CompactPatches(existedOperations []*sdkutils.ValuesPatchOperation, newOpera for path := range patchesTree { paths = append(paths, path) } + sort.Strings(paths) newOps := make([]*sdkutils.ValuesPatchOperation, 0, len(paths)) @@ -260,6 +274,7 @@ func CompactPatches(existedOperations []*sdkutils.ValuesPatchOperation, newOpera } newValuesPatch := ValuesPatch{Operations: newOps} + return newValuesPatch } @@ -286,12 +301,14 @@ func ApplyValuesPatch(values Values, valuesPatch ValuesPatch, mode ApplyPatchMod } var resJSONDoc []byte + switch mode { case Strict: resJSONDoc, err = valuesPatch.ApplyStrict(jsonDoc) case IgnoreNonExistentPaths: resJSONDoc, err = valuesPatch.ApplyIgnoreNonExistentPaths(jsonDoc) } + if err != nil { return nil, false, err } @@ -335,6 +352,7 @@ func ValidateHookValuesPatch(valuesPatch ValuesPatch, permittedRootKey string) e if permittedRootKey == GlobalValuesKey { permittedMessage = fmt.Sprintf("only '%s' and '*Enabled' are permitted", permittedRootKey) } + return fmt.Errorf("unacceptable patch operation for path '%s' (%s): '%s'", rootKey, permittedMessage, op.ToString()) } } @@ -356,6 +374,7 @@ func FilterValuesPatch(valuesPatch ValuesPatch, rootPath string) ValuesPatch { } newValuesPatch := ValuesPatch{Operations: resOps} + return newValuesPatch } @@ -373,6 +392,7 @@ func EnabledFromValuesPatch(valuesPatch ValuesPatch) ValuesPatch { } newValuesPatch := ValuesPatch{Operations: resOps} + return newValuesPatch } @@ -386,13 +406,16 @@ func IsNonExistentPathError(err error) bool { if err == nil { return false } + errStr := err.Error() if strings.HasPrefix(errStr, NonExistentPathErrorMsg) { return true } + if strings.HasPrefix(errStr, MissingPathErrorMsg) { return true } + return false } diff --git a/pkg/values/validation/cel/cel.go b/pkg/values/validation/cel/cel.go index 14c999a91..c546496f7 100644 --- a/pkg/values/validation/cel/cel.go +++ b/pkg/values/validation/cel/cel.go @@ -28,6 +28,7 @@ func Validate(schema *spec.Schema, values any) ([]error, error) { if err != nil { return nil, err } + validationErrs = append(validationErrs, subErrs...) } } @@ -38,10 +39,12 @@ func Validate(schema *spec.Schema, values any) ([]error, error) { if len(validationErrs) > 0 { return validationErrs, nil } + return nil, nil } var rules []rule + switch v := raw.(type) { case []any: for _, entry := range v { @@ -53,6 +56,7 @@ func Validate(schema *spec.Schema, values any) ([]error, error) { if val, ok := mapEntry["expression"]; !ok || len(val.(string)) == 0 { return nil, fmt.Errorf("x-deckhouse-validations invalid: missing expression") } + if val, ok := mapEntry["message"]; !ok || len(val.(string)) == 0 { return nil, fmt.Errorf("x-deckhouse-validations invalid: missing message") } @@ -75,6 +79,7 @@ func Validate(schema *spec.Schema, values any) ([]error, error) { if err != nil { return nil, fmt.Errorf("create CEL env: %w", err) } + for _, r := range rules { ast, issues := env.Compile(r.Expression) if issues.Err() != nil { @@ -91,6 +96,7 @@ func Validate(schema *spec.Schema, values any) ([]error, error) { if strings.Contains(err.Error(), "no such key:") { continue } + return nil, fmt.Errorf("evaluate the '%s' rule: %w", r.Expression, err) } @@ -98,6 +104,7 @@ func Validate(schema *spec.Schema, values any) ([]error, error) { if !ok { return nil, errors.New("rule should return boolean") } + if !pass { validationErrs = append(validationErrs, errors.New(r.Message)) } @@ -113,18 +120,21 @@ func buildCELValueAndType(value any) (*cel.Type, any, error) { if err != nil { return nil, nil, fmt.Errorf("convert values to struct: %w", err) } + return cel.MapType(cel.StringType, cel.DynType), obj, nil case []any: list, err := structpb.NewList(v) if err != nil { return nil, nil, fmt.Errorf("convert array to list: %w", err) } + return cel.ListType(cel.DynType), list, nil default: val, err := structpb.NewValue(v) if err != nil { return nil, nil, fmt.Errorf("convert dyn to value: %w", err) } + return cel.DynType, val, nil } } diff --git a/pkg/values/validation/defaulting.go b/pkg/values/validation/defaulting.go index 831fb8631..be60c89d8 100644 --- a/pkg/values/validation/defaulting.go +++ b/pkg/values/validation/defaulting.go @@ -36,6 +36,7 @@ func ApplyDefaults(obj interface{}, s *spec.Schema) bool { if prop.Default == nil { continue } + if _, found := obj[k]; !found { obj[k] = runtime.DeepCopyJSONValue(prop.Default) res = true diff --git a/pkg/values/validation/schema/additional-properties.go b/pkg/values/validation/schema/additional-properties.go index 9b5dfc190..6988c2633 100644 --- a/pkg/values/validation/schema/additional-properties.go +++ b/pkg/values/validation/schema/additional-properties.go @@ -32,6 +32,7 @@ func (t *AdditionalPropertiesTransformer) Transform(s *spec.Schema) *spec.Schema if s.Items.Schema != nil { s.Items.Schema = t.Transform(s.Items.Schema) } + for i, item := range s.Items.Schemas { ts := item s.Items.Schemas[i] = *t.Transform(&ts) diff --git a/pkg/values/validation/schema/copy.go b/pkg/values/validation/schema/copy.go index 06913354c..020bd18a2 100644 --- a/pkg/values/validation/schema/copy.go +++ b/pkg/values/validation/schema/copy.go @@ -8,5 +8,6 @@ func (t *CopyTransformer) Transform(s *spec.Schema) *spec.Schema { tmpBytes, _ := s.MarshalJSON() res := new(spec.Schema) _ = res.UnmarshalJSON(tmpBytes) + return res } diff --git a/pkg/values/validation/schema/extend.go b/pkg/values/validation/schema/extend.go index fe15a14e3..6a6539ddd 100644 --- a/pkg/values/validation/schema/extend.go +++ b/pkg/values/validation/schema/extend.go @@ -59,6 +59,7 @@ func ExtractExtendSettings(s *spec.Schema) *ExtendSettings { res := new(ExtendSettings) _ = json.Unmarshal(tmpBytes, res) + return res } @@ -86,6 +87,7 @@ func mergeProperties(s *spec.Schema, parent *spec.Schema) map[string]spec.Schema for k, v := range parent.Properties { res[k] = v } + for k, v := range s.Properties { res[k] = v } @@ -99,6 +101,7 @@ func mergePatternProperties(s *spec.Schema, parent *spec.Schema) map[string]spec for k, v := range parent.PatternProperties { res[k] = v } + for k, v := range s.PatternProperties { res[k] = v } @@ -112,6 +115,7 @@ func mergeDefinitions(s *spec.Schema, parent *spec.Schema) spec.Definitions { for k, v := range parent.Definitions { res[k] = v } + for k, v := range s.Definitions { res[k] = v } @@ -125,6 +129,7 @@ func mergeExtensions(s *spec.Schema, parent *spec.Schema) spec.Extensions { for k, v := range parent.Extensions { ext.Add(k, v) } + for k, v := range s.Extensions { ext.Add(k, v) } @@ -136,6 +141,7 @@ func mergeTitle(s *spec.Schema, parent *spec.Schema) string { if s.Title != "" { return s.Title } + return parent.Title } @@ -143,5 +149,6 @@ func mergeDescription(s *spec.Schema, parent *spec.Schema) string { if s.Description != "" { return s.Description } + return parent.Description } diff --git a/pkg/values/validation/schema/required-for-helm.go b/pkg/values/validation/schema/required-for-helm.go index 0b954fec0..86d59bb29 100644 --- a/pkg/values/validation/schema/required-for-helm.go +++ b/pkg/values/validation/schema/required-for-helm.go @@ -17,6 +17,7 @@ func (t *RequiredForHelmTransformer) Transform(s *spec.Schema) *spec.Schema { // Deep transform. transformRequired(s.Properties) + return s } @@ -31,24 +32,29 @@ func transformRequired(props map[string]spec.Schema) { func MergeArrays(ar1 []string, ar2 []string) []string { res := make([]string, 0) m := make(map[string]struct{}) + for _, item := range ar1 { res = append(res, item) m[item] = struct{}{} } + for _, item := range ar2 { if _, ok := m[item]; !ok { res = append(res, item) } } + return res } func MergeRequiredFields(ext spec.Extensions, required []string) []string { var xReqFields []string + _, hasField := ext[XRequiredForHelm] if !hasField { return required } + field, ok := ext.GetString(XRequiredForHelm) if ok { xReqFields = []string{field} diff --git a/pkg/values/validation/schema/transform.go b/pkg/values/validation/schema/transform.go index c46faa048..d4d533922 100644 --- a/pkg/values/validation/schema/transform.go +++ b/pkg/values/validation/schema/transform.go @@ -10,5 +10,6 @@ func TransformSchema(s *spec.Schema, transformers ...SchemaTransformer) *spec.Sc for _, transformer := range transformers { s = transformer.Transform(s) } + return s } diff --git a/pkg/values/validation/schemas.go b/pkg/values/validation/schemas.go index 269d9b542..fb8e10ee2 100644 --- a/pkg/values/validation/schemas.go +++ b/pkg/values/validation/schemas.go @@ -94,6 +94,7 @@ func (st *SchemaStorage) Validate(valuesType SchemaType, moduleName string, valu log.Warn("schema is not found", slog.String(pkg.LogKeyModule, moduleName), slog.String(pkg.LogKeyValuesType, string(valuesType))) + return nil } @@ -111,6 +112,7 @@ func (st *SchemaStorage) Validate(valuesType SchemaType, moduleName string, valu slog.String(pkg.LogKeyValuesType, string(valuesType)), log.Err(validationErr)) } + return validationErr } @@ -140,6 +142,7 @@ func validateObject(dataObj interface{}, s *spec.Schema, rootName string) error if err != nil { return err } + if len(validationErrs) > 0 { return errors.Join(validationErrs...) } @@ -202,6 +205,7 @@ func (st *SchemaStorage) ModuleSchemasDescription() string { if len(types) == 0 { return "No OpenAPI schemas" } + return fmt.Sprintf("OpenAPI schemas: %s.", strings.Join(types, ", ")) } @@ -211,6 +215,7 @@ func (st *SchemaStorage) GlobalSchemasDescription() string { if len(types) == 0 { return "No Global OpenAPI schemas" } + return fmt.Sprintf("Global OpenAPI schemas: %s.", strings.Join(types, ", ")) } @@ -223,12 +228,15 @@ func availableSchemaTypes(schemas map[SchemaType]*spec.Schema) []string { if _, has := schemas[ConfigValuesSchema]; has { types = append(types, "config values") } + if _, has := schemas[ValuesSchema]; has { types = append(types, "values") } + if _, has := schemas[HelmValuesSchema]; has { types = append(types, "helm values") } + return types } @@ -236,6 +244,7 @@ func availableSchemaTypes(schemas map[SchemaType]*spec.Schema) []string { // swag.BytesToYAML uses yaml.MapSlice to unmarshal YAML. This type doesn't support map merge of YAML anchors. func YAMLBytesToJSONDoc(data []byte) (json.RawMessage, error) { var yamlObj interface{} + err := yaml.Unmarshal(data, &yamlObj) if err != nil { return nil, fmt.Errorf("yaml unmarshal: %v", err) @@ -272,11 +281,13 @@ func LoadSchemaFromBytes(openApiContent []byte) (*spec.Schema, error) { // PrepareSchemas loads schemas for config values, values and helm values. func PrepareSchemas(configBytes, valuesBytes []byte) (map[SchemaType]*spec.Schema, error) { res := make(map[SchemaType]*spec.Schema) + if len(configBytes) > 0 { schemaObj, err := LoadSchemaFromBytes(configBytes) if err != nil { return nil, fmt.Errorf("load '%s' schema: %w", ConfigValuesSchema, err) } + res[ConfigValuesSchema] = schema.TransformSchema( schemaObj, &schema.AdditionalPropertiesTransformer{}, diff --git a/sdk/registry.go b/sdk/registry.go index 9395aed69..dfa1fef09 100644 --- a/sdk/registry.go +++ b/sdk/registry.go @@ -48,6 +48,7 @@ func Registry() *HookRegistry { embeddedModuleHooks: make(map[string][]*kind.GoHook), } }) + return instance } @@ -83,20 +84,24 @@ func (h *HookRegistry) Add(hook *kind.GoHook) { hookMeta := &gohook.HookMetadata{} pc := make([]uintptr, 50) + n := runtime.Callers(0, pc) if n == 0 { panic("runtime.Callers is empty") } + pc = pc[:n] // pass only valid pcs to runtime.CallersFrames frames := runtime.CallersFrames(pc) for { frame, more := frames.Next() + matches := globalRe.FindStringSubmatch(frame.File) if matches != nil { hookMeta.Global = true hookMeta.Name = matches[2] hookMeta.Path = matches[1] + break } @@ -105,10 +110,12 @@ func (h *HookRegistry) Add(hook *kind.GoHook) { hookMeta.EmbeddedModule = true hookMeta.Name = matches[2] hookMeta.Path = matches[1] + modNameMatches := moduleNameRe.FindStringSubmatch(matches[3]) if modNameMatches != nil { hookMeta.ModuleName = modNameMatches[1] } + break }