[fix] helm fallback: resource adoption, werf-chart guard and telemetry

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.11.1
 	github.com/tidwall/gjson v1.19.0
-	github.com/werf/nelm v1.26.1
+	github.com/werf/nelm v1.26.2
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.19.5

go.sum

@@ -548,8 +548,8 @@ github.com/werf/lockgate v0.1.1 h1:S400JFYjtWfE4i4LY9FA8zx0fMdfui9DPrBiTciCrx4=
 github.com/werf/lockgate v0.1.1/go.mod h1:0yIFSLq9ausy6ejNxF5uUBf/Ib6daMAfXuCaTMZJzIE=
 github.com/werf/logboek v0.6.1 h1:oEe6FkmlKg0z0n80oZjLplj6sXcBeLleCkjfOOZEL2g=
 github.com/werf/logboek v0.6.1/go.mod h1:Gez5J4bxekyr6MxTmIJyId1F61rpO+0/V4vjCIEIZmk=
-github.com/werf/nelm v1.26.1 h1:0gH8JuJlKZtPLpeRAt4H9mVLRzDQwNqhcllxcr2UTDg=
-github.com/werf/nelm v1.26.1/go.mod h1:D3uU5e1dSBc0l0oo/iB6SjM2sLaXICqRDSWXAYF+vpk=
+github.com/werf/nelm v1.26.2 h1:HKTYpMHGHecfQnHJ3QyK9zu+d3G3GoSjFWXZxhHIVco=
+github.com/werf/nelm v1.26.2/go.mod h1:D3uU5e1dSBc0l0oo/iB6SjM2sLaXICqRDSWXAYF+vpk=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

pkg/helm/fallback_client.go

@@ -1,8 +1,13 @@
 package helm
 
 import (
+	"bytes"
 	"errors"
+	"fmt"
+	"io/fs"
 	"log/slog"
+	"os"
+	"path/filepath"
 
 	"github.com/deckhouse/deckhouse/pkg/log"
 	"github.com/werf/nelm/pkg/action"
@@ -30,11 +35,15 @@ const (
 // that helm3lib is able to recover from: action.ErrBuildPlan and
 // resource.ErrResourceDuplicatesFound.
 //
+// Upgrades of charts that use werf.io/* annotations are never handed to helm3lib: it does
+// not implement those semantics, so nelm's error is reported instead of a wrong deploy.
+//
 // Read-only methods are always served by the primary client, both clients receive
 // label/annotation updates so the fallback is ready to take over at any moment.
 //
-// When MetricStorage is configured the client increments
-// metrics.HelmFallbackTotal{module, operation, error_type} on every fallback.
+// When MetricStorage is configured the client increments both
+// metrics.HelmFallbackTotal{module, operation, error_type} and its telemetry mirror on
+// every fallback.
 type FallbackClient struct {
 	primary       client.HelmClient
 	fallback      client.HelmClient
@@ -65,6 +74,67 @@ func shouldFallback(err error) bool {
 	return errors.Is(err, action.ErrBuildPlan) || errors.Is(err, resource.ErrResourceDuplicatesFound)
 }
 
+// werfAnnotationPrefix marks resources whose deploy semantics only nelm implements:
+// ordering (werf.io/weight, werf.io/deploy-dependency-*), tracking
+// (werf.io/track-termination-mode) and the like. helm3lib ignores such annotations.
+const werfAnnotationPrefix = "werf.io/"
+
+// chartRenderSources are the parts of a chart helm renders manifests from, and therefore the
+// only places an annotation can reach the release from. Hooks and image sources are left out
+// on purpose: a module's hook is a compiled binary, so reading it would be slow and would
+// match werf.io/ inside compiled code, disabling the fallback for a chart that never asked
+// for werf semantics.
+var chartRenderSources = []string{"Chart.yaml", "values.yaml", "templates", "charts"}
+
+// chartUsesWerfAnnotations reports whether the chart declares a werf.io/* annotation. It
+// scans the raw sources rather than rendered manifests: the check runs on the fallback path,
+// where nelm has just failed, so it must not depend on another render. A false positive only
+// disables the fallback, which is the safe way to be wrong.
+func chartUsesWerfAnnotations(chartPath string) (bool, error) {
+	if _, err := os.Stat(chartPath); err != nil {
+		return false, fmt.Errorf("scan chart for werf annotations: %w", err)
+	}
+
+	found := false
+
+	scan := func(path string, entry fs.DirEntry, err error) error {
+		switch {
+		case errors.Is(err, fs.ErrNotExist):
+			// this part of the chart is optional, nothing to scan
+			return fs.SkipDir
+		case err != nil:
+			return err
+		case entry.IsDir():
+			return nil
+		}
+
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		if bytes.Contains(data, []byte(werfAnnotationPrefix)) {
+			found = true
+
+			return fs.SkipAll
+		}
+
+		return nil
+	}
+
+	for _, source := range chartRenderSources {
+		if err := filepath.WalkDir(filepath.Join(chartPath, source), scan); err != nil {
+			return false, fmt.Errorf("scan chart for werf annotations: %w", err)
+		}
+
+		if found {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
 // fallbackErrorType returns a low-cardinality label value describing which
 // recoverable nelm error triggered the fallback.
 func fallbackErrorType(err error) string {
@@ -78,19 +148,23 @@ func fallbackErrorType(err error) string {
 	}
 }
 
-// recordFallback emits the helm_fallback_total counter when metric storage is
-// configured. releaseName is used as the module label since within addon-operator
-// helm release name and module name are equivalent.
+// recordFallback emits the fallback counters when metric storage is configured. The
+// telemetry mirror carries the same value and labels, it only differs in the prefix
+// flant-integration collects by. releaseName is used as the module label since within
+// addon-operator helm release name and module name are equivalent.
 func (c *FallbackClient) recordFallback(operation, releaseName string, err error) {
 	if c.metricStorage == nil {
 		return
 	}
 
-	c.metricStorage.CounterAdd(metrics.HelmFallbackTotal, 1.0, map[string]string{
+	labels := map[string]string{
 		pkg.MetricKeyModule:    releaseName,
 		pkg.MetricKeyOperation: operation,
 		pkg.MetricKeyErrorType: fallbackErrorType(err),
-	})
+	}
+
+	c.metricStorage.CounterAdd(metrics.HelmFallbackTotal, 1.0, labels)
+	c.metricStorage.CounterAdd(metrics.HelmFallbackTelemetryTotal, 1.0, labels)
 }
 
 func (c *FallbackClient) UpgradeRelease(releaseName, chart string, valuesPaths, setValues []string, releaseLabels map[string]string, namespace string) error {
@@ -99,6 +173,30 @@ func (c *FallbackClient) UpgradeRelease(releaseName, chart string, valuesPaths,
 		return err
 	}
 
+	// helm3lib does not implement werf.io/* semantics, so handing such a chart to it would
+	// deploy the release in a different order and track it differently. Report nelm's error
+	// instead of silently deploying the module wrong.
+	usesWerf, scanErr := chartUsesWerfAnnotations(chart)
+	if scanErr != nil {
+		c.logger.Warn("cannot check chart for werf.io annotations, not falling back to helm",
+			slog.String(pkg.LogKeyRelease, releaseName),
+			slog.String(pkg.LogKeyChart, chart),
+			log.Err(scanErr),
+		)
+
+		return err
+	}
+
+	if usesWerf {
+		c.logger.Warn("chart uses werf.io annotations unsupported by helm, not falling back to helm",
+			slog.String(pkg.LogKeyRelease, releaseName),
+			slog.String(pkg.LogKeyChart, chart),
+			log.Err(err),
+		)
+
+		return err
+	}
+
 	c.recordFallback(fallbackOperationUpgrade, releaseName, err)
 
 	c.logger.Warn("nelm UpgradeRelease failed with a recoverable error, falling back to helm",