feat: allow wildcards as part of annotations and labels in allowlists

docs/developer/cli-arguments.md

@@ -60,9 +60,9 @@ Flags:
       --log_file_max_size uint                               Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
       --logtostderr                                          log to standard error instead of files (default true)
       --metric-allowlist string                              Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.
-      --metric-annotations-allowlist string                  Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').
+      --metric-annotations-allowlist string                  Comma-separated list of Kubernetes annotation keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided for resources, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be use to match multiple annotations, i.e., '=pods=[something.example.org/foo-*]' will resolve to '=pods=[something.example.org/foo-bar,something.example.org/foo-baz,...]'
       --metric-denylist string                               Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.
-      --metric-labels-allowlist string                       Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.
+      --metric-labels-allowlist string                       Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be used to match multiple labels, but only a single '*' is supported per key pattern, i.e., '=pods=[something.example.org/foo-*]' will match labels such as 'something.example.org/foo-bar' and 'something.example.org/foo-baz'.
       --metric-opt-in-list string                            Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists
       --namespaces string                                    Comma-separated list of namespaces to be enabled. Defaults to ""
       --namespaces-denylist string                           Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.

internal/store/utils.go

@@ -178,24 +178,52 @@ func isPrefixedNativeResource(name v1.ResourceName) bool {
 // createPrometheusLabelKeysValues takes in passed kubernetes annotations/labels
 // and associated allowed list in kubernetes label format.
 // It returns only those allowed annotations/labels that exist in the list and converts them to Prometheus labels.
+// Full wildcards (*) can only be set as first in the allow list, partial wildcards can appear anywhere in the list
 func createPrometheusLabelKeysValues(prefix string, allKubeData map[string]string, allowList []string) ([]string, []string) {
 	allowedKubeData := make(map[string]string)
 
-	if len(allowList) > 0 {
-		if allowList[0] == options.LabelWildcard {
-			return kubeMapToPrometheusLabels(prefix, allKubeData)
+	for i, l := range allowList {
+		// only the first label can be the wildcard label
+		if l == options.LabelWildcard {
+			if i == 0 {
+				return kubeMapToPrometheusLabels(prefix, allKubeData)
+			}
+			continue
 		}
 
-		for _, l := range allowList {
-			v, found := allKubeData[l]
-			if found {
-				allowedKubeData[l] = v
+		// prepare regular expression based on potential wildcards
+		re, err := regexp.Compile(expandWildcard(l, options.MaxPartialWildcardsPerLabel))
+		if err != nil {
+			continue
+		}
+		for k, v := range allKubeData {
+			if re.MatchString(k) {
+				allowedKubeData[k] = v
 			}
 		}
 	}
+
 	return kubeMapToPrometheusLabels(prefix, allowedKubeData)
 }
 
+// expandWildcard expands wildcards (*) to regular expressions, up to a limited number of wildcards
+func expandWildcard(pattern string, limit uint) string {
+	var result strings.Builder
+	var replacements uint
+	for i, literal := range strings.Split(pattern, options.LabelWildcard) {
+		if i > 0 {
+			result.WriteString(".*")
+			replacements++
+		}
+
+		result.WriteString(regexp.QuoteMeta(literal))
+		if replacements >= limit {
+			break
+		}
+	}
+	return "^" + result.String() + "$"
+}
+
 // mergeKeyValues merges label keys and values slice pairs into a single slice pair.
 // Arguments are passed as equal-length pairs of slices, where the first slice contains keys and second contains values.
 // Example: mergeKeyValues(keys1, values1, keys2, values2) => (keys1+keys2, values1+values2)

internal/store/utils_test.go

@@ -319,3 +319,169 @@ func TestMergeKeyValues(t *testing.T) {
 		})
 	}
 }
+
+func TestCreatePrometheusLabelKeysValues(t *testing.T) {
+	testCases := []struct {
+		name         string
+		kubeData     map[string]string
+		allowList    []string
+		expectKeys   []string
+		expectValues []string
+	}{
+		{
+			name: "allMatches",
+			kubeData: map[string]string{
+				"keyA": "valueA",
+				"keyB": "valueB",
+			},
+			allowList:    []string{"keyA", "keyB"},
+			expectKeys:   []string{"metric_key_a", "metric_key_b"},
+			expectValues: []string{"valueA", "valueB"},
+		},
+		{
+			name: "additionalAllow",
+			kubeData: map[string]string{
+				"keyA": "valueA",
+				"keyB": "valueB",
+			},
+			allowList:    []string{"keyA", "keyB", "keyC"},
+			expectKeys:   []string{"metric_key_a", "metric_key_b"},
+			expectValues: []string{"valueA", "valueB"},
+		},
+		{
+			name: "partialMatches",
+			kubeData: map[string]string{
+				"keyA": "valueA",
+				"keyB": "valueB",
+			},
+			allowList:    []string{"keyA", "keyC"},
+			expectKeys:   []string{"metric_key_a"},
+			expectValues: []string{"valueA"},
+		},
+		{
+			name: "wildcardAsSuffix",
+			kubeData: map[string]string{
+				"keyA":      "valueA",
+				"keyB":      "valueB",
+				"otherKeyA": "valueC",
+				"otherKeyB": "valueD",
+			},
+			allowList:    []string{"key*"},
+			expectKeys:   []string{"metric_key_a", "metric_key_b"},
+			expectValues: []string{"valueA", "valueB"},
+		},
+		{
+			name: "wildcardAsPrefix",
+			kubeData: map[string]string{
+				"keyA":      "valueA",
+				"keyB":      "valueB",
+				"otherKeyA": "valueC",
+				"otherKeyB": "valueD",
+			},
+			allowList:    []string{"*A"},
+			expectKeys:   []string{"metric_key_a", "metric_other_key_a"},
+			expectValues: []string{"valueA", "valueC"},
+		},
+		{
+			name: "onlyFullWildcard",
+			kubeData: map[string]string{
+				"keyA":      "valueA",
+				"keyB":      "valueB",
+				"otherKeyA": "valueC",
+				"otherKeyB": "valueD",
+			},
+			allowList:    []string{"*"},
+			expectKeys:   []string{"metric_key_a", "metric_key_b", "metric_other_key_a", "metric_other_key_b"},
+			expectValues: []string{"valueA", "valueB", "valueC", "valueD"},
+		},
+		{
+			name: "additionalFullWildcard",
+			kubeData: map[string]string{
+				"keyA":      "valueA",
+				"keyB":      "valueB",
+				"otherKeyA": "valueC",
+				"otherKeyB": "valueD",
+			},
+			allowList:    []string{"keyA", "*"},
+			expectKeys:   []string{"metric_key_a"},
+			expectValues: []string{"valueA"},
+		},
+		{
+			name: "multipleWildcards",
+			kubeData: map[string]string{
+				"keyA":      "valueA",
+				"keyB":      "valueB",
+				"otherKeyA": "valueC",
+				"otherKeyB": "valueD",
+			},
+			allowList:    []string{"*key*"},
+			expectKeys:   []string{},
+			expectValues: []string{},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gotKeys, gotValues := createPrometheusLabelKeysValues("metric", tc.kubeData, tc.allowList)
+			if !reflect.DeepEqual(gotKeys, tc.expectKeys) {
+				t.Errorf("createPrometheusLabelKeysValues() got = %v, want %v", gotKeys, tc.expectKeys)
+			}
+			if !reflect.DeepEqual(gotValues, tc.expectValues) {
+				t.Errorf("createPrometheusLabelKeysValues() got1 = %v, want %v", gotValues, tc.expectValues)
+			}
+		})
+	}
+}
+
+func TestExpandWildcard(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+		limit    uint
+	}{
+		{
+			input:    "foo",
+			expected: "^foo$",
+			limit:    1,
+		},
+		{
+			input:    "foo*",
+			expected: "^foo.*$",
+			limit:    1,
+		},
+		{
+			input:    "*foo",
+			expected: "^.*foo$",
+			limit:    1,
+		},
+		{
+			input:    "*foo*",
+			expected: "^.*foo$",
+			limit:    1,
+		},
+		{
+			input:    "*foo*",
+			expected: "^.*foo.*$",
+			limit:    2,
+		},
+		{
+			input:    "*foo*",
+			expected: "^.*foo.*$",
+			limit:    3,
+		},
+		{
+			input:    "*f*o*o*",
+			expected: "^.*f.*o.*o$",
+			limit:    3,
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
+			got := expandWildcard(tc.input, tc.limit)
+			if !reflect.DeepEqual(got, tc.expected) {
+				t.Errorf("expandWildcard() got = %v, want %v", got, tc.expected)
+			}
+		})
+	}
+}

pkg/options/options.go

@@ -168,8 +168,8 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 	o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config YAML file. If this flag is set, the flags defined in the file override the command line flags.")
 	o.cmd.Flags().BoolVar(&o.ContinueWithoutConfig, "continue-without-config", false, "If true, kube-state-metrics continues to run even if the config file specified by --config is not present. This is useful for scenarios where config file is not provided at startup but is provided later, for e.g., via configmap. Kube-state-metrics will not exit with an error if the config file is not found, instead watches and reloads when it is created.")
 	o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.")
-	o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")
-	o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.")
+	o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotation keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided for resources, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be use to match multiple annotations, i.e., '=pods=[something.example.org/foo-*]' will resolve to '=pods=[something.example.org/foo-bar,something.example.org/foo-baz,...]'")
+	o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be used to match multiple labels, but only a single '*' is supported per key pattern, i.e., '=pods=[something.example.org/foo-*]' will match labels such as 'something.example.org/foo-bar' and 'something.example.org/foo-baz'.")
 	o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.")
 	o.cmd.Flags().Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.")
 	o.cmd.Flags().Var(&o.MetricOptInList, "metric-opt-in-list", "Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists")

pkg/options/types.go

@@ -18,6 +18,7 @@ package options
 
 import (
 	"errors"
+	"slices"
 	"sort"
 	"strings"
 
@@ -28,7 +29,16 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var errLabelsAllowListFormat = errors.New("invalid format, metric=[label1,label2,labeln...],metricN=[]")
+const (
+	// MaxPartialWildcardsPerLabel defines the amount of times a wildcard (*) may occur in a Label (or Annotation) passed
+	// with --metric-labels-allowlist or --metric-annotations-allowlist
+	MaxPartialWildcardsPerLabel = 1
+)
+
+var (
+	errLabelsAllowListFormat            = errors.New("invalid format, metric=[label1,label2,labeln...],metricN=[]")
+	errLabelsAllowListMultipleWildcards = errors.New("invalid format, only one wildcard per label allowed")
+)
 
 // MetricSet represents a collection which has a unique set of metrics.
 type MetricSet map[string]struct{}
@@ -291,6 +301,18 @@ func (l *LabelsAllowList) Set(value string) error {
 			firstWordPos = i + 1
 		}
 	}
+
+	// check amount of wildcards per label
+	for _, group := range m {
+		// each label in a group of labels can only contain one partial wildcard
+		// like mylabel/*
+		if slices.ContainsFunc(group, func(label string) bool {
+			return strings.Count(label, "*") > MaxPartialWildcardsPerLabel
+		}) {
+			return errLabelsAllowListMultipleWildcards
+		}
+	}
+
 	*l = m
 	return nil
 }

pkg/options/types_test.go

@@ -293,7 +293,7 @@ func TestLabelsAllowListSet(t *testing.T) {
 		Desc   string
 		Value  string
 		Wanted LabelsAllowList
-		err    bool
+		err    error
 	}{
 		{
 			Desc:   "empty labels list",
@@ -304,26 +304,26 @@ func TestLabelsAllowListSet(t *testing.T) {
 			Desc:   "[invalid] space delimited",
 			Value:  "cronjobs=[somelabel,label2] cronjobs=[label3,label4]",
 			Wanted: LabelsAllowList(map[string][]string{}),
-			err:    true,
+			err:    errLabelsAllowListFormat,
 		},
 		{
 			Desc:   "[invalid] normal missing bracket",
 			Value:  "cronjobs=[somelabel,label2],cronjobs=label3,label4]",
 			Wanted: LabelsAllowList(map[string][]string{}),
-			err:    true,
+			err:    errLabelsAllowListFormat,
 		},
 
 		{
 			Desc:   "[invalid] no comma between metrics",
 			Value:  "cronjobs=[somelabel,label2]cronjobs=[label3,label4]",
 			Wanted: LabelsAllowList(map[string][]string{}),
-			err:    true,
+			err:    errLabelsAllowListFormat,
 		},
 		{
 			Desc:   "[invalid] no '=' between name and label list",
 			Value:  "cronjobs[somelabel,label2]cronjobs=[label3,label4]",
 			Wanted: LabelsAllowList(map[string][]string{}),
-			err:    true,
+			err:    errLabelsAllowListFormat,
 		},
 		{
 			Desc:  "one resource",
@@ -382,12 +382,27 @@ func TestLabelsAllowListSet(t *testing.T) {
 				},
 			}),
 		},
+		{
+			Desc:  "with partial wildcard",
+			Value: "*=[somelabel/*]",
+			Wanted: LabelsAllowList(map[string][]string{
+				"*": {
+					"somelabel/*",
+				},
+			}),
+		},
+		{
+			Desc:   "[invalid] with multiple partial wildcards per label",
+			Value:  "*=[somelabel/*-foo-*]",
+			Wanted: LabelsAllowList(map[string][]string{}),
+			err:    errLabelsAllowListMultipleWildcards,
+		},
 	}
 
 	for _, test := range tests {
 		lal := &LabelsAllowList{}
 		gotError := lal.Set(test.Value)
-		if gotError != nil && !test.err || !reflect.DeepEqual(*lal, test.Wanted) {
+		if gotError != nil && test.err == nil || !reflect.DeepEqual(*lal, test.Wanted) {
 			t.Errorf("Test error for Desc: %s\n Want: \n%+v\n Got: \n%#+v\n Got Error: %#v", test.Desc, test.Wanted, *lal, gotError)
 		}
 	}