diff --git a/docs/developer/cli-arguments.md b/docs/developer/cli-arguments.md index 0d7937e832..a2f48b8b04 100644 --- a/docs/developer/cli-arguments.md +++ b/docs/developer/cli-arguments.md @@ -60,9 +60,9 @@ Flags: --log_file_max_size uint Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) --logtostderr log to standard error instead of files (default true) --metric-allowlist string Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive. - --metric-annotations-allowlist string Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]'). + --metric-annotations-allowlist string Comma-separated list of Kubernetes annotation keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided for resources, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be use to match multiple annotations, i.e., '=pods=[something.example.org/foo-*]' will resolve to '=pods=[something.example.org/foo-bar,something.example.org/foo-baz,...]' --metric-denylist string Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive. - --metric-labels-allowlist string Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. + --metric-labels-allowlist string Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be used to match multiple labels, but only a single '*' is supported per key pattern, i.e., '=pods=[something.example.org/foo-*]' will match labels such as 'something.example.org/foo-bar' and 'something.example.org/foo-baz'. --metric-opt-in-list string Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists --namespaces string Comma-separated list of namespaces to be enabled. Defaults to "" --namespaces-denylist string Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used. diff --git a/internal/store/utils.go b/internal/store/utils.go index 5cebe03ca4..46d5ec4862 100644 --- a/internal/store/utils.go +++ b/internal/store/utils.go @@ -178,24 +178,52 @@ func isPrefixedNativeResource(name v1.ResourceName) bool { // createPrometheusLabelKeysValues takes in passed kubernetes annotations/labels // and associated allowed list in kubernetes label format. // It returns only those allowed annotations/labels that exist in the list and converts them to Prometheus labels. +// Full wildcards (*) can only be set as first in the allow list, partial wildcards can appear anywhere in the list func createPrometheusLabelKeysValues(prefix string, allKubeData map[string]string, allowList []string) ([]string, []string) { allowedKubeData := make(map[string]string) - if len(allowList) > 0 { - if allowList[0] == options.LabelWildcard { - return kubeMapToPrometheusLabels(prefix, allKubeData) + for i, l := range allowList { + // only the first label can be the wildcard label + if l == options.LabelWildcard { + if i == 0 { + return kubeMapToPrometheusLabels(prefix, allKubeData) + } + continue } - for _, l := range allowList { - v, found := allKubeData[l] - if found { - allowedKubeData[l] = v + // prepare regular expression based on potential wildcards + re, err := regexp.Compile(expandWildcard(l, options.MaxPartialWildcardsPerLabel)) + if err != nil { + continue + } + for k, v := range allKubeData { + if re.MatchString(k) { + allowedKubeData[k] = v } } } + return kubeMapToPrometheusLabels(prefix, allowedKubeData) } +// expandWildcard expands wildcards (*) to regular expressions, up to a limited number of wildcards +func expandWildcard(pattern string, limit uint) string { + var result strings.Builder + var replacements uint + for i, literal := range strings.Split(pattern, options.LabelWildcard) { + if i > 0 { + result.WriteString(".*") + replacements++ + } + + result.WriteString(regexp.QuoteMeta(literal)) + if replacements >= limit { + break + } + } + return "^" + result.String() + "$" +} + // mergeKeyValues merges label keys and values slice pairs into a single slice pair. // Arguments are passed as equal-length pairs of slices, where the first slice contains keys and second contains values. // Example: mergeKeyValues(keys1, values1, keys2, values2) => (keys1+keys2, values1+values2) diff --git a/internal/store/utils_test.go b/internal/store/utils_test.go index 6356b1ee5d..70de55daf1 100644 --- a/internal/store/utils_test.go +++ b/internal/store/utils_test.go @@ -319,3 +319,169 @@ func TestMergeKeyValues(t *testing.T) { }) } } + +func TestCreatePrometheusLabelKeysValues(t *testing.T) { + testCases := []struct { + name string + kubeData map[string]string + allowList []string + expectKeys []string + expectValues []string + }{ + { + name: "allMatches", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + }, + allowList: []string{"keyA", "keyB"}, + expectKeys: []string{"metric_key_a", "metric_key_b"}, + expectValues: []string{"valueA", "valueB"}, + }, + { + name: "additionalAllow", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + }, + allowList: []string{"keyA", "keyB", "keyC"}, + expectKeys: []string{"metric_key_a", "metric_key_b"}, + expectValues: []string{"valueA", "valueB"}, + }, + { + name: "partialMatches", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + }, + allowList: []string{"keyA", "keyC"}, + expectKeys: []string{"metric_key_a"}, + expectValues: []string{"valueA"}, + }, + { + name: "wildcardAsSuffix", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + "otherKeyA": "valueC", + "otherKeyB": "valueD", + }, + allowList: []string{"key*"}, + expectKeys: []string{"metric_key_a", "metric_key_b"}, + expectValues: []string{"valueA", "valueB"}, + }, + { + name: "wildcardAsPrefix", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + "otherKeyA": "valueC", + "otherKeyB": "valueD", + }, + allowList: []string{"*A"}, + expectKeys: []string{"metric_key_a", "metric_other_key_a"}, + expectValues: []string{"valueA", "valueC"}, + }, + { + name: "onlyFullWildcard", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + "otherKeyA": "valueC", + "otherKeyB": "valueD", + }, + allowList: []string{"*"}, + expectKeys: []string{"metric_key_a", "metric_key_b", "metric_other_key_a", "metric_other_key_b"}, + expectValues: []string{"valueA", "valueB", "valueC", "valueD"}, + }, + { + name: "additionalFullWildcard", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + "otherKeyA": "valueC", + "otherKeyB": "valueD", + }, + allowList: []string{"keyA", "*"}, + expectKeys: []string{"metric_key_a"}, + expectValues: []string{"valueA"}, + }, + { + name: "multipleWildcards", + kubeData: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + "otherKeyA": "valueC", + "otherKeyB": "valueD", + }, + allowList: []string{"*key*"}, + expectKeys: []string{}, + expectValues: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotKeys, gotValues := createPrometheusLabelKeysValues("metric", tc.kubeData, tc.allowList) + if !reflect.DeepEqual(gotKeys, tc.expectKeys) { + t.Errorf("createPrometheusLabelKeysValues() got = %v, want %v", gotKeys, tc.expectKeys) + } + if !reflect.DeepEqual(gotValues, tc.expectValues) { + t.Errorf("createPrometheusLabelKeysValues() got1 = %v, want %v", gotValues, tc.expectValues) + } + }) + } +} + +func TestExpandWildcard(t *testing.T) { + testCases := []struct { + input string + expected string + limit uint + }{ + { + input: "foo", + expected: "^foo$", + limit: 1, + }, + { + input: "foo*", + expected: "^foo.*$", + limit: 1, + }, + { + input: "*foo", + expected: "^.*foo$", + limit: 1, + }, + { + input: "*foo*", + expected: "^.*foo$", + limit: 1, + }, + { + input: "*foo*", + expected: "^.*foo.*$", + limit: 2, + }, + { + input: "*foo*", + expected: "^.*foo.*$", + limit: 3, + }, + { + input: "*f*o*o*", + expected: "^.*f.*o.*o$", + limit: 3, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { + got := expandWildcard(tc.input, tc.limit) + if !reflect.DeepEqual(got, tc.expected) { + t.Errorf("expandWildcard() got = %v, want %v", got, tc.expected) + } + }) + } +} diff --git a/pkg/options/options.go b/pkg/options/options.go index ce2e960176..5ecf591214 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -168,8 +168,8 @@ func (o *Options) AddFlags(cmd *cobra.Command) { o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config YAML file. If this flag is set, the flags defined in the file override the command line flags.") o.cmd.Flags().BoolVar(&o.ContinueWithoutConfig, "continue-without-config", false, "If true, kube-state-metrics continues to run even if the config file specified by --config is not present. This is useful for scenarios where config file is not provided at startup but is provided later, for e.g., via configmap. Kube-state-metrics will not exit with an error if the config file is not found, instead watches and reloads when it is created.") o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.") - o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').") - o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.") + o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotation keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided for resources, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be use to match multiple annotations, i.e., '=pods=[something.example.org/foo-*]' will resolve to '=pods=[something.example.org/foo-bar,something.example.org/foo-baz,...]'") + o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'. Wildcards can also be used to match multiple labels, but only a single '*' is supported per key pattern, i.e., '=pods=[something.example.org/foo-*]' will match labels such as 'something.example.org/foo-bar' and 'something.example.org/foo-baz'.") o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.") o.cmd.Flags().Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.") o.cmd.Flags().Var(&o.MetricOptInList, "metric-opt-in-list", "Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists") diff --git a/pkg/options/types.go b/pkg/options/types.go index 93f8a7f505..962803fef4 100644 --- a/pkg/options/types.go +++ b/pkg/options/types.go @@ -18,6 +18,7 @@ package options import ( "errors" + "slices" "sort" "strings" @@ -28,7 +29,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var errLabelsAllowListFormat = errors.New("invalid format, metric=[label1,label2,labeln...],metricN=[]") +const ( + // MaxPartialWildcardsPerLabel defines the amount of times a wildcard (*) may occur in a Label (or Annotation) passed + // with --metric-labels-allowlist or --metric-annotations-allowlist + MaxPartialWildcardsPerLabel = 1 +) + +var ( + errLabelsAllowListFormat = errors.New("invalid format, metric=[label1,label2,labeln...],metricN=[]") + errLabelsAllowListMultipleWildcards = errors.New("invalid format, only one wildcard per label allowed") +) // MetricSet represents a collection which has a unique set of metrics. type MetricSet map[string]struct{} @@ -291,6 +301,18 @@ func (l *LabelsAllowList) Set(value string) error { firstWordPos = i + 1 } } + + // check amount of wildcards per label + for _, group := range m { + // each label in a group of labels can only contain one partial wildcard + // like mylabel/* + if slices.ContainsFunc(group, func(label string) bool { + return strings.Count(label, "*") > MaxPartialWildcardsPerLabel + }) { + return errLabelsAllowListMultipleWildcards + } + } + *l = m return nil } diff --git a/pkg/options/types_test.go b/pkg/options/types_test.go index 94f64fba98..a25273fcb5 100644 --- a/pkg/options/types_test.go +++ b/pkg/options/types_test.go @@ -293,7 +293,7 @@ func TestLabelsAllowListSet(t *testing.T) { Desc string Value string Wanted LabelsAllowList - err bool + err error }{ { Desc: "empty labels list", @@ -304,26 +304,26 @@ func TestLabelsAllowListSet(t *testing.T) { Desc: "[invalid] space delimited", Value: "cronjobs=[somelabel,label2] cronjobs=[label3,label4]", Wanted: LabelsAllowList(map[string][]string{}), - err: true, + err: errLabelsAllowListFormat, }, { Desc: "[invalid] normal missing bracket", Value: "cronjobs=[somelabel,label2],cronjobs=label3,label4]", Wanted: LabelsAllowList(map[string][]string{}), - err: true, + err: errLabelsAllowListFormat, }, { Desc: "[invalid] no comma between metrics", Value: "cronjobs=[somelabel,label2]cronjobs=[label3,label4]", Wanted: LabelsAllowList(map[string][]string{}), - err: true, + err: errLabelsAllowListFormat, }, { Desc: "[invalid] no '=' between name and label list", Value: "cronjobs[somelabel,label2]cronjobs=[label3,label4]", Wanted: LabelsAllowList(map[string][]string{}), - err: true, + err: errLabelsAllowListFormat, }, { Desc: "one resource", @@ -382,12 +382,27 @@ func TestLabelsAllowListSet(t *testing.T) { }, }), }, + { + Desc: "with partial wildcard", + Value: "*=[somelabel/*]", + Wanted: LabelsAllowList(map[string][]string{ + "*": { + "somelabel/*", + }, + }), + }, + { + Desc: "[invalid] with multiple partial wildcards per label", + Value: "*=[somelabel/*-foo-*]", + Wanted: LabelsAllowList(map[string][]string{}), + err: errLabelsAllowListMultipleWildcards, + }, } for _, test := range tests { lal := &LabelsAllowList{} gotError := lal.Set(test.Value) - if gotError != nil && !test.err || !reflect.DeepEqual(*lal, test.Wanted) { + if gotError != nil && test.err == nil || !reflect.DeepEqual(*lal, test.Wanted) { t.Errorf("Test error for Desc: %s\n Want: \n%+v\n Got: \n%#+v\n Got Error: %#v", test.Desc, test.Wanted, *lal, gotError) } }