From 7bea0e98b1e9a4671affb9d12358d4651d136363 Mon Sep 17 00:00:00 2001 From: Kazuki Suda Date: Sun, 12 Apr 2026 21:28:43 +0900 Subject: [PATCH] feat: add per-resource label selector support Signed-off-by: Kazuki Suda --- README.md | 33 +++++++++++ README.md.tpl | 33 +++++++++++ docs/developer/cli-arguments.md | 1 + internal/store/builder.go | 50 +++++++++++++++- internal/store/builder_test.go | 93 +++++++++++++++++++++++++++++ internal/store/listwatch.go | 64 ++++++++++++++++++++ pkg/app/server.go | 24 ++++++++ pkg/app/server_test.go | 11 ++++ pkg/options/options.go | 30 ++++++---- pkg/options/options_test.go | 47 +++++++++++++++ pkg/options/types.go | 65 ++++++++++++++++++++- pkg/options/types_test.go | 100 ++++++++++++++++++++++++++++++++ tests/e2e/main_test.go | 1 + 13 files changed, 537 insertions(+), 15 deletions(-) create mode 100644 internal/store/listwatch.go diff --git a/README.md b/README.md index a12ae36259..17d4c989c5 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ are deleted they are no longer visible on the `/metrics` endpoint. * [Horizontal sharding](#horizontal-sharding) * [Automated sharding](#automated-sharding) * [Daemonset sharding for pod metrics](#daemonset-sharding-for-pod-metrics) + * [Per-resource label selectors](#per-resource-label-selectors) * [Resource filtering](#resource-filtering) * [Setup](#setup) * [Building the Docker container](#building-the-docker-container) @@ -304,6 +305,38 @@ spec: Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding). +### Per-resource label selectors + +To restrict which objects kube-state-metrics watches for selected builtin resources, use the repeatable `--label-selector` flag: + +* `--label-selector=nodes=tenant=team-a` +* `--label-selector='pods=app in (frontend,api)'` + +Each flag applies a Kubernetes `LabelSelector` to a single resource type. Resource names use the same plural form as `--resources`, such as `pods`, `nodes`, or `namespaces`. Resources without a matching `--label-selector` flag remain unfiltered. + +Filtered resources expose metrics only for the selected subset of objects and do not preserve cluster-wide semantics. + +`--label-selector=pods=...` can be combined with `--node=$(NODE_NAME)`. In that case both filters apply to Pod metrics. + +Examples: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG + name: kube-state-metrics + args: + - --resources=nodes,pods + - --label-selector=nodes=tenant=team-a + - --label-selector=pods=app=frontend +``` + +This flag currently supports builtin resources only. Custom resource metrics use their own configuration paths. + #### Resource Filtering The `/metrics` endpoint supports filtering by resource type using the `resources` query parameter. This allows you to scrape only the metrics for specific Kubernetes resources, which can be useful for reducing the amount of data scraped or for creating separate scraping jobs for different resource types. diff --git a/README.md.tpl b/README.md.tpl index 69d373816d..5156c96b5f 100644 --- a/README.md.tpl +++ b/README.md.tpl @@ -52,6 +52,7 @@ are deleted they are no longer visible on the `/metrics` endpoint. * [Horizontal sharding](#horizontal-sharding) * [Automated sharding](#automated-sharding) * [Daemonset sharding for pod metrics](#daemonset-sharding-for-pod-metrics) + * [Per-resource label selectors](#per-resource-label-selectors) * [Resource filtering](#resource-filtering) * [Setup](#setup) * [Building the Docker container](#building-the-docker-container) @@ -305,6 +306,38 @@ spec: Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding). +### Per-resource label selectors + +To restrict which objects kube-state-metrics watches for selected builtin resources, use the repeatable `--label-selector` flag: + +* `--label-selector=nodes=tenant=team-a` +* `--label-selector='pods=app in (frontend,api)'` + +Each flag applies a Kubernetes `LabelSelector` to a single resource type. Resource names use the same plural form as `--resources`, such as `pods`, `nodes`, or `namespaces`. Resources without a matching `--label-selector` flag remain unfiltered. + +Filtered resources expose metrics only for the selected subset of objects and do not preserve cluster-wide semantics. + +`--label-selector=pods=...` can be combined with `--node=$(NODE_NAME)`. In that case both filters apply to Pod metrics. + +Examples: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG + name: kube-state-metrics + args: + - --resources=nodes,pods + - --label-selector=nodes=tenant=team-a + - --label-selector=pods=app=frontend +``` + +This flag currently supports builtin resources only. Custom resource metrics use their own configuration paths. + #### Resource Filtering The `/metrics` endpoint supports filtering by resource type using the `resources` query parameter. This allows you to scrape only the metrics for specific Kubernetes resources, which can be useful for reducing the amount of data scraped or for creating separate scraping jobs for different resource types. diff --git a/docs/developer/cli-arguments.md b/docs/developer/cli-arguments.md index 05fe2abf72..47a43186fd 100644 --- a/docs/developer/cli-arguments.md +++ b/docs/developer/cli-arguments.md @@ -54,6 +54,7 @@ Flags: -h, --help Print Help text --host string Host to expose metrics on. (default "::") --kubeconfig string Absolute path to the kubeconfig file + --label-selector string Repeatable resource-specific Kubernetes label selectors in the form 'resource=labelSelector'. Resources use the same plural names as --resources. Examples: '--label-selector=pods=app=frontend' or '--label-selector=nodes=tenant in (team-a,team-b)'. --legacy_stderr_threshold_behavior If true, stderrthreshold is ignored when logtostderr=true (legacy behavior). If false, stderrthreshold is honored even when logtostderr=true (default true) --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) --log_dir string If non-empty, write log files in this directory (no effect when -logtostderr=true) diff --git a/internal/store/builder.go b/internal/store/builder.go index 582c45c8dc..d4aa29bd94 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -38,7 +38,9 @@ import ( policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" @@ -76,6 +78,7 @@ type Builder struct { buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc allowAnnotationsList map[string][]string allowLabelsList map[string][]string + labelSelectorFilters map[string]string utilOptions *options.Options // namespaceFilter is inside fieldSelectorFilter fieldSelectorFilter string @@ -129,6 +132,25 @@ func (b *Builder) WithFieldSelectorFilter(fieldSelectorFilter string) { b.fieldSelectorFilter = fieldSelectorFilter } +// WithLabelSelectorFilters sets resource-specific label selectors for builtin resources. +func (b *Builder) WithLabelSelectorFilters(labelSelectorFilters map[string]string) error { + if len(labelSelectorFilters) == 0 { + b.labelSelectorFilters = nil + return nil + } + + validatedFilters := make(map[string]string, len(labelSelectorFilters)) + for resource, selector := range labelSelectorFilters { + if !resourceExists(resource) { + return fmt.Errorf("resource %s does not exist. Available resources: %s", resource, strings.Join(availableResources(), ",")) + } + validatedFilters[resource] = selector + } + + b.labelSelectorFilters = validatedFilters + return nil +} + // WithNamespaces sets the namespaces property of a Builder. func (b *Builder) WithNamespaces(n options.NamespaceList) { b.namespaces = n @@ -522,6 +544,7 @@ func (b *Builder) buildStores( metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies) composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies) familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies) + labelSelector := b.labelSelectorForExpectedType(expectedType) if b.namespaces.IsAllNamespaces() { store := metricsstore.NewMetricsStore( @@ -531,7 +554,10 @@ func (b *Builder) buildStores( if b.fieldSelectorFilter != "" { klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) } - listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter) + if labelSelector != "" { + klog.InfoS("LabelSelector is used", "labelSelector", labelSelector) + } + listWatcher := withLabelSelector(listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter), labelSelector) b.startReflector(expectedType, store, listWatcher, useAPIServerCache, objectLimit, b.kubeClient) return []cache.Store{store} } @@ -545,7 +571,10 @@ func (b *Builder) buildStores( if b.fieldSelectorFilter != "" { klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) } - listWatcher := listWatchFunc(b.kubeClient, ns, b.fieldSelectorFilter) + if labelSelector != "" { + klog.InfoS("LabelSelector is used", "labelSelector", labelSelector) + } + listWatcher := withLabelSelector(listWatchFunc(b.kubeClient, ns, b.fieldSelectorFilter), labelSelector) b.startReflector(expectedType, store, listWatcher, useAPIServerCache, objectLimit, b.kubeClient) stores = append(stores, store) } @@ -637,3 +666,20 @@ func cacheStoresToMetricStores(cStores []cache.Store) []*metricsstore.MetricsSto return mStores } + +func (b *Builder) labelSelectorForExpectedType(expectedType interface{}) string { + if len(b.labelSelectorFilters) == 0 { + return "" + } + + t := reflect.TypeOf(expectedType) + if t == nil { + return "" + } + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + + resource, _ := meta.UnsafeGuessKindToResource(schema.GroupVersionKind{Kind: t.Name()}) + return b.labelSelectorFilters[resource.Resource] +} diff --git a/internal/store/builder_test.go b/internal/store/builder_test.go index d9ea5813e8..952d62aeb8 100644 --- a/internal/store/builder_test.go +++ b/internal/store/builder_test.go @@ -17,10 +17,17 @@ limitations under the License. package store import ( + "context" "reflect" "slices" "testing" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" + "k8s.io/kube-state-metrics/v2/pkg/options" ) @@ -32,6 +39,21 @@ type expectedError struct { expectedNotEqual bool } +type fakeListerWatcher struct { + listOptions []metav1.ListOptions + watchOptions []metav1.ListOptions +} + +func (r *fakeListerWatcher) ListWithContext(_ context.Context, options metav1.ListOptions) (runtime.Object, error) { + r.listOptions = append(r.listOptions, options) + return &v1.PodList{}, nil +} + +func (r *fakeListerWatcher) WatchWithContext(_ context.Context, options metav1.ListOptions) (watch.Interface, error) { + r.watchOptions = append(r.watchOptions, options) + return watch.NewEmptyWatch(), nil +} + func TestWithAllowLabels(t *testing.T) { tests := []struct { Desc string @@ -258,3 +280,74 @@ func TestWithEnabledResources(t *testing.T) { } } } + +func TestWithLabelSelectorFilters(t *testing.T) { + tests := []struct { + Desc string + LabelSelector map[string]string + Want map[string]string + WantError bool + }{ + { + Desc: "builtin resource selector", + LabelSelector: map[string]string{"pods": "app=frontend", "nodes": "tenant=team-a"}, + Want: map[string]string{"pods": "app=frontend", "nodes": "tenant=team-a"}, + }, + { + Desc: "unknown resource selector", + LabelSelector: map[string]string{"foos": "app=frontend"}, + WantError: true, + }, + } + + for _, test := range tests { + b := NewBuilder() + err := b.WithLabelSelectorFilters(test.LabelSelector) + + if (err != nil) != test.WantError { + t.Fatalf("Test error for Desc: %s. Wanted Error: %v, Got Error: %v", test.Desc, test.WantError, err) + } + if err == nil && !reflect.DeepEqual(b.labelSelectorFilters, test.Want) { + t.Errorf("Test error for Desc: %s\n Want: \n%+v\n Got: \n%#+v", test.Desc, test.Want, b.labelSelectorFilters) + } + } +} + +func TestWithLabelSelector(t *testing.T) { + fakeLW := &fakeListerWatcher{} + baseLW := &cache.ListWatch{ + ListWithContextFunc: fakeLW.ListWithContext, + WatchFuncWithContext: fakeLW.WatchWithContext, + } + labelSelectorLW := withLabelSelector(baseLW, "tenant in (team-a,team-b)") + listerWatcherWithContext := cache.ToListerWatcherWithContext(labelSelectorLW) + + _, err := listerWatcherWithContext.ListWithContext(context.Background(), metav1.ListOptions{FieldSelector: "spec.nodeName=node-a", ResourceVersion: "10"}) + if err != nil { + t.Fatalf("unexpected list error: %v", err) + } + + _, err = listerWatcherWithContext.WatchWithContext(context.Background(), metav1.ListOptions{FieldSelector: "spec.nodeName=node-a"}) + if err != nil { + t.Fatalf("unexpected watch error: %v", err) + } + + if len(fakeLW.listOptions) != 1 { + t.Fatalf("expected 1 list call, got %d", len(fakeLW.listOptions)) + } + if len(fakeLW.watchOptions) != 1 { + t.Fatalf("expected 1 watch call, got %d", len(fakeLW.watchOptions)) + } + if got := fakeLW.listOptions[0].LabelSelector; got != "tenant in (team-a,team-b)" { + t.Fatalf("expected list label selector to be propagated, got %q", got) + } + if got := fakeLW.watchOptions[0].LabelSelector; got != "tenant in (team-a,team-b)" { + t.Fatalf("expected watch label selector to be propagated, got %q", got) + } + if got := fakeLW.listOptions[0].FieldSelector; got != "spec.nodeName=node-a" { + t.Fatalf("expected list field selector to be preserved, got %q", got) + } + if got := fakeLW.listOptions[0].ResourceVersion; got != "10" { + t.Fatalf("expected list resource version to be preserved, got %q", got) + } +} diff --git a/internal/store/listwatch.go b/internal/store/listwatch.go new file mode 100644 index 0000000000..5101c97fca --- /dev/null +++ b/internal/store/listwatch.go @@ -0,0 +1,64 @@ +/* +Copyright 2026 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" +) + +type labelSelectorListerWatcher struct { + *cache.ListWatch + lw cache.ListerWatcher +} + +func withLabelSelector(lw cache.ListerWatcher, labelSelector string) cache.ListerWatcher { + if labelSelector == "" { + return lw + } + + listerWatcherWithContext := cache.ToListerWatcherWithContext(lw) + return &labelSelectorListerWatcher{ + ListWatch: &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { + options.LabelSelector = labelSelector + return listerWatcherWithContext.ListWithContext(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { + options.LabelSelector = labelSelector + return listerWatcherWithContext.WatchWithContext(ctx, options) + }, + }, + lw: lw, + } +} + +func (l *labelSelectorListerWatcher) IsWatchListSemanticsUnSupported() bool { + type unsupported interface { + IsWatchListSemanticsUnSupported() bool + } + + if u, ok := l.lw.(unsupported); ok { + return u.IsWatchListSemanticsUnSupported() + } + + return false +} diff --git a/pkg/app/server.go b/pkg/app/server.go index 30d784153e..fae4f27c4c 100644 --- a/pkg/app/server.go +++ b/pkg/app/server.go @@ -242,6 +242,9 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error { } storeBuilder.WithNamespaces(namespaces) storeBuilder.WithFieldSelectorFilter(merged) + if err := configureLabelSelectorFilters(opts, storeBuilder); err != nil { + return err + } allowDenyList, err := allowdenylist.New(opts.MetricAllowlist, opts.MetricDenylist) if err != nil { @@ -445,6 +448,13 @@ func configureResourcesAndMetrics(opts *options.Options, configFile []byte) *opt } } + if len(config.LabelSelectors) > 0 { + opts.LabelSelectors = options.LabelSelectorSet{} + for resource, selector := range config.LabelSelectors { + opts.LabelSelectors[resource] = selector + } + } + if len(config.AnnotationsAllowList) > 0 { opts.AnnotationsAllowList = options.LabelsAllowList{} for annotation, value := range config.AnnotationsAllowList { @@ -457,6 +467,20 @@ func configureResourcesAndMetrics(opts *options.Options, configFile []byte) *opt return opts } +func configureLabelSelectorFilters(opts *options.Options, storeBuilder *store.Builder) error { + if err := opts.LabelSelectors.Validate(); err != nil { + return fmt.Errorf("failed to validate label selectors: %v", err) + } + if err := storeBuilder.WithLabelSelectorFilters(opts.LabelSelectors); err != nil { + return fmt.Errorf("failed to set up label selectors: %v", err) + } + if len(opts.LabelSelectors) > 0 { + klog.InfoS("Using label selectors", "labelSelectors", opts.LabelSelectors) + } + + return nil +} + func buildTelemetryServer(registry prometheus.Gatherer, authFilter bool, kubeConfig *rest.Config) *http.ServeMux { mux := http.NewServeMux() diff --git a/pkg/app/server_test.go b/pkg/app/server_test.go index 29becc5066..7e44537733 100644 --- a/pkg/app/server_test.go +++ b/pkg/app/server_test.go @@ -999,6 +999,8 @@ func TestConfigureResourcesAndMetrics(t *testing.T) { "kube_pod_labels": {} "metric_opt_in_list": "kube_pod_status_phase": {} +"label_selectors": + "pods": "app=frontend" "labels_allow_list": "labelX": - foo @@ -1014,6 +1016,7 @@ func TestConfigureResourcesAndMetrics(t *testing.T) { opts.MetricDenylist = options.MetricSet{"olddeny": {}} opts.MetricOptInList = options.MetricSet{"oldoptin": {}} opts.LabelsAllowList = options.LabelsAllowList{"oldlabel": {"oldvalue"}} + opts.LabelSelectors = options.LabelSelectorSet{"nodes": "tenant=team-a"} opts.AnnotationsAllowList = options.LabelsAllowList{"oldannotation": {"oldvalue"}} newOpts := configureResourcesAndMetrics(opts, []byte(configYAML)) @@ -1061,6 +1064,14 @@ func TestConfigureResourcesAndMetrics(t *testing.T) { t.Errorf("expected oldlabel to be overwritten, got %v", vals) } + // Check label selectors + if selector, ok := newOpts.LabelSelectors["pods"]; !ok || selector != "app=frontend" { + t.Errorf("expected pods label selector to be overwritten, got %q", selector) + } + if selector, ok := newOpts.LabelSelectors["nodes"]; ok { + t.Errorf("expected old nodes label selector to be overwritten, got %q", selector) + } + // Check annotations allow list if vals, ok := newOpts.AnnotationsAllowList["annotationY"]; !ok || len(vals) != 1 || vals[0] != "baz" { t.Errorf("expected annotationY with value [baz], got %v", vals) diff --git a/pkg/options/options.go b/pkg/options/options.go index e57f4ba4d3..f03093d302 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -40,12 +40,13 @@ var ( // Options are the configurable parameters for kube-state-metrics. type Options struct { - AnnotationsAllowList LabelsAllowList `yaml:"annotations_allow_list"` - LabelsAllowList LabelsAllowList `yaml:"labels_allow_list"` - MetricAllowlist MetricSet `yaml:"metric_allowlist"` - MetricDenylist MetricSet `yaml:"metric_denylist"` - MetricOptInList MetricSet `yaml:"metric_opt_in_list"` - Resources ResourceSet `yaml:"resources"` + AnnotationsAllowList LabelsAllowList `yaml:"annotations_allow_list"` + LabelSelectors LabelSelectorSet `yaml:"label_selectors"` + LabelsAllowList LabelsAllowList `yaml:"labels_allow_list"` + MetricAllowlist MetricSet `yaml:"metric_allowlist"` + MetricDenylist MetricSet `yaml:"metric_denylist"` + MetricOptInList MetricSet `yaml:"metric_opt_in_list"` + Resources ResourceSet `yaml:"resources"` cmd *cobra.Command Apiserver string `yaml:"apiserver"` @@ -98,6 +99,7 @@ func NewOptions() *Options { MetricDenylist: MetricSet{}, MetricOptInList: MetricSet{}, AnnotationsAllowList: LabelsAllowList{}, + LabelSelectors: LabelSelectorSet{}, LabelsAllowList: LabelsAllowList{}, } } @@ -173,6 +175,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config YAML file. If this flag is set, the flags defined in the file override the command line flags.") o.cmd.Flags().BoolVar(&o.ContinueWithoutConfig, "continue-without-config", false, "If true, kube-state-metrics continues to run even if the config file specified by --config is not present. This is useful for scenarios where config file is not provided at startup but is provided later, for e.g., via configmap. Kube-state-metrics will not exit with an error if the config file is not found, instead watches and reloads when it is created.") o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.") + o.cmd.Flags().Var(&o.LabelSelectors, "label-selector", "Repeatable resource-specific Kubernetes label selectors in the form 'resource=labelSelector'. Resources use the same plural names as --resources. Examples: '--label-selector=pods=app=frontend' or '--label-selector=nodes=tenant in (team-a,team-b)'.") o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').") o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.") o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.") @@ -202,12 +205,11 @@ func (o *Options) Usage() { // Validate validates arguments func (o *Options) Validate() error { shardableResource := "pods" - if o.Node == "" { - return nil - } - for _, x := range o.Resources.AsSlice() { - if x != shardableResource { - return fmt.Errorf("resource %s can't be sharded by field selector spec.nodeName", x) + if o.Node != "" { + for _, x := range o.Resources.AsSlice() { + if x != shardableResource { + return fmt.Errorf("resource %s can't be sharded by field selector spec.nodeName", x) + } } } @@ -219,5 +221,9 @@ func (o *Options) Validate() error { return fmt.Errorf("value for --object-limit=%d must be equal or greater than 0", o.ObjectLimit) } + if err := o.LabelSelectors.Validate(); err != nil { + return err + } + return nil } diff --git a/pkg/options/options_test.go b/pkg/options/options_test.go index 0e17c0c5a8..9822814d10 100644 --- a/pkg/options/options_test.go +++ b/pkg/options/options_test.go @@ -63,3 +63,50 @@ func TestOptionsParse(t *testing.T) { }) } } + +func TestOptionsValidateLabelSelectors(t *testing.T) { + tests := []struct { + Desc string + Configure func(*Options) + ExpectsError bool + }{ + { + Desc: "known but disabled resource selector is allowed", + Configure: func(opts *Options) { + opts.Resources = ResourceSet{"nodes": {}} + opts.LabelSelectors = LabelSelectorSet{"pods": "app=frontend"} + }, + ExpectsError: false, + }, + { + Desc: "invalid label selector syntax", + Configure: func(opts *Options) { + opts.LabelSelectors = LabelSelectorSet{"pods": "app in (frontend"} + }, + ExpectsError: true, + }, + { + Desc: "unknown label selector resource", + Configure: func(opts *Options) { + opts.LabelSelectors = LabelSelectorSet{"foos": "app=frontend"} + }, + ExpectsError: false, + }, + } + + for _, test := range tests { + t.Run(test.Desc, func(t *testing.T) { + opts := NewOptions() + opts.AutoGoMemlimitRatio = 0.9 + test.Configure(opts) + + err := opts.Validate() + if !test.ExpectsError && err != nil { + t.Fatalf("unexpected validation error: %v", err) + } + if test.ExpectsError && err == nil { + t.Fatal("expected validation error") + } + }) + } +} diff --git a/pkg/options/types.go b/pkg/options/types.go index 93f8a7f505..4d4d8ffe83 100644 --- a/pkg/options/types.go +++ b/pkg/options/types.go @@ -18,17 +18,22 @@ package options import ( "errors" + "fmt" "sort" "strings" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var errLabelsAllowListFormat = errors.New("invalid format, metric=[label1,label2,labeln...],metricN=[]") +var ( + errLabelsAllowListFormat = errors.New("invalid format, metric=[label1,label2,labeln...],metricN=[]") + errLabelSelectorFormat = errors.New("invalid format, resource=labelSelector") +) // MetricSet represents a collection which has a unique set of metrics. type MetricSet map[string]struct{} @@ -224,6 +229,64 @@ func (n *NamespaceList) Type() string { return "string" } +// LabelSelectorSet represents resource-specific Kubernetes label selectors. +type LabelSelectorSet map[string]string + +func (l *LabelSelectorSet) String() string { + if l == nil { + return "" + } + + entries := make([]string, 0, len(*l)) + for resource, selector := range *l { + entries = append(entries, fmt.Sprintf("%s=%s", resource, selector)) + } + sort.Strings(entries) + + return strings.Join(entries, ";") +} + +// Set converts a repeatable "resource=labelSelector" flag into a map keyed by resource name. +func (l *LabelSelectorSet) Set(value string) error { + resource, selector, found := strings.Cut(value, "=") + if !found { + return errLabelSelectorFormat + } + + resource = strings.TrimSpace(resource) + selector = strings.TrimSpace(selector) + if resource == "" || selector == "" { + return errLabelSelectorFormat + } + + if *l == nil { + *l = LabelSelectorSet{} + } + + if _, ok := (*l)[resource]; ok { + return fmt.Errorf("duplicate label selector for resource %s", resource) + } + + (*l)[resource] = selector + return nil +} + +// Validate checks that all configured selector expressions are syntactically valid. +func (l LabelSelectorSet) Validate() error { + for resource, selector := range l { + if _, err := labels.Parse(selector); err != nil { + return fmt.Errorf("invalid label selector for resource %s: %w", resource, err) + } + } + + return nil +} + +// Type returns a descriptive string about the LabelSelectorSet type. +func (l *LabelSelectorSet) Type() string { + return "string" +} + // LabelWildcard allowlists any label const LabelWildcard = "*" diff --git a/pkg/options/types_test.go b/pkg/options/types_test.go index 94f64fba98..d3ffb1ecf9 100644 --- a/pkg/options/types_test.go +++ b/pkg/options/types_test.go @@ -288,6 +288,106 @@ func TestMetricSetSet(t *testing.T) { } } +func TestLabelSelectorSetSet(t *testing.T) { + tests := []struct { + Desc string + Values []string + Wanted LabelSelectorSet + wantErr bool + }{ + { + Desc: "one resource", + Values: []string{"pods=app=frontend"}, + Wanted: LabelSelectorSet{ + "pods": "app=frontend", + }, + }, + { + Desc: "multiple resources with commas and spaces", + Values: []string{"nodes=tenant in (team-a,team-b)", "pods=app in (frontend,api)"}, + Wanted: LabelSelectorSet{ + "nodes": "tenant in (team-a,team-b)", + "pods": "app in (frontend,api)", + }, + }, + { + Desc: "selector with label key only", + Values: []string{"nodes=node-role.kubernetes.io/control-plane"}, + Wanted: LabelSelectorSet{ + "nodes": "node-role.kubernetes.io/control-plane", + }, + }, + { + Desc: "split only on the first equal sign", + Values: []string{"pods=app.kubernetes.io/name=frontend"}, + Wanted: LabelSelectorSet{ + "pods": "app.kubernetes.io/name=frontend", + }, + }, + { + Desc: "duplicate resource", + Values: []string{"pods=app=frontend", "pods=tier=web"}, + Wanted: LabelSelectorSet{"pods": "app=frontend"}, + wantErr: true, + }, + } + + for _, test := range tests { + labelSelectors := &LabelSelectorSet{} + + var gotErr error + for _, value := range test.Values { + gotErr = labelSelectors.Set(value) + if gotErr != nil { + break + } + } + + if gotErr != nil && !test.wantErr { + t.Errorf("Test error for Desc: %s. Got Error: %v", test.Desc, gotErr) + continue + } + if gotErr == nil && test.wantErr { + t.Errorf("Expected error for Desc: %s", test.Desc) + continue + } + if !reflect.DeepEqual(*labelSelectors, test.Wanted) { + t.Errorf("Test error for Desc: %s\n Want: \n%+v\n Got: \n%#+v", test.Desc, test.Wanted, *labelSelectors) + } + } +} + +func TestLabelSelectorSetValidate(t *testing.T) { + tests := []struct { + Desc string + Selectors LabelSelectorSet + WantedError bool + }{ + { + Desc: "valid selectors", + Selectors: LabelSelectorSet{ + "pods": "app=frontend", + "nodes": "tenant in (team-a,team-b)", + }, + WantedError: false, + }, + { + Desc: "invalid selector syntax", + Selectors: LabelSelectorSet{ + "pods": "app in (frontend", + }, + WantedError: true, + }, + } + + for _, test := range tests { + err := test.Selectors.Validate() + if (err != nil) != test.WantedError { + t.Errorf("Test error for Desc: %s. Wanted Error: %v, Got Error: %v", test.Desc, test.WantedError, err) + } + } +} + func TestLabelsAllowListSet(t *testing.T) { tests := []struct { Desc string diff --git a/tests/e2e/main_test.go b/tests/e2e/main_test.go index 441053fd04..4f3a07a7c6 100644 --- a/tests/e2e/main_test.go +++ b/tests/e2e/main_test.go @@ -290,6 +290,7 @@ func TestDefaultCollectorMetricsAvailable(t *testing.T) { } nonResources := map[string]bool{ "builder": true, + "listwatch": true, "utils": true, "testutils": true, }