History

metric: add KMS-related metrics (#15258)

This commit adds a minimal set of KMS-related metrics: ``` # HELP minio_cluster_kms_online Reports whether the KMS is online (1) or offline (0) # TYPE minio_cluster_kms_online gauge minio_cluster_kms_online{server="127.0.0.1:9000"} 1 # HELP minio_cluster_kms_request_error Number of KMS requests that failed with a well-defined error # TYPE minio_cluster_kms_request_error counter minio_cluster_kms_request_error{server="127.0.0.1:9000"} 16790 # HELP minio_cluster_kms_request_success Number of KMS requests that succeeded # TYPE minio_cluster_kms_request_success counter minio_cluster_kms_request_success{server="127.0.0.1:9000"} 348031 ``` Currently, we report whether the KMS is available and how many requests succeeded/failed. However, KES exposes much more metrics that can be exposed if necessary. See: https://pkg.go.dev/github.com/minio/kes#Metric

Andreas Auernhammer committed 4 years ago

Commit f800cee1 parent b49fc336 files · +130 −6Browse files

Unified · Split

6 files
  1. cmd/metrics-v2.go +111 −1
  2. go.mod +2 −2
  3. go.sum +4 −3
  4. internal/kms/kes.go +4 −0
  5. internal/kms/kms.go +4 −0
  6. internal/kms/single-key.go +5 −0
cmd/metrics-v2.go+111−1
@@ -27,6 +27,7 @@ import ( · 26 unmodified lines
2727 "sync/atomic"
2828 "time"
2929
30+ "github.com/minio/kes"
3031 "github.com/minio/madmin-go"
3132 "github.com/minio/minio/internal/bucket/lifecycle"
3233 "github.com/minio/minio/internal/logger"
@@ -49,6 +50,7 @@ func init() { · 16 unmodified lines
4950 getNodeHealthMetrics(),
5051 getClusterStorageMetrics(),
5152 getClusterTierMetrics(),
53+ getKMSMetrics(),
5254 }
5355
5456 peerMetricsGroups = []*MetricsGroup{
@@ -63,6 +65,7 @@ func init() { · 8 unmodified lines
6365 getILMNodeMetrics(),
6466 getScannerNodeMetrics(),
6567 getIAMNodeMetrics(),
68+ getKMSNodeMetrics(),
6669 }
6770
6871 allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -80,7 +83,6 @@ func init() { · 11 unmodified lines
8083 getMinioVersionMetrics(),
8184 getS3TTFBMetric(),
8285 })
83−
8486 clusterCollector = newMinioClusterCollector(allMetricsGroups)
8587 }
8688
@@ -123,6 +125,7 @@ const ( · 36 unmodified lines
123125 ilmSubsystem MetricSubsystem = "ilm"
124126 scannerSubsystem MetricSubsystem = "scanner"
125127 iamSubsystem MetricSubsystem = "iam"
128+ kmsSubsystem MetricSubsystem = "kms"
126129 )
127130
128131 // MetricName are the individual names for the metric.
@@ -188,6 +191,12 @@ const ( · 59 unmodified lines
188191 transitionedBytes MetricName = "transitioned_bytes"
189192 transitionedObjects MetricName = "transitioned_objects"
190193 transitionedVersions MetricName = "transitioned_versions"
194+
195+ kmsOnline = "online"
196+ kmsRequestsSuccess = "request_success"
197+ kmsRequestsError = "request_error"
198+ kmsRequestsFail = "request_failure"
199+ kmsUptime = "uptime"
191200 )
192201
193202 const (
@@ -1931,6 +1940,107 @@ func getClusterStorageMetrics() *MetricsGroup { · 1737 unmodified lines
19311940 return mg
19321941 }
19331942
1943+func getKMSNodeMetrics() *MetricsGroup {
1944+ mg := &MetricsGroup{
1945+ cacheInterval: 10 * time.Second,
1946+ }
1947+
1948+ mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
1949+ objLayer := newObjectLayerFn()
1950+ // Service not initialized yet
1951+ if objLayer == nil || globalIsGateway || GlobalKMS == nil {
1952+ return
1953+ }
1954+
1955+ const (
1956+ Online = 1
1957+ Offline = 0
1958+ )
1959+ desc := MetricDescription{
1960+ Namespace: clusterMetricNamespace,
1961+ Subsystem: kmsSubsystem,
1962+ Name: kmsOnline,
1963+ Help: "Reports whether the KMS is online (1) or offline (0)",
1964+ Type: gaugeMetric,
1965+ }
1966+ _, err := GlobalKMS.Metrics(ctx)
1967+ if _, ok := kes.IsConnError(err); ok {
1968+ return []Metric{{
1969+ Description: desc,
1970+ Value: float64(Offline),
1971+ }}
1972+ }
1973+ return []Metric{{
1974+ Description: desc,
1975+ Value: float64(Online),
1976+ }}
1977+ })
1978+ return mg
1979+}
1980+
1981+func getKMSMetrics() *MetricsGroup {
1982+ mg := &MetricsGroup{
1983+ cacheInterval: 10 * time.Second,
1984+ }
1985+
1986+ mg.RegisterRead(func(ctx context.Context) []Metric {
1987+ objLayer := newObjectLayerFn()
1988+ // Service not initialized yet
1989+ if objLayer == nil || globalIsGateway || GlobalKMS == nil {
1990+ return []Metric{}
1991+ }
1992+
1993+ metrics := make([]Metric, 0, 4)
1994+ metric, err := GlobalKMS.Metrics(ctx)
1995+ if err != nil {
1996+ return metrics
1997+ }
1998+ metrics = append(metrics, Metric{
1999+ Description: MetricDescription{
2000+ Namespace: clusterMetricNamespace,
2001+ Subsystem: kmsSubsystem,
2002+ Name: kmsRequestsSuccess,
2003+ Help: "Number of KMS requests that succeeded",
2004+ Type: counterMetric,
2005+ },
2006+ Value: float64(metric.RequestOK),
2007+ })
2008+ metrics = append(metrics, Metric{
2009+ Description: MetricDescription{
2010+ Namespace: clusterMetricNamespace,
2011+ Subsystem: kmsSubsystem,
2012+ Name: kmsRequestsError,
2013+ Help: "Number of KMS requests that failed due to some error. (HTTP 4xx status code)",
2014+ Type: counterMetric,
2015+ },
2016+ Value: float64(metric.RequestErr),
2017+ })
2018+ metrics = append(metrics, Metric{
2019+ Description: MetricDescription{
2020+ Namespace: clusterMetricNamespace,
2021+ Subsystem: kmsSubsystem,
2022+ Name: kmsRequestsFail,
2023+ Help: "Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code)",
2024+ Type: counterMetric,
2025+ },
2026+ Value: float64(metric.RequestFail),
2027+ })
2028+ metrics = append(metrics, Metric{
2029+ Description: MetricDescription{
2030+ Namespace: clusterMetricNamespace,
2031+ Subsystem: kmsSubsystem,
2032+ Name: kmsUptime,
2033+ Help: "The time the KMS has been up and running in seconds.",
2034+ Type: counterMetric,
2035+ },
2036+ Value: metric.UpTime.Seconds(),
2037+ })
2038+
2039+ return metrics
2040+ })
2041+ return mg
2042+}
2043+
19342044 type minioClusterCollector struct {
19352045 metricsGroups []*MetricsGroup
19362046 desc *prometheus.Desc
go.mod+2−2
@@ -47,7 +47,7 @@ require ( · 46 unmodified lines
4747 github.com/minio/csvparser v1.0.0
4848 github.com/minio/dperf v0.4.2
4949 github.com/minio/highwayhash v1.0.2
50− github.com/minio/kes v0.19.2
50+ github.com/minio/kes v0.20.0
5151 github.com/minio/madmin-go v1.4.3
5252 github.com/minio/minio-go/v7 v7.0.30
5353 github.com/minio/pkg v1.1.26
@@ -65,7 +65,7 @@ require ( · 11 unmodified lines
6565 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
6666 github.com/pierrec/lz4 v2.6.1+incompatible
6767 github.com/pkg/errors v0.9.1
68− github.com/prometheus/client_golang v1.12.1
68+ github.com/prometheus/client_golang v1.12.2
6969 github.com/prometheus/client_model v0.2.0
7070 github.com/prometheus/common v0.34.0
7171 github.com/prometheus/procfs v0.7.3
go.sum+4−3
@@ -618,8 +618,8 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX · 617 unmodified lines
618618 github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
619619 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
620620 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
621−github.com/minio/kes v0.19.2 h1:0kdMAgLMSkiDA33k8pMHC7d6erDuseuLrZF+N3017SM=
622−github.com/minio/kes v0.19.2/go.mod h1:X2fMkDbAkjbSKDGOQZvyPkHxoG7nuzP6R78Jw+TzXtM=
621+github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE=
622+github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc=
623623 github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
624624 github.com/minio/madmin-go v1.4.3 h1:5/kBHjKTjYOQQHvyznu51weN5hJtFW67LB2VLz+hmzU=
625625 github.com/minio/madmin-go v1.4.3/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
@@ -745,8 +745,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn · 119 unmodified lines
745745 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
746746 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
747747 github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
748−github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
749748 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
749+github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
750+github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
750751 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
751752 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
752753 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
internal/kms/kes.go+4−0
@@ -115,6 +115,10 @@ func (c *kesClient) Stat() (Status, error) { · 114 unmodified lines
115115 }, nil
116116 }
117117
118+func (c *kesClient) Metrics(ctx context.Context) (kes.Metric, error) {
119+ return c.client.Metrics(ctx)
120+}
121+
118122 // CreateKey tries to create a new key at the KMS with the
119123 // given key ID.
120124 //
internal/kms/kms.go+4−0
@@ -23,6 +23,7 @@ import ( · 22 unmodified lines
2323 "encoding/json"
2424
2525 jsoniter "github.com/json-iterator/go"
26+ "github.com/minio/kes"
2627 )
2728
2829 // KMS is the generic interface that abstracts over
@@ -31,6 +32,9 @@ type KMS interface { · 2 unmodified lines
3132 // Stat returns the current KMS status.
3233 Stat() (Status, error)
3334
35+ // Metrics returns a KMS metric snapshot.
36+ Metrics(ctx context.Context) (kes.Metric, error)
37+
3438 // CreateKey creates a new key at the KMS with the given key ID.
3539 CreateKey(keyID string) error
3640
internal/kms/single-key.go+5−0
@@ -33,6 +33,7 @@ import ( · 32 unmodified lines
3333 "golang.org/x/crypto/chacha20"
3434 "golang.org/x/crypto/chacha20poly1305"
3535
36+ "github.com/minio/kes"
3637 "github.com/minio/minio/internal/hash/sha256"
3738 )
3839
@@ -89,6 +90,10 @@ func (kms secretKey) Stat() (Status, error) { · 50 unmodified lines
8990 }, nil
9091 }
9192
93+func (secretKey) Metrics(ctx context.Context) (kes.Metric, error) {
94+ return kes.Metric{}, errors.New("kms: metrics are not supported")
95+}
96+
9297 func (secretKey) CreateKey(string) error {
9398 return errors.New("kms: creating keys is not supported")
9499 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t