History

metric: add KMS-related metrics (#15258)

This commit adds a minimal set of KMS-related metrics: ``` # HELP minio_cluster_kms_online Reports whether the KMS is online (1) or offline (0) # TYPE minio_cluster_kms_online gauge minio_cluster_kms_online{server="127.0.0.1:9000"} 1 # HELP minio_cluster_kms_request_error Number of KMS requests that failed with a well-defined error # TYPE minio_cluster_kms_request_error counter minio_cluster_kms_request_error{server="127.0.0.1:9000"} 16790 # HELP minio_cluster_kms_request_success Number of KMS requests that succeeded # TYPE minio_cluster_kms_request_success counter minio_cluster_kms_request_success{server="127.0.0.1:9000"} 348031 ``` Currently, we report whether the KMS is available and how many requests succeeded/failed. However, KES exposes much more metrics that can be exposed if necessary. See: https://pkg.go.dev/github.com/minio/kes#Metric

Andreas Auernhammer committed 4 years ago

Commit f800cee1 parent b49fc336 files · +130 −6Browse files

Unified · Split

6 files
  1. cmd/metrics-v2.go +111 −1
  2. go.mod +2 −2
  3. go.sum +4 −3
  4. internal/kms/kes.go +4 −0
  5. internal/kms/kms.go +4 −0
  6. internal/kms/single-key.go +5 −0
cmd/metrics-v2.go+111−1
@@ -27,6 +27,7 @@ import ( · 26 unmodified lines
27 "sync/atomic"27 "sync/atomic"
28 "time"28 "time"
29 29
30 "github.com/minio/kes"
30 "github.com/minio/madmin-go"31 "github.com/minio/madmin-go"
31 "github.com/minio/minio/internal/bucket/lifecycle"32 "github.com/minio/minio/internal/bucket/lifecycle"
32 "github.com/minio/minio/internal/logger"33 "github.com/minio/minio/internal/logger"
@@ -49,6 +50,7 @@ func init() { · 16 unmodified lines
49 getNodeHealthMetrics(),50 getNodeHealthMetrics(),
50 getClusterStorageMetrics(),51 getClusterStorageMetrics(),
51 getClusterTierMetrics(),52 getClusterTierMetrics(),
53 getKMSMetrics(),
52 }54 }
53 55
54 peerMetricsGroups = []*MetricsGroup{56 peerMetricsGroups = []*MetricsGroup{
@@ -63,6 +65,7 @@ func init() { · 8 unmodified lines
63 getILMNodeMetrics(),65 getILMNodeMetrics(),
64 getScannerNodeMetrics(),66 getScannerNodeMetrics(),
65 getIAMNodeMetrics(),67 getIAMNodeMetrics(),
68 getKMSNodeMetrics(),
66 }69 }
67 70
68 allMetricsGroups := func() (allMetrics []*MetricsGroup) {71 allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -80,7 +83,6 @@ func init() { · 11 unmodified lines
80 getMinioVersionMetrics(),83 getMinioVersionMetrics(),
81 getS3TTFBMetric(),84 getS3TTFBMetric(),
82 })85 })
83
84 clusterCollector = newMinioClusterCollector(allMetricsGroups)86 clusterCollector = newMinioClusterCollector(allMetricsGroups)
85}87}
86 88
@@ -123,6 +125,7 @@ const ( · 36 unmodified lines
123 ilmSubsystem MetricSubsystem = "ilm"125 ilmSubsystem MetricSubsystem = "ilm"
124 scannerSubsystem MetricSubsystem = "scanner"126 scannerSubsystem MetricSubsystem = "scanner"
125 iamSubsystem MetricSubsystem = "iam"127 iamSubsystem MetricSubsystem = "iam"
128 kmsSubsystem MetricSubsystem = "kms"
126)129)
127 130
128// MetricName are the individual names for the metric.131// MetricName are the individual names for the metric.
@@ -188,6 +191,12 @@ const ( · 59 unmodified lines
188 transitionedBytes MetricName = "transitioned_bytes"191 transitionedBytes MetricName = "transitioned_bytes"
189 transitionedObjects MetricName = "transitioned_objects"192 transitionedObjects MetricName = "transitioned_objects"
190 transitionedVersions MetricName = "transitioned_versions"193 transitionedVersions MetricName = "transitioned_versions"
194
195 kmsOnline = "online"
196 kmsRequestsSuccess = "request_success"
197 kmsRequestsError = "request_error"
198 kmsRequestsFail = "request_failure"
199 kmsUptime = "uptime"
191)200)
192 201
193const (202const (
@@ -1931,6 +1940,107 @@ func getClusterStorageMetrics() *MetricsGroup { · 1737 unmodified lines
1931 return mg1940 return mg
1932}1941}
1933 1942
1943func getKMSNodeMetrics() *MetricsGroup {
1944 mg := &MetricsGroup{
1945 cacheInterval: 10 * time.Second,
1946 }
1947
1948 mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
1949 objLayer := newObjectLayerFn()
1950 // Service not initialized yet
1951 if objLayer == nil || globalIsGateway || GlobalKMS == nil {
1952 return
1953 }
1954
1955 const (
1956 Online = 1
1957 Offline = 0
1958 )
1959 desc := MetricDescription{
1960 Namespace: clusterMetricNamespace,
1961 Subsystem: kmsSubsystem,
1962 Name: kmsOnline,
1963 Help: "Reports whether the KMS is online (1) or offline (0)",
1964 Type: gaugeMetric,
1965 }
1966 _, err := GlobalKMS.Metrics(ctx)
1967 if _, ok := kes.IsConnError(err); ok {
1968 return []Metric{{
1969 Description: desc,
1970 Value: float64(Offline),
1971 }}
1972 }
1973 return []Metric{{
1974 Description: desc,
1975 Value: float64(Online),
1976 }}
1977 })
1978 return mg
1979}
1980
1981func getKMSMetrics() *MetricsGroup {
1982 mg := &MetricsGroup{
1983 cacheInterval: 10 * time.Second,
1984 }
1985
1986 mg.RegisterRead(func(ctx context.Context) []Metric {
1987 objLayer := newObjectLayerFn()
1988 // Service not initialized yet
1989 if objLayer == nil || globalIsGateway || GlobalKMS == nil {
1990 return []Metric{}
1991 }
1992
1993 metrics := make([]Metric, 0, 4)
1994 metric, err := GlobalKMS.Metrics(ctx)
1995 if err != nil {
1996 return metrics
1997 }
1998 metrics = append(metrics, Metric{
1999 Description: MetricDescription{
2000 Namespace: clusterMetricNamespace,
2001 Subsystem: kmsSubsystem,
2002 Name: kmsRequestsSuccess,
2003 Help: "Number of KMS requests that succeeded",
2004 Type: counterMetric,
2005 },
2006 Value: float64(metric.RequestOK),
2007 })
2008 metrics = append(metrics, Metric{
2009 Description: MetricDescription{
2010 Namespace: clusterMetricNamespace,
2011 Subsystem: kmsSubsystem,
2012 Name: kmsRequestsError,
2013 Help: "Number of KMS requests that failed due to some error. (HTTP 4xx status code)",
2014 Type: counterMetric,
2015 },
2016 Value: float64(metric.RequestErr),
2017 })
2018 metrics = append(metrics, Metric{
2019 Description: MetricDescription{
2020 Namespace: clusterMetricNamespace,
2021 Subsystem: kmsSubsystem,
2022 Name: kmsRequestsFail,
2023 Help: "Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code)",
2024 Type: counterMetric,
2025 },
2026 Value: float64(metric.RequestFail),
2027 })
2028 metrics = append(metrics, Metric{
2029 Description: MetricDescription{
2030 Namespace: clusterMetricNamespace,
2031 Subsystem: kmsSubsystem,
2032 Name: kmsUptime,
2033 Help: "The time the KMS has been up and running in seconds.",
2034 Type: counterMetric,
2035 },
2036 Value: metric.UpTime.Seconds(),
2037 })
2038
2039 return metrics
2040 })
2041 return mg
2042}
2043
1934type minioClusterCollector struct {2044type minioClusterCollector struct {
1935 metricsGroups []*MetricsGroup2045 metricsGroups []*MetricsGroup
1936 desc *prometheus.Desc2046 desc *prometheus.Desc
go.mod+2−2
@@ -47,7 +47,7 @@ require ( · 46 unmodified lines
47 github.com/minio/csvparser v1.0.047 github.com/minio/csvparser v1.0.0
48 github.com/minio/dperf v0.4.248 github.com/minio/dperf v0.4.2
49 github.com/minio/highwayhash v1.0.249 github.com/minio/highwayhash v1.0.2
50 github.com/minio/kes v0.19.250 github.com/minio/kes v0.20.0
51 github.com/minio/madmin-go v1.4.351 github.com/minio/madmin-go v1.4.3
52 github.com/minio/minio-go/v7 v7.0.3052 github.com/minio/minio-go/v7 v7.0.30
53 github.com/minio/pkg v1.1.2653 github.com/minio/pkg v1.1.26
@@ -65,7 +65,7 @@ require ( · 11 unmodified lines
65 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb965 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
66 github.com/pierrec/lz4 v2.6.1+incompatible66 github.com/pierrec/lz4 v2.6.1+incompatible
67 github.com/pkg/errors v0.9.167 github.com/pkg/errors v0.9.1
68 github.com/prometheus/client_golang v1.12.168 github.com/prometheus/client_golang v1.12.2
69 github.com/prometheus/client_model v0.2.069 github.com/prometheus/client_model v0.2.0
70 github.com/prometheus/common v0.34.070 github.com/prometheus/common v0.34.0
71 github.com/prometheus/procfs v0.7.371 github.com/prometheus/procfs v0.7.3
go.sum+4−3
@@ -618,8 +618,8 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX · 617 unmodified lines
618github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=618github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
619github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=619github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
620github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=620github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
621github.com/minio/kes v0.19.2 h1:0kdMAgLMSkiDA33k8pMHC7d6erDuseuLrZF+N3017SM=621github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE=
622github.com/minio/kes v0.19.2/go.mod h1:X2fMkDbAkjbSKDGOQZvyPkHxoG7nuzP6R78Jw+TzXtM=622github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc=
623github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=623github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
624github.com/minio/madmin-go v1.4.3 h1:5/kBHjKTjYOQQHvyznu51weN5hJtFW67LB2VLz+hmzU=624github.com/minio/madmin-go v1.4.3 h1:5/kBHjKTjYOQQHvyznu51weN5hJtFW67LB2VLz+hmzU=
625github.com/minio/madmin-go v1.4.3/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=625github.com/minio/madmin-go v1.4.3/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
@@ -745,8 +745,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn · 119 unmodified lines
745github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=745github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
746github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=746github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
747github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=747github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
748github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
749github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=748github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
749github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
750github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
750github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=751github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
751github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=752github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
752github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=753github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
internal/kms/kes.go+4−0
@@ -115,6 +115,10 @@ func (c *kesClient) Stat() (Status, error) { · 114 unmodified lines
115 }, nil115 }, nil
116}116}
117 117
118func (c *kesClient) Metrics(ctx context.Context) (kes.Metric, error) {
119 return c.client.Metrics(ctx)
120}
121
118// CreateKey tries to create a new key at the KMS with the122// CreateKey tries to create a new key at the KMS with the
119// given key ID.123// given key ID.
120//124//
internal/kms/kms.go+4−0
@@ -23,6 +23,7 @@ import ( · 22 unmodified lines
23 "encoding/json"23 "encoding/json"
24 24
25 jsoniter "github.com/json-iterator/go"25 jsoniter "github.com/json-iterator/go"
26 "github.com/minio/kes"
26)27)
27 28
28// KMS is the generic interface that abstracts over29// KMS is the generic interface that abstracts over
@@ -31,6 +32,9 @@ type KMS interface { · 2 unmodified lines
31 // Stat returns the current KMS status.32 // Stat returns the current KMS status.
32 Stat() (Status, error)33 Stat() (Status, error)
33 34
35 // Metrics returns a KMS metric snapshot.
36 Metrics(ctx context.Context) (kes.Metric, error)
37
34 // CreateKey creates a new key at the KMS with the given key ID.38 // CreateKey creates a new key at the KMS with the given key ID.
35 CreateKey(keyID string) error39 CreateKey(keyID string) error
36 40
internal/kms/single-key.go+5−0
@@ -33,6 +33,7 @@ import ( · 32 unmodified lines
33 "golang.org/x/crypto/chacha20"33 "golang.org/x/crypto/chacha20"
34 "golang.org/x/crypto/chacha20poly1305"34 "golang.org/x/crypto/chacha20poly1305"
35 35
36 "github.com/minio/kes"
36 "github.com/minio/minio/internal/hash/sha256"37 "github.com/minio/minio/internal/hash/sha256"
37)38)
38 39
@@ -89,6 +90,10 @@ func (kms secretKey) Stat() (Status, error) { · 50 unmodified lines
89 }, nil90 }, nil
90}91}
91 92
93func (secretKey) Metrics(ctx context.Context) (kes.Metric, error) {
94 return kes.Metric{}, errors.New("kms: metrics are not supported")
95}
96
92func (secretKey) CreateKey(string) error {97func (secretKey) CreateKey(string) error {
93 return errors.New("kms: creating keys is not supported")98 return errors.New("kms: creating keys is not supported")
94}99}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t