History

ceph: periodically send perf metrics to MDSes

This will send the caps/read/write/metadata metrics to any available MDS once per second, which will be the same as the userland client. It will skip the MDS sessions which don't support the metric collection, as the MDSs will close socket connections when they get an unknown type message. We can disable the metric sending via the disable_send_metrics module parameter. [ jlayton: fix up endianness bug in ceph_mdsc_send_metrics() ]

Xiubo Li committed 6 years ago

Commit 18f473b1 parent aaf5a477 files · +276 −1Browse files

Unified · Split

7 files
  1. fs/ceph/mds_client.c +3 −0
  2. fs/ceph/mds_client.h +3 −1
  3. fs/ceph/metric.c +148 −0
  4. fs/ceph/metric.h +77 −0
  5. fs/ceph/super.c +42 −0
  6. fs/ceph/super.h +2 −0
  7. include/linux/ceph/ceph_fs.h +1 −0
fs/ceph/mds_client.c+3−0
@@ -3334,6 +3334,8 @@ static void handle_session(struct ceph_mds_session *session, · 3333 unmodified lines
3334 session->s_state = CEPH_MDS_SESSION_OPEN;3334 session->s_state = CEPH_MDS_SESSION_OPEN;
3335 session->s_features = features;3335 session->s_features = features;
3336 renewed_caps(mdsc, session, 0);3336 renewed_caps(mdsc, session, 0);
3337 if (test_bit(CEPHFS_FEATURE_METRIC_COLLECT, &session->s_features))
3338 metric_schedule_delayed(&mdsc->metric);
3337 wake = 1;3339 wake = 1;
3338 if (mdsc->stopping)3340 if (mdsc->stopping)
3339 __close_session(mdsc, session);3341 __close_session(mdsc, session);
@@ -4725,6 +4727,7 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc) · 1385 unmodified lines
4725 4727
4726 ceph_metric_destroy(&mdsc->metric);4728 ceph_metric_destroy(&mdsc->metric);
4727 4729
4730 flush_delayed_work(&mdsc->metric.delayed_work);
4728 fsc->mdsc = NULL;4731 fsc->mdsc = NULL;
4729 kfree(mdsc);4732 kfree(mdsc);
4730 dout("mdsc_destroy %p done\n", mdsc);4733 dout("mdsc_destroy %p done\n", mdsc);
fs/ceph/mds_client.h+3−1
@@ -28,8 +28,9 @@ enum ceph_feature_type { · 27 unmodified lines
28 CEPHFS_FEATURE_LAZY_CAP_WANTED,28 CEPHFS_FEATURE_LAZY_CAP_WANTED,
29 CEPHFS_FEATURE_MULTI_RECONNECT,29 CEPHFS_FEATURE_MULTI_RECONNECT,
30 CEPHFS_FEATURE_DELEG_INO,30 CEPHFS_FEATURE_DELEG_INO,
31 CEPHFS_FEATURE_METRIC_COLLECT,
31 32
32 CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_DELEG_INO,33 CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_METRIC_COLLECT,
33};34};
34 35
35/*36/*
@@ -43,6 +44,7 @@ enum ceph_feature_type { · 7 unmodified lines
43 CEPHFS_FEATURE_LAZY_CAP_WANTED, \44 CEPHFS_FEATURE_LAZY_CAP_WANTED, \
44 CEPHFS_FEATURE_MULTI_RECONNECT, \45 CEPHFS_FEATURE_MULTI_RECONNECT, \
45 CEPHFS_FEATURE_DELEG_INO, \46 CEPHFS_FEATURE_DELEG_INO, \
47 CEPHFS_FEATURE_METRIC_COLLECT, \
46 \48 \
47 CEPHFS_FEATURE_MAX, \49 CEPHFS_FEATURE_MAX, \
48}50}
fs/ceph/metric.c+148−0
@@ -1,10 +1,150 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/ceph/ceph_debug.h>
2 3
3#include <linux/types.h>4#include <linux/types.h>
4#include <linux/percpu_counter.h>5#include <linux/percpu_counter.h>
5#include <linux/math64.h>6#include <linux/math64.h>
6 7
7#include "metric.h"8#include "metric.h"
9#include "mds_client.h"
10
11static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc,
12 struct ceph_mds_session *s)
13{
14 struct ceph_metric_head *head;
15 struct ceph_metric_cap *cap;
16 struct ceph_metric_read_latency *read;
17 struct ceph_metric_write_latency *write;
18 struct ceph_metric_metadata_latency *meta;
19 struct ceph_client_metric *m = &mdsc->metric;
20 u64 nr_caps = atomic64_read(&m->total_caps);
21 struct ceph_msg *msg;
22 struct timespec64 ts;
23 s64 sum;
24 s32 items = 0;
25 s32 len;
26
27 len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write)
28 + sizeof(*meta);
29
30 msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true);
31 if (!msg) {
32 pr_err("send metrics to mds%d, failed to allocate message\n",
33 s->s_mds);
34 return false;
35 }
36
37 head = msg->front.iov_base;
38
39 /* encode the cap metric */
40 cap = (struct ceph_metric_cap *)(head + 1);
41 cap->type = cpu_to_le32(CLIENT_METRIC_TYPE_CAP_INFO);
42 cap->ver = 1;
43 cap->compat = 1;
44 cap->data_len = cpu_to_le32(sizeof(*cap) - 10);
45 cap->hit = cpu_to_le64(percpu_counter_sum(&mdsc->metric.i_caps_hit));
46 cap->mis = cpu_to_le64(percpu_counter_sum(&mdsc->metric.i_caps_mis));
47 cap->total = cpu_to_le64(nr_caps);
48 items++;
49
50 /* encode the read latency metric */
51 read = (struct ceph_metric_read_latency *)(cap + 1);
52 read->type = cpu_to_le32(CLIENT_METRIC_TYPE_READ_LATENCY);
53 read->ver = 1;
54 read->compat = 1;
55 read->data_len = cpu_to_le32(sizeof(*read) - 10);
56 sum = m->read_latency_sum;
57 jiffies_to_timespec64(sum, &ts);
58 read->sec = cpu_to_le32(ts.tv_sec);
59 read->nsec = cpu_to_le32(ts.tv_nsec);
60 items++;
61
62 /* encode the write latency metric */
63 write = (struct ceph_metric_write_latency *)(read + 1);
64 write->type = cpu_to_le32(CLIENT_METRIC_TYPE_WRITE_LATENCY);
65 write->ver = 1;
66 write->compat = 1;
67 write->data_len = cpu_to_le32(sizeof(*write) - 10);
68 sum = m->write_latency_sum;
69 jiffies_to_timespec64(sum, &ts);
70 write->sec = cpu_to_le32(ts.tv_sec);
71 write->nsec = cpu_to_le32(ts.tv_nsec);
72 items++;
73
74 /* encode the metadata latency metric */
75 meta = (struct ceph_metric_metadata_latency *)(write + 1);
76 meta->type = cpu_to_le32(CLIENT_METRIC_TYPE_METADATA_LATENCY);
77 meta->ver = 1;
78 meta->compat = 1;
79 meta->data_len = cpu_to_le32(sizeof(*meta) - 10);
80 sum = m->metadata_latency_sum;
81 jiffies_to_timespec64(sum, &ts);
82 meta->sec = cpu_to_le32(ts.tv_sec);
83 meta->nsec = cpu_to_le32(ts.tv_nsec);
84 items++;
85
86 put_unaligned_le32(items, &head->num);
87 msg->front.iov_len = len;
88 msg->hdr.version = cpu_to_le16(1);
89 msg->hdr.compat_version = cpu_to_le16(1);
90 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
91 dout("client%llu send metrics to mds%d\n",
92 ceph_client_gid(mdsc->fsc->client), s->s_mds);
93 ceph_con_send(&s->s_con, msg);
94
95 return true;
96}
97
98
99static void metric_get_session(struct ceph_mds_client *mdsc)
100{
101 struct ceph_mds_session *s;
102 int i;
103
104 mutex_lock(&mdsc->mutex);
105 for (i = 0; i < mdsc->max_sessions; i++) {
106 s = __ceph_lookup_mds_session(mdsc, i);
107 if (!s)
108 continue;
109
110 /*
111 * Skip it if MDS doesn't support the metric collection,
112 * or the MDS will close the session's socket connection
113 * directly when it get this message.
114 */
115 if (check_session_state(s) &&
116 test_bit(CEPHFS_FEATURE_METRIC_COLLECT, &s->s_features)) {
117 mdsc->metric.session = s;
118 break;
119 }
120
121 ceph_put_mds_session(s);
122 }
123 mutex_unlock(&mdsc->mutex);
124}
125
126static void metric_delayed_work(struct work_struct *work)
127{
128 struct ceph_client_metric *m =
129 container_of(work, struct ceph_client_metric, delayed_work.work);
130 struct ceph_mds_client *mdsc =
131 container_of(m, struct ceph_mds_client, metric);
132
133 if (mdsc->stopping)
134 return;
135
136 if (!m->session || !check_session_state(m->session)) {
137 if (m->session) {
138 ceph_put_mds_session(m->session);
139 m->session = NULL;
140 }
141 metric_get_session(mdsc);
142 }
143 if (m->session) {
144 ceph_mdsc_send_metrics(mdsc, m->session);
145 metric_schedule_delayed(m);
146 }
147}
8 148
9int ceph_metric_init(struct ceph_client_metric *m)149int ceph_metric_init(struct ceph_client_metric *m)
10{150{
@@ -52,6 +192,9 @@ int ceph_metric_init(struct ceph_client_metric *m) · 41 unmodified lines
52 m->total_metadatas = 0;192 m->total_metadatas = 0;
53 m->metadata_latency_sum = 0;193 m->metadata_latency_sum = 0;
54 194
195 m->session = NULL;
196 INIT_DELAYED_WORK(&m->delayed_work, metric_delayed_work);
197
55 return 0;198 return 0;
56 199
57err_i_caps_mis:200err_i_caps_mis:
@@ -73,6 +216,11 @@ void ceph_metric_destroy(struct ceph_client_metric *m) · 15 unmodified lines
73 percpu_counter_destroy(&m->i_caps_hit);216 percpu_counter_destroy(&m->i_caps_hit);
74 percpu_counter_destroy(&m->d_lease_mis);217 percpu_counter_destroy(&m->d_lease_mis);
75 percpu_counter_destroy(&m->d_lease_hit);218 percpu_counter_destroy(&m->d_lease_hit);
219
220 cancel_delayed_work_sync(&m->delayed_work);
221
222 if (m->session)
223 ceph_put_mds_session(m->session);
76}224}
77 225
78static inline void __update_latency(ktime_t *totalp, ktime_t *lsump,226static inline void __update_latency(ktime_t *totalp, ktime_t *lsump,
fs/ceph/metric.h+77−0
@@ -6,6 +6,71 @@ · 5 unmodified lines
6#include <linux/percpu_counter.h>6#include <linux/percpu_counter.h>
7#include <linux/ktime.h>7#include <linux/ktime.h>
8 8
9extern bool disable_send_metrics;
10
11enum ceph_metric_type {
12 CLIENT_METRIC_TYPE_CAP_INFO,
13 CLIENT_METRIC_TYPE_READ_LATENCY,
14 CLIENT_METRIC_TYPE_WRITE_LATENCY,
15 CLIENT_METRIC_TYPE_METADATA_LATENCY,
16 CLIENT_METRIC_TYPE_DENTRY_LEASE,
17
18 CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE,
19};
20
21/* metric caps header */
22struct ceph_metric_cap {
23 __le32 type; /* ceph metric type */
24
25 __u8 ver;
26 __u8 compat;
27
28 __le32 data_len; /* length of sizeof(hit + mis + total) */
29 __le64 hit;
30 __le64 mis;
31 __le64 total;
32} __packed;
33
34/* metric read latency header */
35struct ceph_metric_read_latency {
36 __le32 type; /* ceph metric type */
37
38 __u8 ver;
39 __u8 compat;
40
41 __le32 data_len; /* length of sizeof(sec + nsec) */
42 __le32 sec;
43 __le32 nsec;
44} __packed;
45
46/* metric write latency header */
47struct ceph_metric_write_latency {
48 __le32 type; /* ceph metric type */
49
50 __u8 ver;
51 __u8 compat;
52
53 __le32 data_len; /* length of sizeof(sec + nsec) */
54 __le32 sec;
55 __le32 nsec;
56} __packed;
57
58/* metric metadata latency header */
59struct ceph_metric_metadata_latency {
60 __le32 type; /* ceph metric type */
61
62 __u8 ver;
63 __u8 compat;
64
65 __le32 data_len; /* length of sizeof(sec + nsec) */
66 __le32 sec;
67 __le32 nsec;
68} __packed;
69
70struct ceph_metric_head {
71 __le32 num; /* the number of metrics that will be sent */
72} __packed;
73
9/* This is the global metrics */74/* This is the global metrics */
10struct ceph_client_metric {75struct ceph_client_metric {
11 atomic64_t total_dentries;76 atomic64_t total_dentries;
@@ -36,8 +101,20 @@ struct ceph_client_metric { · 24 unmodified lines
36 ktime_t metadata_latency_sq_sum;101 ktime_t metadata_latency_sq_sum;
37 ktime_t metadata_latency_min;102 ktime_t metadata_latency_min;
38 ktime_t metadata_latency_max;103 ktime_t metadata_latency_max;
104
105 struct ceph_mds_session *session;
106 struct delayed_work delayed_work; /* delayed work */
39};107};
40 108
109static inline void metric_schedule_delayed(struct ceph_client_metric *m)
110{
111 if (disable_send_metrics)
112 return;
113
114 /* per second */
115 schedule_delayed_work(&m->delayed_work, round_jiffies_relative(HZ));
116}
117
41extern int ceph_metric_init(struct ceph_client_metric *m);118extern int ceph_metric_init(struct ceph_client_metric *m);
42extern void ceph_metric_destroy(struct ceph_client_metric *m);119extern void ceph_metric_destroy(struct ceph_client_metric *m);
43 120
fs/ceph/super.c+42−0
@@ -27,6 +27,9 @@ · 26 unmodified lines
27#include <linux/ceph/auth.h>27#include <linux/ceph/auth.h>
28#include <linux/ceph/debugfs.h>28#include <linux/ceph/debugfs.h>
29 29
30static DEFINE_SPINLOCK(ceph_fsc_lock);
31static LIST_HEAD(ceph_fsc_list);
32
30/*33/*
31 * Ceph superblock operations34 * Ceph superblock operations
32 *35 *
@@ -691,6 +694,10 @@ static struct ceph_fs_client *create_fs_client(struct ceph_mount_options *fsopt, · 658 unmodified lines
691 if (!fsc->wb_pagevec_pool)694 if (!fsc->wb_pagevec_pool)
692 goto fail_cap_wq;695 goto fail_cap_wq;
693 696
697 spin_lock(&ceph_fsc_lock);
698 list_add_tail(&fsc->metric_wakeup, &ceph_fsc_list);
699 spin_unlock(&ceph_fsc_lock);
700
694 return fsc;701 return fsc;
695 702
696fail_cap_wq:703fail_cap_wq:
@@ -717,6 +724,10 @@ static void destroy_fs_client(struct ceph_fs_client *fsc) · 20 unmodified lines
717{724{
718 dout("destroy_fs_client %p\n", fsc);725 dout("destroy_fs_client %p\n", fsc);
719 726
727 spin_lock(&ceph_fsc_lock);
728 list_del(&fsc->metric_wakeup);
729 spin_unlock(&ceph_fsc_lock);
730
720 ceph_mdsc_destroy(fsc);731 ceph_mdsc_destroy(fsc);
721 destroy_workqueue(fsc->inode_wq);732 destroy_workqueue(fsc->inode_wq);
722 destroy_workqueue(fsc->cap_wq);733 destroy_workqueue(fsc->cap_wq);
@@ -1282,6 +1293,37 @@ static void __exit exit_ceph(void) · 559 unmodified lines
1282 destroy_caches();1293 destroy_caches();
1283}1294}
1284 1295
1296static int param_set_metrics(const char *val, const struct kernel_param *kp)
1297{
1298 struct ceph_fs_client *fsc;
1299 int ret;
1300
1301 ret = param_set_bool(val, kp);
1302 if (ret) {
1303 pr_err("Failed to parse sending metrics switch value '%s'\n",
1304 val);
1305 return ret;
1306 } else if (!disable_send_metrics) {
1307 // wake up all the mds clients
1308 spin_lock(&ceph_fsc_lock);
1309 list_for_each_entry(fsc, &ceph_fsc_list, metric_wakeup) {
1310 metric_schedule_delayed(&fsc->mdsc->metric);
1311 }
1312 spin_unlock(&ceph_fsc_lock);
1313 }
1314
1315 return 0;
1316}
1317
1318static const struct kernel_param_ops param_ops_metrics = {
1319 .set = param_set_metrics,
1320 .get = param_get_bool,
1321};
1322
1323bool disable_send_metrics = false;
1324module_param_cb(disable_send_metrics, &param_ops_metrics, &disable_send_metrics, 0644);
1325MODULE_PARM_DESC(disable_send_metrics, "Enable sending perf metrics to ceph cluster (default: on)");
1326
1285module_init(init_ceph);1327module_init(init_ceph);
1286module_exit(exit_ceph);1328module_exit(exit_ceph);
1287 1329
fs/ceph/super.h+2−0
@@ -101,6 +101,8 @@ struct ceph_mount_options { · 100 unmodified lines
101struct ceph_fs_client {101struct ceph_fs_client {
102 struct super_block *sb;102 struct super_block *sb;
103 103
104 struct list_head metric_wakeup;
105
104 struct ceph_mount_options *mount_options;106 struct ceph_mount_options *mount_options;
105 struct ceph_client *client;107 struct ceph_client *client;
106 108
include/linux/ceph/ceph_fs.h+1−0
@@ -130,6 +130,7 @@ struct ceph_dir_layout { · 129 unmodified lines
130#define CEPH_MSG_CLIENT_REQUEST 24130#define CEPH_MSG_CLIENT_REQUEST 24
131#define CEPH_MSG_CLIENT_REQUEST_FORWARD 25131#define CEPH_MSG_CLIENT_REQUEST_FORWARD 25
132#define CEPH_MSG_CLIENT_REPLY 26132#define CEPH_MSG_CLIENT_REPLY 26
133#define CEPH_MSG_CLIENT_METRICS 29
133#define CEPH_MSG_CLIENT_CAPS 0x310134#define CEPH_MSG_CLIENT_CAPS 0x310
134#define CEPH_MSG_CLIENT_LEASE 0x311135#define CEPH_MSG_CLIENT_LEASE 0x311
135#define CEPH_MSG_CLIENT_SNAP 0x312136#define CEPH_MSG_CLIENT_SNAP 0x312

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t