History

io-wq: use private CPU mask

In preparation for allowing user specific CPU masks for IO thread creation, switch to using a mask embedded in the per-node wqe structure.

Jens Axboe committed 5 years ago

Commit 0e034961 parent ec16d351 file · +43 −7Browse files

Unified · Split

fs/io-wq.c+43−7
@@ -94,6 +94,8 @@ struct io_wqe { · 93 unmodified lines
94 94
95 struct io_wq *wq;95 struct io_wq *wq;
96 struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];96 struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];
97
98 cpumask_var_t cpu_mask;
97};99};
98 100
99/*101/*
@@ -638,7 +640,7 @@ fail: · 538 unmodified lines
638 640
639 tsk->pf_io_worker = worker;641 tsk->pf_io_worker = worker;
640 worker->task = tsk;642 worker->task = tsk;
641 set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node));643 set_cpus_allowed_ptr(tsk, wqe->cpu_mask);
642 tsk->flags |= PF_NO_SETAFFINITY;644 tsk->flags |= PF_NO_SETAFFINITY;
643 645
644 raw_spin_lock_irq(&wqe->lock);646 raw_spin_lock_irq(&wqe->lock);
@@ -922,6 +924,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 277 unmodified lines
922 wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node);924 wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node);
923 if (!wqe)925 if (!wqe)
924 goto err;926 goto err;
927 if (!alloc_cpumask_var(&wqe->cpu_mask, GFP_KERNEL))
928 goto err;
929 cpumask_copy(wqe->cpu_mask, cpumask_of_node(node));
925 wq->wqes[node] = wqe;930 wq->wqes[node] = wqe;
926 wqe->node = alloc_node;931 wqe->node = alloc_node;
927 wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND;932 wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND;
@@ -947,8 +952,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 19 unmodified lines
947err:952err:
948 io_wq_put_hash(data->hash);953 io_wq_put_hash(data->hash);
949 cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);954 cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
950 for_each_node(node)955 for_each_node(node) {
956 if (!wq->wqes[node])
957 continue;
958 free_cpumask_var(wq->wqes[node]->cpu_mask);
951 kfree(wq->wqes[node]);959 kfree(wq->wqes[node]);
960 }
952err_wq:961err_wq:
953 kfree(wq);962 kfree(wq);
954 return ERR_PTR(ret);963 return ERR_PTR(ret);
@@ -1018,6 +1027,7 @@ static void io_wq_destroy(struct io_wq *wq) · 63 unmodified lines
1018 .cancel_all = true,1027 .cancel_all = true,
1019 };1028 };
1020 io_wqe_cancel_pending_work(wqe, &match);1029 io_wqe_cancel_pending_work(wqe, &match);
1030 free_cpumask_var(wqe->cpu_mask);
1021 kfree(wqe);1031 kfree(wqe);
1022 }1032 }
1023 io_wq_put_hash(wq->hash);1033 io_wq_put_hash(wq->hash);
@@ -1032,31 +1042,57 @@ void io_wq_put_and_exit(struct io_wq *wq) · 8 unmodified lines
1032 io_wq_destroy(wq);1042 io_wq_destroy(wq);
1033}1043}
1034 1044
1045struct online_data {
1046 unsigned int cpu;
1047 bool online;
1048};
1049
1035static bool io_wq_worker_affinity(struct io_worker *worker, void *data)1050static bool io_wq_worker_affinity(struct io_worker *worker, void *data)
1036{1051{
1037 set_cpus_allowed_ptr(worker->task, cpumask_of_node(worker->wqe->node));1052 struct online_data *od = data;
1038 1053
1054 if (od->online)
1055 cpumask_set_cpu(od->cpu, worker->wqe->cpu_mask);
1056 else
1057 cpumask_clear_cpu(od->cpu, worker->wqe->cpu_mask);
1039 return false;1058 return false;
1040}1059}
1041 1060
1042static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node)1061static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online)
1043{1062{
1044 struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);1063 struct online_data od = {
1064 .cpu = cpu,
1065 .online = online
1066 };
1045 int i;1067 int i;
1046 1068
1047 rcu_read_lock();1069 rcu_read_lock();
1048 for_each_node(i)1070 for_each_node(i)
1049 io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL);1071 io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, &od);
1050 rcu_read_unlock();1072 rcu_read_unlock();
1051 return 0;1073 return 0;
1052}1074}
1053 1075
1076static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node)
1077{
1078 struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
1079
1080 return __io_wq_cpu_online(wq, cpu, true);
1081}
1082
1083static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node)
1084{
1085 struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
1086
1087 return __io_wq_cpu_online(wq, cpu, false);
1088}
1089
1054static __init int io_wq_init(void)1090static __init int io_wq_init(void)
1055{1091{
1056 int ret;1092 int ret;
1057 1093
1058 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online",1094 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online",
1059 io_wq_cpu_online, NULL);1095 io_wq_cpu_online, io_wq_cpu_offline);
1060 if (ret < 0)1096 if (ret < 0)
1061 return ret;1097 return ret;
1062 io_wq_online = ret;1098 io_wq_online = ret;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t