History

io-wq: use private CPU mask

In preparation for allowing user specific CPU masks for IO thread creation, switch to using a mask embedded in the per-node wqe structure.

Jens Axboe committed 5 years ago

Commit 0e034961 parent ec16d351 file · +43 −7Browse files

Unified · Split

fs/io-wq.c+43−7
@@ -94,6 +94,8 @@ struct io_wqe { · 93 unmodified lines
9494
9595 struct io_wq *wq;
9696 struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];
97+
98+ cpumask_var_t cpu_mask;
9799 };
98100
99101 /*
@@ -638,7 +640,7 @@ fail: · 538 unmodified lines
638640
639641 tsk->pf_io_worker = worker;
640642 worker->task = tsk;
641− set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node));
643+ set_cpus_allowed_ptr(tsk, wqe->cpu_mask);
642644 tsk->flags |= PF_NO_SETAFFINITY;
643645
644646 raw_spin_lock_irq(&wqe->lock);
@@ -922,6 +924,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 277 unmodified lines
922924 wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node);
923925 if (!wqe)
924926 goto err;
927+ if (!alloc_cpumask_var(&wqe->cpu_mask, GFP_KERNEL))
928+ goto err;
929+ cpumask_copy(wqe->cpu_mask, cpumask_of_node(node));
925930 wq->wqes[node] = wqe;
926931 wqe->node = alloc_node;
927932 wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND;
@@ -947,8 +952,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 19 unmodified lines
947952 err:
948953 io_wq_put_hash(data->hash);
949954 cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
950− for_each_node(node)
955+ for_each_node(node) {
956+ if (!wq->wqes[node])
957+ continue;
958+ free_cpumask_var(wq->wqes[node]->cpu_mask);
951959 kfree(wq->wqes[node]);
960+ }
952961 err_wq:
953962 kfree(wq);
954963 return ERR_PTR(ret);
@@ -1018,6 +1027,7 @@ static void io_wq_destroy(struct io_wq *wq) · 63 unmodified lines
10181027 .cancel_all = true,
10191028 };
10201029 io_wqe_cancel_pending_work(wqe, &match);
1030+ free_cpumask_var(wqe->cpu_mask);
10211031 kfree(wqe);
10221032 }
10231033 io_wq_put_hash(wq->hash);
@@ -1032,31 +1042,57 @@ void io_wq_put_and_exit(struct io_wq *wq) · 8 unmodified lines
10321042 io_wq_destroy(wq);
10331043 }
10341044
1045+struct online_data {
1046+ unsigned int cpu;
1047+ bool online;
1048+};
1049+
10351050 static bool io_wq_worker_affinity(struct io_worker *worker, void *data)
10361051 {
1037− set_cpus_allowed_ptr(worker->task, cpumask_of_node(worker->wqe->node));
1052+ struct online_data *od = data;
10381053
1054+ if (od->online)
1055+ cpumask_set_cpu(od->cpu, worker->wqe->cpu_mask);
1056+ else
1057+ cpumask_clear_cpu(od->cpu, worker->wqe->cpu_mask);
10391058 return false;
10401059 }
10411060
1042−static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node)
1061+static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online)
10431062 {
1044− struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
1063+ struct online_data od = {
1064+ .cpu = cpu,
1065+ .online = online
1066+ };
10451067 int i;
10461068
10471069 rcu_read_lock();
10481070 for_each_node(i)
1049− io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL);
1071+ io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, &od);
10501072 rcu_read_unlock();
10511073 return 0;
10521074 }
10531075
1076+static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node)
1077+{
1078+ struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
1079+
1080+ return __io_wq_cpu_online(wq, cpu, true);
1081+}
1082+
1083+static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node)
1084+{
1085+ struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
1086+
1087+ return __io_wq_cpu_online(wq, cpu, false);
1088+}
1089+
10541090 static __init int io_wq_init(void)
10551091 {
10561092 int ret;
10571093
10581094 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online",
1059− io_wq_cpu_online, NULL);
1095+ io_wq_cpu_online, io_wq_cpu_offline);
10601096 if (ret < 0)
10611097 return ret;
10621098 io_wq_online = ret;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t