In preparation for allowing user specific CPU masks for IO thread creation, switch to using a mask embedded in the per-node wqe structure.
Unified · Split
| @@ -94,6 +94,8 @@ struct io_wqe { · 93 unmodified lines | ||
| 94 | 94 | |
| 95 | 95 | struct io_wq *wq; |
| 96 | 96 | struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; |
| 97 | + | |
| 98 | + cpumask_var_t cpu_mask; | |
| 97 | 99 | }; |
| 98 | 100 | |
| 99 | 101 | /* |
| @@ -638,7 +640,7 @@ fail: · 538 unmodified lines | ||
| 638 | 640 | |
| 639 | 641 | tsk->pf_io_worker = worker; |
| 640 | 642 | worker->task = tsk; |
| 641 | − set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node)); | |
| 643 | + set_cpus_allowed_ptr(tsk, wqe->cpu_mask); | |
| 642 | 644 | tsk->flags |= PF_NO_SETAFFINITY; |
| 643 | 645 | |
| 644 | 646 | raw_spin_lock_irq(&wqe->lock); |
| @@ -922,6 +924,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 277 unmodified lines | ||
| 922 | 924 | wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); |
| 923 | 925 | if (!wqe) |
| 924 | 926 | goto err; |
| 927 | + if (!alloc_cpumask_var(&wqe->cpu_mask, GFP_KERNEL)) | |
| 928 | + goto err; | |
| 929 | + cpumask_copy(wqe->cpu_mask, cpumask_of_node(node)); | |
| 925 | 930 | wq->wqes[node] = wqe; |
| 926 | 931 | wqe->node = alloc_node; |
| 927 | 932 | wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND; |
| @@ -947,8 +952,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 19 unmodified lines | ||
| 947 | 952 | err: |
| 948 | 953 | io_wq_put_hash(data->hash); |
| 949 | 954 | cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); |
| 950 | − for_each_node(node) | |
| 955 | + for_each_node(node) { | |
| 956 | + if (!wq->wqes[node]) | |
| 957 | + continue; | |
| 958 | + free_cpumask_var(wq->wqes[node]->cpu_mask); | |
| 951 | 959 | kfree(wq->wqes[node]); |
| 960 | + } | |
| 952 | 961 | err_wq: |
| 953 | 962 | kfree(wq); |
| 954 | 963 | return ERR_PTR(ret); |
| @@ -1018,6 +1027,7 @@ static void io_wq_destroy(struct io_wq *wq) · 63 unmodified lines | ||
| 1018 | 1027 | .cancel_all = true, |
| 1019 | 1028 | }; |
| 1020 | 1029 | io_wqe_cancel_pending_work(wqe, &match); |
| 1030 | + free_cpumask_var(wqe->cpu_mask); | |
| 1021 | 1031 | kfree(wqe); |
| 1022 | 1032 | } |
| 1023 | 1033 | io_wq_put_hash(wq->hash); |
| @@ -1032,31 +1042,57 @@ void io_wq_put_and_exit(struct io_wq *wq) · 8 unmodified lines | ||
| 1032 | 1042 | io_wq_destroy(wq); |
| 1033 | 1043 | } |
| 1034 | 1044 | |
| 1045 | +struct online_data { | |
| 1046 | + unsigned int cpu; | |
| 1047 | + bool online; | |
| 1048 | +}; | |
| 1049 | + | |
| 1035 | 1050 | static bool io_wq_worker_affinity(struct io_worker *worker, void *data) |
| 1036 | 1051 | { |
| 1037 | − set_cpus_allowed_ptr(worker->task, cpumask_of_node(worker->wqe->node)); | |
| 1052 | + struct online_data *od = data; | |
| 1038 | 1053 | |
| 1054 | + if (od->online) | |
| 1055 | + cpumask_set_cpu(od->cpu, worker->wqe->cpu_mask); | |
| 1056 | + else | |
| 1057 | + cpumask_clear_cpu(od->cpu, worker->wqe->cpu_mask); | |
| 1039 | 1058 | return false; |
| 1040 | 1059 | } |
| 1041 | 1060 | |
| 1042 | −static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) | |
| 1061 | +static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online) | |
| 1043 | 1062 | { |
| 1044 | − struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); | |
| 1063 | + struct online_data od = { | |
| 1064 | + .cpu = cpu, | |
| 1065 | + .online = online | |
| 1066 | + }; | |
| 1045 | 1067 | int i; |
| 1046 | 1068 | |
| 1047 | 1069 | rcu_read_lock(); |
| 1048 | 1070 | for_each_node(i) |
| 1049 | − io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL); | |
| 1071 | + io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, &od); | |
| 1050 | 1072 | rcu_read_unlock(); |
| 1051 | 1073 | return 0; |
| 1052 | 1074 | } |
| 1053 | 1075 | |
| 1076 | +static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) | |
| 1077 | +{ | |
| 1078 | + struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); | |
| 1079 | + | |
| 1080 | + return __io_wq_cpu_online(wq, cpu, true); | |
| 1081 | +} | |
| 1082 | + | |
| 1083 | +static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node) | |
| 1084 | +{ | |
| 1085 | + struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); | |
| 1086 | + | |
| 1087 | + return __io_wq_cpu_online(wq, cpu, false); | |
| 1088 | +} | |
| 1089 | + | |
| 1054 | 1090 | static __init int io_wq_init(void) |
| 1055 | 1091 | { |
| 1056 | 1092 | int ret; |
| 1057 | 1093 | |
| 1058 | 1094 | ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", |
| 1059 | − io_wq_cpu_online, NULL); | |
| 1095 | + io_wq_cpu_online, io_wq_cpu_offline); | |
| 1060 | 1096 | if (ret < 0) |
| 1061 | 1097 | return ret; |
| 1062 | 1098 | io_wq_online = ret; |