In preparation for allowing user specific CPU masks for IO thread creation, switch to using a mask embedded in the per-node wqe structure.
Unified · Split
| @@ -94,6 +94,8 @@ struct io_wqe { · 93 unmodified lines | |||
| 94 | 94 | ||
| 95 | struct io_wq *wq; | 95 | struct io_wq *wq; |
| 96 | struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; | 96 | struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS]; |
| 97 | |||
| 98 | cpumask_var_t cpu_mask; | ||
| 97 | }; | 99 | }; |
| 98 | 100 | ||
| 99 | /* | 101 | /* |
| @@ -638,7 +640,7 @@ fail: · 538 unmodified lines | |||
| 638 | 640 | ||
| 639 | tsk->pf_io_worker = worker; | 641 | tsk->pf_io_worker = worker; |
| 640 | worker->task = tsk; | 642 | worker->task = tsk; |
| 641 | set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node)); | 643 | set_cpus_allowed_ptr(tsk, wqe->cpu_mask); |
| 642 | tsk->flags |= PF_NO_SETAFFINITY; | 644 | tsk->flags |= PF_NO_SETAFFINITY; |
| 643 | 645 | ||
| 644 | raw_spin_lock_irq(&wqe->lock); | 646 | raw_spin_lock_irq(&wqe->lock); |
| @@ -922,6 +924,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 277 unmodified lines | |||
| 922 | wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); | 924 | wqe = kzalloc_node(sizeof(struct io_wqe), GFP_KERNEL, alloc_node); |
| 923 | if (!wqe) | 925 | if (!wqe) |
| 924 | goto err; | 926 | goto err; |
| 927 | if (!alloc_cpumask_var(&wqe->cpu_mask, GFP_KERNEL)) | ||
| 928 | goto err; | ||
| 929 | cpumask_copy(wqe->cpu_mask, cpumask_of_node(node)); | ||
| 925 | wq->wqes[node] = wqe; | 930 | wq->wqes[node] = wqe; |
| 926 | wqe->node = alloc_node; | 931 | wqe->node = alloc_node; |
| 927 | wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND; | 932 | wqe->acct[IO_WQ_ACCT_BOUND].index = IO_WQ_ACCT_BOUND; |
| @@ -947,8 +952,12 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 19 unmodified lines | |||
| 947 | err: | 952 | err: |
| 948 | io_wq_put_hash(data->hash); | 953 | io_wq_put_hash(data->hash); |
| 949 | cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); | 954 | cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node); |
| 950 | for_each_node(node) | 955 | for_each_node(node) { |
| 956 | if (!wq->wqes[node]) | ||
| 957 | continue; | ||
| 958 | free_cpumask_var(wq->wqes[node]->cpu_mask); | ||
| 951 | kfree(wq->wqes[node]); | 959 | kfree(wq->wqes[node]); |
| 960 | } | ||
| 952 | err_wq: | 961 | err_wq: |
| 953 | kfree(wq); | 962 | kfree(wq); |
| 954 | return ERR_PTR(ret); | 963 | return ERR_PTR(ret); |
| @@ -1018,6 +1027,7 @@ static void io_wq_destroy(struct io_wq *wq) · 63 unmodified lines | |||
| 1018 | .cancel_all = true, | 1027 | .cancel_all = true, |
| 1019 | }; | 1028 | }; |
| 1020 | io_wqe_cancel_pending_work(wqe, &match); | 1029 | io_wqe_cancel_pending_work(wqe, &match); |
| 1030 | free_cpumask_var(wqe->cpu_mask); | ||
| 1021 | kfree(wqe); | 1031 | kfree(wqe); |
| 1022 | } | 1032 | } |
| 1023 | io_wq_put_hash(wq->hash); | 1033 | io_wq_put_hash(wq->hash); |
| @@ -1032,31 +1042,57 @@ void io_wq_put_and_exit(struct io_wq *wq) · 8 unmodified lines | |||
| 1032 | io_wq_destroy(wq); | 1042 | io_wq_destroy(wq); |
| 1033 | } | 1043 | } |
| 1034 | 1044 | ||
| 1045 | struct online_data { | ||
| 1046 | unsigned int cpu; | ||
| 1047 | bool online; | ||
| 1048 | }; | ||
| 1049 | |||
| 1035 | static bool io_wq_worker_affinity(struct io_worker *worker, void *data) | 1050 | static bool io_wq_worker_affinity(struct io_worker *worker, void *data) |
| 1036 | { | 1051 | { |
| 1037 | set_cpus_allowed_ptr(worker->task, cpumask_of_node(worker->wqe->node)); | 1052 | struct online_data *od = data; |
| 1038 | 1053 | ||
| 1054 | if (od->online) | ||
| 1055 | cpumask_set_cpu(od->cpu, worker->wqe->cpu_mask); | ||
| 1056 | else | ||
| 1057 | cpumask_clear_cpu(od->cpu, worker->wqe->cpu_mask); | ||
| 1039 | return false; | 1058 | return false; |
| 1040 | } | 1059 | } |
| 1041 | 1060 | ||
| 1042 | static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) | 1061 | static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online) |
| 1043 | { | 1062 | { |
| 1044 | struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); | 1063 | struct online_data od = { |
| 1064 | .cpu = cpu, | ||
| 1065 | .online = online | ||
| 1066 | }; | ||
| 1045 | int i; | 1067 | int i; |
| 1046 | 1068 | ||
| 1047 | rcu_read_lock(); | 1069 | rcu_read_lock(); |
| 1048 | for_each_node(i) | 1070 | for_each_node(i) |
| 1049 | io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, NULL); | 1071 | io_wq_for_each_worker(wq->wqes[i], io_wq_worker_affinity, &od); |
| 1050 | rcu_read_unlock(); | 1072 | rcu_read_unlock(); |
| 1051 | return 0; | 1073 | return 0; |
| 1052 | } | 1074 | } |
| 1053 | 1075 | ||
| 1076 | static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node) | ||
| 1077 | { | ||
| 1078 | struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); | ||
| 1079 | |||
| 1080 | return __io_wq_cpu_online(wq, cpu, true); | ||
| 1081 | } | ||
| 1082 | |||
| 1083 | static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node) | ||
| 1084 | { | ||
| 1085 | struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node); | ||
| 1086 | |||
| 1087 | return __io_wq_cpu_online(wq, cpu, false); | ||
| 1088 | } | ||
| 1089 | |||
| 1054 | static __init int io_wq_init(void) | 1090 | static __init int io_wq_init(void) |
| 1055 | { | 1091 | { |
| 1056 | int ret; | 1092 | int ret; |
| 1057 | 1093 | ||
| 1058 | ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", | 1094 | ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online", |
| 1059 | io_wq_cpu_online, NULL); | 1095 | io_wq_cpu_online, io_wq_cpu_offline); |
| 1060 | if (ret < 0) | 1096 | if (ret < 0) |
| 1061 | return ret; | 1097 | return ret; |
| 1062 | io_wq_online = ret; | 1098 | io_wq_online = ret; |