Add a count of the number of RCU users (currently 1) of the task struct so that we can later add the scheduler case and get rid of the very subtle task_rcu_dereference(), and just use rcu_dereference(). As suggested by Oleg have the count overlap rcu_head so that no additional space in task_struct is required.
Unified · Split
| @@ -1147,7 +1147,10 @@ struct task_struct { · 1146 unmodified lines | |||
| 1147 | 1147 | ||
| 1148 | struct tlbflush_unmap_batch tlb_ubc; | 1148 | struct tlbflush_unmap_batch tlb_ubc; |
| 1149 | 1149 | ||
| 1150 | struct rcu_head rcu; | 1150 | union { |
| 1151 | refcount_t rcu_users; | ||
| 1152 | struct rcu_head rcu; | ||
| 1153 | }; | ||
| 1151 | 1154 | ||
| 1152 | /* Cache last used pipe for splice(): */ | 1155 | /* Cache last used pipe for splice(): */ |
| 1153 | struct pipe_inode_info *splice_pipe; | 1156 | struct pipe_inode_info *splice_pipe; |
| @@ -120,6 +120,7 @@ static inline void put_task_struct(struct task_struct *t) · 119 unmodified lines | |||
| 120 | } | 120 | } |
| 121 | 121 | ||
| 122 | struct task_struct *task_rcu_dereference(struct task_struct **ptask); | 122 | struct task_struct *task_rcu_dereference(struct task_struct **ptask); |
| 123 | void put_task_struct_rcu_user(struct task_struct *task); | ||
| 123 | 124 | ||
| 124 | #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT | 125 | #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT |
| 125 | extern int arch_task_struct_size __read_mostly; | 126 | extern int arch_task_struct_size __read_mostly; |
| @@ -182,6 +182,11 @@ static void delayed_put_task_struct(struct rcu_head *rhp) · 181 unmodified lines | |||
| 182 | put_task_struct(tsk); | 182 | put_task_struct(tsk); |
| 183 | } | 183 | } |
| 184 | 184 | ||
| 185 | void put_task_struct_rcu_user(struct task_struct *task) | ||
| 186 | { | ||
| 187 | if (refcount_dec_and_test(&task->rcu_users)) | ||
| 188 | call_rcu(&task->rcu, delayed_put_task_struct); | ||
| 189 | } | ||
| 185 | 190 | ||
| 186 | void release_task(struct task_struct *p) | 191 | void release_task(struct task_struct *p) |
| 187 | { | 192 | { |
| @@ -222,7 +227,7 @@ repeat: · 34 unmodified lines | |||
| 222 | 227 | ||
| 223 | write_unlock_irq(&tasklist_lock); | 228 | write_unlock_irq(&tasklist_lock); |
| 224 | release_thread(p); | 229 | release_thread(p); |
| 225 | call_rcu(&p->rcu, delayed_put_task_struct); | 230 | put_task_struct_rcu_user(p); |
| 226 | 231 | ||
| 227 | p = leader; | 232 | p = leader; |
| 228 | if (unlikely(zap_leader)) | 233 | if (unlikely(zap_leader)) |
| @@ -902,10 +902,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) · 901 unmodified lines | |||
| 902 | if (orig->cpus_ptr == &orig->cpus_mask) | 902 | if (orig->cpus_ptr == &orig->cpus_mask) |
| 903 | tsk->cpus_ptr = &tsk->cpus_mask; | 903 | tsk->cpus_ptr = &tsk->cpus_mask; |
| 904 | 904 | ||
| 905 | /* | 905 | /* One for the user space visible state that goes away when reaped. */ |
| 906 | * One for us, one for whoever does the "release_task()" (usually | 906 | refcount_set(&tsk->rcu_users, 1); |
| 907 | * parent) | 907 | /* One for the rcu users, and one for the scheduler */ |
| 908 | */ | ||
| 909 | refcount_set(&tsk->usage, 2); | 908 | refcount_set(&tsk->usage, 2); |
| 910 | #ifdef CONFIG_BLK_DEV_IO_TRACE | 909 | #ifdef CONFIG_BLK_DEV_IO_TRACE |
| 911 | tsk->btrace_seq = 0; | 910 | tsk->btrace_seq = 0; |