Since be44562613851 ("cgroup: remove synchronize_rcu() from cgroup_diput()"), cgroup destruction path makes use of workqueue. css freeing is performed from a work item from that point on and a later commit, ea15f8ccdb430 ("cgroup: split cgroup destruction into two steps"), moves css offlining to workqueue too. As cgroup destruction isn't depended upon for memory reclaim, the destruction work items were put on the system_wq; unfortunately, some controller may block in the destruction path for considerable duration while holding cgroup_mutex. As large part of destruction path is synchronized through cgroup_mutex, when combined with high rate of cgroup removals, this has potential to fill up system_wq's max_active of 256. Also, it turns out that memcg's css destruction path ends up queueing and waiting for work items on system_wq through work_on_cpu(). If such operation happens while system_wq is fully occupied by cgroup destruction work items, work_on_cpu() can't make forward progress because system_wq is full and other destruction work items on system_wq can't make forward progress because the work item waiting for work_on_cpu() is holding cgroup_mutex, leading to deadlock. This can be fixed by queueing destruction work items on a separate workqueue. This patch creates a dedicated workqueue - cgroup_destroy_wq - for this purpose. As these work items shouldn't have inter-dependencies and mostly serialized by cgroup_mutex anyway, giving high concurrency level doesn't buy anything and the workqueue's @max_active is set to 1 so that destruction work items are executed one by one on each CPU. Hugh Dickins: Because cgroup_init() is run before init_workqueues(), cgroup_destroy_wq can't be allocated from cgroup_init(). Do it from a separate core_initcall(). In the future, we probably want to reorder so that workqueue init happens before cgroup_init().
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -89,6 +89,14 @@ static DEFINE_MUTEX(cgroup_mutex); · 88 unmodified lines | ||
| 89 | 89 | |
| 90 | 90 | static DEFINE_MUTEX(cgroup_root_mutex); |
| 91 | 91 | |
| 92 | +/* | |
| 93 | + * cgroup destruction makes heavy use of work items and there can be a lot | |
| 94 | + * of concurrent destructions. Use a separate workqueue so that cgroup | |
| 95 | + * destruction work items don't end up filling up max_active of system_wq | |
| 96 | + * which may lead to deadlock. | |
| 97 | + */ | |
| 98 | +static struct workqueue_struct *cgroup_destroy_wq; | |
| 99 | + | |
| 92 | 100 | /* |
| 93 | 101 | * Generate an array of cgroup subsystem pointers. At boot time, this is |
| 94 | 102 | * populated with the built in subsystems, and modular subsystems are |
| @@ -871,7 +879,7 @@ static void cgroup_free_rcu(struct rcu_head *head) · 776 unmodified lines | ||
| 871 | 879 | struct cgroup *cgrp = container_of(head, struct cgroup, rcu_head); |
| 872 | 880 | |
| 873 | 881 | INIT_WORK(&cgrp->destroy_work, cgroup_free_fn); |
| 874 | − schedule_work(&cgrp->destroy_work); | |
| 882 | + queue_work(cgroup_destroy_wq, &cgrp->destroy_work); | |
| 875 | 883 | } |
| 876 | 884 | |
| 877 | 885 | static void cgroup_diput(struct dentry *dentry, struct inode *inode) |
| @@ -4249,7 +4257,7 @@ static void css_free_rcu_fn(struct rcu_head *rcu_head) · 3371 unmodified lines | ||
| 4249 | 4257 | * css_put(). dput() requires process context which we don't have. |
| 4250 | 4258 | */ |
| 4251 | 4259 | INIT_WORK(&css->destroy_work, css_free_work_fn); |
| 4252 | − schedule_work(&css->destroy_work); | |
| 4260 | + queue_work(cgroup_destroy_wq, &css->destroy_work); | |
| 4253 | 4261 | } |
| 4254 | 4262 | |
| 4255 | 4263 | static void css_release(struct percpu_ref *ref) |
| @@ -4539,7 +4547,7 @@ static void css_killed_ref_fn(struct percpu_ref *ref) · 283 unmodified lines | ||
| 4539 | 4547 | container_of(ref, struct cgroup_subsys_state, refcnt); |
| 4540 | 4548 | |
| 4541 | 4549 | INIT_WORK(&css->destroy_work, css_killed_work_fn); |
| 4542 | − schedule_work(&css->destroy_work); | |
| 4550 | + queue_work(cgroup_destroy_wq, &css->destroy_work); | |
| 4543 | 4551 | } |
| 4544 | 4552 | |
| 4545 | 4553 | /** |
| @@ -5063,6 +5071,22 @@ out: · 517 unmodified lines | ||
| 5063 | 5071 | return err; |
| 5064 | 5072 | } |
| 5065 | 5073 | |
| 5074 | +static int __init cgroup_wq_init(void) | |
| 5075 | +{ | |
| 5076 | + /* | |
| 5077 | + * There isn't much point in executing destruction path in | |
| 5078 | + * parallel. Good chunk is serialized with cgroup_mutex anyway. | |
| 5079 | + * Use 1 for @max_active. | |
| 5080 | + * | |
| 5081 | + * We would prefer to do this in cgroup_init() above, but that | |
| 5082 | + * is called before init_workqueues(): so leave this until after. | |
| 5083 | + */ | |
| 5084 | + cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); | |
| 5085 | + BUG_ON(!cgroup_destroy_wq); | |
| 5086 | + return 0; | |
| 5087 | +} | |
| 5088 | +core_initcall(cgroup_wq_init); | |
| 5089 | + | |
| 5066 | 5090 | /* |
| 5067 | 5091 | * proc_cgroup_show() |
| 5068 | 5092 | * - Print task's cgroup paths into seq_file, one line for each hierarchy |