Rstat currently only supports the default hierarchy in cgroup2. In order to replace memcg's private stats infrastructure - used in both cgroup1 and cgroup2 - with rstat, the latter needs to support cgroup1. The initialization and destruction callbacks for regular cgroups are already in place. Remove the cgroup_on_dfl() guards to handle cgroup1. The initialization of the root cgroup is currently hardcoded to only handle cgrp_dfl_root.cgrp. Move those callbacks to cgroup_setup_root() and cgroup_destroy_root() to handle the default root as well as the various cgroup1 roots we may set up during mounting. The linking of css to cgroups happens in code shared between cgroup1 and cgroup2 as well. Simply remove the cgroup_on_dfl() guard. Linkage of the root css to the root cgroup is a bit trickier: per default, the root css of a subsystem controller belongs to the default hierarchy (i.e. the cgroup2 root). When a controller is mounted in its cgroup1 version, the root css is stolen and moved to the cgroup1 root; on unmount, the css moves back to the default hierarchy. Annotate rebind_subsystems() to move the root css linkage along between roots.
Showing the changes to kernel (2 of 2 files). Show all
Unified · Split
| @@ -1339,6 +1339,7 @@ static void cgroup_destroy_root(struct cgroup_root *root) · 1338 unmodified lines | |||
| 1339 | 1339 | ||
| 1340 | mutex_unlock(&cgroup_mutex); | 1340 | mutex_unlock(&cgroup_mutex); |
| 1341 | 1341 | ||
| 1342 | cgroup_rstat_exit(cgrp); | ||
| 1342 | kernfs_destroy_root(root->kf_root); | 1343 | kernfs_destroy_root(root->kf_root); |
| 1343 | cgroup_free_root(root); | 1344 | cgroup_free_root(root); |
| 1344 | } | 1345 | } |
| @@ -1751,6 +1752,12 @@ int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask) · 406 unmodified lines | |||
| 1751 | &dcgrp->e_csets[ss->id]); | 1752 | &dcgrp->e_csets[ss->id]); |
| 1752 | spin_unlock_irq(&css_set_lock); | 1753 | spin_unlock_irq(&css_set_lock); |
| 1753 | 1754 | ||
| 1755 | if (ss->css_rstat_flush) { | ||
| 1756 | list_del_rcu(&css->rstat_css_node); | ||
| 1757 | list_add_rcu(&css->rstat_css_node, | ||
| 1758 | &dcgrp->rstat_css_list); | ||
| 1759 | } | ||
| 1760 | |||
| 1754 | /* default hierarchy doesn't enable controllers by default */ | 1761 | /* default hierarchy doesn't enable controllers by default */ |
| 1755 | dst_root->subsys_mask |= 1 << ssid; | 1762 | dst_root->subsys_mask |= 1 << ssid; |
| 1756 | if (dst_root == &cgrp_dfl_root) { | 1763 | if (dst_root == &cgrp_dfl_root) { |
| @@ -1971,10 +1978,14 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) · 214 unmodified lines | |||
| 1971 | if (ret) | 1978 | if (ret) |
| 1972 | goto destroy_root; | 1979 | goto destroy_root; |
| 1973 | 1980 | ||
| 1974 | ret = rebind_subsystems(root, ss_mask); | 1981 | ret = cgroup_rstat_init(root_cgrp); |
| 1975 | if (ret) | 1982 | if (ret) |
| 1976 | goto destroy_root; | 1983 | goto destroy_root; |
| 1977 | 1984 | ||
| 1985 | ret = rebind_subsystems(root, ss_mask); | ||
| 1986 | if (ret) | ||
| 1987 | goto exit_stats; | ||
| 1988 | |||
| 1978 | ret = cgroup_bpf_inherit(root_cgrp); | 1989 | ret = cgroup_bpf_inherit(root_cgrp); |
| 1979 | WARN_ON_ONCE(ret); | 1990 | WARN_ON_ONCE(ret); |
| 1980 | 1991 | ||
| @@ -2006,6 +2017,8 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) · 25 unmodified lines | |||
| 2006 | ret = 0; | 2017 | ret = 0; |
| 2007 | goto out; | 2018 | goto out; |
| 2008 | 2019 | ||
| 2020 | exit_stats: | ||
| 2021 | cgroup_rstat_exit(root_cgrp); | ||
| 2009 | destroy_root: | 2022 | destroy_root: |
| 2010 | kernfs_destroy_root(root->kf_root); | 2023 | kernfs_destroy_root(root->kf_root); |
| 2011 | root->kf_root = NULL; | 2024 | root->kf_root = NULL; |
| @@ -4934,8 +4947,7 @@ static void css_free_rwork_fn(struct work_struct *work) · 2922 unmodified lines | |||
| 4934 | cgroup_put(cgroup_parent(cgrp)); | 4947 | cgroup_put(cgroup_parent(cgrp)); |
| 4935 | kernfs_put(cgrp->kn); | 4948 | kernfs_put(cgrp->kn); |
| 4936 | psi_cgroup_free(cgrp); | 4949 | psi_cgroup_free(cgrp); |
| 4937 | if (cgroup_on_dfl(cgrp)) | 4950 | cgroup_rstat_exit(cgrp); |
| 4938 | cgroup_rstat_exit(cgrp); | ||
| 4939 | kfree(cgrp); | 4951 | kfree(cgrp); |
| 4940 | } else { | 4952 | } else { |
| 4941 | /* | 4953 | /* |
| @@ -4976,8 +4988,7 @@ static void css_release_work_fn(struct work_struct *work) · 34 unmodified lines | |||
| 4976 | /* cgroup release path */ | 4988 | /* cgroup release path */ |
| 4977 | TRACE_CGROUP_PATH(release, cgrp); | 4989 | TRACE_CGROUP_PATH(release, cgrp); |
| 4978 | 4990 | ||
| 4979 | if (cgroup_on_dfl(cgrp)) | 4991 | cgroup_rstat_flush(cgrp); |
| 4980 | cgroup_rstat_flush(cgrp); | ||
| 4981 | 4992 | ||
| 4982 | spin_lock_irq(&css_set_lock); | 4993 | spin_lock_irq(&css_set_lock); |
| 4983 | for (tcgrp = cgroup_parent(cgrp); tcgrp; | 4994 | for (tcgrp = cgroup_parent(cgrp); tcgrp; |
| @@ -5034,7 +5045,7 @@ static void init_and_link_css(struct cgroup_subsys_state *css, · 50 unmodified lines | |||
| 5034 | css_get(css->parent); | 5045 | css_get(css->parent); |
| 5035 | } | 5046 | } |
| 5036 | 5047 | ||
| 5037 | if (cgroup_on_dfl(cgrp) && ss->css_rstat_flush) | 5048 | if (ss->css_rstat_flush) |
| 5038 | list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list); | 5049 | list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list); |
| 5039 | 5050 | ||
| 5040 | BUG_ON(cgroup_css(cgrp, ss)); | 5051 | BUG_ON(cgroup_css(cgrp, ss)); |
| @@ -5159,11 +5170,9 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, · 118 unmodified lines | |||
| 5159 | if (ret) | 5170 | if (ret) |
| 5160 | goto out_free_cgrp; | 5171 | goto out_free_cgrp; |
| 5161 | 5172 | ||
| 5162 | if (cgroup_on_dfl(parent)) { | 5173 | ret = cgroup_rstat_init(cgrp); |
| 5163 | ret = cgroup_rstat_init(cgrp); | 5174 | if (ret) |
| 5164 | if (ret) | 5175 | goto out_cancel_ref; |
| 5165 | goto out_cancel_ref; | ||
| 5166 | } | ||
| 5167 | 5176 | ||
| 5168 | /* create the directory */ | 5177 | /* create the directory */ |
| 5169 | kn = kernfs_create_dir(parent->kn, name, mode, cgrp); | 5178 | kn = kernfs_create_dir(parent->kn, name, mode, cgrp); |
| @@ -5250,8 +5259,7 @@ out_psi_free: · 80 unmodified lines | |||
| 5250 | out_kernfs_remove: | 5259 | out_kernfs_remove: |
| 5251 | kernfs_remove(cgrp->kn); | 5260 | kernfs_remove(cgrp->kn); |
| 5252 | out_stat_exit: | 5261 | out_stat_exit: |
| 5253 | if (cgroup_on_dfl(parent)) | 5262 | cgroup_rstat_exit(cgrp); |
| 5254 | cgroup_rstat_exit(cgrp); | ||
| 5255 | out_cancel_ref: | 5263 | out_cancel_ref: |
| 5256 | percpu_ref_exit(&cgrp->self.refcnt); | 5264 | percpu_ref_exit(&cgrp->self.refcnt); |
| 5257 | out_free_cgrp: | 5265 | out_free_cgrp: |
| @@ -285,8 +285,6 @@ void __init cgroup_rstat_boot(void) · 284 unmodified lines | |||
| 285 | 285 | ||
| 286 | for_each_possible_cpu(cpu) | 286 | for_each_possible_cpu(cpu) |
| 287 | raw_spin_lock_init(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); | 287 | raw_spin_lock_init(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); |
| 288 | |||
| 289 | BUG_ON(cgroup_rstat_init(&cgrp_dfl_root.cgrp)); | ||
| 290 | } | 288 | } |
| 291 | 289 | ||
| 292 | /* | 290 | /* |