Recent optimizations of sd->shared assignment moved to allocating a single instance of per-CPU sched_domain_shared objects per s_data. Recent optimizations to select_idle_capacity() moved the sd->shared assignments to "sd_asym" domain when ASYM_CPUCAPACITY is detected but cache-aware scheduling mandates the presence of "sd_llc_shared" to compute and cache per-LLC statistics. Use an "alloc_flags" union in sched_domain_shared to claim a sched_domain_shared object per sched_domain. Allocation starts searching for an available / matching sched_domain_shared instance from the first CPU of sched_domain_span(sd) (sd can be sd_llc, or sd_asym). If the shared object is claimed by another domain, the instance corresponding to next CPU in the domain span is explored until a matching / available instance is found. In case of a single CPU in sched_domain_span(), the domain will be degenerated and a temporary overlap of ->shared objects across different domains is acceptable. "alloc_flags" forms a union with "nr_idle_scan" and the stale flags are left as is when the sd->shared is published. The expectation is for the first load balancing instance to correct the value just like the current behavior, except the initial value is no longer 0.
Showing the changes to kernel (1 of 2 files). Show all
Unified · Split
| @@ -623,6 +623,12 @@ static void free_sched_groups(struct sched_group *sg, int free_sgc) · 622 unmodified lines | ||
| 623 | 623 | } while (sg != first); |
| 624 | 624 | } |
| 625 | 625 | |
| 626 | +static void free_sched_domain_shared(struct sched_domain_shared *sds) | |
| 627 | +{ | |
| 628 | + if (sds && atomic_dec_and_test(&sds->ref)) | |
| 629 | + kfree(sds); | |
| 630 | +} | |
| 631 | + | |
| 626 | 632 | static void destroy_sched_domain(struct sched_domain *sd) |
| 627 | 633 | { |
| 628 | 634 | /* |
| @@ -631,9 +637,7 @@ static void destroy_sched_domain(struct sched_domain *sd) · 2 unmodified lines | ||
| 631 | 637 | * dropping group/capacity references, freeing where none remain. |
| 632 | 638 | */ |
| 633 | 639 | free_sched_groups(sd->groups, 1); |
| 634 | − | |
| 635 | − if (sd->shared && atomic_dec_and_test(&sd->shared->ref)) | |
| 636 | − kfree(sd->shared); | |
| 640 | + free_sched_domain_shared(sd->shared); | |
| 637 | 641 | |
| 638 | 642 | #ifdef CONFIG_SCHED_CACHE |
| 639 | 643 | /* only the bottom sd has llc_counts array */ |
| @@ -755,7 +759,14 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) · 115 unmodified lines | ||
| 755 | 759 | |
| 756 | 760 | /* Pick reference to parent->shared. */ |
| 757 | 761 | if (parent->shared) { |
| 758 | − WARN_ON_ONCE(tmp->shared); | |
| 762 | + /* | |
| 763 | + * It is safe to free a sd->shared that | |
| 764 | + * has not been published yet. If a | |
| 765 | + * sd->shared was published, the refcount | |
| 766 | + * will end up being non-zero and it will | |
| 767 | + * not be freed here. | |
| 768 | + */ | |
| 769 | + free_sched_domain_shared(tmp->shared); | |
| 759 | 770 | tmp->shared = parent->shared; |
| 760 | 771 | parent->shared = NULL; |
| 761 | 772 | } |
| @@ -2916,11 +2927,45 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc) · 2154 unmodified lines | ||
| 2916 | 2927 | } |
| 2917 | 2928 | } |
| 2918 | 2929 | |
| 2919 | −static void init_sched_domain_shared(struct s_data *d, struct sched_domain *sd) | |
| 2930 | +static void | |
| 2931 | +init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags) | |
| 2920 | 2932 | { |
| 2921 | − int sd_id = cpumask_first(sched_domain_span(sd)); | |
| 2933 | + struct sched_domain_shared *sds = NULL; | |
| 2934 | + int cpu; | |
| 2935 | + | |
| 2936 | + /* | |
| 2937 | + * Multiple domains can try to claim a shared object like | |
| 2938 | + * SD_ASYM_CPUCAPACITY and SD_SHARE_LLC which can alias to | |
| 2939 | + * same cpumask_first(sched_domain_span(sd)) CPU and can | |
| 2940 | + * cause "nr_idle_scan" to be populated incorrectly during | |
| 2941 | + * load balancing. | |
| 2942 | + * | |
| 2943 | + * Find the first CPU in sched_domain_span(sd) with an | |
| 2944 | + * unclaimed domain (!alloc_flags) or where the alloc_flag | |
| 2945 | + * matches the requested flag (SD_* flag) | |
| 2946 | + * | |
| 2947 | + * If the domain only has single CPU, allow temporary overlap | |
| 2948 | + * in allocation since the domains will be degenerated later. | |
| 2949 | + */ | |
| 2950 | + for_each_cpu(cpu, sched_domain_span(sd)) { | |
| 2951 | + sds = *per_cpu_ptr(d->sds, cpu); | |
| 2952 | + | |
| 2953 | + if (!sds->alloc_flags || | |
| 2954 | + sd->span_weight == 1 || | |
| 2955 | + sds->alloc_flags == flags) { | |
| 2956 | + sds->alloc_flags = flags; | |
| 2957 | + sd->shared = sds; | |
| 2958 | + break; | |
| 2959 | + } | |
| 2960 | + } | |
| 2961 | + | |
| 2962 | + /* | |
| 2963 | + * Use the sd_shared corresponding to the last | |
| 2964 | + * CPU in the span if none are avaialable. | |
| 2965 | + */ | |
| 2966 | + if (WARN_ON_ONCE(!sd->shared)) | |
| 2967 | + sd->shared = sds; | |
| 2922 | 2968 | |
| 2923 | − sd->shared = *per_cpu_ptr(d->sds, sd_id); | |
| 2924 | 2969 | /* |
| 2925 | 2970 | * nr_busy_cpus is consumed only by the NOHZ kick path via |
| 2926 | 2971 | * sd_balance_shared; on the asym-capacity path it is initialized but |
| @@ -2960,7 +3005,7 @@ static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu) · 33 unmodified lines | ||
| 2960 | 3005 | if (!sd_asym || (sd_asym->flags & SD_NUMA)) |
| 2961 | 3006 | return false; |
| 2962 | 3007 | |
| 2963 | − init_sched_domain_shared(d, sd_asym); | |
| 3008 | + init_sched_domain_shared(d, sd_asym, SD_ASYM_CPUCAPACITY); | |
| 2964 | 3009 | return true; |
| 2965 | 3010 | } |
| 2966 | 3011 | |
| @@ -3115,7 +3160,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att · 148 unmodified lines | ||
| 3115 | 3160 | sd = sd->parent; |
| 3116 | 3161 | |
| 3117 | 3162 | if (sd->flags & SD_SHARE_LLC) { |
| 3118 | − init_sched_domain_shared(&d, sd); | |
| 3163 | + init_sched_domain_shared(&d, sd, SD_SHARE_LLC); | |
| 3119 | 3164 | |
| 3120 | 3165 | /* |
| 3121 | 3166 | * In presence of higher domains, adjust the |