Recent optimizations of sd->shared assignment moved to allocating a single instance of per-CPU sched_domain_shared objects per s_data. Recent optimizations to select_idle_capacity() moved the sd->shared assignments to "sd_asym" domain when ASYM_CPUCAPACITY is detected but cache-aware scheduling mandates the presence of "sd_llc_shared" to compute and cache per-LLC statistics. Use an "alloc_flags" union in sched_domain_shared to claim a sched_domain_shared object per sched_domain. Allocation starts searching for an available / matching sched_domain_shared instance from the first CPU of sched_domain_span(sd) (sd can be sd_llc, or sd_asym). If the shared object is claimed by another domain, the instance corresponding to next CPU in the domain span is explored until a matching / available instance is found. In case of a single CPU in sched_domain_span(), the domain will be degenerated and a temporary overlap of ->shared objects across different domains is acceptable. "alloc_flags" forms a union with "nr_idle_scan" and the stale flags are left as is when the sd->shared is published. The expectation is for the first load balancing instance to correct the value just like the current behavior, except the initial value is no longer 0.
Unified · Split
| @@ -67,7 +67,21 @@ struct sched_domain_shared { · 66 unmodified lines | ||
| 67 | 67 | atomic_t ref; |
| 68 | 68 | atomic_t nr_busy_cpus; |
| 69 | 69 | int has_idle_cores; |
| 70 | − int nr_idle_scan; | |
| 70 | + union { | |
| 71 | + int nr_idle_scan; | |
| 72 | + /* | |
| 73 | + * Used during allocation to claim the sched_domain_shared | |
| 74 | + * object at multiple levels. | |
| 75 | + * | |
| 76 | + * Note: between build and the first periodic LB tick, which | |
| 77 | + * rewrites the union via update_idle_cpu_scan(), readers of | |
| 78 | + * nr_idle_scan may observe the transient SD_* flag value as | |
| 79 | + * the scan bound. The flag bits are small positive integers, | |
| 80 | + * so the effect is just a slightly relaxed scan bound for one | |
| 81 | + * window and self-heals on the first tick. | |
| 82 | + */ | |
| 83 | + int alloc_flags; | |
| 84 | + }; | |
| 71 | 85 | #ifdef CONFIG_SCHED_CACHE |
| 72 | 86 | unsigned long util_avg; |
| 73 | 87 | unsigned long capacity; |
| @@ -623,6 +623,12 @@ static void free_sched_groups(struct sched_group *sg, int free_sgc) · 622 unmodified lines | ||
| 623 | 623 | } while (sg != first); |
| 624 | 624 | } |
| 625 | 625 | |
| 626 | +static void free_sched_domain_shared(struct sched_domain_shared *sds) | |
| 627 | +{ | |
| 628 | + if (sds && atomic_dec_and_test(&sds->ref)) | |
| 629 | + kfree(sds); | |
| 630 | +} | |
| 631 | + | |
| 626 | 632 | static void destroy_sched_domain(struct sched_domain *sd) |
| 627 | 633 | { |
| 628 | 634 | /* |
| @@ -631,9 +637,7 @@ static void destroy_sched_domain(struct sched_domain *sd) · 2 unmodified lines | ||
| 631 | 637 | * dropping group/capacity references, freeing where none remain. |
| 632 | 638 | */ |
| 633 | 639 | free_sched_groups(sd->groups, 1); |
| 634 | − | |
| 635 | − if (sd->shared && atomic_dec_and_test(&sd->shared->ref)) | |
| 636 | − kfree(sd->shared); | |
| 640 | + free_sched_domain_shared(sd->shared); | |
| 637 | 641 | |
| 638 | 642 | #ifdef CONFIG_SCHED_CACHE |
| 639 | 643 | /* only the bottom sd has llc_counts array */ |
| @@ -755,7 +759,14 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) · 115 unmodified lines | ||
| 755 | 759 | |
| 756 | 760 | /* Pick reference to parent->shared. */ |
| 757 | 761 | if (parent->shared) { |
| 758 | − WARN_ON_ONCE(tmp->shared); | |
| 762 | + /* | |
| 763 | + * It is safe to free a sd->shared that | |
| 764 | + * has not been published yet. If a | |
| 765 | + * sd->shared was published, the refcount | |
| 766 | + * will end up being non-zero and it will | |
| 767 | + * not be freed here. | |
| 768 | + */ | |
| 769 | + free_sched_domain_shared(tmp->shared); | |
| 759 | 770 | tmp->shared = parent->shared; |
| 760 | 771 | parent->shared = NULL; |
| 761 | 772 | } |
| @@ -2916,11 +2927,45 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc) · 2154 unmodified lines | ||
| 2916 | 2927 | } |
| 2917 | 2928 | } |
| 2918 | 2929 | |
| 2919 | −static void init_sched_domain_shared(struct s_data *d, struct sched_domain *sd) | |
| 2930 | +static void | |
| 2931 | +init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags) | |
| 2920 | 2932 | { |
| 2921 | − int sd_id = cpumask_first(sched_domain_span(sd)); | |
| 2933 | + struct sched_domain_shared *sds = NULL; | |
| 2934 | + int cpu; | |
| 2935 | + | |
| 2936 | + /* | |
| 2937 | + * Multiple domains can try to claim a shared object like | |
| 2938 | + * SD_ASYM_CPUCAPACITY and SD_SHARE_LLC which can alias to | |
| 2939 | + * same cpumask_first(sched_domain_span(sd)) CPU and can | |
| 2940 | + * cause "nr_idle_scan" to be populated incorrectly during | |
| 2941 | + * load balancing. | |
| 2942 | + * | |
| 2943 | + * Find the first CPU in sched_domain_span(sd) with an | |
| 2944 | + * unclaimed domain (!alloc_flags) or where the alloc_flag | |
| 2945 | + * matches the requested flag (SD_* flag) | |
| 2946 | + * | |
| 2947 | + * If the domain only has single CPU, allow temporary overlap | |
| 2948 | + * in allocation since the domains will be degenerated later. | |
| 2949 | + */ | |
| 2950 | + for_each_cpu(cpu, sched_domain_span(sd)) { | |
| 2951 | + sds = *per_cpu_ptr(d->sds, cpu); | |
| 2952 | + | |
| 2953 | + if (!sds->alloc_flags || | |
| 2954 | + sd->span_weight == 1 || | |
| 2955 | + sds->alloc_flags == flags) { | |
| 2956 | + sds->alloc_flags = flags; | |
| 2957 | + sd->shared = sds; | |
| 2958 | + break; | |
| 2959 | + } | |
| 2960 | + } | |
| 2961 | + | |
| 2962 | + /* | |
| 2963 | + * Use the sd_shared corresponding to the last | |
| 2964 | + * CPU in the span if none are avaialable. | |
| 2965 | + */ | |
| 2966 | + if (WARN_ON_ONCE(!sd->shared)) | |
| 2967 | + sd->shared = sds; | |
| 2922 | 2968 | |
| 2923 | − sd->shared = *per_cpu_ptr(d->sds, sd_id); | |
| 2924 | 2969 | /* |
| 2925 | 2970 | * nr_busy_cpus is consumed only by the NOHZ kick path via |
| 2926 | 2971 | * sd_balance_shared; on the asym-capacity path it is initialized but |
| @@ -2960,7 +3005,7 @@ static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu) · 33 unmodified lines | ||
| 2960 | 3005 | if (!sd_asym || (sd_asym->flags & SD_NUMA)) |
| 2961 | 3006 | return false; |
| 2962 | 3007 | |
| 2963 | − init_sched_domain_shared(d, sd_asym); | |
| 3008 | + init_sched_domain_shared(d, sd_asym, SD_ASYM_CPUCAPACITY); | |
| 2964 | 3009 | return true; |
| 2965 | 3010 | } |
| 2966 | 3011 | |
| @@ -3115,7 +3160,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att · 148 unmodified lines | ||
| 3115 | 3160 | sd = sd->parent; |
| 3116 | 3161 | |
| 3117 | 3162 | if (sd->flags & SD_SHARE_LLC) { |
| 3118 | − init_sched_domain_shared(&d, sd); | |
| 3163 | + init_sched_domain_shared(&d, sd, SD_SHARE_LLC); | |
| 3119 | 3164 | |
| 3120 | 3165 | /* |
| 3121 | 3166 | * In presence of higher domains, adjust the |