Recent optimizations of sd->shared assignment moved to allocating a single instance of per-CPU sched_domain_shared objects per s_data. Recent optimizations to select_idle_capacity() moved the sd->shared assignments to "sd_asym" domain when ASYM_CPUCAPACITY is detected but cache-aware scheduling mandates the presence of "sd_llc_shared" to compute and cache per-LLC statistics. Use an "alloc_flags" union in sched_domain_shared to claim a sched_domain_shared object per sched_domain. Allocation starts searching for an available / matching sched_domain_shared instance from the first CPU of sched_domain_span(sd) (sd can be sd_llc, or sd_asym). If the shared object is claimed by another domain, the instance corresponding to next CPU in the domain span is explored until a matching / available instance is found. In case of a single CPU in sched_domain_span(), the domain will be degenerated and a temporary overlap of ->shared objects across different domains is acceptable. "alloc_flags" forms a union with "nr_idle_scan" and the stale flags are left as is when the sd->shared is published. The expectation is for the first load balancing instance to correct the value just like the current behavior, except the initial value is no longer 0.
Unified · Split
| @@ -67,7 +67,21 @@ struct sched_domain_shared { · 66 unmodified lines | |||
| 67 | atomic_t ref; | 67 | atomic_t ref; |
| 68 | atomic_t nr_busy_cpus; | 68 | atomic_t nr_busy_cpus; |
| 69 | int has_idle_cores; | 69 | int has_idle_cores; |
| 70 | int nr_idle_scan; | 70 | union { |
| 71 | int nr_idle_scan; | ||
| 72 | /* | ||
| 73 | * Used during allocation to claim the sched_domain_shared | ||
| 74 | * object at multiple levels. | ||
| 75 | * | ||
| 76 | * Note: between build and the first periodic LB tick, which | ||
| 77 | * rewrites the union via update_idle_cpu_scan(), readers of | ||
| 78 | * nr_idle_scan may observe the transient SD_* flag value as | ||
| 79 | * the scan bound. The flag bits are small positive integers, | ||
| 80 | * so the effect is just a slightly relaxed scan bound for one | ||
| 81 | * window and self-heals on the first tick. | ||
| 82 | */ | ||
| 83 | int alloc_flags; | ||
| 84 | }; | ||
| 71 | #ifdef CONFIG_SCHED_CACHE | 85 | #ifdef CONFIG_SCHED_CACHE |
| 72 | unsigned long util_avg; | 86 | unsigned long util_avg; |
| 73 | unsigned long capacity; | 87 | unsigned long capacity; |
| @@ -623,6 +623,12 @@ static void free_sched_groups(struct sched_group *sg, int free_sgc) · 622 unmodified lines | |||
| 623 | } while (sg != first); | 623 | } while (sg != first); |
| 624 | } | 624 | } |
| 625 | 625 | ||
| 626 | static void free_sched_domain_shared(struct sched_domain_shared *sds) | ||
| 627 | { | ||
| 628 | if (sds && atomic_dec_and_test(&sds->ref)) | ||
| 629 | kfree(sds); | ||
| 630 | } | ||
| 631 | |||
| 626 | static void destroy_sched_domain(struct sched_domain *sd) | 632 | static void destroy_sched_domain(struct sched_domain *sd) |
| 627 | { | 633 | { |
| 628 | /* | 634 | /* |
| @@ -631,9 +637,7 @@ static void destroy_sched_domain(struct sched_domain *sd) · 2 unmodified lines | |||
| 631 | * dropping group/capacity references, freeing where none remain. | 637 | * dropping group/capacity references, freeing where none remain. |
| 632 | */ | 638 | */ |
| 633 | free_sched_groups(sd->groups, 1); | 639 | free_sched_groups(sd->groups, 1); |
| 634 | 640 | free_sched_domain_shared(sd->shared); | |
| 635 | if (sd->shared && atomic_dec_and_test(&sd->shared->ref)) | ||
| 636 | kfree(sd->shared); | ||
| 637 | 641 | ||
| 638 | #ifdef CONFIG_SCHED_CACHE | 642 | #ifdef CONFIG_SCHED_CACHE |
| 639 | /* only the bottom sd has llc_counts array */ | 643 | /* only the bottom sd has llc_counts array */ |
| @@ -755,7 +759,14 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) · 115 unmodified lines | |||
| 755 | 759 | ||
| 756 | /* Pick reference to parent->shared. */ | 760 | /* Pick reference to parent->shared. */ |
| 757 | if (parent->shared) { | 761 | if (parent->shared) { |
| 758 | WARN_ON_ONCE(tmp->shared); | 762 | /* |
| 763 | * It is safe to free a sd->shared that | ||
| 764 | * has not been published yet. If a | ||
| 765 | * sd->shared was published, the refcount | ||
| 766 | * will end up being non-zero and it will | ||
| 767 | * not be freed here. | ||
| 768 | */ | ||
| 769 | free_sched_domain_shared(tmp->shared); | ||
| 759 | tmp->shared = parent->shared; | 770 | tmp->shared = parent->shared; |
| 760 | parent->shared = NULL; | 771 | parent->shared = NULL; |
| 761 | } | 772 | } |
| @@ -2916,11 +2927,45 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc) · 2154 unmodified lines | |||
| 2916 | } | 2927 | } |
| 2917 | } | 2928 | } |
| 2918 | 2929 | ||
| 2919 | static void init_sched_domain_shared(struct s_data *d, struct sched_domain *sd) | 2930 | static void |
| 2931 | init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags) | ||
| 2920 | { | 2932 | { |
| 2921 | int sd_id = cpumask_first(sched_domain_span(sd)); | 2933 | struct sched_domain_shared *sds = NULL; |
| 2934 | int cpu; | ||
| 2935 | |||
| 2936 | /* | ||
| 2937 | * Multiple domains can try to claim a shared object like | ||
| 2938 | * SD_ASYM_CPUCAPACITY and SD_SHARE_LLC which can alias to | ||
| 2939 | * same cpumask_first(sched_domain_span(sd)) CPU and can | ||
| 2940 | * cause "nr_idle_scan" to be populated incorrectly during | ||
| 2941 | * load balancing. | ||
| 2942 | * | ||
| 2943 | * Find the first CPU in sched_domain_span(sd) with an | ||
| 2944 | * unclaimed domain (!alloc_flags) or where the alloc_flag | ||
| 2945 | * matches the requested flag (SD_* flag) | ||
| 2946 | * | ||
| 2947 | * If the domain only has single CPU, allow temporary overlap | ||
| 2948 | * in allocation since the domains will be degenerated later. | ||
| 2949 | */ | ||
| 2950 | for_each_cpu(cpu, sched_domain_span(sd)) { | ||
| 2951 | sds = *per_cpu_ptr(d->sds, cpu); | ||
| 2952 | |||
| 2953 | if (!sds->alloc_flags || | ||
| 2954 | sd->span_weight == 1 || | ||
| 2955 | sds->alloc_flags == flags) { | ||
| 2956 | sds->alloc_flags = flags; | ||
| 2957 | sd->shared = sds; | ||
| 2958 | break; | ||
| 2959 | } | ||
| 2960 | } | ||
| 2961 | |||
| 2962 | /* | ||
| 2963 | * Use the sd_shared corresponding to the last | ||
| 2964 | * CPU in the span if none are avaialable. | ||
| 2965 | */ | ||
| 2966 | if (WARN_ON_ONCE(!sd->shared)) | ||
| 2967 | sd->shared = sds; | ||
| 2922 | 2968 | ||
| 2923 | sd->shared = *per_cpu_ptr(d->sds, sd_id); | ||
| 2924 | /* | 2969 | /* |
| 2925 | * nr_busy_cpus is consumed only by the NOHZ kick path via | 2970 | * nr_busy_cpus is consumed only by the NOHZ kick path via |
| 2926 | * sd_balance_shared; on the asym-capacity path it is initialized but | 2971 | * sd_balance_shared; on the asym-capacity path it is initialized but |
| @@ -2960,7 +3005,7 @@ static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu) · 33 unmodified lines | |||
| 2960 | if (!sd_asym || (sd_asym->flags & SD_NUMA)) | 3005 | if (!sd_asym || (sd_asym->flags & SD_NUMA)) |
| 2961 | return false; | 3006 | return false; |
| 2962 | 3007 | ||
| 2963 | init_sched_domain_shared(d, sd_asym); | 3008 | init_sched_domain_shared(d, sd_asym, SD_ASYM_CPUCAPACITY); |
| 2964 | return true; | 3009 | return true; |
| 2965 | } | 3010 | } |
| 2966 | 3011 | ||
| @@ -3115,7 +3160,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att · 148 unmodified lines | |||
| 3115 | sd = sd->parent; | 3160 | sd = sd->parent; |
| 3116 | 3161 | ||
| 3117 | if (sd->flags & SD_SHARE_LLC) { | 3162 | if (sd->flags & SD_SHARE_LLC) { |
| 3118 | init_sched_domain_shared(&d, sd); | 3163 | init_sched_domain_shared(&d, sd, SD_SHARE_LLC); |
| 3119 | 3164 | ||
| 3120 | /* | 3165 | /* |
| 3121 | * In presence of higher domains, adjust the | 3166 | * In presence of higher domains, adjust the |