History

Merge tag 'v5.17-rc5' into sched/core, to resolve conflicts

New conflicts in sched/core due to the following upstream fixes: 44585f7bc0cb ("psi: fix "defined but not used" warnings when CONFIG_PROC_FS=n") a06247c6804f ("psi: Fix uaf issue when psi trigger is destroyed while being polled") Conflicts: include/linux/psi_types.h kernel/sched/psi.c

Ingo Molnar committed 4 years ago

Commit 6255b482 parents 1b2d345 cfb92441212 files · +13287 −6877Browse files

Showing the changes to kernel (37 of 1212 files). Show all

Unified · Split

37 files
  1. kernel/async.c +0 −3
  2. kernel/audit.c +43 −19
  3. kernel/auditsc.c +1 −1
  4. kernel/bpf/bpf_lsm.c +1 −1
  5. kernel/bpf/ringbuf.c +1 −1
  6. kernel/bpf/stackmap.c +3 −2
  7. kernel/bpf/trampoline.c +3 −2
  8. kernel/cgroup/cgroup-v1.c +14 −0
  9. kernel/cgroup/cgroup.c +8 −3
  10. kernel/cgroup/cpuset.c +51 −14
  11. kernel/cred.c +2 −7
  12. kernel/events/core.c +183 −106
  13. kernel/fork.c +20 −10
  14. kernel/locking/lockdep.c +2 −2
  15. kernel/module.c +5 −20
  16. kernel/module_decompress.c +2 −0
  17. kernel/power/main.c +4 −1
  18. kernel/power/process.c +1 −1
  19. kernel/power/snapshot.c +7 −14
  20. kernel/power/suspend.c +0 −2
  21. kernel/power/wakelock.c +4 −7
  22. kernel/printk/sysctl.c +1 −1
  23. kernel/rcu/tasks.h +8 −4
  24. kernel/sched/core.c +20 −18
  25. kernel/sched/core_sched.c +1 −1
  26. kernel/sched/fair.c +77 −41
  27. kernel/sched/membarrier.c +5 −4
  28. kernel/sched/pelt.h +3 −1
  29. kernel/sched/psi.c +29 −37
  30. kernel/seccomp.c +10 −0
  31. kernel/signal.c +3 −2
  32. kernel/stackleak.c +2 −3
  33. kernel/sys.c +14 −6
  34. kernel/trace/Kconfig +7 −1
  35. kernel/trace/trace.c +6 −1
  36. kernel/trace/trace_events_hist.c +8 −2
  37. kernel/ucount.c +4 −1
kernel/async.c+0−3
@@ -205,9 +205,6 @@ async_cookie_t async_schedule_node_domain(async_func_t func, void *data, · 204 unmodified lines
205205 atomic_inc(&entry_count);
206206 spin_unlock_irqrestore(&async_lock, flags);
207207
208− /* mark that this task has queued an async job, used by module init */
209− current->flags |= PF_USED_ASYNC;
210−
211208 /* schedule for execution */
212209 queue_work_node(node, system_unbound_wq, &entry->work);
213210
kernel/audit.c+43−19
@@ -541,20 +541,22 @@ static void kauditd_printk_skb(struct sk_buff *skb) · 540 unmodified lines
541541 /**
542542 * kauditd_rehold_skb - Handle a audit record send failure in the hold queue
543543 * @skb: audit record
544+ * @error: error code (unused)
544545 *
545546 * Description:
546547 * This should only be used by the kauditd_thread when it fails to flush the
547548 * hold queue.
548549 */
549−static void kauditd_rehold_skb(struct sk_buff *skb)
550+static void kauditd_rehold_skb(struct sk_buff *skb, __always_unused int error)
550551 {
551− /* put the record back in the queue at the same place */
552− skb_queue_head(&audit_hold_queue, skb);
552+ /* put the record back in the queue */
553+ skb_queue_tail(&audit_hold_queue, skb);
553554 }
554555
555556 /**
556557 * kauditd_hold_skb - Queue an audit record, waiting for auditd
557558 * @skb: audit record
559+ * @error: error code
558560 *
559561 * Description:
560562 * Queue the audit record, waiting for an instance of auditd. When this
@@ -564,19 +566,31 @@ static void kauditd_rehold_skb(struct sk_buff *skb) · 3 unmodified lines
564566 * and queue it, if we have room. If we want to hold on to the record, but we
565567 * don't have room, record a record lost message.
566568 */
567−static void kauditd_hold_skb(struct sk_buff *skb)
569+static void kauditd_hold_skb(struct sk_buff *skb, int error)
568570 {
569571 /* at this point it is uncertain if we will ever send this to auditd so
570572 * try to send the message via printk before we go any further */
571573 kauditd_printk_skb(skb);
572574
573575 /* can we just silently drop the message? */
574− if (!audit_default) {
575− kfree_skb(skb);
576− return;
576+ if (!audit_default)
577+ goto drop;
578+
579+ /* the hold queue is only for when the daemon goes away completely,
580+ * not -EAGAIN failures; if we are in a -EAGAIN state requeue the
581+ * record on the retry queue unless it's full, in which case drop it
582+ */
583+ if (error == -EAGAIN) {
584+ if (!audit_backlog_limit ||
585+ skb_queue_len(&audit_retry_queue) < audit_backlog_limit) {
586+ skb_queue_tail(&audit_retry_queue, skb);
587+ return;
588+ }
589+ audit_log_lost("kauditd retry queue overflow");
590+ goto drop;
577591 }
578592
579− /* if we have room, queue the message */
593+ /* if we have room in the hold queue, queue the message */
580594 if (!audit_backlog_limit ||
581595 skb_queue_len(&audit_hold_queue) < audit_backlog_limit) {
582596 skb_queue_tail(&audit_hold_queue, skb);
@@ -585,24 +599,32 @@ static void kauditd_hold_skb(struct sk_buff *skb) · 2 unmodified lines
585599
586600 /* we have no other options - drop the message */
587601 audit_log_lost("kauditd hold queue overflow");
602+drop:
588603 kfree_skb(skb);
589604 }
590605
591606 /**
592607 * kauditd_retry_skb - Queue an audit record, attempt to send again to auditd
593608 * @skb: audit record
609+ * @error: error code (unused)
594610 *
595611 * Description:
596612 * Not as serious as kauditd_hold_skb() as we still have a connected auditd,
597613 * but for some reason we are having problems sending it audit records so
598614 * queue the given record and attempt to resend.
599615 */
600−static void kauditd_retry_skb(struct sk_buff *skb)
616+static void kauditd_retry_skb(struct sk_buff *skb, __always_unused int error)
601617 {
602− /* NOTE: because records should only live in the retry queue for a
603− * short period of time, before either being sent or moved to the hold
604− * queue, we don't currently enforce a limit on this queue */
605− skb_queue_tail(&audit_retry_queue, skb);
618+ if (!audit_backlog_limit ||
619+ skb_queue_len(&audit_retry_queue) < audit_backlog_limit) {
620+ skb_queue_tail(&audit_retry_queue, skb);
621+ return;
622+ }
623+
624+ /* we have to drop the record, send it via printk as a last effort */
625+ kauditd_printk_skb(skb);
626+ audit_log_lost("kauditd retry queue overflow");
627+ kfree_skb(skb);
606628 }
607629
608630 /**
@@ -640,7 +662,7 @@ static void auditd_reset(const struct auditd_connection *ac) · 31 unmodified lines
640662 /* flush the retry queue to the hold queue, but don't touch the main
641663 * queue since we need to process that normally for multicast */
642664 while ((skb = skb_dequeue(&audit_retry_queue)))
643− kauditd_hold_skb(skb);
665+ kauditd_hold_skb(skb, -ECONNREFUSED);
644666 }
645667
646668 /**
@@ -714,16 +736,18 @@ static int kauditd_send_queue(struct sock *sk, u32 portid, · 67 unmodified lines
714736 struct sk_buff_head *queue,
715737 unsigned int retry_limit,
716738 void (*skb_hook)(struct sk_buff *skb),
717− void (*err_hook)(struct sk_buff *skb))
739+ void (*err_hook)(struct sk_buff *skb, int error))
718740 {
719741 int rc = 0;
720− struct sk_buff *skb;
742+ struct sk_buff *skb = NULL;
743+ struct sk_buff *skb_tail;
721744 unsigned int failed = 0;
722745
723746 /* NOTE: kauditd_thread takes care of all our locking, we just use
724747 * the netlink info passed to us (e.g. sk and portid) */
725748
726− while ((skb = skb_dequeue(queue))) {
749+ skb_tail = skb_peek_tail(queue);
750+ while ((skb != skb_tail) && (skb = skb_dequeue(queue))) {
727751 /* call the skb_hook for each skb we touch */
728752 if (skb_hook)
729753 (*skb_hook)(skb);
@@ -731,7 +755,7 @@ static int kauditd_send_queue(struct sock *sk, u32 portid, · 1 unmodified line
731755 /* can we send to anyone via unicast? */
732756 if (!sk) {
733757 if (err_hook)
734− (*err_hook)(skb);
758+ (*err_hook)(skb, -ECONNREFUSED);
735759 continue;
736760 }
737761
@@ -745,7 +769,7 @@ retry: · 7 unmodified lines
745769 rc == -ECONNREFUSED || rc == -EPERM) {
746770 sk = NULL;
747771 if (err_hook)
748− (*err_hook)(skb);
772+ (*err_hook)(skb, rc);
749773 if (rc == -EAGAIN)
750774 rc = 0;
751775 /* continue to drain the queue */
kernel/auditsc.c+1−1
@@ -185,7 +185,7 @@ static int audit_match_perm(struct audit_context *ctx, int mask) · 184 unmodified lines
185185 case AUDITSC_EXECVE:
186186 return mask & AUDIT_PERM_EXEC;
187187 case AUDITSC_OPENAT2:
188− return mask & ACC_MODE((u32)((struct open_how *)ctx->argv[2])->flags);
188+ return mask & ACC_MODE((u32)ctx->openat2.flags);
189189 default:
190190 return 0;
191191 }
kernel/bpf/bpf_lsm.c+1−1
@@ -207,7 +207,7 @@ BTF_ID(func, bpf_lsm_socket_socketpair) · 206 unmodified lines
207207
208208 BTF_ID(func, bpf_lsm_syslog)
209209 BTF_ID(func, bpf_lsm_task_alloc)
210−BTF_ID(func, bpf_lsm_task_getsecid_subj)
210+BTF_ID(func, bpf_lsm_current_getsecid_subj)
211211 BTF_ID(func, bpf_lsm_task_getsecid_obj)
212212 BTF_ID(func, bpf_lsm_task_prctl)
213213 BTF_ID(func, bpf_lsm_task_setscheduler)
kernel/bpf/ringbuf.c+1−1
@@ -104,7 +104,7 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node) · 103 unmodified lines
104104 }
105105
106106 rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,
107− VM_ALLOC | VM_USERMAP, PAGE_KERNEL);
107+ VM_MAP | VM_USERMAP, PAGE_KERNEL);
108108 if (rb) {
109109 kmemleak_not_leak(pages);
110110 rb->pages = pages;
kernel/bpf/stackmap.c+3−2
@@ -472,13 +472,14 @@ BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf, · 471 unmodified lines
472472 u32, size, u64, flags)
473473 {
474474 struct pt_regs *regs;
475− long res;
475+ long res = -EINVAL;
476476
477477 if (!try_get_task_stack(task))
478478 return -EFAULT;
479479
480480 regs = task_pt_regs(task);
481− res = __bpf_get_stack(regs, task, NULL, buf, size, flags);
481+ if (regs)
482+ res = __bpf_get_stack(regs, task, NULL, buf, size, flags);
482483 put_task_stack(task);
483484
484485 return res;
kernel/bpf/trampoline.c+3−2
@@ -550,11 +550,12 @@ static __always_inline u64 notrace bpf_prog_start_time(void) · 549 unmodified lines
550550 static void notrace inc_misses_counter(struct bpf_prog *prog)
551551 {
552552 struct bpf_prog_stats *stats;
553+ unsigned int flags;
553554
554555 stats = this_cpu_ptr(prog->stats);
555− u64_stats_update_begin(&stats->syncp);
556+ flags = u64_stats_update_begin_irqsave(&stats->syncp);
556557 u64_stats_inc(&stats->misses);
557− u64_stats_update_end(&stats->syncp);
558+ u64_stats_update_end_irqrestore(&stats->syncp, flags);
558559 }
559560
560561 /* The logic is similar to bpf_prog_run(), but with an explicit
kernel/cgroup/cgroup-v1.c+14−0
@@ -549,6 +549,14 @@ static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, · 548 unmodified lines
549549
550550 BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
551551
552+ /*
553+ * Release agent gets called with all capabilities,
554+ * require capabilities to set release agent.
555+ */
556+ if ((of->file->f_cred->user_ns != &init_user_ns) ||
557+ !capable(CAP_SYS_ADMIN))
558+ return -EPERM;
559+
552560 cgrp = cgroup_kn_lock_live(of->kn, false);
553561 if (!cgrp)
554562 return -ENODEV;
@@ -954,6 +962,12 @@ int cgroup1_parse_param(struct fs_context *fc, struct fs_parameter *param) · 399 unmodified lines
954962 /* Specifying two release agents is forbidden */
955963 if (ctx->release_agent)
956964 return invalfc(fc, "release_agent respecified");
965+ /*
966+ * Release agent gets called with all capabilities,
967+ * require capabilities to set release agent.
968+ */
969+ if ((fc->user_ns != &init_user_ns) || !capable(CAP_SYS_ADMIN))
970+ return invalfc(fc, "Setting release_agent not allowed");
957971 ctx->release_agent = param->string;
958972 param->string = NULL;
959973 break;
kernel/cgroup/cgroup.c+8−3
@@ -3643,6 +3643,12 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, · 3642 unmodified lines
36433643 cgroup_get(cgrp);
36443644 cgroup_kn_unlock(of->kn);
36453645
3646+ /* Allow only one trigger per file descriptor */
3647+ if (ctx->psi.trigger) {
3648+ cgroup_put(cgrp);
3649+ return -EBUSY;
3650+ }
3651+
36463652 psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi;
36473653 new = psi_trigger_create(psi, buf, nbytes, res);
36483654 if (IS_ERR(new)) {
@@ -3650,8 +3656,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, · 1 unmodified line
36503656 return PTR_ERR(new);
36513657 }
36523658
3653− psi_trigger_replace(&ctx->psi.trigger, new);
3654−
3659+ smp_store_release(&ctx->psi.trigger, new);
36553660 cgroup_put(cgrp);
36563661
36573662 return nbytes;
@@ -3690,7 +3695,7 @@ static void cgroup_pressure_release(struct kernfs_open_file *of) · 32 unmodified lines
36903695 {
36913696 struct cgroup_file_ctx *ctx = of->priv;
36923697
3693− psi_trigger_replace(&ctx->psi.trigger, NULL);
3698+ psi_trigger_destroy(ctx->psi.trigger);
36943699 }
36953700
36963701 bool cgroup_psi_enabled(void)
kernel/cgroup/cpuset.c+51−14
@@ -590,6 +590,35 @@ static inline void free_cpuset(struct cpuset *cs) · 589 unmodified lines
590590 kfree(cs);
591591 }
592592
593+/*
594+ * validate_change_legacy() - Validate conditions specific to legacy (v1)
595+ * behavior.
596+ */
597+static int validate_change_legacy(struct cpuset *cur, struct cpuset *trial)
598+{
599+ struct cgroup_subsys_state *css;
600+ struct cpuset *c, *par;
601+ int ret;
602+
603+ WARN_ON_ONCE(!rcu_read_lock_held());
604+
605+ /* Each of our child cpusets must be a subset of us */
606+ ret = -EBUSY;
607+ cpuset_for_each_child(c, css, cur)
608+ if (!is_cpuset_subset(c, trial))
609+ goto out;
610+
611+ /* On legacy hierarchy, we must be a subset of our parent cpuset. */
612+ ret = -EACCES;
613+ par = parent_cs(cur);
614+ if (par && !is_cpuset_subset(trial, par))
615+ goto out;
616+
617+ ret = 0;
618+out:
619+ return ret;
620+}
621+
593622 /*
594623 * validate_change() - Used to validate that any proposed cpuset change
595624 * follows the structural rules for cpusets.
@@ -614,20 +643,21 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) · 18 unmodified lines
614643 {
615644 struct cgroup_subsys_state *css;
616645 struct cpuset *c, *par;
617− int ret;
618−
619− /* The checks don't apply to root cpuset */
620− if (cur == &top_cpuset)
621− return 0;
646+ int ret = 0;
622647
623648 rcu_read_lock();
624− par = parent_cs(cur);
625649
626− /* On legacy hierarchy, we must be a subset of our parent cpuset. */
627− ret = -EACCES;
628− if (!is_in_v2_mode() && !is_cpuset_subset(trial, par))
650+ if (!is_in_v2_mode())
651+ ret = validate_change_legacy(cur, trial);
652+ if (ret)
653+ goto out;
654+
655+ /* Remaining checks don't apply to root cpuset */
656+ if (cur == &top_cpuset)
629657 goto out;
630658
659+ par = parent_cs(cur);
660+
631661 /*
632662 * If either I or some sibling (!= me) is exclusive, we can't
633663 * overlap
@@ -1175,9 +1205,7 @@ enum subparts_cmd { · 541 unmodified lines
11751205 *
11761206 * Because of the implicit cpu exclusive nature of a partition root,
11771207 * cpumask changes that violates the cpu exclusivity rule will not be
1178− * permitted when checked by validate_change(). The validate_change()
1179− * function will also prevent any changes to the cpu list if it is not
1180− * a superset of children's cpu lists.
1208+ * permitted when checked by validate_change().
11811209 */
11821210 static int update_parent_subparts_cpumask(struct cpuset *cpuset, int cmd,
11831211 struct cpumask *newmask,
@@ -1522,10 +1550,15 @@ static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, · 338 unmodified lines
15221550 struct cpuset *sibling;
15231551 struct cgroup_subsys_state *pos_css;
15241552
1553+ percpu_rwsem_assert_held(&cpuset_rwsem);
1554+
15251555 /*
15261556 * Check all its siblings and call update_cpumasks_hier()
15271557 * if their use_parent_ecpus flag is set in order for them
15281558 * to use the right effective_cpus value.
1559+ *
1560+ * The update_cpumasks_hier() function may sleep. So we have to
1561+ * release the RCU read lock before calling it.
15291562 */
15301563 rcu_read_lock();
15311564 cpuset_for_each_child(sibling, pos_css, parent) {
@@ -1533,8 +1566,13 @@ static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, · 1 unmodified line
15331566 continue;
15341567 if (!sibling->use_parent_ecpus)
15351568 continue;
1569+ if (!css_tryget_online(&sibling->css))
1570+ continue;
15361571
1572+ rcu_read_unlock();
15371573 update_cpumasks_hier(sibling, tmp);
1574+ rcu_read_lock();
1575+ css_put(&sibling->css);
15381576 }
15391577 rcu_read_unlock();
15401578 }
@@ -1607,8 +1645,7 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, · 66 unmodified lines
16071645 * Make sure that subparts_cpus is a subset of cpus_allowed.
16081646 */
16091647 if (cs->nr_subparts_cpus) {
1610− cpumask_andnot(cs->subparts_cpus, cs->subparts_cpus,
1611− cs->cpus_allowed);
1648+ cpumask_and(cs->subparts_cpus, cs->subparts_cpus, cs->cpus_allowed);
16121649 cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus);
16131650 }
16141651 spin_unlock_irq(&callback_lock);
kernel/cred.c+2−7
@@ -665,21 +665,16 @@ EXPORT_SYMBOL(cred_fscmp); · 664 unmodified lines
665665
666666 int set_cred_ucounts(struct cred *new)
667667 {
668− struct task_struct *task = current;
669− const struct cred *old = task->real_cred;
670668 struct ucounts *new_ucounts, *old_ucounts = new->ucounts;
671669
672− if (new->user == old->user && new->user_ns == old->user_ns)
673− return 0;
674−
675670 /*
676671 * This optimization is needed because alloc_ucounts() uses locks
677672 * for table lookups.
678673 */
679− if (old_ucounts->ns == new->user_ns && uid_eq(old_ucounts->uid, new->euid))
674+ if (old_ucounts->ns == new->user_ns && uid_eq(old_ucounts->uid, new->uid))
680675 return 0;
681676
682− if (!(new_ucounts = alloc_ucounts(new->user_ns, new->euid)))
677+ if (!(new_ucounts = alloc_ucounts(new->user_ns, new->uid)))
683678 return -EAGAIN;
684679
685680 new->ucounts = new_ucounts;
kernel/events/core.c+183−106
@@ -674,6 +674,23 @@ perf_event_set_state(struct perf_event *event, enum perf_event_state state) · 673 unmodified lines
674674 WRITE_ONCE(event->state, state);
675675 }
676676
677+/*
678+ * UP store-release, load-acquire
679+ */
680+
681+#define __store_release(ptr, val) \
682+do { \
683+ barrier(); \
684+ WRITE_ONCE(*(ptr), (val)); \
685+} while (0)
686+
687+#define __load_acquire(ptr) \
688+({ \
689+ __unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr)); \
690+ barrier(); \
691+ ___p; \
692+})
693+
677694 #ifdef CONFIG_CGROUP_PERF
678695
679696 static inline bool
@@ -719,34 +736,51 @@ static inline u64 perf_cgroup_event_time(struct perf_event *event) · 39 unmodified lines
719736 return t->time;
720737 }
721738
722−static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
739+static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
723740 {
724− struct perf_cgroup_info *info;
725− u64 now;
726−
727− now = perf_clock();
741+ struct perf_cgroup_info *t;
728742
729− info = this_cpu_ptr(cgrp->info);
743+ t = per_cpu_ptr(event->cgrp->info, event->cpu);
744+ if (!__load_acquire(&t->active))
745+ return t->time;
746+ now += READ_ONCE(t->timeoffset);
747+ return now;
748+}
730749
731− info->time += now - info->timestamp;
750+static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
751+{
752+ if (adv)
753+ info->time += now - info->timestamp;
732754 info->timestamp = now;
755+ /*
756+ * see update_context_time()
757+ */
758+ WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
733759 }
734760
735−static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
761+static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
736762 {
737763 struct perf_cgroup *cgrp = cpuctx->cgrp;
738764 struct cgroup_subsys_state *css;
765+ struct perf_cgroup_info *info;
739766
740767 if (cgrp) {
768+ u64 now = perf_clock();
769+
741770 for (css = &cgrp->css; css; css = css->parent) {
742771 cgrp = container_of(css, struct perf_cgroup, css);
743− __update_cgrp_time(cgrp);
772+ info = this_cpu_ptr(cgrp->info);
773+
774+ __update_cgrp_time(info, now, true);
775+ if (final)
776+ __store_release(&info->active, 0);
744777 }
745778 }
746779 }
747780
748781 static inline void update_cgrp_time_from_event(struct perf_event *event)
749782 {
783+ struct perf_cgroup_info *info;
750784 struct perf_cgroup *cgrp;
751785
752786 /*
@@ -760,8 +794,10 @@ static inline void update_cgrp_time_from_event(struct perf_event *event) · 7 unmodified lines
760794 /*
761795 * Do not update time when cgroup is not active
762796 */
763− if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
764− __update_cgrp_time(event->cgrp);
797+ if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) {
798+ info = this_cpu_ptr(event->cgrp->info);
799+ __update_cgrp_time(info, perf_clock(), true);
800+ }
765801 }
766802
767803 static inline void
@@ -785,7 +821,8 @@ perf_cgroup_set_timestamp(struct task_struct *task, · 17 unmodified lines
785821 for (css = &cgrp->css; css; css = css->parent) {
786822 cgrp = container_of(css, struct perf_cgroup, css);
787823 info = this_cpu_ptr(cgrp->info);
788− info->timestamp = ctx->timestamp;
824+ __update_cgrp_time(info, ctx->timestamp, false);
825+ __store_release(&info->active, 1);
789826 }
790827 }
791828
@@ -802,7 +839,7 @@ static DEFINE_PER_CPU(struct list_head, cgrp_cpuctx_list); · 10 unmodified lines
802839 */
803840 static void perf_cgroup_switch(struct task_struct *task, int mode)
804841 {
805− struct perf_cpu_context *cpuctx;
842+ struct perf_cpu_context *cpuctx, *tmp;
806843 struct list_head *list;
807844 unsigned long flags;
808845
@@ -813,7 +850,7 @@ static void perf_cgroup_switch(struct task_struct *task, int mode) · 4 unmodified lines
813850 local_irq_save(flags);
814851
815852 list = this_cpu_ptr(&cgrp_cpuctx_list);
816− list_for_each_entry(cpuctx, list, cgrp_cpuctx_entry) {
853+ list_for_each_entry_safe(cpuctx, tmp, list, cgrp_cpuctx_entry) {
817854 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
818855
819856 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
@@ -981,14 +1018,6 @@ out: · 161 unmodified lines
9811018 return ret;
9821019 }
9831020
984−static inline void
985−perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
986−{
987− struct perf_cgroup_info *t;
988− t = per_cpu_ptr(event->cgrp->info, event->cpu);
989− event->shadow_ctx_time = now - t->timestamp;
990−}
991−
9921021 static inline void
9931022 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
9941023 {
@@ -1066,7 +1095,8 @@ static inline void update_cgrp_time_from_event(struct perf_event *event) · 71 unmodified lines
10661095 {
10671096 }
10681097
1069−static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
1098+static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
1099+ bool final)
10701100 {
10711101 }
10721102
@@ -1098,12 +1128,12 @@ perf_cgroup_switch(struct task_struct *task, struct task_struct *next) · 25 unmodified lines
10981128 {
10991129 }
11001130
1101−static inline void
1102−perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
1131+static inline u64 perf_cgroup_event_time(struct perf_event *event)
11031132 {
1133+ return 0;
11041134 }
11051135
1106−static inline u64 perf_cgroup_event_time(struct perf_event *event)
1136+static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
11071137 {
11081138 return 0;
11091139 }
@@ -1525,22 +1555,59 @@ static void perf_unpin_context(struct perf_event_context *ctx) · 415 unmodified lines
15251555 /*
15261556 * Update the record of the current time in a context.
15271557 */
1528−static void update_context_time(struct perf_event_context *ctx)
1558+static void __update_context_time(struct perf_event_context *ctx, bool adv)
15291559 {
15301560 u64 now = perf_clock();
15311561
1532− ctx->time += now - ctx->timestamp;
1562+ if (adv)
1563+ ctx->time += now - ctx->timestamp;
15331564 ctx->timestamp = now;
1565+
1566+ /*
1567+ * The above: time' = time + (now - timestamp), can be re-arranged
1568+ * into: time` = now + (time - timestamp), which gives a single value
1569+ * offset to compute future time without locks on.
1570+ *
1571+ * See perf_event_time_now(), which can be used from NMI context where
1572+ * it's (obviously) not possible to acquire ctx->lock in order to read
1573+ * both the above values in a consistent manner.
1574+ */
1575+ WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
1576+}
1577+
1578+static void update_context_time(struct perf_event_context *ctx)
1579+{
1580+ __update_context_time(ctx, true);
15341581 }
15351582
15361583 static u64 perf_event_time(struct perf_event *event)
15371584 {
15381585 struct perf_event_context *ctx = event->ctx;
15391586
1587+ if (unlikely(!ctx))
1588+ return 0;
1589+
15401590 if (is_cgroup_event(event))
15411591 return perf_cgroup_event_time(event);
15421592
1543− return ctx ? ctx->time : 0;
1593+ return ctx->time;
1594+}
1595+
1596+static u64 perf_event_time_now(struct perf_event *event, u64 now)
1597+{
1598+ struct perf_event_context *ctx = event->ctx;
1599+
1600+ if (unlikely(!ctx))
1601+ return 0;
1602+
1603+ if (is_cgroup_event(event))
1604+ return perf_cgroup_event_time_now(event, now);
1605+
1606+ if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
1607+ return ctx->time;
1608+
1609+ now += READ_ONCE(ctx->timeoffset);
1610+ return now;
15441611 }
15451612
15461613 static enum event_type_t get_event_type(struct perf_event *event)
@@ -2350,7 +2417,7 @@ __perf_remove_from_context(struct perf_event *event, · 803 unmodified lines
23502417
23512418 if (ctx->is_active & EVENT_TIME) {
23522419 update_context_time(ctx);
2353− update_cgrp_time_from_cpuctx(cpuctx);
2420+ update_cgrp_time_from_cpuctx(cpuctx, false);
23542421 }
23552422
23562423 event_sched_out(event, cpuctx, ctx);
@@ -2361,6 +2428,9 @@ __perf_remove_from_context(struct perf_event *event, · 4 unmodified lines
23612428 list_del_event(event, ctx);
23622429
23632430 if (!ctx->nr_events && ctx->is_active) {
2431+ if (ctx == &cpuctx->ctx)
2432+ update_cgrp_time_from_cpuctx(cpuctx, true);
2433+
23642434 ctx->is_active = 0;
23652435 ctx->rotate_necessary = 0;
23662436 if (ctx->task) {
@@ -2392,7 +2462,11 @@ static void perf_remove_from_context(struct perf_event *event, unsigned long fla · 25 unmodified lines
23922462 * event_function_call() user.
23932463 */
23942464 raw_spin_lock_irq(&ctx->lock);
2395− if (!ctx->is_active) {
2465+ /*
2466+ * Cgroup events are per-cpu events, and must IPI because of
2467+ * cgrp_cpuctx_list.
2468+ */
2469+ if (!ctx->is_active && !is_cgroup_event(event)) {
23962470 __perf_remove_from_context(event, __get_cpu_context(ctx),
23972471 ctx, (void *)flags);
23982472 raw_spin_unlock_irq(&ctx->lock);
@@ -2482,40 +2556,6 @@ void perf_event_disable_inatomic(struct perf_event *event) · 83 unmodified lines
24822556 irq_work_queue(&event->pending);
24832557 }
24842558
2485−static void perf_set_shadow_time(struct perf_event *event,
2486− struct perf_event_context *ctx)
2487−{
2488− /*
2489− * use the correct time source for the time snapshot
2490− *
2491− * We could get by without this by leveraging the
2492− * fact that to get to this function, the caller
2493− * has most likely already called update_context_time()
2494− * and update_cgrp_time_xx() and thus both timestamp
2495− * are identical (or very close). Given that tstamp is,
2496− * already adjusted for cgroup, we could say that:
2497− * tstamp - ctx->timestamp
2498− * is equivalent to
2499− * tstamp - cgrp->timestamp.
2500− *
2501− * Then, in perf_output_read(), the calculation would
2502− * work with no changes because:
2503− * - event is guaranteed scheduled in
2504− * - no scheduled out in between
2505− * - thus the timestamp would be the same
2506− *
2507− * But this is a bit hairy.
2508− *
2509− * So instead, we have an explicit cgroup call to remain
2510− * within the time source all along. We believe it
2511− * is cleaner and simpler to understand.
2512− */
2513− if (is_cgroup_event(event))
2514− perf_cgroup_set_shadow_time(event, event->tstamp);
2515− else
2516− event->shadow_ctx_time = event->tstamp - ctx->timestamp;
2517−}
2518−
25192559 #define MAX_INTERRUPTS (~0ULL)
25202560
25212561 static void perf_log_throttle(struct perf_event *event, int enable);
@@ -2556,8 +2596,6 @@ event_sched_in(struct perf_event *event, · 34 unmodified lines
25562596
25572597 perf_pmu_disable(event->pmu);
25582598
2559− perf_set_shadow_time(event, ctx);
2560−
25612599 perf_log_itrace_start(event);
25622600
25632601 if (event->pmu->add(event, PERF_EF_START)) {
@@ -2861,11 +2899,14 @@ perf_install_in_context(struct perf_event_context *ctx, · 297 unmodified lines
28612899 * perf_event_attr::disabled events will not run and can be initialized
28622900 * without IPI. Except when this is the first event for the context, in
28632901 * that case we need the magic of the IPI to set ctx->is_active.
2902+ * Similarly, cgroup events for the context also needs the IPI to
2903+ * manipulate the cgrp_cpuctx_list.
28642904 *
28652905 * The IOC_ENABLE that is sure to follow the creation of a disabled
28662906 * event will issue the IPI and reprogram the hardware.
28672907 */
2868− if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF && ctx->nr_events) {
2908+ if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
2909+ ctx->nr_events && !is_cgroup_event(event)) {
28692910 raw_spin_lock_irq(&ctx->lock);
28702911 if (ctx->task == TASK_TOMBSTONE) {
28712912 raw_spin_unlock_irq(&ctx->lock);
@@ -3197,6 +3238,15 @@ static int perf_event_modify_breakpoint(struct perf_event *bp, · 325 unmodified lines
31973238 return err;
31983239 }
31993240
3241+/*
3242+ * Copy event-type-independent attributes that may be modified.
3243+ */
3244+static void perf_event_modify_copy_attr(struct perf_event_attr *to,
3245+ const struct perf_event_attr *from)
3246+{
3247+ to->sig_data = from->sig_data;
3248+}
3249+
32003250 static int perf_event_modify_attr(struct perf_event *event,
32013251 struct perf_event_attr *attr)
32023252 {
@@ -3219,10 +3269,17 @@ static int perf_event_modify_attr(struct perf_event *event, · 16 unmodified lines
32193269 WARN_ON_ONCE(event->ctx->parent_ctx);
32203270
32213271 mutex_lock(&event->child_mutex);
3272+ /*
3273+ * Event-type-independent attributes must be copied before event-type
3274+ * modification, which will validate that final attributes match the
3275+ * source attributes after all relevant attributes have been copied.
3276+ */
3277+ perf_event_modify_copy_attr(&event->attr, attr);
32223278 err = func(event, attr);
32233279 if (err)
32243280 goto out;
32253281 list_for_each_entry(child, &event->child_list, child_list) {
3282+ perf_event_modify_copy_attr(&child->attr, attr);
32263283 err = func(child, attr);
32273284 if (err)
32283285 goto out;
@@ -3251,16 +3308,6 @@ static void ctx_sched_out(struct perf_event_context *ctx, · 22 unmodified lines
32513308 return;
32523309 }
32533310
3254− ctx->is_active &= ~event_type;
3255− if (!(ctx->is_active & EVENT_ALL))
3256− ctx->is_active = 0;
3257−
3258− if (ctx->task) {
3259− WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3260− if (!ctx->is_active)
3261− cpuctx->task_ctx = NULL;
3262− }
3263−
32643311 /*
32653312 * Always update time if it was set; not only when it changes.
32663313 * Otherwise we can 'forget' to update time for any but the last
@@ -3274,7 +3321,22 @@ static void ctx_sched_out(struct perf_event_context *ctx, · 7 unmodified lines
32743321 if (is_active & EVENT_TIME) {
32753322 /* update (and stop) ctx time */
32763323 update_context_time(ctx);
3277− update_cgrp_time_from_cpuctx(cpuctx);
3324+ update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx);
3325+ /*
3326+ * CPU-release for the below ->is_active store,
3327+ * see __load_acquire() in perf_event_time_now()
3328+ */
3329+ barrier();
3330+ }
3331+
3332+ ctx->is_active &= ~event_type;
3333+ if (!(ctx->is_active & EVENT_ALL))
3334+ ctx->is_active = 0;
3335+
3336+ if (ctx->task) {
3337+ WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3338+ if (!ctx->is_active)
3339+ cpuctx->task_ctx = NULL;
32783340 }
32793341
32803342 is_active ^= ctx->is_active; /* changed bits */
@@ -3711,13 +3773,19 @@ static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx, · 430 unmodified lines
37113773 return 0;
37123774 }
37133775
3776+/*
3777+ * Because the userpage is strictly per-event (there is no concept of context,
3778+ * so there cannot be a context indirection), every userpage must be updated
3779+ * when context time starts :-(
3780+ *
3781+ * IOW, we must not miss EVENT_TIME edges.
3782+ */
37143783 static inline bool event_update_userpage(struct perf_event *event)
37153784 {
37163785 if (likely(!atomic_read(&event->mmap_count)))
37173786 return false;
37183787
37193788 perf_event_update_time(event);
3720− perf_set_shadow_time(event, event->ctx);
37213789 perf_event_update_userpage(event);
37223790
37233791 return true;
@@ -3801,13 +3869,23 @@ ctx_sched_in(struct perf_event_context *ctx, · 77 unmodified lines
38013869 struct task_struct *task)
38023870 {
38033871 int is_active = ctx->is_active;
3804− u64 now;
38053872
38063873 lockdep_assert_held(&ctx->lock);
38073874
38083875 if (likely(!ctx->nr_events))
38093876 return;
38103877
3878+ if (is_active ^ EVENT_TIME) {
3879+ /* start ctx time */
3880+ __update_context_time(ctx, false);
3881+ perf_cgroup_set_timestamp(task, ctx);
3882+ /*
3883+ * CPU-release for the below ->is_active store,
3884+ * see __load_acquire() in perf_event_time_now()
3885+ */
3886+ barrier();
3887+ }
3888+
38113889 ctx->is_active |= (event_type | EVENT_TIME);
38123890 if (ctx->task) {
38133891 if (!is_active)
@@ -3818,13 +3896,6 @@ ctx_sched_in(struct perf_event_context *ctx, · 4 unmodified lines
38183896
38193897 is_active ^= ctx->is_active; /* changed bits */
38203898
3821− if (is_active & EVENT_TIME) {
3822− /* start ctx time */
3823− now = perf_clock();
3824− ctx->timestamp = now;
3825− perf_cgroup_set_timestamp(task, ctx);
3826− }
3827−
38283899 /*
38293900 * First go through the list and put on any pinned groups
38303901 * in order to give them the best chance of going on.
@@ -4418,6 +4489,18 @@ static inline u64 perf_event_count(struct perf_event *event) · 587 unmodified lines
44184489 return local64_read(&event->count) + atomic64_read(&event->child_count);
44194490 }
44204491
4492+static void calc_timer_values(struct perf_event *event,
4493+ u64 *now,
4494+ u64 *enabled,
4495+ u64 *running)
4496+{
4497+ u64 ctx_time;
4498+
4499+ *now = perf_clock();
4500+ ctx_time = perf_event_time_now(event, *now);
4501+ __perf_update_times(event, ctx_time, enabled, running);
4502+}
4503+
44214504 /*
44224505 * NMI-safe method to read a local event, that is an event that
44234506 * is:
@@ -4477,10 +4560,9 @@ int perf_event_read_local(struct perf_event *event, u64 *value, · 53 unmodified lines
44774560
44784561 *value = local64_read(&event->count);
44794562 if (enabled || running) {
4480− u64 now = event->shadow_ctx_time + perf_clock();
4481− u64 __enabled, __running;
4563+ u64 __enabled, __running, __now;;
44824564
4483− __perf_update_times(event, now, &__enabled, &__running);
4565+ calc_timer_values(event, &__now, &__enabled, &__running);
44844566 if (enabled)
44854567 *enabled = __enabled;
44864568 if (running)
@@ -5802,18 +5884,6 @@ static int perf_event_index(struct perf_event *event) · 1315 unmodified lines
58025884 return event->pmu->event_idx(event);
58035885 }
58045886
5805−static void calc_timer_values(struct perf_event *event,
5806− u64 *now,
5807− u64 *enabled,
5808− u64 *running)
5809−{
5810− u64 ctx_time;
5811−
5812− *now = perf_clock();
5813− ctx_time = event->shadow_ctx_time + *now;
5814− __perf_update_times(event, ctx_time, enabled, running);
5815−}
5816−
58175887 static void perf_event_init_userpage(struct perf_event *event)
58185888 {
58195889 struct perf_event_mmap_page *userpg;
@@ -5938,6 +6008,8 @@ static void ring_buffer_attach(struct perf_event *event, · 118 unmodified lines
59386008 struct perf_buffer *old_rb = NULL;
59396009 unsigned long flags;
59406010
6011+ WARN_ON_ONCE(event->parent);
6012+
59416013 if (event->rb) {
59426014 /*
59436015 * Should be impossible, we set this when removing
@@ -5995,6 +6067,9 @@ static void ring_buffer_wakeup(struct perf_event *event) · 51 unmodified lines
59956067 {
59966068 struct perf_buffer *rb;
59976069
6070+ if (event->parent)
6071+ event = event->parent;
6072+
59986073 rcu_read_lock();
59996074 rb = rcu_dereference(event->rb);
60006075 if (rb) {
@@ -6008,6 +6083,9 @@ struct perf_buffer *ring_buffer_get(struct perf_event *event) · 7 unmodified lines
60086083 {
60096084 struct perf_buffer *rb;
60106085
6086+ if (event->parent)
6087+ event = event->parent;
6088+
60116089 rcu_read_lock();
60126090 rb = rcu_dereference(event->rb);
60136091 if (rb) {
@@ -6353,7 +6431,6 @@ accounting: · 339 unmodified lines
63536431 ring_buffer_attach(event, rb);
63546432
63556433 perf_event_update_time(event);
6356− perf_set_shadow_time(event, event->ctx);
63576434 perf_event_init_userpage(event);
63586435 perf_event_update_userpage(event);
63596436 } else {
@@ -6717,7 +6794,7 @@ static unsigned long perf_prepare_sample_aux(struct perf_event *event, · 357 unmodified lines
67176794 if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
67186795 goto out;
67196796
6720− rb = ring_buffer_get(sampler->parent ? sampler->parent : sampler);
6797+ rb = ring_buffer_get(sampler);
67216798 if (!rb)
67226799 goto out;
67236800
@@ -6783,7 +6860,7 @@ static void perf_aux_sample_output(struct perf_event *event, · 59 unmodified lines
67836860 if (WARN_ON_ONCE(!sampler || !data->aux_size))
67846861 return;
67856862
6786− rb = ring_buffer_get(sampler->parent ? sampler->parent : sampler);
6863+ rb = ring_buffer_get(sampler);
67876864 if (!rb)
67886865 return;
67896866
kernel/fork.c+20−10
@@ -2021,18 +2021,18 @@ static __latent_entropy struct task_struct *copy_process( · 2020 unmodified lines
20212021 #ifdef CONFIG_PROVE_LOCKING
20222022 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
20232023 #endif
2024+ retval = copy_creds(p, clone_flags);
2025+ if (retval < 0)
2026+ goto bad_fork_free;
2027+
20242028 retval = -EAGAIN;
20252029 if (is_ucounts_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) {
20262030 if (p->real_cred->user != INIT_USER &&
20272031 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
2028− goto bad_fork_free;
2032+ goto bad_fork_cleanup_count;
20292033 }
20302034 current->flags &= ~PF_NPROC_EXCEEDED;
20312035
2032− retval = copy_creds(p, clone_flags);
2033− if (retval < 0)
2034− goto bad_fork_free;
2035−
20362036 /*
20372037 * If multiple threads are within copy_process(), then this check
20382038 * triggers too late. This doesn't hurt, the check is only there
@@ -2266,6 +2266,17 @@ static __latent_entropy struct task_struct *copy_process( · 227 unmodified lines
22662266 if (retval)
22672267 goto bad_fork_put_pidfd;
22682268
2269+ /*
2270+ * Now that the cgroups are pinned, re-clone the parent cgroup and put
2271+ * the new task on the correct runqueue. All this *before* the task
2272+ * becomes visible.
2273+ *
2274+ * This isn't part of ->can_fork() because while the re-cloning is
2275+ * cgroup specific, it unconditionally needs to place the task on a
2276+ * runqueue.
2277+ */
2278+ sched_cgroup_fork(p, args);
2279+
22692280 /*
22702281 * From this point on we must avoid any synchronous user-space
22712282 * communication until we take the tasklist-lock. In particular, we do
@@ -2323,10 +2334,6 @@ static __latent_entropy struct task_struct *copy_process( · 51 unmodified lines
23232334 goto bad_fork_cancel_cgroup;
23242335 }
23252336
2326− /* past the last point of failure */
2327− if (pidfile)
2328− fd_install(pidfd, pidfile);
2329−
23302337 init_task_pid_links(p);
23312338 if (likely(p->pid)) {
23322339 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
@@ -2375,8 +2382,11 @@ static __latent_entropy struct task_struct *copy_process( · 42 unmodified lines
23752382 syscall_tracepoint_update(p);
23762383 write_unlock_irq(&tasklist_lock);
23772384
2385+ if (pidfile)
2386+ fd_install(pidfd, pidfile);
2387+
23782388 proc_fork_connector(p);
2379− sched_post_fork(p, args);
2389+ sched_post_fork(p);
23802390 cgroup_post_fork(p, args);
23812391 perf_event_fork(p);
23822392
kernel/locking/lockdep.c+2−2
@@ -3462,7 +3462,7 @@ struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i) · 3461 unmodified lines
34623462 u16 chain_hlock = chain_hlocks[chain->base + i];
34633463 unsigned int class_idx = chain_hlock_class_idx(chain_hlock);
34643464
3465− return lock_classes + class_idx - 1;
3465+ return lock_classes + class_idx;
34663466 }
34673467
34683468 /*
@@ -3530,7 +3530,7 @@ static void print_chain_keys_chain(struct lock_chain *chain) · 61 unmodified lines
35303530 hlock_id = chain_hlocks[chain->base + i];
35313531 chain_key = print_chain_key_iteration(hlock_id, chain_key);
35323532
3533− print_lock_name(lock_classes + chain_hlock_class_idx(hlock_id) - 1);
3533+ print_lock_name(lock_classes + chain_hlock_class_idx(hlock_id));
35343534 printk("\n");
35353535 }
35363536 }
kernel/module.c+5−20
@@ -3725,12 +3725,6 @@ static noinline int do_init_module(struct module *mod) · 3724 unmodified lines
37253725 }
37263726 freeinit->module_init = mod->init_layout.base;
37273727
3728− /*
3729− * We want to find out whether @mod uses async during init. Clear
3730− * PF_USED_ASYNC. async_schedule*() will set it.
3731− */
3732− current->flags &= ~PF_USED_ASYNC;
3733−
37343728 do_mod_ctors(mod);
37353729 /* Start the module */
37363730 if (mod->init != NULL)
@@ -3756,22 +3750,13 @@ static noinline int do_init_module(struct module *mod) · 19 unmodified lines
37563750
37573751 /*
37583752 * We need to finish all async code before the module init sequence
3759− * is done. This has potential to deadlock. For example, a newly
3760− * detected block device can trigger request_module() of the
3761− * default iosched from async probing task. Once userland helper
3762− * reaches here, async_synchronize_full() will wait on the async
3763− * task waiting on request_module() and deadlock.
3764− *
3765− * This deadlock is avoided by perfomring async_synchronize_full()
3766− * iff module init queued any async jobs. This isn't a full
3767− * solution as it will deadlock the same if module loading from
3768− * async jobs nests more than once; however, due to the various
3769− * constraints, this hack seems to be the best option for now.
3770− * Please refer to the following thread for details.
3753+ * is done. This has potential to deadlock if synchronous module
3754+ * loading is requested from async (which is not allowed!).
37713755 *
3772− * http://thread.gmane.org/gmane.linux.kernel/1420814
3756+ * See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
3757+ * request_module() from async workers") for more details.
37733758 */
3774− if (!mod->async_probe_requested && (current->flags & PF_USED_ASYNC))
3759+ if (!mod->async_probe_requested)
37753760 async_synchronize_full();
37763761
37773762 ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base +
kernel/module_decompress.c+2−0
@@ -250,6 +250,7 @@ void module_decompress_cleanup(struct load_info *info) · 249 unmodified lines
250250 info->max_pages = info->used_pages = 0;
251251 }
252252
253+#ifdef CONFIG_SYSFS
253254 static ssize_t compression_show(struct kobject *kobj,
254255 struct kobj_attribute *attr, char *buf)
255256 {
@@ -269,3 +270,4 @@ static int __init module_decompress_sysfs_init(void) · 13 unmodified lines
269270 return 0;
270271 }
271272 late_initcall(module_decompress_sysfs_init);
273+#endif
kernel/power/main.c+4−1
@@ -504,7 +504,10 @@ static ssize_t pm_wakeup_irq_show(struct kobject *kobj, · 503 unmodified lines
504504 struct kobj_attribute *attr,
505505 char *buf)
506506 {
507− return pm_wakeup_irq ? sprintf(buf, "%u\n", pm_wakeup_irq) : -ENODATA;
507+ if (!pm_wakeup_irq())
508+ return -ENODATA;
509+
510+ return sprintf(buf, "%u\n", pm_wakeup_irq());
508511 }
509512
510513 power_attr_ro(pm_wakeup_irq);
kernel/power/process.c+1−1
@@ -134,7 +134,7 @@ int freeze_processes(void) · 133 unmodified lines
134134 if (!pm_freezing)
135135 atomic_inc(&system_freezing_cnt);
136136
137− pm_wakeup_clear(true);
137+ pm_wakeup_clear(0);
138138 pr_info("Freezing user space processes ... ");
139139 pm_freezing = true;
140140 error = try_to_freeze_tasks(true);
kernel/power/snapshot.c+7−14
@@ -978,8 +978,7 @@ static void memory_bm_recycle(struct memory_bitmap *bm) · 977 unmodified lines
978978 * Register a range of page frames the contents of which should not be saved
979979 * during hibernation (to be used in the early initialization code).
980980 */
981−void __init __register_nosave_region(unsigned long start_pfn,
982− unsigned long end_pfn, int use_kmalloc)
981+void __init register_nosave_region(unsigned long start_pfn, unsigned long end_pfn)
983982 {
984983 struct nosave_region *region;
985984
@@ -995,18 +994,12 @@ void __init __register_nosave_region(unsigned long start_pfn, · 9 unmodified lines
995994 goto Report;
996995 }
997996 }
998− if (use_kmalloc) {
999− /* During init, this shouldn't fail */
1000− region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
1001− BUG_ON(!region);
1002− } else {
1003− /* This allocation cannot fail */
1004− region = memblock_alloc(sizeof(struct nosave_region),
1005− SMP_CACHE_BYTES);
1006− if (!region)
1007− panic("%s: Failed to allocate %zu bytes\n", __func__,
1008− sizeof(struct nosave_region));
1009− }
997+ /* This allocation cannot fail */
998+ region = memblock_alloc(sizeof(struct nosave_region),
999+ SMP_CACHE_BYTES);
1000+ if (!region)
1001+ panic("%s: Failed to allocate %zu bytes\n", __func__,
1002+ sizeof(struct nosave_region));
10101003 region->start_pfn = start_pfn;
10111004 region->end_pfn = end_pfn;
10121005 list_add_tail(&region->list, &nosave_regions);
kernel/power/suspend.c+0−2
@@ -136,8 +136,6 @@ static void s2idle_loop(void) · 135 unmodified lines
136136 break;
137137 }
138138
139− pm_wakeup_clear(false);
140−
141139 s2idle_enter();
142140 }
143141
kernel/power/wakelock.c+4−7
@@ -39,23 +39,20 @@ ssize_t pm_show_wakelocks(char *buf, bool show_active) · 38 unmodified lines
3939 {
4040 struct rb_node *node;
4141 struct wakelock *wl;
42− char *str = buf;
43− char *end = buf + PAGE_SIZE;
42+ int len = 0;
4443
4544 mutex_lock(&wakelocks_lock);
4645
4746 for (node = rb_first(&wakelocks_tree); node; node = rb_next(node)) {
4847 wl = rb_entry(node, struct wakelock, node);
4948 if (wl->ws->active == show_active)
50− str += scnprintf(str, end - str, "%s ", wl->name);
49+ len += sysfs_emit_at(buf, len, "%s ", wl->name);
5150 }
52− if (str > buf)
53− str--;
5451
55− str += scnprintf(str, end - str, "\n");
52+ len += sysfs_emit_at(buf, len, "\n");
5653
5754 mutex_unlock(&wakelocks_lock);
58− return (str - buf);
55+ return len;
5956 }
6057
6158 #if CONFIG_PM_WAKELOCKS_LIMIT > 0
kernel/printk/sysctl.c+1−1
@@ -12,7 +12,7 @@ · 11 unmodified lines
1212 static const int ten_thousand = 10000;
1313
1414 static int proc_dointvec_minmax_sysadmin(struct ctl_table *table, int write,
15− void __user *buffer, size_t *lenp, loff_t *ppos)
15+ void *buffer, size_t *lenp, loff_t *ppos)
1616 {
1717 if (write && !capable(CAP_SYS_ADMIN))
1818 return -EPERM;
kernel/rcu/tasks.h+8−4
@@ -123,7 +123,7 @@ static struct rcu_tasks rt_name = \ · 122 unmodified lines
123123 .call_func = call, \
124124 .rtpcpu = &rt_name ## __percpu, \
125125 .name = n, \
126− .percpu_enqueue_shift = ilog2(CONFIG_NR_CPUS), \
126+ .percpu_enqueue_shift = ilog2(CONFIG_NR_CPUS) + 1, \
127127 .percpu_enqueue_lim = 1, \
128128 .percpu_dequeue_lim = 1, \
129129 .barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \
@@ -216,6 +216,7 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 86 unmodified lines
216216 int cpu;
217217 unsigned long flags;
218218 int lim;
219+ int shift;
219220
220221 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
221222 if (rcu_task_enqueue_lim < 0) {
@@ -229,7 +230,10 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 7 unmodified lines
229230
230231 if (lim > nr_cpu_ids)
231232 lim = nr_cpu_ids;
232− WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids / lim));
233+ shift = ilog2(nr_cpu_ids / lim);
234+ if (((nr_cpu_ids - 1) >> shift) >= lim)
235+ shift++;
236+ WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
233237 WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
234238 smp_store_release(&rtp->percpu_enqueue_lim, lim);
235239 for_each_possible_cpu(cpu) {
@@ -298,7 +302,7 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, · 62 unmodified lines
298302 if (unlikely(needadjust)) {
299303 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
300304 if (rtp->percpu_enqueue_lim != nr_cpu_ids) {
301− WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));
305+ WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids) + 1);
302306 WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids);
303307 smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids);
304308 pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
@@ -413,7 +417,7 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) · 108 unmodified lines
413417 if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
414418 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
415419 if (rtp->percpu_enqueue_lim > 1) {
416− WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));
420+ WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids) + 1);
417421 smp_store_release(&rtp->percpu_enqueue_lim, 1);
418422 rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
419423 pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name);
kernel/sched/core.c+20−18
@@ -4426,6 +4426,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) · 4425 unmodified lines
44264426
44274427 init_entity_runnable_average(&p->se);
44284428
4429+
44294430 #ifdef CONFIG_SCHED_INFO
44304431 if (likely(sched_info_on()))
44314432 memset(&p->sched_info, 0, sizeof(p->sched_info));
@@ -4441,18 +4442,23 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) · 9 unmodified lines
44414442 return 0;
44424443 }
44434444
4444−void sched_post_fork(struct task_struct *p, struct kernel_clone_args *kargs)
4445+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
44454446 {
44464447 unsigned long flags;
4447−#ifdef CONFIG_CGROUP_SCHED
4448− struct task_group *tg;
4449−#endif
44504448
4449+ /*
4450+ * Because we're not yet on the pid-hash, p->pi_lock isn't strictly
4451+ * required yet, but lockdep gets upset if rules are violated.
4452+ */
44514453 raw_spin_lock_irqsave(&p->pi_lock, flags);
44524454 #ifdef CONFIG_CGROUP_SCHED
4453− tg = container_of(kargs->cset->subsys[cpu_cgrp_id],
4454− struct task_group, css);
4455− p->sched_task_group = autogroup_task_group(p, tg);
4455+ if (1) {
4456+ struct task_group *tg;
4457+ tg = container_of(kargs->cset->subsys[cpu_cgrp_id],
4458+ struct task_group, css);
4459+ tg = autogroup_task_group(p, tg);
4460+ p->sched_task_group = tg;
4461+ }
44564462 #endif
44574463 rseq_migrate(p);
44584464 /*
@@ -4463,7 +4469,10 @@ void sched_post_fork(struct task_struct *p, struct kernel_clone_args *kargs) · 4 unmodified lines
44634469 if (p->sched_class->task_fork)
44644470 p->sched_class->task_fork(p);
44654471 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4472+}
44664473
4474+void sched_post_fork(struct task_struct *p)
4475+{
44674476 uclamp_post_fork(p);
44684477 }
44694478
@@ -5824,8 +5833,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) · 1354 unmodified lines
58245833 }
58255834
58265835 if (schedstat_enabled() && rq->core->core_forceidle_count) {
5827− if (cookie)
5828− rq->core->core_forceidle_start = rq_clock(rq->core);
5836+ rq->core->core_forceidle_start = rq_clock(rq->core);
58295837 rq->core->core_forceidle_occupation = occ;
58305838 }
58315839
@@ -8139,9 +8147,7 @@ int __cond_resched_lock(spinlock_t *lock) · 2307 unmodified lines
81398147
81408148 if (spin_needbreak(lock) || resched) {
81418149 spin_unlock(lock);
8142− if (resched)
8143− preempt_schedule_common();
8144− else
8150+ if (!_cond_resched())
81458151 cpu_relax();
81468152 ret = 1;
81478153 spin_lock(lock);
@@ -8159,9 +8165,7 @@ int __cond_resched_rwlock_read(rwlock_t *lock) · 11 unmodified lines
81598165
81608166 if (rwlock_needbreak(lock) || resched) {
81618167 read_unlock(lock);
8162− if (resched)
8163− preempt_schedule_common();
8164− else
8168+ if (!_cond_resched())
81658169 cpu_relax();
81668170 ret = 1;
81678171 read_lock(lock);
@@ -8179,9 +8183,7 @@ int __cond_resched_rwlock_write(rwlock_t *lock) · 11 unmodified lines
81798183
81808184 if (rwlock_needbreak(lock) || resched) {
81818185 write_unlock(lock);
8182− if (resched)
8183− preempt_schedule_common();
8184− else
8186+ if (!_cond_resched())
81858187 cpu_relax();
81868188 ret = 1;
81878189 write_lock(lock);
kernel/sched/core_sched.c+1−1
@@ -277,7 +277,7 @@ void __sched_core_account_forceidle(struct rq *rq) · 276 unmodified lines
277277 rq_i = cpu_rq(i);
278278 p = rq_i->core_pick ?: rq_i->curr;
279279
280− if (!p->core_cookie)
280+ if (p == rq_i->idle)
281281 continue;
282282
283283 __schedstat_add(p->stats.core_forceidle_sum, delta);
kernel/sched/fair.c+77−41
@@ -3049,9 +3049,11 @@ enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) · 3048 unmodified lines
30493049 static inline void
30503050 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
30513051 {
3052− u32 divider = get_pelt_divider(&se->avg);
30533052 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
3054− cfs_rq->avg.load_sum = cfs_rq->avg.load_avg * divider;
3053+ sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);
3054+ /* See update_cfs_rq_load_avg() */
3055+ cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
3056+ cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
30553057 }
30563058 #else
30573059 static inline void
@@ -3402,7 +3404,6 @@ void set_task_rq_fair(struct sched_entity *se, · 344 unmodified lines
34023404 se->avg.last_update_time = n_last_update_time;
34033405 }
34043406
3405−
34063407 /*
34073408 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
34083409 * propagate its contribution. The key to this propagation is the invariant
@@ -3470,15 +3471,14 @@ void set_task_rq_fair(struct sched_entity *se, · 61 unmodified lines
34703471 * XXX: only do this for the part of runnable > running ?
34713472 *
34723473 */
3473−
34743474 static inline void
34753475 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
34763476 {
3477− long delta = gcfs_rq->avg.util_avg - se->avg.util_avg;
3478− u32 divider;
3477+ long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg;
3478+ u32 new_sum, divider;
34793479
34803480 /* Nothing to update */
3481− if (!delta)
3481+ if (!delta_avg)
34823482 return;
34833483
34843484 /*
@@ -3487,23 +3487,30 @@ update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq · 2 unmodified lines
34873487 */
34883488 divider = get_pelt_divider(&cfs_rq->avg);
34893489
3490+
34903491 /* Set new sched_entity's utilization */
34913492 se->avg.util_avg = gcfs_rq->avg.util_avg;
3492− se->avg.util_sum = se->avg.util_avg * divider;
3493+ new_sum = se->avg.util_avg * divider;
3494+ delta_sum = (long)new_sum - (long)se->avg.util_sum;
3495+ se->avg.util_sum = new_sum;
34933496
34943497 /* Update parent cfs_rq utilization */
3495− add_positive(&cfs_rq->avg.util_avg, delta);
3496− cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * divider;
3498+ add_positive(&cfs_rq->avg.util_avg, delta_avg);
3499+ add_positive(&cfs_rq->avg.util_sum, delta_sum);
3500+
3501+ /* See update_cfs_rq_load_avg() */
3502+ cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
3503+ cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
34973504 }
34983505
34993506 static inline void
35003507 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
35013508 {
3502− long delta = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
3503− u32 divider;
3509+ long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
3510+ u32 new_sum, divider;
35043511
35053512 /* Nothing to update */
3506− if (!delta)
3513+ if (!delta_avg)
35073514 return;
35083515
35093516 /*
@@ -3514,19 +3521,25 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf · 4 unmodified lines
35143521
35153522 /* Set new sched_entity's runnable */
35163523 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;
3517− se->avg.runnable_sum = se->avg.runnable_avg * divider;
3524+ new_sum = se->avg.runnable_avg * divider;
3525+ delta_sum = (long)new_sum - (long)se->avg.runnable_sum;
3526+ se->avg.runnable_sum = new_sum;
35183527
35193528 /* Update parent cfs_rq runnable */
3520− add_positive(&cfs_rq->avg.runnable_avg, delta);
3521− cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * divider;
3529+ add_positive(&cfs_rq->avg.runnable_avg, delta_avg);
3530+ add_positive(&cfs_rq->avg.runnable_sum, delta_sum);
3531+ /* See update_cfs_rq_load_avg() */
3532+ cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
3533+ cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
35223534 }
35233535
35243536 static inline void
35253537 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
35263538 {
3527− long delta, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
3539+ long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
35283540 unsigned long load_avg;
35293541 u64 load_sum = 0;
3542+ s64 delta_sum;
35303543 u32 divider;
35313544
35323545 if (!runnable_sum)
@@ -3553,7 +3566,7 @@ update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq · 20 unmodified lines
35533566 * assuming all tasks are equally runnable.
35543567 */
35553568 if (scale_load_down(gcfs_rq->load.weight)) {
3556− load_sum = div_s64(gcfs_rq->avg.load_sum,
3569+ load_sum = div_u64(gcfs_rq->avg.load_sum,
35573570 scale_load_down(gcfs_rq->load.weight));
35583571 }
35593572
@@ -3570,19 +3583,22 @@ update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq · 10 unmodified lines
35703583 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;
35713584 runnable_sum = max(runnable_sum, running_sum);
35723585
3573− load_sum = (s64)se_weight(se) * runnable_sum;
3574− load_avg = div_s64(load_sum, divider);
3575−
3576− se->avg.load_sum = runnable_sum;
3586+ load_sum = se_weight(se) * runnable_sum;
3587+ load_avg = div_u64(load_sum, divider);
35773588
3578− delta = load_avg - se->avg.load_avg;
3579− if (!delta)
3589+ delta_avg = load_avg - se->avg.load_avg;
3590+ if (!delta_avg)
35803591 return;
35813592
3582− se->avg.load_avg = load_avg;
3593+ delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum;
35833594
3584− add_positive(&cfs_rq->avg.load_avg, delta);
3585− cfs_rq->avg.load_sum = cfs_rq->avg.load_avg * divider;
3595+ se->avg.load_sum = runnable_sum;
3596+ se->avg.load_avg = load_avg;
3597+ add_positive(&cfs_rq->avg.load_avg, delta_avg);
3598+ add_positive(&cfs_rq->avg.load_sum, delta_sum);
3599+ /* See update_cfs_rq_load_avg() */
3600+ cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
3601+ cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
35863602 }
35873603
35883604 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
@@ -3673,7 +3689,7 @@ static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum · 84 unmodified lines
36733689 *
36743690 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example.
36753691 *
3676− * Returns true if the load decayed or we removed load.
3692+ * Return: true if the load decayed or we removed load.
36773693 *
36783694 * Since both these conditions indicate a changed cfs_rq->avg.load we should
36793695 * call update_tg_load_avg() when this function returns true.
@@ -3698,15 +3714,32 @@ update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) · 18 unmodified lines
36983714
36993715 r = removed_load;
37003716 sub_positive(&sa->load_avg, r);
3701− sa->load_sum = sa->load_avg * divider;
3717+ sub_positive(&sa->load_sum, r * divider);
3718+ /* See sa->util_sum below */
3719+ sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER);
37023720
37033721 r = removed_util;
37043722 sub_positive(&sa->util_avg, r);
3705− sa->util_sum = sa->util_avg * divider;
3723+ sub_positive(&sa->util_sum, r * divider);
3724+ /*
3725+ * Because of rounding, se->util_sum might ends up being +1 more than
3726+ * cfs->util_sum. Although this is not a problem by itself, detaching
3727+ * a lot of tasks with the rounding problem between 2 updates of
3728+ * util_avg (~1ms) can make cfs->util_sum becoming null whereas
3729+ * cfs_util_avg is not.
3730+ * Check that util_sum is still above its lower bound for the new
3731+ * util_avg. Given that period_contrib might have moved since the last
3732+ * sync, we are only sure that util_sum must be above or equal to
3733+ * util_avg * minimum possible divider
3734+ */
3735+ sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER);
37063736
37073737 r = removed_runnable;
37083738 sub_positive(&sa->runnable_avg, r);
3709− sa->runnable_sum = sa->runnable_avg * divider;
3739+ sub_positive(&sa->runnable_sum, r * divider);
3740+ /* See sa->util_sum above */
3741+ sa->runnable_sum = max_t(u32, sa->runnable_sum,
3742+ sa->runnable_avg * PELT_MIN_DIVIDER);
37103743
37113744 /*
37123745 * removed_runnable is the unweighted version of removed_load so we
@@ -3793,17 +3826,18 @@ static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s · 80 unmodified lines
37933826 */
37943827 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
37953828 {
3796− /*
3797− * cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
3798− * See ___update_load_avg() for details.
3799− */
3800− u32 divider = get_pelt_divider(&cfs_rq->avg);
3801−
38023829 dequeue_load_avg(cfs_rq, se);
38033830 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
3804− cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * divider;
3831+ sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);
3832+ /* See update_cfs_rq_load_avg() */
3833+ cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
3834+ cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
3835+
38053836 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg);
3806− cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * divider;
3837+ sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum);
3838+ /* See update_cfs_rq_load_avg() */
3839+ cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
3840+ cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
38073841
38083842 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);
38093843
@@ -8560,6 +8594,8 @@ group_type group_classify(unsigned int imbalance_pct, · 4750 unmodified lines
85608594 *
85618595 * If @sg does not have SMT siblings, only pull tasks if all of the SMT siblings
85628596 * of @dst_cpu are idle and @sg has lower priority.
8597+ *
8598+ * Return: true if @dst_cpu can pull tasks, false otherwise.
85638599 */
85648600 static bool asym_smt_can_pull_tasks(int dst_cpu, struct sd_lb_stats *sds,
85658601 struct sg_lb_stats *sgs,
@@ -8635,6 +8671,7 @@ sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs · 69 unmodified lines
86358671 /**
86368672 * update_sg_lb_stats - Update sched_group's statistics for load balancing.
86378673 * @env: The load balancing environment.
8674+ * @sds: Load-balancing data with statistics of the local group.
86388675 * @group: sched_group whose statistics are to be updated.
86398676 * @sgs: variable to hold the statistics for this group.
86408677 * @sg_status: Holds flag indicating the status of the sched_group
@@ -9443,12 +9480,11 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s · 802 unmodified lines
94439480 /**
94449481 * find_busiest_group - Returns the busiest group within the sched_domain
94459482 * if there is an imbalance.
9483+ * @env: The load balancing environment.
94469484 *
94479485 * Also calculates the amount of runnable load which should be moved
94489486 * to restore balance.
94499487 *
9450− * @env: The load balancing environment.
9451− *
94529488 * Return: - The busiest group if imbalance exists.
94539489 */
94549490 static struct sched_group *find_busiest_group(struct lb_env *env)
kernel/sched/membarrier.c+5−4
@@ -147,11 +147,11 @@ · 146 unmodified lines
147147 #endif
148148
149149 #ifdef CONFIG_RSEQ
150−#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK \
150+#define MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK \
151151 (MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ \
152− | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ_BITMASK)
152+ | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ)
153153 #else
154−#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK 0
154+#define MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK 0
155155 #endif
156156
157157 #define MEMBARRIER_CMD_BITMASK \
@@ -159,7 +159,8 @@ · 1 unmodified line
159159 | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \
160160 | MEMBARRIER_CMD_PRIVATE_EXPEDITED \
161161 | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED \
162− | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK)
162+ | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK \
163+ | MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK)
163164
164165 static void ipi_mb(void *info)
165166 {
kernel/sched/pelt.h+3−1
@@ -37,9 +37,11 @@ update_irq_load_avg(struct rq *rq, u64 running) · 36 unmodified lines
3737 }
3838 #endif
3939
40+#define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024)
41+
4042 static inline u32 get_pelt_divider(struct sched_avg *avg)
4143 {
42− return LOAD_AVG_MAX - 1024 + avg->period_contrib;
44+ return PELT_MIN_DIVIDER + avg->period_contrib;
4345 }
4446
4547 static inline void cfs_se_util_change(struct sched_avg *avg)
kernel/sched/psi.c+29−37
@@ -1137,7 +1137,6 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, · 1136 unmodified lines
11371137 t->event = 0;
11381138 t->last_event_time = 0;
11391139 init_waitqueue_head(&t->event_wait);
1140− kref_init(&t->refcount);
11411140 t->pending_event = false;
11421141
11431142 mutex_lock(&group->trigger_lock);
@@ -1167,15 +1166,19 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, · 23 unmodified lines
11671166 return t;
11681167 }
11691168
1170−static void psi_trigger_destroy(struct kref *ref)
1169+void psi_trigger_destroy(struct psi_trigger *t)
11711170 {
1172− struct psi_trigger *t = container_of(ref, struct psi_trigger, refcount);
1173− struct psi_group *group = t->group;
1171+ struct psi_group *group;
11741172 struct task_struct *task_to_destroy = NULL;
11751173
1176− if (static_branch_likely(&psi_disabled))
1174+ /*
1175+ * We do not check psi_disabled since it might have been disabled after
1176+ * the trigger got created.
1177+ */
1178+ if (!t)
11771179 return;
11781180
1181+ group = t->group;
11791182 /*
11801183 * Wakeup waiters to stop polling. Can happen if cgroup is deleted
11811184 * from under a polling process.
@@ -1211,9 +1214,9 @@ static void psi_trigger_destroy(struct kref *ref) · 29 unmodified lines
12111214 mutex_unlock(&group->trigger_lock);
12121215
12131216 /*
1214− * Wait for both *trigger_ptr from psi_trigger_replace and
1215− * poll_task RCUs to complete their read-side critical sections
1216− * before destroying the trigger and optionally the poll_task
1217+ * Wait for psi_schedule_poll_work RCU to complete its read-side
1218+ * critical section before destroying the trigger and optionally the
1219+ * poll_task.
12171220 */
12181221 synchronize_rcu();
12191222 /*
@@ -1230,18 +1233,6 @@ static void psi_trigger_destroy(struct kref *ref) · 10 unmodified lines
12301233 kfree(t);
12311234 }
12321235
1233−void psi_trigger_replace(void **trigger_ptr, struct psi_trigger *new)
1234−{
1235− struct psi_trigger *old = *trigger_ptr;
1236−
1237− if (static_branch_likely(&psi_disabled))
1238− return;
1239−
1240− rcu_assign_pointer(*trigger_ptr, new);
1241− if (old)
1242− kref_put(&old->refcount, psi_trigger_destroy);
1243−}
1244−
12451236 __poll_t psi_trigger_poll(void **trigger_ptr,
12461237 struct file *file, poll_table *wait)
12471238 {
@@ -1251,24 +1242,15 @@ __poll_t psi_trigger_poll(void **trigger_ptr, · 3 unmodified lines
12511242 if (static_branch_likely(&psi_disabled))
12521243 return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI;
12531244
1254− rcu_read_lock();
1255−
1256− t = rcu_dereference(*(void __rcu __force **)trigger_ptr);
1257− if (!t) {
1258− rcu_read_unlock();
1245+ t = smp_load_acquire(trigger_ptr);
1246+ if (!t)
12591247 return DEFAULT_POLLMASK | EPOLLERR | EPOLLPRI;
1260− }
1261− kref_get(&t->refcount);
1262−
1263− rcu_read_unlock();
12641248
12651249 poll_wait(file, &t->event_wait, wait);
12661250
12671251 if (cmpxchg(&t->event, 1, 0) == 1)
12681252 ret |= EPOLLPRI;
12691253
1270− kref_put(&t->refcount, psi_trigger_destroy);
1271−
12721254 return ret;
12731255 }
12741256
@@ -1331,14 +1313,24 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, · 56 unmodified lines
13311313
13321314 buf[buf_size - 1] = '\0';
13331315
1334− new = psi_trigger_create(&psi_system, buf, nbytes, res);
1335− if (IS_ERR(new))
1336− return PTR_ERR(new);
1337−
13381316 seq = file->private_data;
1317+
13391318 /* Take seq->lock to protect seq->private from concurrent writes */
13401319 mutex_lock(&seq->lock);
1341− psi_trigger_replace(&seq->private, new);
1320+
1321+ /* Allow only one trigger per file descriptor */
1322+ if (seq->private) {
1323+ mutex_unlock(&seq->lock);
1324+ return -EBUSY;
1325+ }
1326+
1327+ new = psi_trigger_create(&psi_system, buf, nbytes, res);
1328+ if (IS_ERR(new)) {
1329+ mutex_unlock(&seq->lock);
1330+ return PTR_ERR(new);
1331+ }
1332+
1333+ smp_store_release(&seq->private, new);
13421334 mutex_unlock(&seq->lock);
13431335
13441336 return nbytes;
@@ -1373,7 +1365,7 @@ static int psi_fop_release(struct inode *inode, struct file *file) · 28 unmodified lines
13731365 {
13741366 struct seq_file *seq = file->private_data;
13751367
1376− psi_trigger_replace(&seq->private, NULL);
1368+ psi_trigger_destroy(seq->private);
13771369 return single_release(inode, file);
13781370 }
13791371
kernel/seccomp.c+10−0
@@ -29,6 +29,9 @@ · 28 unmodified lines
2929 #include <linux/syscalls.h>
3030 #include <linux/sysctl.h>
3131
32+/* Not exposed in headers: strictly internal use only. */
33+#define SECCOMP_MODE_DEAD (SECCOMP_MODE_FILTER + 1)
34+
3235 #ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER
3336 #include <asm/syscall.h>
3437 #endif
@@ -1010,6 +1013,7 @@ static void __secure_computing_strict(int this_syscall) · 975 unmodified lines
10101013 #ifdef SECCOMP_DEBUG
10111014 dump_stack();
10121015 #endif
1016+ current->seccomp.mode = SECCOMP_MODE_DEAD;
10131017 seccomp_log(this_syscall, SIGKILL, SECCOMP_RET_KILL_THREAD, true);
10141018 do_exit(SIGKILL);
10151019 }
@@ -1261,6 +1265,7 @@ static int __seccomp_filter(int this_syscall, const struct seccomp_data *sd, · 245 unmodified lines
12611265 case SECCOMP_RET_KILL_THREAD:
12621266 case SECCOMP_RET_KILL_PROCESS:
12631267 default:
1268+ current->seccomp.mode = SECCOMP_MODE_DEAD;
12641269 seccomp_log(this_syscall, SIGSYS, action, true);
12651270 /* Dump core only if this is the last remaining thread. */
12661271 if (action != SECCOMP_RET_KILL_THREAD ||
@@ -1309,6 +1314,11 @@ int __secure_computing(const struct seccomp_data *sd) · 42 unmodified lines
13091314 return 0;
13101315 case SECCOMP_MODE_FILTER:
13111316 return __seccomp_filter(this_syscall, sd, false);
1317+ /* Surviving SECCOMP_RET_KILL_* must be proactively impossible. */
1318+ case SECCOMP_MODE_DEAD:
1319+ WARN_ON_ONCE(1);
1320+ do_exit(SIGKILL);
1321+ return -1;
13121322 default:
13131323 BUG();
13141324 }
kernel/signal.c+3−2
@@ -1342,9 +1342,10 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t, · 1341 unmodified lines
13421342 }
13431343 /*
13441344 * Don't clear SIGNAL_UNKILLABLE for traced tasks, users won't expect
1345− * debugging to leave init killable.
1345+ * debugging to leave init killable. But HANDLER_EXIT is always fatal.
13461346 */
1347− if (action->sa.sa_handler == SIG_DFL && !t->ptrace)
1347+ if (action->sa.sa_handler == SIG_DFL &&
1348+ (!t->ptrace || (handler == HANDLER_EXIT)))
13481349 t->signal->flags &= ~SIGNAL_UNKILLABLE;
13491350 ret = send_signal(sig, info, t, PIDTYPE_PID);
13501351 spin_unlock_irqrestore(&t->sighand->siglock, flags);
kernel/stackleak.c+2−3
@@ -70,7 +70,7 @@ late_initcall(stackleak_sysctls_init); · 69 unmodified lines
7070 #define skip_erasing() false
7171 #endif /* CONFIG_STACKLEAK_RUNTIME_DISABLE */
7272
73−asmlinkage void notrace stackleak_erase(void)
73+asmlinkage void noinstr stackleak_erase(void)
7474 {
7575 /* It would be nice not to have 'kstack_ptr' and 'boundary' on stack */
7676 unsigned long kstack_ptr = current->lowest_stack;
@@ -124,9 +124,8 @@ asmlinkage void notrace stackleak_erase(void) · 47 unmodified lines
124124 /* Reset the 'lowest_stack' value for the next syscall */
125125 current->lowest_stack = current_top_of_stack() - THREAD_SIZE/64;
126126 }
127−NOKPROBE_SYMBOL(stackleak_erase);
128127
129−void __used __no_caller_saved_registers notrace stackleak_track_stack(void)
128+void __used __no_caller_saved_registers noinstr stackleak_track_stack(void)
130129 {
131130 unsigned long sp = current_stack_pointer;
132131
kernel/sys.c+14−6
@@ -472,6 +472,16 @@ static int set_user(struct cred *new) · 471 unmodified lines
472472 if (!new_user)
473473 return -EAGAIN;
474474
475+ free_uid(new->user);
476+ new->user = new_user;
477+ return 0;
478+}
479+
480+static void flag_nproc_exceeded(struct cred *new)
481+{
482+ if (new->ucounts == current_ucounts())
483+ return;
484+
475485 /*
476486 * We don't fail in case of NPROC limit excess here because too many
477487 * poorly written programs don't check set*uid() return code, assuming
@@ -480,15 +490,10 @@ static int set_user(struct cred *new) · 2 unmodified lines
480490 * failure to the execve() stage.
481491 */
482492 if (is_ucounts_overlimit(new->ucounts, UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC)) &&
483− new_user != INIT_USER &&
484− !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
493+ new->user != INIT_USER)
485494 current->flags |= PF_NPROC_EXCEEDED;
486495 else
487496 current->flags &= ~PF_NPROC_EXCEEDED;
488−
489− free_uid(new->user);
490− new->user = new_user;
491− return 0;
492497 }
493498
494499 /*
@@ -563,6 +568,7 @@ long __sys_setreuid(uid_t ruid, uid_t euid) · 68 unmodified lines
563568 if (retval < 0)
564569 goto error;
565570
571+ flag_nproc_exceeded(new);
566572 return commit_creds(new);
567573
568574 error:
@@ -625,6 +631,7 @@ long __sys_setuid(uid_t uid) · 56 unmodified lines
625631 if (retval < 0)
626632 goto error;
627633
634+ flag_nproc_exceeded(new);
628635 return commit_creds(new);
629636
630637 error:
@@ -704,6 +711,7 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) · 73 unmodified lines
704711 if (retval < 0)
705712 goto error;
706713
714+ flag_nproc_exceeded(new);
707715 return commit_creds(new);
708716
709717 error:
kernel/trace/Kconfig+7−1
@@ -70,10 +70,16 @@ config HAVE_C_RECORDMCOUNT · 69 unmodified lines
7070 help
7171 C version of recordmcount available?
7272
73+config HAVE_BUILDTIME_MCOUNT_SORT
74+ bool
75+ help
76+ An architecture selects this if it sorts the mcount_loc section
77+ at build time.
78+
7379 config BUILDTIME_MCOUNT_SORT
7480 bool
7581 default y
76− depends on BUILDTIME_TABLE_SORT && !S390
82+ depends on HAVE_BUILDTIME_MCOUNT_SORT && DYNAMIC_FTRACE
7783 help
7884 Sort the mcount_loc section at build time.
7985
kernel/trace/trace.c+6−1
@@ -252,6 +252,10 @@ __setup("trace_clock=", set_trace_boot_clock); · 251 unmodified lines
252252
253253 static int __init set_tracepoint_printk(char *str)
254254 {
255+ /* Ignore the "tp_printk_stop_on_boot" param */
256+ if (*str == '_')
257+ return 0;
258+
255259 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
256260 tracepoint_printk = 1;
257261 return 1;
@@ -7740,7 +7744,8 @@ static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr) · 7482 unmodified lines
77407744 err = kzalloc(sizeof(*err), GFP_KERNEL);
77417745 if (!err)
77427746 err = ERR_PTR(-ENOMEM);
7743− tr->n_err_log_entries++;
7747+ else
7748+ tr->n_err_log_entries++;
77447749
77457750 return err;
77467751 }
kernel/trace/trace_events_hist.c+8−2
@@ -2503,6 +2503,8 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, · 2502 unmodified lines
25032503 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
25042504 expr->fn = hist_field_unary_minus;
25052505 expr->operands[0] = operand1;
2506+ expr->size = operand1->size;
2507+ expr->is_signed = operand1->is_signed;
25062508 expr->operator = FIELD_OP_UNARY_MINUS;
25072509 expr->name = expr_str(expr, 0);
25082510 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
@@ -2719,6 +2721,7 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, · 210 unmodified lines
27192721
27202722 /* The operand sizes should be the same, so just pick one */
27212723 expr->size = operand1->size;
2724+ expr->is_signed = operand1->is_signed;
27222725
27232726 expr->operator = field_op;
27242727 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
@@ -3935,6 +3938,7 @@ static int trace_action_create(struct hist_trigger_data *hist_data, · 1210 unmodified lines
39353938
39363939 var_ref_idx = find_var_ref_idx(hist_data, var_ref);
39373940 if (WARN_ON(var_ref_idx < 0)) {
3941+ kfree(p);
39383942 ret = var_ref_idx;
39393943 goto err;
39403944 }
@@ -6163,7 +6167,9 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, · 2222 unmodified lines
61636167
61646168 lockdep_assert_held(&event_mutex);
61656169
6166− if (glob && strlen(glob)) {
6170+ WARN_ON(!glob);
6171+
6172+ if (strlen(glob)) {
61676173 hist_err_clear();
61686174 last_cmd_set(file, param);
61696175 }
@@ -6196,7 +6202,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, · 26 unmodified lines
61966202 continue;
61976203 }
61986204 break;
6199− } while (p);
6205+ } while (1);
62006206
62016207 if (!p)
62026208 param = NULL;
kernel/ucount.c+4−1
@@ -190,6 +190,7 @@ struct ucounts *alloc_ucounts(struct user_namespace *ns, kuid_t uid) · 189 unmodified lines
190190 kfree(new);
191191 } else {
192192 hlist_add_head(&new->node, hashent);
193+ get_user_ns(new->ns);
193194 spin_unlock_irq(&ucounts_lock);
194195 return new;
195196 }
@@ -210,6 +211,7 @@ void put_ucounts(struct ucounts *ucounts) · 14 unmodified lines
210211 if (atomic_dec_and_lock_irqsave(&ucounts->count, &ucounts_lock, flags)) {
211212 hlist_del_init(&ucounts->node);
212213 spin_unlock_irqrestore(&ucounts_lock, flags);
214+ put_user_ns(ucounts->ns);
213215 kfree(ucounts);
214216 }
215217 }
@@ -348,7 +350,8 @@ bool is_ucounts_overlimit(struct ucounts *ucounts, enum ucount_type type, unsign · 132 unmodified lines
348350 if (rlimit > LONG_MAX)
349351 max = LONG_MAX;
350352 for (iter = ucounts; iter; iter = iter->ns->ucounts) {
351− if (get_ucounts_value(iter, type) > max)
353+ long val = get_ucounts_value(iter, type);
354+ if (val < 0 || val > max)
352355 return true;
353356 max = READ_ONCE(iter->ns->ucount_max[type]);
354357 }