History

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 and cross-arch updates from Catalin Marinas: "Here's a slightly wider-spread set of updates for 5.9. Going outside the usual arch/arm64/ area is the removal of read_barrier_depends() series from Will and the MSI/IOMMU ID translation series from Lorenzo. The notable arm64 updates include ARMv8.4 TLBI range operations and translation level hint, time namespace support, and perf. Summary: - Removal of the tremendously unpopular read_barrier_depends() barrier, which is a NOP on all architectures apart from Alpha, in favour of allowing architectures to override READ_ONCE() and do whatever dance they need to do to ensure address dependencies provide LOAD -> LOAD/STORE ordering. This work also offers a potential solution if compilers are shown to convert LOAD -> LOAD address dependencies into control dependencies (e.g. under LTO), as weakly ordered architectures will effectively be able to upgrade READ_ONCE() to smp_load_acquire(). The latter case is not used yet, but will be discussed further at LPC. - Make the MSI/IOMMU input/output ID translation PCI agnostic, augment the MSI/IOMMU ACPI/OF ID mapping APIs to accept an input ID bus-specific parameter and apply the resulting changes to the device ID space provided by the Freescale FSL bus. - arm64 support for TLBI range operations and translation table level hints (part of the ARMv8.4 architecture version). - Time namespace support for arm64. - Export the virtual and physical address sizes in vmcoreinfo for makedumpfile and crash utilities. - CPU feature handling cleanups and checks for programmer errors (overlapping bit-fields). - ACPI updates for arm64: disallow AML accesses to EFI code regions and kernel memory. - perf updates for arm64. - Miscellaneous fixes and cleanups, most notably PLT counting optimisation for module loading, recordmcount fix to ignore relocations other than R_AARCH64_CALL26, CMA areas reserved for gigantic pages on 16K and 64K configurations. - Trivial typos, duplicate words" Link: http://lkml.kernel.org/r/20200710165203.31284-1-will@kernel.org Link: http://lkml.kernel.org/r/20200619082013.13661-1-lorenzo.pieralisi@arm.com * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (82 commits) arm64: use IRQ_STACK_SIZE instead of THREAD_SIZE for irq stack arm64/mm: save memory access in check_and_switch_context() fast switch path arm64: sigcontext.h: delete duplicated word arm64: ptrace.h: delete duplicated word arm64: pgtable-hwdef.h: delete duplicated words bus: fsl-mc: Add ACPI support for fsl-mc bus/fsl-mc: Refactor the MSI domain creation in the DPRC driver of/irq: Make of_msi_map_rid() PCI bus agnostic of/irq: make of_msi_map_get_device_domain() bus agnostic dt-bindings: arm: fsl: Add msi-map device-tree binding for fsl-mc bus of/device: Add input id to of_dma_configure() of/iommu: Make of_map_rid() PCI agnostic ACPI/IORT: Add an input ID to acpi_dma_configure() ACPI/IORT: Remove useless PCI bus walk ACPI/IORT: Make iort_msi_map_rid() PCI agnostic ACPI/IORT: Make iort_get_device_domain IRQ domain agnostic ACPI/IORT: Make iort_match_node_callback walk the ACPI namespace for NC arm64: enable time namespace support arm64/vdso: Restrict splitting VVAR VMA arm64/vdso: Handle faults on timens page ...

Linus Torvalds committed 6 years ago

Commit 145ff1e2 parents 8c4e1c0 0e4cd9f98 files · +1892 −1091Browse files

Showing the changes to include (20 of 98 files). Show all

Unified · Split

20 files
  1. include/acpi/acpi_bus.h +7 −2
  2. include/asm-generic/Kbuild +1 −0
  3. include/asm-generic/barrier.h +1 −18
  4. include/asm-generic/rwonce.h +90 −0
  5. include/asm-generic/tlb.h +40 −15
  6. include/linux/acpi.h +7 −0
  7. include/linux/acpi_iort.h +11 −9
  8. include/linux/arm-smccc.h +22 −22
  9. include/linux/compiler.h +2 −132
  10. include/linux/compiler_types.h +41 −0
  11. include/linux/nospec.h +2 −0
  12. include/linux/of.h +2 −2
  13. include/linux/of_device.h +14 −2
  14. include/linux/of_iommu.h +4 −2
  15. include/linux/of_irq.h +7 −6
  16. include/linux/percpu-refcount.h +1 −1
  17. include/linux/ptr_ring.h +1 −1
  18. include/linux/sched_clock.h +28 −0
  19. include/uapi/linux/perf_event.h +20 −3
  20. include/vdso/datapage.h +1 −0
include/acpi/acpi_bus.h+7−2
@@ -588,8 +588,13 @@ bool acpi_dma_supported(struct acpi_device *adev); · 587 unmodified lines
588588 enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev);
589589 int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset,
590590 u64 *size);
591−int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr);
592−
591+int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr,
592+ const u32 *input_id);
593+static inline int acpi_dma_configure(struct device *dev,
594+ enum dev_dma_attr attr)
595+{
596+ return acpi_dma_configure_id(dev, attr, NULL);
597+}
593598 struct acpi_device *acpi_find_child_device(struct acpi_device *parent,
594599 u64 address, bool check_children);
595600 int acpi_is_root_bridge(acpi_handle);
include/asm-generic/Kbuild+1−0
@@ -45,6 +45,7 @@ mandatory-y += pci.h · 44 unmodified lines
4545 mandatory-y += percpu.h
4646 mandatory-y += pgalloc.h
4747 mandatory-y += preempt.h
48+mandatory-y += rwonce.h
4849 mandatory-y += sections.h
4950 mandatory-y += serial.h
5051 mandatory-y += shmparam.h
include/asm-generic/barrier.h+1−18
@@ -13,7 +13,7 @@ · 12 unmodified lines
1313
1414 #ifndef __ASSEMBLY__
1515
16−#include <linux/compiler.h>
16+#include <asm/rwonce.h>
1717
1818 #ifndef nop
1919 #define nop() asm volatile ("nop")
@@ -46,10 +46,6 @@ · 26 unmodified lines
4646 #define dma_wmb() wmb()
4747 #endif
4848
49−#ifndef read_barrier_depends
50−#define read_barrier_depends() do { } while (0)
51−#endif
52−
5349 #ifndef __smp_mb
5450 #define __smp_mb() mb()
5551 #endif
@@ -62,10 +58,6 @@ · 6 unmodified lines
6258 #define __smp_wmb() wmb()
6359 #endif
6460
65−#ifndef __smp_read_barrier_depends
66−#define __smp_read_barrier_depends() read_barrier_depends()
67−#endif
68−
6961 #ifdef CONFIG_SMP
7062
7163 #ifndef smp_mb
@@ -80,10 +72,6 @@ · 8 unmodified lines
8072 #define smp_wmb() __smp_wmb()
8173 #endif
8274
83−#ifndef smp_read_barrier_depends
84−#define smp_read_barrier_depends() __smp_read_barrier_depends()
85−#endif
86−
8775 #else /* !CONFIG_SMP */
8876
8977 #ifndef smp_mb
@@ -98,10 +86,6 @@ · 8 unmodified lines
9886 #define smp_wmb() barrier()
9987 #endif
10088
101−#ifndef smp_read_barrier_depends
102−#define smp_read_barrier_depends() do { } while (0)
103−#endif
104−
10589 #endif /* CONFIG_SMP */
10690
10791 #ifndef __smp_store_mb
@@ -196,7 +180,6 @@ do { \ · 88 unmodified lines
196180 #define virt_mb() __smp_mb()
197181 #define virt_rmb() __smp_rmb()
198182 #define virt_wmb() __smp_wmb()
199−#define virt_read_barrier_depends() __smp_read_barrier_depends()
200183 #define virt_store_mb(var, value) __smp_store_mb(var, value)
201184 #define virt_mb__before_atomic() __smp_mb__before_atomic()
202185 #define virt_mb__after_atomic() __smp_mb__after_atomic()
include/asm-generic/rwonce.h+90−0
@@ -0,0 +1,90 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Prevent the compiler from merging or refetching reads or writes. The
4+ * compiler is also forbidden from reordering successive instances of
5+ * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some
6+ * particular ordering. One way to make the compiler aware of ordering is to
7+ * put the two invocations of READ_ONCE or WRITE_ONCE in different C
8+ * statements.
9+ *
10+ * These two macros will also work on aggregate data types like structs or
11+ * unions.
12+ *
13+ * Their two major use cases are: (1) Mediating communication between
14+ * process-level code and irq/NMI handlers, all running on the same CPU,
15+ * and (2) Ensuring that the compiler does not fold, spindle, or otherwise
16+ * mutilate accesses that either do not require ordering or that interact
17+ * with an explicit memory barrier or atomic instruction that provides the
18+ * required ordering.
19+ */
20+#ifndef __ASM_GENERIC_RWONCE_H
21+#define __ASM_GENERIC_RWONCE_H
22+
23+#ifndef __ASSEMBLY__
24+
25+#include <linux/compiler_types.h>
26+#include <linux/kasan-checks.h>
27+#include <linux/kcsan-checks.h>
28+
29+/*
30+ * Yes, this permits 64-bit accesses on 32-bit architectures. These will
31+ * actually be atomic in some cases (namely Armv7 + LPAE), but for others we
32+ * rely on the access being split into 2x32-bit accesses for a 32-bit quantity
33+ * (e.g. a virtual address) and a strong prevailing wind.
34+ */
35+#define compiletime_assert_rwonce_type(t) \
36+ compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
37+ "Unsupported access size for {READ,WRITE}_ONCE().")
38+
39+/*
40+ * Use __READ_ONCE() instead of READ_ONCE() if you do not require any
41+ * atomicity. Note that this may result in tears!
42+ */
43+#ifndef __READ_ONCE
44+#define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
45+#endif
46+
47+#define READ_ONCE(x) \
48+({ \
49+ compiletime_assert_rwonce_type(x); \
50+ __READ_ONCE(x); \
51+})
52+
53+#define __WRITE_ONCE(x, val) \
54+do { \
55+ *(volatile typeof(x) *)&(x) = (val); \
56+} while (0)
57+
58+#define WRITE_ONCE(x, val) \
59+do { \
60+ compiletime_assert_rwonce_type(x); \
61+ __WRITE_ONCE(x, val); \
62+} while (0)
63+
64+static __no_sanitize_or_inline
65+unsigned long __read_once_word_nocheck(const void *addr)
66+{
67+ return __READ_ONCE(*(unsigned long *)addr);
68+}
69+
70+/*
71+ * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a
72+ * word from memory atomically but without telling KASAN/KCSAN. This is
73+ * usually used by unwinding code when walking the stack of a running process.
74+ */
75+#define READ_ONCE_NOCHECK(x) \
76+({ \
77+ compiletime_assert(sizeof(x) == sizeof(unsigned long), \
78+ "Unsupported access size for READ_ONCE_NOCHECK()."); \
79+ (typeof(x))__read_once_word_nocheck(&(x)); \
80+})
81+
82+static __no_kasan_or_inline
83+unsigned long read_word_at_a_time(const void *addr)
84+{
85+ kasan_check_read(addr, 1);
86+ return *(unsigned long *)addr;
87+}
88+
89+#endif /* __ASSEMBLY__ */
90+#endif /* __ASM_GENERIC_RWONCE_H */
include/asm-generic/tlb.h+40−15
@@ -512,6 +512,38 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 511 unmodified lines
512512 }
513513 #endif
514514
515+/*
516+ * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,
517+ * and set corresponding cleared_*.
518+ */
519+static inline void tlb_flush_pte_range(struct mmu_gather *tlb,
520+ unsigned long address, unsigned long size)
521+{
522+ __tlb_adjust_range(tlb, address, size);
523+ tlb->cleared_ptes = 1;
524+}
525+
526+static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,
527+ unsigned long address, unsigned long size)
528+{
529+ __tlb_adjust_range(tlb, address, size);
530+ tlb->cleared_pmds = 1;
531+}
532+
533+static inline void tlb_flush_pud_range(struct mmu_gather *tlb,
534+ unsigned long address, unsigned long size)
535+{
536+ __tlb_adjust_range(tlb, address, size);
537+ tlb->cleared_puds = 1;
538+}
539+
540+static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,
541+ unsigned long address, unsigned long size)
542+{
543+ __tlb_adjust_range(tlb, address, size);
544+ tlb->cleared_p4ds = 1;
545+}
546+
515547 #ifndef __tlb_remove_tlb_entry
516548 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
517549 #endif
@@ -525,19 +557,17 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 7 unmodified lines
525557 */
526558 #define tlb_remove_tlb_entry(tlb, ptep, address) \
527559 do { \
528− __tlb_adjust_range(tlb, address, PAGE_SIZE); \
529− tlb->cleared_ptes = 1; \
560+ tlb_flush_pte_range(tlb, address, PAGE_SIZE); \
530561 __tlb_remove_tlb_entry(tlb, ptep, address); \
531562 } while (0)
532563
533564 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
534565 do { \
535566 unsigned long _sz = huge_page_size(h); \
536− __tlb_adjust_range(tlb, address, _sz); \
537567 if (_sz == PMD_SIZE) \
538− tlb->cleared_pmds = 1; \
568+ tlb_flush_pmd_range(tlb, address, _sz); \
539569 else if (_sz == PUD_SIZE) \
540− tlb->cleared_puds = 1; \
570+ tlb_flush_pud_range(tlb, address, _sz); \
541571 __tlb_remove_tlb_entry(tlb, ptep, address); \
542572 } while (0)
543573
@@ -551,8 +581,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 7 unmodified lines
551581
552582 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
553583 do { \
554− __tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \
555− tlb->cleared_pmds = 1; \
584+ tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE); \
556585 __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
557586 } while (0)
558587
@@ -566,8 +595,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 7 unmodified lines
566595
567596 #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \
568597 do { \
569− __tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \
570− tlb->cleared_puds = 1; \
598+ tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE); \
571599 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \
572600 } while (0)
573601
@@ -592,9 +620,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 18 unmodified lines
592620 #ifndef pte_free_tlb
593621 #define pte_free_tlb(tlb, ptep, address) \
594622 do { \
595− __tlb_adjust_range(tlb, address, PAGE_SIZE); \
623+ tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \
596624 tlb->freed_tables = 1; \
597− tlb->cleared_pmds = 1; \
598625 __pte_free_tlb(tlb, ptep, address); \
599626 } while (0)
600627 #endif
@@ -602,9 +629,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 1 unmodified line
602629 #ifndef pmd_free_tlb
603630 #define pmd_free_tlb(tlb, pmdp, address) \
604631 do { \
605− __tlb_adjust_range(tlb, address, PAGE_SIZE); \
632+ tlb_flush_pud_range(tlb, address, PAGE_SIZE); \
606633 tlb->freed_tables = 1; \
607− tlb->cleared_puds = 1; \
608634 __pmd_free_tlb(tlb, pmdp, address); \
609635 } while (0)
610636 #endif
@@ -612,9 +638,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 1 unmodified line
612638 #ifndef pud_free_tlb
613639 #define pud_free_tlb(tlb, pudp, address) \
614640 do { \
615− __tlb_adjust_range(tlb, address, PAGE_SIZE); \
641+ tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \
616642 tlb->freed_tables = 1; \
617− tlb->cleared_p4ds = 1; \
618643 __pud_free_tlb(tlb, pudp, address); \
619644 } while (0)
620645 #endif
include/linux/acpi.h+7−0
@@ -905,6 +905,13 @@ static inline int acpi_dma_configure(struct device *dev, · 904 unmodified lines
905905 return 0;
906906 }
907907
908+static inline int acpi_dma_configure_id(struct device *dev,
909+ enum dev_dma_attr attr,
910+ const u32 *input_id)
911+{
912+ return 0;
913+}
914+
908915 #define ACPI_PTR(_ptr) (NULL)
909916
910917 static inline void acpi_device_set_enumerated(struct acpi_device *adev)
include/linux/acpi_iort.h+11−9
@@ -28,27 +28,29 @@ void iort_deregister_domain_token(int trans_id); · 27 unmodified lines
2828 struct fwnode_handle *iort_find_domain_token(int trans_id);
2929 #ifdef CONFIG_ACPI_IORT
3030 void acpi_iort_init(void);
31−u32 iort_msi_map_rid(struct device *dev, u32 req_id);
32−struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id);
31+u32 iort_msi_map_id(struct device *dev, u32 id);
32+struct irq_domain *iort_get_device_domain(struct device *dev, u32 id,
33+ enum irq_domain_bus_token bus_token);
3334 void acpi_configure_pmsi_domain(struct device *dev);
3435 int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id);
3536 /* IOMMU interface */
3637 void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *size);
37−const struct iommu_ops *iort_iommu_configure(struct device *dev);
38+const struct iommu_ops *iort_iommu_configure_id(struct device *dev,
39+ const u32 *id_in);
3840 int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head);
3941 #else
4042 static inline void acpi_iort_init(void) { }
41−static inline u32 iort_msi_map_rid(struct device *dev, u32 req_id)
42−{ return req_id; }
43−static inline struct irq_domain *iort_get_device_domain(struct device *dev,
44− u32 req_id)
43+static inline u32 iort_msi_map_id(struct device *dev, u32 id)
44+{ return id; }
45+static inline struct irq_domain *iort_get_device_domain(
46+ struct device *dev, u32 id, enum irq_domain_bus_token bus_token)
4547 { return NULL; }
4648 static inline void acpi_configure_pmsi_domain(struct device *dev) { }
4749 /* IOMMU interface */
4850 static inline void iort_dma_setup(struct device *dev, u64 *dma_addr,
4951 u64 *size) { }
50−static inline const struct iommu_ops *iort_iommu_configure(
51− struct device *dev)
52+static inline const struct iommu_ops *iort_iommu_configure_id(
53+ struct device *dev, const u32 *id_in)
5254 { return NULL; }
5355 static inline
5456 int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head)
include/linux/arm-smccc.h+22−22
@@ -81,6 +81,28 @@ · 80 unmodified lines
8181 ARM_SMCCC_SMC_32, \
8282 0, 0x7fff)
8383
84+/* Paravirtualised time calls (defined by ARM DEN0057A) */
85+#define ARM_SMCCC_HV_PV_TIME_FEATURES \
86+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
87+ ARM_SMCCC_SMC_64, \
88+ ARM_SMCCC_OWNER_STANDARD_HYP, \
89+ 0x20)
90+
91+#define ARM_SMCCC_HV_PV_TIME_ST \
92+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
93+ ARM_SMCCC_SMC_64, \
94+ ARM_SMCCC_OWNER_STANDARD_HYP, \
95+ 0x21)
96+
97+/*
98+ * Return codes defined in ARM DEN 0070A
99+ * ARM DEN 0070A is now merged/consolidated into ARM DEN 0028 C
100+ */
101+#define SMCCC_RET_SUCCESS 0
102+#define SMCCC_RET_NOT_SUPPORTED -1
103+#define SMCCC_RET_NOT_REQUIRED -2
104+#define SMCCC_RET_INVALID_PARAMETER -3
105+
84106 #ifndef __ASSEMBLY__
85107
86108 #include <linux/linkage.h>
@@ -331,15 +353,6 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 244 unmodified lines
331353 */
332354 #define arm_smccc_1_1_hvc(...) __arm_smccc_1_1(SMCCC_HVC_INST, __VA_ARGS__)
333355
334−/*
335− * Return codes defined in ARM DEN 0070A
336− * ARM DEN 0070A is now merged/consolidated into ARM DEN 0028 C
337− */
338−#define SMCCC_RET_SUCCESS 0
339−#define SMCCC_RET_NOT_SUPPORTED -1
340−#define SMCCC_RET_NOT_REQUIRED -2
341−#define SMCCC_RET_INVALID_PARAMETER -3
342−
343356 /*
344357 * Like arm_smccc_1_1* but always returns SMCCC_RET_NOT_SUPPORTED.
345358 * Used when the SMCCC conduit is not defined. The empty asm statement
@@ -385,18 +398,5 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 39 unmodified lines
385398 method; \
386399 })
387400
388−/* Paravirtualised time calls (defined by ARM DEN0057A) */
389−#define ARM_SMCCC_HV_PV_TIME_FEATURES \
390− ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
391− ARM_SMCCC_SMC_64, \
392− ARM_SMCCC_OWNER_STANDARD_HYP, \
393− 0x20)
394−
395−#define ARM_SMCCC_HV_PV_TIME_ST \
396− ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
397− ARM_SMCCC_SMC_64, \
398− ARM_SMCCC_OWNER_STANDARD_HYP, \
399− 0x21)
400−
401401 #endif /*__ASSEMBLY__*/
402402 #endif /*__LINUX_ARM_SMCCC_H*/
include/linux/compiler.h+2−132
@@ -230,28 +230,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 229 unmodified lines
230230 # define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
231231 #endif
232232
233−/*
234− * Prevent the compiler from merging or refetching reads or writes. The
235− * compiler is also forbidden from reordering successive instances of
236− * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some
237− * particular ordering. One way to make the compiler aware of ordering is to
238− * put the two invocations of READ_ONCE or WRITE_ONCE in different C
239− * statements.
240− *
241− * These two macros will also work on aggregate data types like structs or
242− * unions.
243− *
244− * Their two major use cases are: (1) Mediating communication between
245− * process-level code and irq/NMI handlers, all running on the same CPU,
246− * and (2) Ensuring that the compiler does not fold, spindle, or otherwise
247− * mutilate accesses that either do not require ordering or that interact
248− * with an explicit memory barrier or atomic instruction that provides the
249− * required ordering.
250− */
251−#include <asm/barrier.h>
252−#include <linux/kasan-checks.h>
253−#include <linux/kcsan-checks.h>
254−
255233 /**
256234 * data_race - mark an expression as containing intentional data races
257235 *
@@ -272,65 +250,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 14 unmodified lines
272250 __v; \
273251 })
274252
275−/*
276− * Use __READ_ONCE() instead of READ_ONCE() if you do not require any
277− * atomicity or dependency ordering guarantees. Note that this may result
278− * in tears!
279− */
280−#define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x))
281−
282−#define __READ_ONCE_SCALAR(x) \
283−({ \
284− __unqual_scalar_typeof(x) __x = __READ_ONCE(x); \
285− smp_read_barrier_depends(); \
286− (typeof(x))__x; \
287−})
288−
289−#define READ_ONCE(x) \
290−({ \
291− compiletime_assert_rwonce_type(x); \
292− __READ_ONCE_SCALAR(x); \
293−})
294−
295−#define __WRITE_ONCE(x, val) \
296−do { \
297− *(volatile typeof(x) *)&(x) = (val); \
298−} while (0)
299−
300−#define WRITE_ONCE(x, val) \
301−do { \
302− compiletime_assert_rwonce_type(x); \
303− __WRITE_ONCE(x, val); \
304−} while (0)
305−
306−static __no_sanitize_or_inline
307−unsigned long __read_once_word_nocheck(const void *addr)
308−{
309− return __READ_ONCE(*(unsigned long *)addr);
310−}
311−
312−/*
313− * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a
314− * word from memory atomically but without telling KASAN/KCSAN. This is
315− * usually used by unwinding code when walking the stack of a running process.
316− */
317−#define READ_ONCE_NOCHECK(x) \
318−({ \
319− unsigned long __x; \
320− compiletime_assert(sizeof(x) == sizeof(__x), \
321− "Unsupported access size for READ_ONCE_NOCHECK()."); \
322− __x = __read_once_word_nocheck(&(x)); \
323− smp_read_barrier_depends(); \
324− (typeof(x))__x; \
325−})
326−
327−static __no_kasan_or_inline
328−unsigned long read_word_at_a_time(const void *addr)
329−{
330− kasan_check_read(addr, 1);
331− return *(unsigned long *)addr;
332−}
333−
334253 #endif /* __KERNEL__ */
335254
336255 /*
@@ -354,57 +273,6 @@ static inline void *offset_to_ptr(const int *off) · 17 unmodified lines
354273
355274 #endif /* __ASSEMBLY__ */
356275
357−/* Compile time object size, -1 for unknown */
358−#ifndef __compiletime_object_size
359−# define __compiletime_object_size(obj) -1
360−#endif
361−#ifndef __compiletime_warning
362−# define __compiletime_warning(message)
363−#endif
364−#ifndef __compiletime_error
365−# define __compiletime_error(message)
366−#endif
367−
368−#ifdef __OPTIMIZE__
369−# define __compiletime_assert(condition, msg, prefix, suffix) \
370− do { \
371− extern void prefix ## suffix(void) __compiletime_error(msg); \
372− if (!(condition)) \
373− prefix ## suffix(); \
374− } while (0)
375−#else
376−# define __compiletime_assert(condition, msg, prefix, suffix) do { } while (0)
377−#endif
378−
379−#define _compiletime_assert(condition, msg, prefix, suffix) \
380− __compiletime_assert(condition, msg, prefix, suffix)
381−
382−/**
383− * compiletime_assert - break build and emit msg if condition is false
384− * @condition: a compile-time constant condition to check
385− * @msg: a message to emit if condition is false
386− *
387− * In tradition of POSIX assert, this macro will break the build if the
388− * supplied condition is *false*, emitting the supplied error message if the
389− * compiler has support to do so.
390− */
391−#define compiletime_assert(condition, msg) \
392− _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
393−
394−#define compiletime_assert_atomic_type(t) \
395− compiletime_assert(__native_word(t), \
396− "Need native word sized stores/loads for atomicity.")
397−
398−/*
399− * Yes, this permits 64-bit accesses on 32-bit architectures. These will
400− * actually be atomic in some cases (namely Armv7 + LPAE), but for others we
401− * rely on the access being split into 2x32-bit accesses for a 32-bit quantity
402− * (e.g. a virtual address) and a strong prevailing wind.
403− */
404−#define compiletime_assert_rwonce_type(t) \
405− compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
406− "Unsupported access size for {READ,WRITE}_ONCE().")
407−
408276 /* &a[0] degrades to a pointer: a different type from an array */
409277 #define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))
410278
@@ -414,4 +282,6 @@ static inline void *offset_to_ptr(const int *off) · 3 unmodified lines
414282 */
415283 #define prevent_tail_call_optimization() mb()
416284
285+#include <asm/rwonce.h>
286+
417287 #endif /* __LINUX_COMPILER_H */
include/linux/compiler_types.h+41−0
@@ -275,6 +275,47 @@ struct ftrace_likely_data { · 274 unmodified lines
275275 (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || \
276276 sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
277277
278+/* Compile time object size, -1 for unknown */
279+#ifndef __compiletime_object_size
280+# define __compiletime_object_size(obj) -1
281+#endif
282+#ifndef __compiletime_warning
283+# define __compiletime_warning(message)
284+#endif
285+#ifndef __compiletime_error
286+# define __compiletime_error(message)
287+#endif
288+
289+#ifdef __OPTIMIZE__
290+# define __compiletime_assert(condition, msg, prefix, suffix) \
291+ do { \
292+ extern void prefix ## suffix(void) __compiletime_error(msg); \
293+ if (!(condition)) \
294+ prefix ## suffix(); \
295+ } while (0)
296+#else
297+# define __compiletime_assert(condition, msg, prefix, suffix) do { } while (0)
298+#endif
299+
300+#define _compiletime_assert(condition, msg, prefix, suffix) \
301+ __compiletime_assert(condition, msg, prefix, suffix)
302+
303+/**
304+ * compiletime_assert - break build and emit msg if condition is false
305+ * @condition: a compile-time constant condition to check
306+ * @msg: a message to emit if condition is false
307+ *
308+ * In tradition of POSIX assert, this macro will break the build if the
309+ * supplied condition is *false*, emitting the supplied error message if the
310+ * compiler has support to do so.
311+ */
312+#define compiletime_assert(condition, msg) \
313+ _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
314+
315+#define compiletime_assert_atomic_type(t) \
316+ compiletime_assert(__native_word(t), \
317+ "Need native word sized stores/loads for atomicity.")
318+
278319 /* Helpers for emitting diagnostics in pragmas. */
279320 #ifndef __diag
280321 #define __diag(string)
include/linux/nospec.h+2−0
@@ -5,6 +5,8 @@ · 4 unmodified lines
55
66 #ifndef _LINUX_NOSPEC_H
77 #define _LINUX_NOSPEC_H
8+
9+#include <linux/compiler.h>
810 #include <asm/barrier.h>
911
1012 struct task_struct;
include/linux/of.h+2−2
@@ -554,7 +554,7 @@ bool of_console_check(struct device_node *dn, char *name, int index); · 553 unmodified lines
554554
555555 extern int of_cpu_node_to_id(struct device_node *np);
556556
557−int of_map_rid(struct device_node *np, u32 rid,
557+int of_map_id(struct device_node *np, u32 id,
558558 const char *map_name, const char *map_mask_name,
559559 struct device_node **target, u32 *id_out);
560560
@@ -978,7 +978,7 @@ static inline int of_cpu_node_to_id(struct device_node *np) · 417 unmodified lines
978978 return -ENODEV;
979979 }
980980
981−static inline int of_map_rid(struct device_node *np, u32 rid,
981+static inline int of_map_id(struct device_node *np, u32 id,
982982 const char *map_name, const char *map_mask_name,
983983 struct device_node **target, u32 *id_out)
984984 {
include/linux/of_device.h+14−2
@@ -55,9 +55,15 @@ static inline struct device_node *of_cpu_device_node_get(int cpu) · 54 unmodified lines
5555 return of_node_get(cpu_dev->of_node);
5656 }
5757
58−int of_dma_configure(struct device *dev,
58+int of_dma_configure_id(struct device *dev,
5959 struct device_node *np,
60− bool force_dma);
60+ bool force_dma, const u32 *id);
61+static inline int of_dma_configure(struct device *dev,
62+ struct device_node *np,
63+ bool force_dma)
64+{
65+ return of_dma_configure_id(dev, np, force_dma, NULL);
66+}
6167 #else /* CONFIG_OF */
6268
6369 static inline int of_driver_match_device(struct device *dev,
@@ -106,6 +112,12 @@ static inline struct device_node *of_cpu_device_node_get(int cpu) · 42 unmodified lines
106112 return NULL;
107113 }
108114
115+static inline int of_dma_configure_id(struct device *dev,
116+ struct device_node *np,
117+ bool force_dma)
118+{
119+ return 0;
120+}
109121 static inline int of_dma_configure(struct device *dev,
110122 struct device_node *np,
111123 bool force_dma)
include/linux/of_iommu.h+4−2
@@ -13,7 +13,8 @@ extern int of_get_dma_window(struct device_node *dn, const char *prefix, · 12 unmodified lines
1313 size_t *size);
1414
1515 extern const struct iommu_ops *of_iommu_configure(struct device *dev,
16− struct device_node *master_np);
16+ struct device_node *master_np,
17+ const u32 *id);
1718
1819 #else
1920
@@ -25,7 +26,8 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix, · 5 unmodified lines
2526 }
2627
2728 static inline const struct iommu_ops *of_iommu_configure(struct device *dev,
28− struct device_node *master_np)
29+ struct device_node *master_np,
30+ const u32 *id)
2931 {
3032 return NULL;
3133 }
include/linux/of_irq.h+7−6
@@ -52,9 +52,10 @@ extern struct irq_domain *of_msi_get_domain(struct device *dev, · 51 unmodified lines
5252 struct device_node *np,
5353 enum irq_domain_bus_token token);
5454 extern struct irq_domain *of_msi_map_get_device_domain(struct device *dev,
55− u32 rid);
55+ u32 id,
56+ u32 bus_token);
5657 extern void of_msi_configure(struct device *dev, struct device_node *np);
57−u32 of_msi_map_rid(struct device *dev, struct device_node *msi_np, u32 rid_in);
58+u32 of_msi_map_id(struct device *dev, struct device_node *msi_np, u32 id_in);
5859 #else
5960 static inline int of_irq_count(struct device_node *dev)
6061 {
@@ -85,17 +86,17 @@ static inline struct irq_domain *of_msi_get_domain(struct device *dev, · 24 unmodified lines
8586 return NULL;
8687 }
8788 static inline struct irq_domain *of_msi_map_get_device_domain(struct device *dev,
88− u32 rid)
89+ u32 id, u32 bus_token)
8990 {
9091 return NULL;
9192 }
9293 static inline void of_msi_configure(struct device *dev, struct device_node *np)
9394 {
9495 }
95−static inline u32 of_msi_map_rid(struct device *dev,
96− struct device_node *msi_np, u32 rid_in)
96+static inline u32 of_msi_map_id(struct device *dev,
97+ struct device_node *msi_np, u32 id_in)
9798 {
98− return rid_in;
99+ return id_in;
99100 }
100101 #endif
101102
include/linux/percpu-refcount.h+1−1
@@ -155,7 +155,7 @@ static inline bool __ref_is_percpu(struct percpu_ref *ref, · 154 unmodified lines
155155 * between contaminating the pointer value, meaning that
156156 * READ_ONCE() is required when fetching it.
157157 *
158− * The smp_read_barrier_depends() implied by READ_ONCE() pairs
158+ * The dependency ordering from the READ_ONCE() pairs
159159 * with smp_store_release() in __percpu_ref_switch_to_percpu().
160160 */
161161 percpu_ptr = READ_ONCE(ref->percpu_count_ptr);
include/linux/ptr_ring.h+1−1
@@ -107,7 +107,7 @@ static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) · 106 unmodified lines
107107 return -ENOSPC;
108108
109109 /* Make sure the pointer we are storing points to a valid data. */
110− /* Pairs with smp_read_barrier_depends in __ptr_ring_consume. */
110+ /* Pairs with the dependency ordering in __ptr_ring_consume. */
111111 smp_wmb();
112112
113113 WRITE_ONCE(r->queue[r->producer++], ptr);
include/linux/sched_clock.h+28−0
@@ -6,6 +6,34 @@ · 5 unmodified lines
66 #define LINUX_SCHED_CLOCK
77
88 #ifdef CONFIG_GENERIC_SCHED_CLOCK
9+/**
10+ * struct clock_read_data - data required to read from sched_clock()
11+ *
12+ * @epoch_ns: sched_clock() value at last update
13+ * @epoch_cyc: Clock cycle value at last update.
14+ * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit
15+ * clocks.
16+ * @read_sched_clock: Current clock source (or dummy source when suspended).
17+ * @mult: Multipler for scaled math conversion.
18+ * @shift: Shift value for scaled math conversion.
19+ *
20+ * Care must be taken when updating this structure; it is read by
21+ * some very hot code paths. It occupies <=40 bytes and, when combined
22+ * with the seqcount used to synchronize access, comfortably fits into
23+ * a 64 byte cache line.
24+ */
25+struct clock_read_data {
26+ u64 epoch_ns;
27+ u64 epoch_cyc;
28+ u64 sched_clock_mask;
29+ u64 (*read_sched_clock)(void);
30+ u32 mult;
31+ u32 shift;
32+};
33+
34+extern struct clock_read_data *sched_clock_read_begin(unsigned int *seq);
35+extern int sched_clock_read_retry(unsigned int seq);
36+
937 extern void generic_sched_clock_init(void);
1038
1139 extern void sched_clock_register(u64 (*read)(void), int bits,
include/uapi/linux/perf_event.h+20−3
@@ -532,9 +532,10 @@ struct perf_event_mmap_page { · 531 unmodified lines
532532 cap_bit0_is_deprecated : 1, /* Always 1, signals that bit 0 is zero */
533533
534534 cap_user_rdpmc : 1, /* The RDPMC instruction can be used to read counts */
535− cap_user_time : 1, /* The time_* fields are used */
535+ cap_user_time : 1, /* The time_{shift,mult,offset} fields are used */
536536 cap_user_time_zero : 1, /* The time_zero field is used */
537− cap_____res : 59;
537+ cap_user_time_short : 1, /* the time_{cycle,mask} fields are used */
538+ cap_____res : 58;
538539 };
539540 };
540541
@@ -593,13 +594,29 @@ struct perf_event_mmap_page { · 52 unmodified lines
593594 * ((rem * time_mult) >> time_shift);
594595 */
595596 __u64 time_zero;
597+
596598 __u32 size; /* Header size up to __reserved[] fields. */
599+ __u32 __reserved_1;
600+
601+ /*
602+ * If cap_usr_time_short, the hardware clock is less than 64bit wide
603+ * and we must compute the 'cyc' value, as used by cap_usr_time, as:
604+ *
605+ * cyc = time_cycles + ((cyc - time_cycles) & time_mask)
606+ *
607+ * NOTE: this form is explicitly chosen such that cap_usr_time_short
608+ * is a correction on top of cap_usr_time, and code that doesn't
609+ * know about cap_usr_time_short still works under the assumption
610+ * the counter doesn't wrap.
611+ */
612+ __u64 time_cycles;
613+ __u64 time_mask;
597614
598615 /*
599616 * Hole for extension of the self monitor capabilities
600617 */
601618
602− __u8 __reserved[118*8+4]; /* align to 1k. */
619+ __u8 __reserved[116*8]; /* align to 1k. */
603620
604621 /*
605622 * Control data for the mmap() data buffer.
include/vdso/datapage.h+1−0
@@ -109,6 +109,7 @@ struct vdso_data { · 108 unmodified lines
109109 * relocation, and this is what we need.
110110 */
111111 extern struct vdso_data _vdso_data[CS_BASES] __attribute__((visibility("hidden")));
112+extern struct vdso_data _timens_data[CS_BASES] __attribute__((visibility("hidden")));
112113
113114 /*
114115 * The generic vDSO implementation requires that gettimeofday.h

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t