Merge branch 'for-7.0' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound into asoc-7.1 for both ASoC and general bug fixes to support testing.
Unified · Split
| @@ -316,6 +316,7 @@ Hans Verkuil <hverkuil@kernel.org> <hverkuil-cisco@xs4all.nl> · 315 unmodified lines | ||
| 316 | 316 | Hans Verkuil <hverkuil@kernel.org> <hansverk@cisco.com> |
| 317 | 317 | Hao Ge <hao.ge@linux.dev> <gehao@kylinos.cn> |
| 318 | 318 | Harry Yoo <harry.yoo@oracle.com> <42.hyeyoo@gmail.com> |
| 319 | +Harry Yoo <harry@kernel.org> <harry.yoo@oracle.com> | |
| 319 | 320 | Heiko Carstens <hca@linux.ibm.com> <h.carstens@de.ibm.com> |
| 320 | 321 | Heiko Carstens <hca@linux.ibm.com> <heiko.carstens@de.ibm.com> |
| 321 | 322 | Heiko Stuebner <heiko@sntech.de> <heiko.stuebner@bqreaders.com> |
| @@ -587,6 +588,7 @@ Morten Welinder <terra@gnome.org> · 265 unmodified lines | ||
| 587 | 588 | Morten Welinder <welinder@anemone.rentec.com> |
| 588 | 589 | Morten Welinder <welinder@darter.rentec.com> |
| 589 | 590 | Morten Welinder <welinder@troll.com> |
| 591 | +Muhammad Usama Anjum <usama.anjum@arm.com> <usama.anjum@collabora.com> | |
| 590 | 592 | Mukesh Ojha <quic_mojha@quicinc.com> <mojha@codeaurora.org> |
| 591 | 593 | Muna Sinada <quic_msinada@quicinc.com> <msinada@codeaurora.org> |
| 592 | 594 | Murali Nalajala <quic_mnalajal@quicinc.com> <mnalajal@codeaurora.org> |
| @@ -85,6 +85,16 @@ In the example, 'Requester ID' means the ID of the device that sent · 84 unmodified lines | ||
| 85 | 85 | the error message to the Root Port. Please refer to PCIe specs for other |
| 86 | 86 | fields. |
| 87 | 87 | |
| 88 | +The 'TLP Header' is the prefix/header of the TLP that caused the error | |
| 89 | +in raw hex format. To decode the TLP Header into human-readable form | |
| 90 | +one may use tlp-tool: | |
| 91 | + | |
| 92 | +https://github.com/mmpg-x86/tlp-tool | |
| 93 | + | |
| 94 | +Example usage:: | |
| 95 | + | |
| 96 | + curl -L https://git.kernel.org/linus/2ca1c94ce0b6 | rtlp-tool --aer | |
| 97 | + | |
| 88 | 98 | AER Ratelimits |
| 89 | 99 | -------------- |
| 90 | 100 | |
| @@ -149,11 +149,33 @@ For architectures that require cache flushing for DMA coherence · 148 unmodified lines | ||
| 149 | 149 | DMA_ATTR_MMIO will not perform any cache flushing. The address |
| 150 | 150 | provided must never be mapped cacheable into the CPU. |
| 151 | 151 | |
| 152 | −DMA_ATTR_CPU_CACHE_CLEAN | |
| 153 | −------------------------ | |
| 154 | − | |
| 155 | −This attribute indicates the CPU will not dirty any cacheline overlapping this | |
| 156 | −DMA_FROM_DEVICE/DMA_BIDIRECTIONAL buffer while it is mapped. This allows | |
| 157 | −multiple small buffers to safely share a cacheline without risk of data | |
| 158 | −corruption, suppressing DMA debug warnings about overlapping mappings. | |
| 159 | −All mappings sharing a cacheline should have this attribute. | |
| 152 | +DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | |
| 153 | +------------------------------------ | |
| 154 | + | |
| 155 | +This attribute indicates that CPU cache lines may overlap for buffers mapped | |
| 156 | +with DMA_FROM_DEVICE or DMA_BIDIRECTIONAL. | |
| 157 | + | |
| 158 | +Such overlap may occur when callers map multiple small buffers that reside | |
| 159 | +within the same cache line. In this case, callers must guarantee that the CPU | |
| 160 | +will not dirty these cache lines after the mappings are established. When this | |
| 161 | +condition is met, multiple buffers can safely share a cache line without risking | |
| 162 | +data corruption. | |
| 163 | + | |
| 164 | +All mappings that share a cache line must set this attribute to suppress DMA | |
| 165 | +debug warnings about overlapping mappings. | |
| 166 | + | |
| 167 | +DMA_ATTR_REQUIRE_COHERENT | |
| 168 | +------------------------- | |
| 169 | + | |
| 170 | +DMA mapping requests with the DMA_ATTR_REQUIRE_COHERENT fail on any | |
| 171 | +system where SWIOTLB or cache management is required. This should only | |
| 172 | +be used to support uAPI designs that require continuous HW DMA | |
| 173 | +coherence with userspace processes, for example RDMA and DRM. At a | |
| 174 | +minimum the memory being mapped must be userspace memory from | |
| 175 | +pin_user_pages() or similar. | |
| 176 | + | |
| 177 | +Drivers should consider using dma_mmap_pages() instead of this | |
| 178 | +interface when building their uAPIs, when possible. | |
| 179 | + | |
| 180 | +It must never be used in an in-kernel driver that only works with | |
| 181 | +kernel memory. | |
| @@ -783,6 +783,56 @@ controlled by the "uuid" mount option, which supports these values: · 782 unmodified lines | ||
| 783 | 783 | mounted with "uuid=on". |
| 784 | 784 | |
| 785 | 785 | |
| 786 | +Durability and copy up | |
| 787 | +---------------------- | |
| 788 | + | |
| 789 | +The fsync(2) system call ensures that the data and metadata of a file | |
| 790 | +are safely written to the backing storage, which is expected to | |
| 791 | +guarantee the existence of the information post system crash. | |
| 792 | + | |
| 793 | +Without an fsync(2) call, there is no guarantee that the observed | |
| 794 | +data after a system crash will be either the old or the new data, but | |
| 795 | +in practice, the observed data after crash is often the old or new data | |
| 796 | +or a mix of both. | |
| 797 | + | |
| 798 | +When an overlayfs file is modified for the first time, copy up will | |
| 799 | +create a copy of the lower file and its parent directories in the upper | |
| 800 | +layer. Since the Linux filesystem API does not enforce any particular | |
| 801 | +ordering on storing changes without explicit fsync(2) calls, in case | |
| 802 | +of a system crash, the upper file could end up with no data at all | |
| 803 | +(i.e. zeros), which would be an unusual outcome. To avoid this | |
| 804 | +experience, overlayfs calls fsync(2) on the upper file before completing | |
| 805 | +data copy up with rename(2) or link(2) to make the copy up "atomic". | |
| 806 | + | |
| 807 | +By default, overlayfs does not explicitly call fsync(2) on copied up | |
| 808 | +directories or on metadata-only copy up, so it provides no guarantee to | |
| 809 | +persist the user's modification unless the user calls fsync(2). | |
| 810 | +The fsync during copy up only guarantees that if a copy up is observed | |
| 811 | +after a crash, the observed data is not zeroes or intermediate values | |
| 812 | +from the copy up staging area. | |
| 813 | + | |
| 814 | +On traditional local filesystems with a single journal (e.g. ext4, xfs), | |
| 815 | +fsync on a file also persists the parent directory changes, because they | |
| 816 | +are usually modified in the same transaction, so metadata durability during | |
| 817 | +data copy up effectively comes for free. Overlayfs further limits risk by | |
| 818 | +disallowing network filesystems as upper layer. | |
| 819 | + | |
| 820 | +Overlayfs can be tuned to prefer performance or durability when storing | |
| 821 | +to the underlying upper layer. This is controlled by the "fsync" mount | |
| 822 | +option, which supports these values: | |
| 823 | + | |
| 824 | +- "auto": (default) | |
| 825 | + Call fsync(2) on upper file before completion of data copy up. | |
| 826 | + No explicit fsync(2) on directory or metadata-only copy up. | |
| 827 | +- "strict": | |
| 828 | + Call fsync(2) on upper file and directories before completion of any | |
| 829 | + copy up. | |
| 830 | +- "volatile": [*] | |
| 831 | + Prefer performance over durability (see `Volatile mount`_) | |
| 832 | + | |
| 833 | +[*] The mount option "volatile" is an alias to "fsync=volatile". | |
| 834 | + | |
| 835 | + | |
| 786 | 836 | Volatile mount |
| 787 | 837 | -------------- |
| 788 | 838 | |
| @@ -27,10 +27,10 @@ for details. · 26 unmodified lines | ||
| 27 | 27 | Sysfs entries |
| 28 | 28 | ------------- |
| 29 | 29 | |
| 30 | −The following attributes are supported. Current maxim attribute | |
| 30 | +The following attributes are supported. Current maximum attribute | |
| 31 | 31 | is read-write, all other attributes are read-only. |
| 32 | 32 | |
| 33 | −in0_input Measured voltage in microvolts. | |
| 33 | +in0_input Measured voltage in millivolts. | |
| 34 | 34 | |
| 35 | −curr1_input Measured current in microamperes. | |
| 36 | −curr1_max_alarm Overcurrent alarm in microamperes. | |
| 35 | +curr1_input Measured current in milliamperes. | |
| 36 | +curr1_max Overcurrent shutdown threshold in milliamperes. | |
| @@ -51,8 +51,9 @@ temp1_max Provides thermal control temperature of the CPU package · 50 unmodified lines | ||
| 51 | 51 | temp1_crit Provides shutdown temperature of the CPU package which |
| 52 | 52 | is also known as the maximum processor junction |
| 53 | 53 | temperature, Tjmax or Tprochot. |
| 54 | −temp1_crit_hyst Provides the hysteresis value from Tcontrol to Tjmax of | |
| 55 | − the CPU package. | |
| 54 | +temp1_crit_hyst Provides the hysteresis temperature of the CPU | |
| 55 | + package. Returns Tcontrol, the temperature at which | |
| 56 | + the critical condition clears. | |
| 56 | 57 | |
| 57 | 58 | temp2_label "DTS" |
| 58 | 59 | temp2_input Provides current temperature of the CPU package scaled |
| @@ -62,8 +63,9 @@ temp2_max Provides thermal control temperature of the CPU package · 3 unmodified lines | ||
| 62 | 63 | temp2_crit Provides shutdown temperature of the CPU package which |
| 63 | 64 | is also known as the maximum processor junction |
| 64 | 65 | temperature, Tjmax or Tprochot. |
| 65 | −temp2_crit_hyst Provides the hysteresis value from Tcontrol to Tjmax of | |
| 66 | − the CPU package. | |
| 66 | +temp2_crit_hyst Provides the hysteresis temperature of the CPU | |
| 67 | + package. Returns Tcontrol, the temperature at which | |
| 68 | + the critical condition clears. | |
| 67 | 69 | |
| 68 | 70 | temp3_label "Tcontrol" |
| 69 | 71 | temp3_input Provides current Tcontrol temperature of the CPU |
| @@ -8,7 +8,7 @@ Landlock: unprivileged access control · 7 unmodified lines | ||
| 8 | 8 | ===================================== |
| 9 | 9 | |
| 10 | 10 | :Author: Mickaël Salaün |
| 11 | −:Date: January 2026 | |
| 11 | +:Date: March 2026 | |
| 12 | 12 | |
| 13 | 13 | The goal of Landlock is to enable restriction of ambient rights (e.g. global |
| 14 | 14 | filesystem or network access) for a set of processes. Because Landlock |
| @@ -197,12 +197,27 @@ similar backwards compatibility check is needed for the restrict flags · 182 unmodified lines | ||
| 197 | 197 | |
| 198 | 198 | .. code-block:: c |
| 199 | 199 | |
| 200 | − __u32 restrict_flags = LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON; | |
| 201 | − if (abi < 7) { | |
| 202 | − /* Clear logging flags unsupported before ABI 7. */ | |
| 200 | + __u32 restrict_flags = | |
| 201 | + LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON | | |
| 202 | + LANDLOCK_RESTRICT_SELF_TSYNC; | |
| 203 | + switch (abi) { | |
| 204 | + case 1 ... 6: | |
| 205 | + /* Removes logging flags for ABI < 7 */ | |
| 203 | 206 | restrict_flags &= ~(LANDLOCK_RESTRICT_SELF_LOG_SAME_EXEC_OFF | |
| 204 | 207 | LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON | |
| 205 | 208 | LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF); |
| 209 | + __attribute__((fallthrough)); | |
| 210 | + case 7: | |
| 211 | + /* | |
| 212 | + * Removes multithreaded enforcement flag for ABI < 8 | |
| 213 | + * | |
| 214 | + * WARNING: Without this flag, calling landlock_restrict_self(2) is | |
| 215 | + * only equivalent if the calling process is single-threaded. Below | |
| 216 | + * ABI v8 (and as of ABI v8, when not using this flag), a Landlock | |
| 217 | + * policy would only be enforced for the calling thread and its | |
| 218 | + * children (and not for all threads, including parents and siblings). | |
| 219 | + */ | |
| 220 | + restrict_flags &= ~LANDLOCK_RESTRICT_SELF_TSYNC; | |
| 206 | 221 | } |
| 207 | 222 | |
| 208 | 223 | The next step is to restrict the current thread from gaining more privileges |
| @@ -3986,7 +3986,7 @@ F: drivers/hwmon/asus-ec-sensors.c · 3985 unmodified lines | ||
| 3986 | 3986 | ASUS NOTEBOOKS AND EEEPC ACPI/WMI EXTRAS DRIVERS |
| 3987 | 3987 | M: Corentin Chary <corentin.chary@gmail.com> |
| 3988 | 3988 | M: Luke D. Jones <luke@ljones.dev> |
| 3989 | −M: Denis Benato <benato.denis96@gmail.com> | |
| 3989 | +M: Denis Benato <denis.benato@linux.dev> | |
| 3990 | 3990 | L: platform-driver-x86@vger.kernel.org |
| 3991 | 3991 | S: Maintained |
| 3992 | 3992 | W: https://asus-linux.org/ |
| @@ -8628,8 +8628,14 @@ F: drivers/gpu/drm/lima/ · 4635 unmodified lines | ||
| 8628 | 8628 | F: include/uapi/drm/lima_drm.h |
| 8629 | 8629 | |
| 8630 | 8630 | DRM DRIVERS FOR LOONGSON |
| 8631 | +M: Jianmin Lv <lvjianmin@loongson.cn> | |
| 8632 | +M: Qianhai Wu <wuqianhai@loongson.cn> | |
| 8633 | +R: Huacai Chen <chenhuacai@kernel.org> | |
| 8634 | +R: Mingcong Bai <jeffbai@aosc.io> | |
| 8635 | +R: Xi Ruoyao <xry111@xry111.site> | |
| 8636 | +R: Icenowy Zheng <zhengxingda@iscas.ac.cn> | |
| 8631 | 8637 | L: dri-devel@lists.freedesktop.org |
| 8632 | −S: Orphan | |
| 8638 | +S: Maintained | |
| 8633 | 8639 | T: git https://gitlab.freedesktop.org/drm/misc/kernel.git |
| 8634 | 8640 | F: drivers/gpu/drm/loongson/ |
| 8635 | 8641 | |
| @@ -9613,7 +9619,12 @@ F: include/linux/ext2* · 977 unmodified lines | ||
| 9613 | 9619 | |
| 9614 | 9620 | EXT4 FILE SYSTEM |
| 9615 | 9621 | M: "Theodore Ts'o" <tytso@mit.edu> |
| 9616 | −M: Andreas Dilger <adilger.kernel@dilger.ca> | |
| 9622 | +R: Andreas Dilger <adilger.kernel@dilger.ca> | |
| 9623 | +R: Baokun Li <libaokun@linux.alibaba.com> | |
| 9624 | +R: Jan Kara <jack@suse.cz> | |
| 9625 | +R: Ojaswin Mujoo <ojaswin@linux.ibm.com> | |
| 9626 | +R: Ritesh Harjani (IBM) <ritesh.list@gmail.com> | |
| 9627 | +R: Zhang Yi <yi.zhang@huawei.com> | |
| 9617 | 9628 | L: linux-ext4@vger.kernel.org |
| 9618 | 9629 | S: Maintained |
| 9619 | 9630 | W: http://ext4.wiki.kernel.org |
| @@ -12009,7 +12020,6 @@ I2C SUBSYSTEM · 2389 unmodified lines | ||
| 12009 | 12020 | M: Wolfram Sang <wsa+renesas@sang-engineering.com> |
| 12010 | 12021 | L: linux-i2c@vger.kernel.org |
| 12011 | 12022 | S: Maintained |
| 12012 | −W: https://i2c.wiki.kernel.org/ | |
| 12013 | 12023 | Q: https://patchwork.ozlabs.org/project/linux-i2c/list/ |
| 12014 | 12024 | T: git git://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git |
| 12015 | 12025 | F: Documentation/i2c/ |
| @@ -12035,7 +12045,6 @@ I2C SUBSYSTEM HOST DRIVERS · 19 unmodified lines | ||
| 12035 | 12045 | M: Andi Shyti <andi.shyti@kernel.org> |
| 12036 | 12046 | L: linux-i2c@vger.kernel.org |
| 12037 | 12047 | S: Maintained |
| 12038 | −W: https://i2c.wiki.kernel.org/ | |
| 12039 | 12048 | Q: https://patchwork.ozlabs.org/project/linux-i2c/list/ |
| 12040 | 12049 | T: git git://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git |
| 12041 | 12050 | F: Documentation/devicetree/bindings/i2c/ |
| @@ -16877,7 +16886,7 @@ M: Lorenzo Stoakes <ljs@kernel.org> · 4835 unmodified lines | ||
| 16877 | 16886 | R: Rik van Riel <riel@surriel.com> |
| 16878 | 16887 | R: Liam R. Howlett <Liam.Howlett@oracle.com> |
| 16879 | 16888 | R: Vlastimil Babka <vbabka@kernel.org> |
| 16880 | −R: Harry Yoo <harry.yoo@oracle.com> | |
| 16889 | +R: Harry Yoo <harry@kernel.org> | |
| 16881 | 16890 | R: Jann Horn <jannh@google.com> |
| 16882 | 16891 | L: linux-mm@kvack.org |
| 16883 | 16892 | S: Maintained |
| @@ -24343,7 +24352,7 @@ F: drivers/nvmem/layouts/sl28vpd.c · 7459 unmodified lines | ||
| 24343 | 24352 | |
| 24344 | 24353 | SLAB ALLOCATOR |
| 24345 | 24354 | M: Vlastimil Babka <vbabka@kernel.org> |
| 24346 | −M: Harry Yoo <harry.yoo@oracle.com> | |
| 24355 | +M: Harry Yoo <harry@kernel.org> | |
| 24347 | 24356 | M: Andrew Morton <akpm@linux-foundation.org> |
| 24348 | 24357 | R: Hao Li <hao.li@linux.dev> |
| 24349 | 24358 | R: Christoph Lameter <cl@gentwo.org> |
| @@ -2,7 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | VERSION = 7 |
| 3 | 3 | PATCHLEVEL = 0 |
| 4 | 4 | SUBLEVEL = 0 |
| 5 | −EXTRAVERSION = -rc5 | |
| 5 | +EXTRAVERSION = -rc6 | |
| 6 | 6 | NAME = Baby Opossum Posse |
| 7 | 7 | |
| 8 | 8 | # *DOCUMENTATION* |
| @@ -1654,7 +1654,7 @@ CLEAN_FILES += vmlinux.symvers modules-only.symvers \ · 1645 unmodified lines | ||
| 1654 | 1654 | modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ |
| 1655 | 1655 | compile_commands.json rust/test \ |
| 1656 | 1656 | rust-project.json .vmlinux.objs .vmlinux.export.c \ |
| 1657 | − .builtin-dtbs-list .builtin-dtb.S | |
| 1657 | + .builtin-dtbs-list .builtin-dtbs.S | |
| 1658 | 1658 | |
| 1659 | 1659 | # Directories & files removed with 'make mrproper' |
| 1660 | 1660 | MRPROPER_FILES += include/config include/generated \ |
| @@ -1753,7 +1753,7 @@ int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new) · 1752 unmodified lines | ||
| 1753 | 1753 | if (!writable) |
| 1754 | 1754 | return -EPERM; |
| 1755 | 1755 | |
| 1756 | − ptep = (u64 __user *)hva + offset; | |
| 1756 | + ptep = (void __user *)hva + offset; | |
| 1757 | 1757 | if (cpus_have_final_cap(ARM64_HAS_LSE_ATOMICS)) |
| 1758 | 1758 | r = __lse_swap_desc(ptep, old, new); |
| 1759 | 1759 | else |
| @@ -247,6 +247,20 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) · 246 unmodified lines | ||
| 247 | 247 | kvm_vcpu_set_be(vcpu); |
| 248 | 248 | |
| 249 | 249 | *vcpu_pc(vcpu) = target_pc; |
| 250 | + | |
| 251 | + /* | |
| 252 | + * We may come from a state where either a PC update was | |
| 253 | + * pending (SMC call resulting in PC being increpented to | |
| 254 | + * skip the SMC) or a pending exception. Make sure we get | |
| 255 | + * rid of all that, as this cannot be valid out of reset. | |
| 256 | + * | |
| 257 | + * Note that clearing the exception mask also clears PC | |
| 258 | + * updates, but that's an implementation detail, and we | |
| 259 | + * really want to make it explicit. | |
| 260 | + */ | |
| 261 | + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); | |
| 262 | + vcpu_clear_flag(vcpu, EXCEPT_MASK); | |
| 263 | + vcpu_clear_flag(vcpu, INCREMENT_PC); | |
| 250 | 264 | vcpu_set_reg(vcpu, 0, reset_state.r0); |
| 251 | 265 | } |
| 252 | 266 | |
| @@ -41,4 +41,40 @@ · 40 unmodified lines | ||
| 41 | 41 | .cfi_endproc; \ |
| 42 | 42 | SYM_END(name, SYM_T_NONE) |
| 43 | 43 | |
| 44 | +/* | |
| 45 | + * This is for the signal handler trampoline, which is used as the return | |
| 46 | + * address of the signal handlers in userspace instead of called normally. | |
| 47 | + * The long standing libgcc bug https://gcc.gnu.org/PR124050 requires a | |
| 48 | + * nop between .cfi_startproc and the actual address of the trampoline, so | |
| 49 | + * we cannot simply use SYM_FUNC_START. | |
| 50 | + * | |
| 51 | + * This wrapper also contains all the .cfi_* directives for recovering | |
| 52 | + * the content of the GPRs and the "return address" (where the rt_sigreturn | |
| 53 | + * syscall will jump to), assuming there is a struct rt_sigframe (where | |
| 54 | + * a struct sigcontext containing those information we need to recover) at | |
| 55 | + * $sp. The "DWARF for the LoongArch(TM) Architecture" manual states | |
| 56 | + * column 0 is for $zero, but it does not make too much sense to | |
| 57 | + * save/restore the hardware zero register. Repurpose this column here | |
| 58 | + * for the return address (here it's not the content of $ra we cannot use | |
| 59 | + * the default column 3). | |
| 60 | + */ | |
| 61 | +#define SYM_SIGFUNC_START(name) \ | |
| 62 | + .cfi_startproc; \ | |
| 63 | + .cfi_signal_frame; \ | |
| 64 | + .cfi_def_cfa 3, RT_SIGFRAME_SC; \ | |
| 65 | + .cfi_return_column 0; \ | |
| 66 | + .cfi_offset 0, SC_PC; \ | |
| 67 | + \ | |
| 68 | + .irp num, 1, 2, 3, 4, 5, 6, 7, 8, \ | |
| 69 | + 9, 10, 11, 12, 13, 14, 15, 16, \ | |
| 70 | + 17, 18, 19, 20, 21, 22, 23, 24, \ | |
| 71 | + 25, 26, 27, 28, 29, 30, 31; \ | |
| 72 | + .cfi_offset \num, SC_REGS + \num * SZREG; \ | |
| 73 | + .endr; \ | |
| 74 | + \ | |
| 75 | + nop; \ | |
| 76 | + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) | |
| 77 | + | |
| 78 | +#define SYM_SIGFUNC_END(name) SYM_FUNC_END(name) | |
| 79 | + | |
| 44 | 80 | #endif |
| @@ -0,0 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0+ */ | |
| 2 | + | |
| 3 | +#include <asm/siginfo.h> | |
| 4 | +#include <asm/ucontext.h> | |
| 5 | + | |
| 6 | +struct rt_sigframe { | |
| 7 | + struct siginfo rs_info; | |
| 8 | + struct ucontext rs_uctx; | |
| 9 | +}; | |
| @@ -16,6 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <asm/ptrace.h> |
| 17 | 17 | #include <asm/processor.h> |
| 18 | 18 | #include <asm/ftrace.h> |
| 19 | +#include <asm/sigframe.h> | |
| 19 | 20 | #include <vdso/datapage.h> |
| 20 | 21 | |
| 21 | 22 | static void __used output_ptreg_defines(void) |
| @@ -220,6 +221,7 @@ static void __used output_sc_defines(void) · 198 unmodified lines | ||
| 220 | 221 | COMMENT("Linux sigcontext offsets."); |
| 221 | 222 | OFFSET(SC_REGS, sigcontext, sc_regs); |
| 222 | 223 | OFFSET(SC_PC, sigcontext, sc_pc); |
| 224 | + OFFSET(RT_SIGFRAME_SC, rt_sigframe, rs_uctx.uc_mcontext); | |
| 223 | 225 | BLANK(); |
| 224 | 226 | } |
| 225 | 227 | |
| @@ -42,16 +42,15 @@ static int __init init_cpu_fullname(void) · 41 unmodified lines | ||
| 42 | 42 | int cpu, ret; |
| 43 | 43 | char *cpuname; |
| 44 | 44 | const char *model; |
| 45 | − struct device_node *root; | |
| 46 | 45 | |
| 47 | 46 | /* Parsing cpuname from DTS model property */ |
| 48 | − root = of_find_node_by_path("/"); | |
| 49 | − ret = of_property_read_string(root, "model", &model); | |
| 47 | + ret = of_property_read_string(of_root, "model", &model); | |
| 50 | 48 | if (ret == 0) { |
| 51 | 49 | cpuname = kstrdup(model, GFP_KERNEL); |
| 50 | + if (!cpuname) | |
| 51 | + return -ENOMEM; | |
| 52 | 52 | loongson_sysconf.cpuname = strsep(&cpuname, " "); |
| 53 | 53 | } |
| 54 | − of_node_put(root); | |
| 55 | 54 | |
| 56 | 55 | if (loongson_sysconf.cpuname && !strncmp(loongson_sysconf.cpuname, "Loongson", 8)) { |
| 57 | 56 | for (cpu = 0; cpu < NR_CPUS; cpu++) |
| @@ -35,6 +35,7 @@ · 34 unmodified lines | ||
| 35 | 35 | #include <asm/cpu-features.h> |
| 36 | 36 | #include <asm/fpu.h> |
| 37 | 37 | #include <asm/lbt.h> |
| 38 | +#include <asm/sigframe.h> | |
| 38 | 39 | #include <asm/ucontext.h> |
| 39 | 40 | #include <asm/vdso.h> |
| 40 | 41 | |
| @@ -51,11 +52,6 @@ · 10 unmodified lines | ||
| 51 | 52 | #define lock_lbt_owner() ({ preempt_disable(); pagefault_disable(); }) |
| 52 | 53 | #define unlock_lbt_owner() ({ pagefault_enable(); preempt_enable(); }) |
| 53 | 54 | |
| 54 | −struct rt_sigframe { | |
| 55 | − struct siginfo rs_info; | |
| 56 | − struct ucontext rs_uctx; | |
| 57 | −}; | |
| 58 | − | |
| 59 | 55 | struct _ctx_layout { |
| 60 | 56 | struct sctx_info *addr; |
| 61 | 57 | unsigned int size; |
| @@ -83,7 +83,7 @@ static inline void eiointc_update_sw_coremap(struct loongarch_eiointc *s, · 82 unmodified lines | ||
| 83 | 83 | |
| 84 | 84 | if (!(s->status & BIT(EIOINTC_ENABLE_CPU_ENCODE))) { |
| 85 | 85 | cpuid = ffs(cpuid) - 1; |
| 86 | − cpuid = (cpuid >= 4) ? 0 : cpuid; | |
| 86 | + cpuid = ((cpuid < 0) || (cpuid >= 4)) ? 0 : cpuid; | |
| 87 | 87 | } |
| 88 | 88 | |
| 89 | 89 | vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid); |
| @@ -472,34 +472,34 @@ static int kvm_eiointc_regs_access(struct kvm_device *dev, · 382 unmodified lines | ||
| 472 | 472 | switch (addr) { |
| 473 | 473 | case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: |
| 474 | 474 | offset = (addr - EIOINTC_NODETYPE_START) / 4; |
| 475 | − p = s->nodetype + offset * 4; | |
| 475 | + p = (void *)s->nodetype + offset * 4; | |
| 476 | 476 | break; |
| 477 | 477 | case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: |
| 478 | 478 | offset = (addr - EIOINTC_IPMAP_START) / 4; |
| 479 | − p = &s->ipmap + offset * 4; | |
| 479 | + p = (void *)&s->ipmap + offset * 4; | |
| 480 | 480 | break; |
| 481 | 481 | case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: |
| 482 | 482 | offset = (addr - EIOINTC_ENABLE_START) / 4; |
| 483 | − p = s->enable + offset * 4; | |
| 483 | + p = (void *)s->enable + offset * 4; | |
| 484 | 484 | break; |
| 485 | 485 | case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: |
| 486 | 486 | offset = (addr - EIOINTC_BOUNCE_START) / 4; |
| 487 | − p = s->bounce + offset * 4; | |
| 487 | + p = (void *)s->bounce + offset * 4; | |
| 488 | 488 | break; |
| 489 | 489 | case EIOINTC_ISR_START ... EIOINTC_ISR_END: |
| 490 | 490 | offset = (addr - EIOINTC_ISR_START) / 4; |
| 491 | − p = s->isr + offset * 4; | |
| 491 | + p = (void *)s->isr + offset * 4; | |
| 492 | 492 | break; |
| 493 | 493 | case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: |
| 494 | 494 | if (cpu >= s->num_cpu) |
| 495 | 495 | return -EINVAL; |
| 496 | 496 | |
| 497 | 497 | offset = (addr - EIOINTC_COREISR_START) / 4; |
| 498 | − p = s->coreisr[cpu] + offset * 4; | |
| 498 | + p = (void *)s->coreisr[cpu] + offset * 4; | |
| 499 | 499 | break; |
| 500 | 500 | case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: |
| 501 | 501 | offset = (addr - EIOINTC_COREMAP_START) / 4; |
| 502 | − p = s->coremap + offset * 4; | |
| 502 | + p = (void *)s->coremap + offset * 4; | |
| 503 | 503 | break; |
| 504 | 504 | default: |
| 505 | 505 | kvm_err("%s: unknown eiointc register, addr = %d\n", __func__, addr); |
| @@ -588,6 +588,9 @@ struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid) · 587 unmodified lines | ||
| 588 | 588 | { |
| 589 | 589 | struct kvm_phyid_map *map; |
| 590 | 590 | |
| 591 | + if (cpuid < 0) | |
| 592 | + return NULL; | |
| 593 | + | |
| 591 | 594 | if (cpuid >= KVM_MAX_PHYID) |
| 592 | 595 | return NULL; |
| 593 | 596 | |
| @@ -5,9 +5,11 @@ · 4 unmodified lines | ||
| 5 | 5 | #include <linux/kernel.h> |
| 6 | 6 | #include <linux/init.h> |
| 7 | 7 | #include <linux/acpi.h> |
| 8 | +#include <linux/delay.h> | |
| 8 | 9 | #include <linux/types.h> |
| 9 | 10 | #include <linux/pci.h> |
| 10 | 11 | #include <linux/vgaarb.h> |
| 12 | +#include <linux/io-64-nonatomic-lo-hi.h> | |
| 11 | 13 | #include <asm/cacheflush.h> |
| 12 | 14 | #include <asm/loongson.h> |
| 13 | 15 | |
| @@ -15,6 +17,9 @@ · 1 unmodified line | ||
| 15 | 17 | #define PCI_DEVICE_ID_LOONGSON_DC1 0x7a06 |
| 16 | 18 | #define PCI_DEVICE_ID_LOONGSON_DC2 0x7a36 |
| 17 | 19 | #define PCI_DEVICE_ID_LOONGSON_DC3 0x7a46 |
| 20 | +#define PCI_DEVICE_ID_LOONGSON_GPU1 0x7a15 | |
| 21 | +#define PCI_DEVICE_ID_LOONGSON_GPU2 0x7a25 | |
| 22 | +#define PCI_DEVICE_ID_LOONGSON_GPU3 0x7a35 | |
| 18 | 23 | |
| 19 | 24 | int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn, |
| 20 | 25 | int reg, int len, u32 *val) |
| @@ -99,3 +104,78 @@ static void pci_fixup_vgadev(struct pci_dev *pdev) · 78 unmodified lines | ||
| 99 | 104 | DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC1, pci_fixup_vgadev); |
| 100 | 105 | DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC2, pci_fixup_vgadev); |
| 101 | 106 | DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC3, pci_fixup_vgadev); |
| 107 | + | |
| 108 | +#define CRTC_NUM_MAX 2 | |
| 109 | +#define CRTC_OUTPUT_ENABLE 0x100 | |
| 110 | + | |
| 111 | +static void loongson_gpu_fixup_dma_hang(struct pci_dev *pdev, bool on) | |
| 112 | +{ | |
| 113 | + u32 i, val, count, crtc_offset, device; | |
| 114 | + void __iomem *crtc_reg, *base, *regbase; | |
| 115 | + static u32 crtc_status[CRTC_NUM_MAX] = { 0 }; | |
| 116 | + | |
| 117 | + base = pdev->bus->ops->map_bus(pdev->bus, pdev->devfn + 1, 0); | |
| 118 | + device = readw(base + PCI_DEVICE_ID); | |
| 119 | + | |
| 120 | + regbase = ioremap(readq(base + PCI_BASE_ADDRESS_0) & ~0xffull, SZ_64K); | |
| 121 | + if (!regbase) { | |
| 122 | + pci_err(pdev, "Failed to ioremap()\n"); | |
| 123 | + return; | |
| 124 | + } | |
| 125 | + | |
| 126 | + switch (device) { | |
| 127 | + case PCI_DEVICE_ID_LOONGSON_DC2: | |
| 128 | + crtc_reg = regbase + 0x1240; | |
| 129 | + crtc_offset = 0x10; | |
| 130 | + break; | |
| 131 | + case PCI_DEVICE_ID_LOONGSON_DC3: | |
| 132 | + crtc_reg = regbase; | |
| 133 | + crtc_offset = 0x400; | |
| 134 | + break; | |
| 135 | + } | |
| 136 | + | |
| 137 | + for (i = 0; i < CRTC_NUM_MAX; i++, crtc_reg += crtc_offset) { | |
| 138 | + val = readl(crtc_reg); | |
| 139 | + | |
| 140 | + if (!on) | |
| 141 | + crtc_status[i] = val; | |
| 142 | + | |
| 143 | + /* No need to fixup if the status is off at startup. */ | |
| 144 | + if (!(crtc_status[i] & CRTC_OUTPUT_ENABLE)) | |
| 145 | + continue; | |
| 146 | + | |
| 147 | + if (on) | |
| 148 | + val |= CRTC_OUTPUT_ENABLE; | |
| 149 | + else | |
| 150 | + val &= ~CRTC_OUTPUT_ENABLE; | |
| 151 | + | |
| 152 | + mb(); | |
| 153 | + writel(val, crtc_reg); | |
| 154 | + | |
| 155 | + for (count = 0; count < 40; count++) { | |
| 156 | + val = readl(crtc_reg) & CRTC_OUTPUT_ENABLE; | |
| 157 | + if ((on && val) || (!on && !val)) | |
| 158 | + break; | |
| 159 | + udelay(1000); | |
| 160 | + } | |
| 161 | + | |
| 162 | + pci_info(pdev, "DMA hang fixup at reg[0x%lx]: 0x%x\n", | |
| 163 | + (unsigned long)crtc_reg & 0xffff, readl(crtc_reg)); | |
| 164 | + } | |
| 165 | + | |
| 166 | + iounmap(regbase); | |
| 167 | +} | |
| 168 | + | |
| 169 | +static void pci_fixup_dma_hang_early(struct pci_dev *pdev) | |
| 170 | +{ | |
| 171 | + loongson_gpu_fixup_dma_hang(pdev, false); | |
| 172 | +} | |
| 173 | +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_early); | |
| 174 | +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_early); | |
| 175 | + | |
| 176 | +static void pci_fixup_dma_hang_final(struct pci_dev *pdev) | |
| 177 | +{ | |
| 178 | + loongson_gpu_fixup_dma_hang(pdev, true); | |
| 179 | +} | |
| 180 | +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_final); | |
| 181 | +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_final); | |
| @@ -26,7 +26,7 @@ cflags-vdso := $(ccflags-vdso) \ · 25 unmodified lines | ||
| 26 | 26 | $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ |
| 27 | 27 | -std=gnu11 -fms-extensions -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \ |
| 28 | 28 | -fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \ |
| 29 | − $(call cc-option, -fno-asynchronous-unwind-tables) \ | |
| 29 | + $(call cc-option, -fasynchronous-unwind-tables) \ | |
| 30 | 30 | $(call cc-option, -fno-stack-protector) |
| 31 | 31 | aflags-vdso := $(ccflags-vdso) \ |
| 32 | 32 | -D__ASSEMBLY__ -Wa,-gdwarf-2 |
| @@ -41,7 +41,7 @@ endif · 8 unmodified lines | ||
| 41 | 41 | |
| 42 | 42 | # VDSO linker flags. |
| 43 | 43 | ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ |
| 44 | − $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id -T | |
| 44 | + $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id --eh-frame-hdr -T | |
| 45 | 45 | |
| 46 | 46 | # |
| 47 | 47 | # Shared build commands. |
| @@ -12,13 +12,13 @@ · 11 unmodified lines | ||
| 12 | 12 | |
| 13 | 13 | #include <asm/regdef.h> |
| 14 | 14 | #include <asm/asm.h> |
| 15 | +#include <asm/asm-offsets.h> | |
| 15 | 16 | |
| 16 | 17 | .section .text |
| 17 | − .cfi_sections .debug_frame | |
| 18 | 18 | |
| 19 | −SYM_FUNC_START(__vdso_rt_sigreturn) | |
| 19 | +SYM_SIGFUNC_START(__vdso_rt_sigreturn) | |
| 20 | 20 | |
| 21 | 21 | li.w a7, __NR_rt_sigreturn |
| 22 | 22 | syscall 0 |
| 23 | 23 | |
| 24 | −SYM_FUNC_END(__vdso_rt_sigreturn) | |
| 24 | +SYM_SIGFUNC_END(__vdso_rt_sigreturn) | |
| @@ -62,8 +62,8 @@ do { \ · 61 unmodified lines | ||
| 62 | 62 | * @size: number of elements in array |
| 63 | 63 | */ |
| 64 | 64 | #define array_index_mask_nospec array_index_mask_nospec |
| 65 | −static inline unsigned long array_index_mask_nospec(unsigned long index, | |
| 66 | − unsigned long size) | |
| 65 | +static __always_inline unsigned long array_index_mask_nospec(unsigned long index, | |
| 66 | + unsigned long size) | |
| 67 | 67 | { |
| 68 | 68 | unsigned long mask; |
| 69 | 69 | |
| @@ -710,6 +710,9 @@ void kvm_arch_crypto_clear_masks(struct kvm *kvm); · 709 unmodified lines | ||
| 710 | 710 | void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, |
| 711 | 711 | unsigned long *aqm, unsigned long *adm); |
| 712 | 712 | |
| 713 | +#define SIE64_RETURN_NORMAL 0 | |
| 714 | +#define SIE64_RETURN_MCCK 1 | |
| 715 | + | |
| 713 | 716 | int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa, |
| 714 | 717 | unsigned long gasce); |
| 715 | 718 | |
| @@ -62,7 +62,7 @@ struct stack_frame { · 61 unmodified lines | ||
| 62 | 62 | struct { |
| 63 | 63 | unsigned long sie_control_block; |
| 64 | 64 | unsigned long sie_savearea; |
| 65 | − unsigned long sie_reason; | |
| 65 | + unsigned long sie_return; | |
| 66 | 66 | unsigned long sie_flags; |
| 67 | 67 | unsigned long sie_control_block_phys; |
| 68 | 68 | unsigned long sie_guest_asce; |
| @@ -63,7 +63,7 @@ int main(void) · 62 unmodified lines | ||
| 63 | 63 | OFFSET(__SF_EMPTY, stack_frame, empty[0]); |
| 64 | 64 | OFFSET(__SF_SIE_CONTROL, stack_frame, sie_control_block); |
| 65 | 65 | OFFSET(__SF_SIE_SAVEAREA, stack_frame, sie_savearea); |
| 66 | − OFFSET(__SF_SIE_REASON, stack_frame, sie_reason); | |
| 66 | + OFFSET(__SF_SIE_RETURN, stack_frame, sie_return); | |
| 67 | 67 | OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags); |
| 68 | 68 | OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys); |
| 69 | 69 | OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce); |
| @@ -200,7 +200,7 @@ SYM_FUNC_START(__sie64a) · 199 unmodified lines | ||
| 200 | 200 | stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses |
| 201 | 201 | stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area |
| 202 | 202 | stg %r5,__SF_SIE_GUEST_ASCE(%r15) # save guest asce |
| 203 | − xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0 | |
| 203 | + xc __SF_SIE_RETURN(8,%r15),__SF_SIE_RETURN(%r15) # return code = 0 | |
| 204 | 204 | mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r14) # copy thread flags |
| 205 | 205 | lmg %r0,%r13,0(%r4) # load guest gprs 0-13 |
| 206 | 206 | mvi __TI_sie(%r14),1 |
| @@ -237,7 +237,7 @@ SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL) · 30 unmodified lines | ||
| 237 | 237 | xgr %r4,%r4 |
| 238 | 238 | xgr %r5,%r5 |
| 239 | 239 | lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers |
| 240 | − lg %r2,__SF_SIE_REASON(%r15) # return exit reason code | |
| 240 | + lg %r2,__SF_SIE_RETURN(%r15) # return sie return code | |
| 241 | 241 | BR_EX %r14 |
| 242 | 242 | SYM_FUNC_END(__sie64a) |
| 243 | 243 | EXPORT_SYMBOL(__sie64a) |
| @@ -271,6 +271,7 @@ SYM_CODE_START(system_call) · 27 unmodified lines | ||
| 271 | 271 | xgr %r9,%r9 |
| 272 | 272 | xgr %r10,%r10 |
| 273 | 273 | xgr %r11,%r11 |
| 274 | + xgr %r12,%r12 | |
| 274 | 275 | la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs |
| 275 | 276 | mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13) |
| 276 | 277 | MBEAR %r2,%r13 |
| @@ -407,6 +408,7 @@ SYM_CODE_START(\name) · 130 unmodified lines | ||
| 407 | 408 | xgr %r6,%r6 |
| 408 | 409 | xgr %r7,%r7 |
| 409 | 410 | xgr %r10,%r10 |
| 411 | + xgr %r12,%r12 | |
| 410 | 412 | xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) |
| 411 | 413 | mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) |
| 412 | 414 | MBEAR %r11,%r13 |
| @@ -496,6 +498,7 @@ SYM_CODE_START(mcck_int_handler) · 83 unmodified lines | ||
| 496 | 498 | xgr %r6,%r6 |
| 497 | 499 | xgr %r7,%r7 |
| 498 | 500 | xgr %r10,%r10 |
| 501 | + xgr %r12,%r12 | |
| 499 | 502 | stmg %r8,%r9,__PT_PSW(%r11) |
| 500 | 503 | xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) |
| 501 | 504 | xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) |
| @@ -487,8 +487,8 @@ void notrace s390_do_machine_check(struct pt_regs *regs) · 486 unmodified lines | ||
| 487 | 487 | mcck_dam_code = (mci.val & MCIC_SUBCLASS_MASK); |
| 488 | 488 | if (test_cpu_flag(CIF_MCCK_GUEST) && |
| 489 | 489 | (mcck_dam_code & MCCK_CODE_NO_GUEST) != mcck_dam_code) { |
| 490 | − /* Set exit reason code for host's later handling */ | |
| 491 | − *((long *)(regs->gprs[15] + __SF_SIE_REASON)) = -EINTR; | |
| 490 | + /* Set sie return code for host's later handling */ | |
| 491 | + ((struct stack_frame *)regs->gprs[15])->sie_return = SIE64_RETURN_MCCK; | |
| 492 | 492 | } |
| 493 | 493 | clear_cpu_flag(CIF_MCCK_GUEST); |
| 494 | 494 | |
| @@ -13,6 +13,7 @@ · 12 unmodified lines | ||
| 13 | 13 | */ |
| 14 | 14 | |
| 15 | 15 | #include <linux/cpufeature.h> |
| 16 | +#include <linux/nospec.h> | |
| 16 | 17 | #include <linux/errno.h> |
| 17 | 18 | #include <linux/sched.h> |
| 18 | 19 | #include <linux/mm.h> |
| @@ -131,8 +132,10 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) · 112 unmodified lines | ||
| 131 | 132 | if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET))) |
| 132 | 133 | goto out; |
| 133 | 134 | regs->gprs[2] = -ENOSYS; |
| 134 | − if (likely(nr < NR_syscalls)) | |
| 135 | + if (likely(nr < NR_syscalls)) { | |
| 136 | + nr = array_index_nospec(nr, NR_syscalls); | |
| 135 | 137 | regs->gprs[2] = sys_call_table[nr](regs); |
| 138 | + } | |
| 136 | 139 | out: |
| 137 | 140 | syscall_exit_to_user_mode(regs); |
| 138 | 141 | } |
| @@ -134,32 +134,6 @@ int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newt · 133 unmodified lines | ||
| 134 | 134 | return 0; |
| 135 | 135 | } |
| 136 | 136 | |
| 137 | −/** | |
| 138 | − * dat_crstep_xchg() - Exchange a gmap CRSTE with another. | |
| 139 | − * @crstep: Pointer to the CRST entry | |
| 140 | − * @new: Replacement entry. | |
| 141 | − * @gfn: The affected guest address. | |
| 142 | − * @asce: The ASCE of the address space. | |
| 143 | − * | |
| 144 | − * Context: This function is assumed to be called with kvm->mmu_lock held. | |
| 145 | − */ | |
| 146 | −void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce) | |
| 147 | −{ | |
| 148 | − if (crstep->h.i) { | |
| 149 | − WRITE_ONCE(*crstep, new); | |
| 150 | − return; | |
| 151 | − } else if (cpu_has_edat2()) { | |
| 152 | − crdte_crste(crstep, *crstep, new, gfn, asce); | |
| 153 | − return; | |
| 154 | − } | |
| 155 | − | |
| 156 | − if (machine_has_tlb_guest()) | |
| 157 | − idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL); | |
| 158 | − else | |
| 159 | − idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL); | |
| 160 | − WRITE_ONCE(*crstep, new); | |
| 161 | −} | |
| 162 | − | |
| 163 | 137 | /** |
| 164 | 138 | * dat_crstep_xchg_atomic() - Atomically exchange a gmap CRSTE with another. |
| 165 | 139 | * @crstep: Pointer to the CRST entry. |
| @@ -175,8 +149,8 @@ void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce · 9 unmodified lines | ||
| 175 | 149 | * |
| 176 | 150 | * Return: %true if the exchange was successful. |
| 177 | 151 | */ |
| 178 | −bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn, | |
| 179 | − union asce asce) | |
| 152 | +bool __must_check dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, | |
| 153 | + gfn_t gfn, union asce asce) | |
| 180 | 154 | { |
| 181 | 155 | if (old.h.i) |
| 182 | 156 | return arch_try_cmpxchg((long *)crstep, &old.val, new.val); |
| @@ -292,6 +266,7 @@ static int dat_split_ste(struct kvm_s390_mmu_cache *mc, union pmd *pmdp, gfn_t g · 109 unmodified lines | ||
| 292 | 266 | pt->ptes[i].val = init.val | i * PAGE_SIZE; |
| 293 | 267 | /* No need to take locks as the page table is not installed yet. */ |
| 294 | 268 | pgste_init.prefix_notif = old.s.fc1.prefix_notif; |
| 269 | + pgste_init.vsie_notif = old.s.fc1.vsie_notif; | |
| 295 | 270 | pgste_init.pcl = uses_skeys && init.h.i; |
| 296 | 271 | dat_init_pgstes(pt, pgste_init.val); |
| 297 | 272 | } else { |
| @@ -893,7 +868,8 @@ static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct d · 595 unmodified lines | ||
| 893 | 868 | |
| 894 | 869 | /* This table entry needs to be updated. */ |
| 895 | 870 | if (walk->start <= gfn && walk->end >= next) { |
| 896 | − dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce); | |
| 871 | + if (!dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce)) | |
| 872 | + return -EINVAL; | |
| 897 | 873 | /* A lower level table was present, needs to be freed. */ |
| 898 | 874 | if (!crste.h.fc && !crste.h.i) { |
| 899 | 875 | if (is_pmd(crste)) |
| @@ -1021,67 +997,21 @@ bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end) · 121 unmodified lines | ||
| 1021 | 997 | return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0; |
| 1022 | 998 | } |
| 1023 | 999 | |
| 1024 | −int dat_link(struct kvm_s390_mmu_cache *mc, union asce asce, int level, | |
| 1025 | − bool uses_skeys, struct guest_fault *f) | |
| 1026 | −{ | |
| 1027 | − union crste oldval, newval; | |
| 1028 | − union pte newpte, oldpte; | |
| 1029 | − union pgste pgste; | |
| 1030 | − int rc = 0; | |
| 1031 | − | |
| 1032 | − rc = dat_entry_walk(mc, f->gfn, asce, DAT_WALK_ALLOC_CONTINUE, level, &f->crstep, &f->ptep); | |
| 1033 | − if (rc == -EINVAL || rc == -ENOMEM) | |
| 1034 | − return rc; | |
| 1035 | − if (rc) | |
| 1036 | − return -EAGAIN; | |
| 1037 | − | |
| 1038 | − if (WARN_ON_ONCE(unlikely(get_level(f->crstep, f->ptep) > level))) | |
| 1039 | − return -EINVAL; | |
| 1040 | − | |
| 1041 | − if (f->ptep) { | |
| 1042 | − pgste = pgste_get_lock(f->ptep); | |
| 1043 | − oldpte = *f->ptep; | |
| 1044 | − newpte = _pte(f->pfn, f->writable, f->write_attempt | oldpte.s.d, !f->page); | |
| 1045 | − newpte.s.sd = oldpte.s.sd; | |
| 1046 | − oldpte.s.sd = 0; | |
| 1047 | − if (oldpte.val == _PTE_EMPTY.val || oldpte.h.pfra == f->pfn) { | |
| 1048 | − pgste = __dat_ptep_xchg(f->ptep, pgste, newpte, f->gfn, asce, uses_skeys); | |
| 1049 | − if (f->callback) | |
| 1050 | − f->callback(f); | |
| 1051 | − } else { | |
| 1052 | − rc = -EAGAIN; | |
| 1053 | − } | |
| 1054 | − pgste_set_unlock(f->ptep, pgste); | |
| 1055 | − } else { | |
| 1056 | − oldval = READ_ONCE(*f->crstep); | |
| 1057 | − newval = _crste_fc1(f->pfn, oldval.h.tt, f->writable, | |
| 1058 | − f->write_attempt | oldval.s.fc1.d); | |
| 1059 | − newval.s.fc1.sd = oldval.s.fc1.sd; | |
| 1060 | − if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && | |
| 1061 | − crste_origin_large(oldval) != crste_origin_large(newval)) | |
| 1062 | − return -EAGAIN; | |
| 1063 | − if (!dat_crstep_xchg_atomic(f->crstep, oldval, newval, f->gfn, asce)) | |
| 1064 | − return -EAGAIN; | |
| 1065 | − if (f->callback) | |
| 1066 | − f->callback(f); | |
| 1067 | − } | |
| 1068 | − | |
| 1069 | − return rc; | |
| 1070 | −} | |
| 1071 | − | |
| 1072 | 1000 | static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) |
| 1073 | 1001 | { |
| 1074 | − union crste crste = READ_ONCE(*crstep); | |
| 1002 | + union crste newcrste, oldcrste; | |
| 1075 | 1003 | int *n = walk->priv; |
| 1076 | 1004 | |
| 1077 | − if (!crste.h.fc || crste.h.i || crste.h.p) | |
| 1078 | − return 0; | |
| 1079 | − | |
| 1005 | + do { | |
| 1006 | + oldcrste = READ_ONCE(*crstep); | |
| 1007 | + if (!oldcrste.h.fc || oldcrste.h.i || oldcrste.h.p) | |
| 1008 | + return 0; | |
| 1009 | + if (oldcrste.s.fc1.prefix_notif) | |
| 1010 | + break; | |
| 1011 | + newcrste = oldcrste; | |
| 1012 | + newcrste.s.fc1.prefix_notif = 1; | |
| 1013 | + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, walk->asce)); | |
| 1080 | 1014 | *n = 2; |
| 1081 | − if (crste.s.fc1.prefix_notif) | |
| 1082 | − return 0; | |
| 1083 | − crste.s.fc1.prefix_notif = 1; | |
| 1084 | − dat_crstep_xchg(crstep, crste, gfn, walk->asce); | |
| 1085 | 1015 | return 0; |
| 1086 | 1016 | } |
| 1087 | 1017 | |
| @@ -160,14 +160,14 @@ union pmd { · 159 unmodified lines | ||
| 160 | 160 | unsigned long :44; /* HW */ |
| 161 | 161 | unsigned long : 3; /* Unused */ |
| 162 | 162 | unsigned long : 1; /* HW */ |
| 163 | + unsigned long s : 1; /* Special */ | |
| 163 | 164 | unsigned long w : 1; /* Writable soft-bit */ |
| 164 | 165 | unsigned long r : 1; /* Readable soft-bit */ |
| 165 | 166 | unsigned long d : 1; /* Dirty */ |
| 166 | 167 | unsigned long y : 1; /* Young */ |
| 167 | − unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ | |
| 168 | 168 | unsigned long : 3; /* HW */ |
| 169 | + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ | |
| 169 | 170 | unsigned long vsie_notif : 1; /* Referenced in a shadow table */ |
| 170 | − unsigned long : 1; /* Unused */ | |
| 171 | 171 | unsigned long : 4; /* HW */ |
| 172 | 172 | unsigned long sd : 1; /* Soft-Dirty */ |
| 173 | 173 | unsigned long pr : 1; /* Present */ |
| @@ -183,14 +183,14 @@ union pud { · 9 unmodified lines | ||
| 183 | 183 | unsigned long :33; /* HW */ |
| 184 | 184 | unsigned long :14; /* Unused */ |
| 185 | 185 | unsigned long : 1; /* HW */ |
| 186 | + unsigned long s : 1; /* Special */ | |
| 186 | 187 | unsigned long w : 1; /* Writable soft-bit */ |
| 187 | 188 | unsigned long r : 1; /* Readable soft-bit */ |
| 188 | 189 | unsigned long d : 1; /* Dirty */ |
| 189 | 190 | unsigned long y : 1; /* Young */ |
| 190 | − unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ | |
| 191 | 191 | unsigned long : 3; /* HW */ |
| 192 | + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ | |
| 192 | 193 | unsigned long vsie_notif : 1; /* Referenced in a shadow table */ |
| 193 | − unsigned long : 1; /* Unused */ | |
| 194 | 194 | unsigned long : 4; /* HW */ |
| 195 | 195 | unsigned long sd : 1; /* Soft-Dirty */ |
| 196 | 196 | unsigned long pr : 1; /* Present */ |
| @@ -254,14 +254,14 @@ union crste { · 57 unmodified lines | ||
| 254 | 254 | struct { |
| 255 | 255 | unsigned long :47; |
| 256 | 256 | unsigned long : 1; /* HW (should be 0) */ |
| 257 | + unsigned long s : 1; /* Special */ | |
| 257 | 258 | unsigned long w : 1; /* Writable */ |
| 258 | 259 | unsigned long r : 1; /* Readable */ |
| 259 | 260 | unsigned long d : 1; /* Dirty */ |
| 260 | 261 | unsigned long y : 1; /* Young */ |
| 261 | − unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ | |
| 262 | 262 | unsigned long : 3; /* HW */ |
| 263 | + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ | |
| 263 | 264 | unsigned long vsie_notif : 1; /* Referenced in a shadow table */ |
| 264 | − unsigned long : 1; | |
| 265 | 265 | unsigned long : 4; /* HW */ |
| 266 | 266 | unsigned long sd : 1; /* Soft-Dirty */ |
| 267 | 267 | unsigned long pr : 1; /* Present */ |
| @@ -540,8 +540,6 @@ int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gf · 272 unmodified lines | ||
| 540 | 540 | u16 type, u16 param); |
| 541 | 541 | int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); |
| 542 | 542 | bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); |
| 543 | −int dat_link(struct kvm_s390_mmu_cache *mc, union asce asce, int level, | |
| 544 | − bool uses_skeys, struct guest_fault *f); | |
| 545 | 543 | |
| 546 | 544 | int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty); |
| 547 | 545 | long dat_reset_cmma(union asce asce, gfn_t start_gfn); |
| @@ -938,11 +936,14 @@ static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pu · 390 unmodified lines | ||
| 938 | 936 | return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce); |
| 939 | 937 | } |
| 940 | 938 | |
| 941 | −static inline void dat_crstep_clear(union crste *crstep, gfn_t gfn, union asce asce) | |
| 939 | +static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce) | |
| 942 | 940 | { |
| 943 | − union crste newcrste = _CRSTE_EMPTY(crstep->h.tt); | |
| 941 | + union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt); | |
| 944 | 942 | |
| 945 | − dat_crstep_xchg(crstep, newcrste, gfn, asce); | |
| 943 | + do { | |
| 944 | + oldcrste = READ_ONCE(*crstep); | |
| 945 | + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, empty, gfn, asce)); | |
| 946 | + return oldcrste; | |
| 946 | 947 | } |
| 947 | 948 | |
| 948 | 949 | static inline int get_level(union crste *crstep, union pte *ptep) |
| @@ -1434,17 +1434,27 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union · 1433 unmodified lines | ||
| 1434 | 1434 | if (rc) |
| 1435 | 1435 | return rc; |
| 1436 | 1436 | |
| 1437 | − pgste = pgste_get_lock(ptep_h); | |
| 1438 | − newpte = _pte(f->pfn, f->writable, !p, 0); | |
| 1439 | − newpte.s.d |= ptep->s.d; | |
| 1440 | − newpte.s.sd |= ptep->s.sd; | |
| 1441 | − newpte.h.p &= ptep->h.p; | |
| 1442 | − pgste = _gmap_ptep_xchg(sg->parent, ptep_h, newpte, pgste, f->gfn, false); | |
| 1443 | − pgste.vsie_notif = 1; | |
| 1437 | + if (!pgste_get_trylock(ptep_h, &pgste)) | |
| 1438 | + return -EAGAIN; | |
| 1439 | + newpte = _pte(f->pfn, f->writable, !p, ptep_h->s.s); | |
| 1440 | + newpte.s.d |= ptep_h->s.d; | |
| 1441 | + newpte.s.sd |= ptep_h->s.sd; | |
| 1442 | + newpte.h.p &= ptep_h->h.p; | |
| 1443 | + if (!newpte.h.p && !f->writable) { | |
| 1444 | + rc = -EOPNOTSUPP; | |
| 1445 | + } else { | |
| 1446 | + pgste = _gmap_ptep_xchg(sg->parent, ptep_h, newpte, pgste, f->gfn, false); | |
| 1447 | + pgste.vsie_notif = 1; | |
| 1448 | + } | |
| 1444 | 1449 | pgste_set_unlock(ptep_h, pgste); |
| 1450 | + if (rc) | |
| 1451 | + return rc; | |
| 1452 | + if (!sg->parent) | |
| 1453 | + return -EAGAIN; | |
| 1445 | 1454 | |
| 1446 | 1455 | newpte = _pte(f->pfn, 0, !p, 0); |
| 1447 | − pgste = pgste_get_lock(ptep); | |
| 1456 | + if (!pgste_get_trylock(ptep, &pgste)) | |
| 1457 | + return -EAGAIN; | |
| 1448 | 1458 | pgste = __dat_ptep_xchg(ptep, pgste, newpte, gpa_to_gfn(raddr), sg->asce, uses_skeys(sg)); |
| 1449 | 1459 | pgste_set_unlock(ptep, pgste); |
| 1450 | 1460 | |
| @@ -1454,7 +1464,7 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union · 3 unmodified lines | ||
| 1454 | 1464 | static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, union crste *table, |
| 1455 | 1465 | struct guest_fault *f, bool p) |
| 1456 | 1466 | { |
| 1457 | − union crste newcrste; | |
| 1467 | + union crste newcrste, oldcrste; | |
| 1458 | 1468 | gfn_t gfn; |
| 1459 | 1469 | int rc; |
| 1460 | 1470 | |
| @@ -1467,16 +1477,28 @@ static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, uni · 6 unmodified lines | ||
| 1467 | 1477 | if (rc) |
| 1468 | 1478 | return rc; |
| 1469 | 1479 | |
| 1470 | − newcrste = _crste_fc1(f->pfn, host->h.tt, f->writable, !p); | |
| 1471 | − newcrste.s.fc1.d |= host->s.fc1.d; | |
| 1472 | − newcrste.s.fc1.sd |= host->s.fc1.sd; | |
| 1473 | − newcrste.h.p &= host->h.p; | |
| 1474 | − newcrste.s.fc1.vsie_notif = 1; | |
| 1475 | − newcrste.s.fc1.prefix_notif = host->s.fc1.prefix_notif; | |
| 1476 | − _gmap_crstep_xchg(sg->parent, host, newcrste, f->gfn, false); | |
| 1480 | + do { | |
| 1481 | + /* _gmap_crstep_xchg_atomic() could have unshadowed this shadow gmap */ | |
| 1482 | + if (!sg->parent) | |
| 1483 | + return -EAGAIN; | |
| 1484 | + oldcrste = READ_ONCE(*host); | |
| 1485 | + newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, f->writable, !p); | |
| 1486 | + newcrste.s.fc1.d |= oldcrste.s.fc1.d; | |
| 1487 | + newcrste.s.fc1.sd |= oldcrste.s.fc1.sd; | |
| 1488 | + newcrste.h.p &= oldcrste.h.p; | |
| 1489 | + newcrste.s.fc1.vsie_notif = 1; | |
| 1490 | + newcrste.s.fc1.prefix_notif = oldcrste.s.fc1.prefix_notif; | |
| 1491 | + newcrste.s.fc1.s = oldcrste.s.fc1.s; | |
| 1492 | + if (!newcrste.h.p && !f->writable) | |
| 1493 | + return -EOPNOTSUPP; | |
| 1494 | + } while (!_gmap_crstep_xchg_atomic(sg->parent, host, oldcrste, newcrste, f->gfn, false)); | |
| 1495 | + if (!sg->parent) | |
| 1496 | + return -EAGAIN; | |
| 1477 | 1497 | |
| 1478 | − newcrste = _crste_fc1(f->pfn, host->h.tt, 0, !p); | |
| 1479 | − dat_crstep_xchg(table, newcrste, gpa_to_gfn(raddr), sg->asce); | |
| 1498 | + newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, 0, !p); | |
| 1499 | + gfn = gpa_to_gfn(raddr); | |
| 1500 | + while (!dat_crstep_xchg_atomic(table, READ_ONCE(*table), newcrste, gfn, sg->asce)) | |
| 1501 | + ; | |
| 1480 | 1502 | return 0; |
| 1481 | 1503 | } |
| 1482 | 1504 | |
| @@ -1500,21 +1522,31 @@ static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, · 17 unmodified lines | ||
| 1500 | 1522 | if (rc) |
| 1501 | 1523 | return rc; |
| 1502 | 1524 | |
| 1503 | − /* A race occourred. The shadow mapping is already valid, nothing to do */ | |
| 1504 | − if ((ptep && !ptep->h.i) || (!ptep && crste_leaf(*table))) | |
| 1525 | + /* A race occurred. The shadow mapping is already valid, nothing to do */ | |
| 1526 | + if ((ptep && !ptep->h.i && ptep->h.p == w->p) || | |
| 1527 | + (!ptep && crste_leaf(*table) && !table->h.i && table->h.p == w->p)) | |
| 1505 | 1528 | return 0; |
| 1506 | 1529 | |
| 1507 | 1530 | gl = get_level(table, ptep); |
| 1508 | 1531 | |
| 1532 | + /* In case of a real address space */ | |
| 1533 | + if (w->level <= LEVEL_MEM) { | |
| 1534 | + l = TABLE_TYPE_PAGE_TABLE; | |
| 1535 | + hl = TABLE_TYPE_REGION1; | |
| 1536 | + goto real_address_space; | |
| 1537 | + } | |
| 1538 | + | |
| 1509 | 1539 | /* |
| 1510 | 1540 | * Skip levels that are already protected. For each level, protect |
| 1511 | 1541 | * only the page containing the entry, not the whole table. |
| 1512 | 1542 | */ |
| 1513 | 1543 | for (i = gl ; i >= w->level; i--) { |
| 1514 | − rc = gmap_protect_rmap(mc, sg, entries[i - 1].gfn, gpa_to_gfn(saddr), | |
| 1515 | − entries[i - 1].pfn, i, entries[i - 1].writable); | |
| 1544 | + rc = gmap_protect_rmap(mc, sg, entries[i].gfn, gpa_to_gfn(saddr), | |
| 1545 | + entries[i].pfn, i + 1, entries[i].writable); | |
| 1516 | 1546 | if (rc) |
| 1517 | 1547 | return rc; |
| 1548 | + if (!sg->parent) | |
| 1549 | + return -EAGAIN; | |
| 1518 | 1550 | } |
| 1519 | 1551 | |
| 1520 | 1552 | rc = dat_entry_walk(NULL, entries[LEVEL_MEM].gfn, sg->parent->asce, DAT_WALK_LEAF, |
| @@ -1526,6 +1558,7 @@ static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, · 5 unmodified lines | ||
| 1526 | 1558 | /* Get the smallest granularity */ |
| 1527 | 1559 | l = min3(gl, hl, w->level); |
| 1528 | 1560 | |
| 1561 | +real_address_space: | |
| 1529 | 1562 | flags = DAT_WALK_SPLIT_ALLOC | (uses_skeys(sg->parent) ? DAT_WALK_USES_SKEYS : 0); |
| 1530 | 1563 | /* If necessary, create the shadow mapping */ |
| 1531 | 1564 | if (l < gl) { |
| @@ -313,13 +313,16 @@ static long gmap_clear_young_crste(union crste *crstep, gfn_t gfn, gfn_t end, st · 312 unmodified lines | ||
| 313 | 313 | struct clear_young_pte_priv *priv = walk->priv; |
| 314 | 314 | union crste crste, new; |
| 315 | 315 | |
| 316 | − crste = READ_ONCE(*crstep); | |
| 316 | + do { | |
| 317 | + crste = READ_ONCE(*crstep); | |
| 318 | + | |
| 319 | + if (!crste.h.fc) | |
| 320 | + return 0; | |
| 321 | + if (!crste.s.fc1.y && crste.h.i) | |
| 322 | + return 0; | |
| 323 | + if (crste_prefix(crste) && !gmap_mkold_prefix(priv->gmap, gfn, end)) | |
| 324 | + break; | |
| 317 | 325 | |
| 318 | − if (!crste.h.fc) | |
| 319 | − return 0; | |
| 320 | − if (!crste.s.fc1.y && crste.h.i) | |
| 321 | − return 0; | |
| 322 | − if (!crste_prefix(crste) || gmap_mkold_prefix(priv->gmap, gfn, end)) { | |
| 323 | 326 | new = crste; |
| 324 | 327 | new.h.i = 1; |
| 325 | 328 | new.s.fc1.y = 0; |
| @@ -328,8 +331,8 @@ static long gmap_clear_young_crste(union crste *crstep, gfn_t gfn, gfn_t end, st · 2 unmodified lines | ||
| 328 | 331 | folio_set_dirty(phys_to_folio(crste_origin_large(crste))); |
| 329 | 332 | new.s.fc1.d = 0; |
| 330 | 333 | new.h.p = 1; |
| 331 | − dat_crstep_xchg(crstep, new, gfn, walk->asce); | |
| 332 | − } | |
| 334 | + } while (!dat_crstep_xchg_atomic(crstep, crste, new, gfn, walk->asce)); | |
| 335 | + | |
| 333 | 336 | priv->young = 1; |
| 334 | 337 | return 0; |
| 335 | 338 | } |
| @@ -391,14 +394,18 @@ static long _gmap_unmap_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct · 55 unmodified lines | ||
| 391 | 394 | { |
| 392 | 395 | struct gmap_unmap_priv *priv = walk->priv; |
| 393 | 396 | struct folio *folio = NULL; |
| 397 | + union crste old = *crstep; | |
| 394 | 398 | |
| 395 | − if (crstep->h.fc) { | |
| 396 | − if (crstep->s.fc1.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags)) | |
| 397 | − folio = phys_to_folio(crste_origin_large(*crstep)); | |
| 398 | − gmap_crstep_xchg(priv->gmap, crstep, _CRSTE_EMPTY(crstep->h.tt), gfn); | |
| 399 | − if (folio) | |
| 400 | − uv_convert_from_secure_folio(folio); | |
| 401 | − } | |
| 399 | + if (!old.h.fc) | |
| 400 | + return 0; | |
| 401 | + | |
| 402 | + if (old.s.fc1.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags)) | |
| 403 | + folio = phys_to_folio(crste_origin_large(old)); | |
| 404 | + /* No races should happen because kvm->mmu_lock is held in write mode */ | |
| 405 | + KVM_BUG_ON(!gmap_crstep_xchg_atomic(priv->gmap, crstep, old, _CRSTE_EMPTY(old.h.tt), gfn), | |
| 406 | + priv->gmap->kvm); | |
| 407 | + if (folio) | |
| 408 | + uv_convert_from_secure_folio(folio); | |
| 402 | 409 | |
| 403 | 410 | return 0; |
| 404 | 411 | } |
| @@ -474,23 +481,24 @@ static long _crste_test_and_clear_softdirty(union crste *table, gfn_t gfn, gfn_t · 69 unmodified lines | ||
| 474 | 481 | |
| 475 | 482 | if (fatal_signal_pending(current)) |
| 476 | 483 | return 1; |
| 477 | − crste = READ_ONCE(*table); | |
| 478 | − if (!crste.h.fc) | |
| 479 | − return 0; | |
| 480 | − if (crste.h.p && !crste.s.fc1.sd) | |
| 481 | − return 0; | |
| 484 | + do { | |
| 485 | + crste = READ_ONCE(*table); | |
| 486 | + if (!crste.h.fc) | |
| 487 | + return 0; | |
| 488 | + if (crste.h.p && !crste.s.fc1.sd) | |
| 489 | + return 0; | |
| 482 | 490 | |
| 483 | − /* | |
| 484 | − * If this large page contains one or more prefixes of vCPUs that are | |
| 485 | − * currently running, do not reset the protection, leave it marked as | |
| 486 | − * dirty. | |
| 487 | − */ | |
| 488 | − if (!crste.s.fc1.prefix_notif || gmap_mkold_prefix(gmap, gfn, end)) { | |
| 491 | + /* | |
| 492 | + * If this large page contains one or more prefixes of vCPUs that are | |
| 493 | + * currently running, do not reset the protection, leave it marked as | |
| 494 | + * dirty. | |
| 495 | + */ | |
| 496 | + if (crste.s.fc1.prefix_notif && !gmap_mkold_prefix(gmap, gfn, end)) | |
| 497 | + break; | |
| 489 | 498 | new = crste; |
| 490 | 499 | new.h.p = 1; |
| 491 | 500 | new.s.fc1.sd = 0; |
| 492 | − gmap_crstep_xchg(gmap, table, new, gfn); | |
| 493 | − } | |
| 501 | + } while (!gmap_crstep_xchg_atomic(gmap, table, crste, new, gfn)); | |
| 494 | 502 | |
| 495 | 503 | for ( ; gfn < end; gfn++) |
| 496 | 504 | mark_page_dirty(gmap->kvm, gfn); |
| @@ -511,7 +519,7 @@ void gmap_sync_dirty_log(struct gmap *gmap, gfn_t start, gfn_t end) · 14 unmodified lines | ||
| 511 | 519 | _dat_walk_gfn_range(start, end, gmap->asce, &walk_ops, 0, gmap); |
| 512 | 520 | } |
| 513 | 521 | |
| 514 | −static int gmap_handle_minor_crste_fault(union asce asce, struct guest_fault *f) | |
| 522 | +static int gmap_handle_minor_crste_fault(struct gmap *gmap, struct guest_fault *f) | |
| 515 | 523 | { |
| 516 | 524 | union crste newcrste, oldcrste = READ_ONCE(*f->crstep); |
| 517 | 525 | |
| @@ -536,10 +544,8 @@ static int gmap_handle_minor_crste_fault(union asce asce, struct guest_fault *f) · 18 unmodified lines | ||
| 536 | 544 | newcrste.s.fc1.d = 1; |
| 537 | 545 | newcrste.s.fc1.sd = 1; |
| 538 | 546 | } |
| 539 | − if (!oldcrste.s.fc1.d && newcrste.s.fc1.d) | |
| 540 | − SetPageDirty(phys_to_page(crste_origin_large(newcrste))); | |
| 541 | 547 | /* In case of races, let the slow path deal with it. */ |
| 542 | − return !dat_crstep_xchg_atomic(f->crstep, oldcrste, newcrste, f->gfn, asce); | |
| 548 | + return !gmap_crstep_xchg_atomic(gmap, f->crstep, oldcrste, newcrste, f->gfn); | |
| 543 | 549 | } |
| 544 | 550 | /* Trying to write on a read-only page, let the slow path deal with it. */ |
| 545 | 551 | return 1; |
| @@ -568,8 +574,6 @@ static int _gmap_handle_minor_pte_fault(struct gmap *gmap, union pgste *pgste, · 22 unmodified lines | ||
| 568 | 574 | newpte.s.d = 1; |
| 569 | 575 | newpte.s.sd = 1; |
| 570 | 576 | } |
| 571 | − if (!oldpte.s.d && newpte.s.d) | |
| 572 | − SetPageDirty(pfn_to_page(newpte.h.pfra)); | |
| 573 | 577 | *pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, *pgste, f->gfn); |
| 574 | 578 | |
| 575 | 579 | return 0; |
| @@ -606,7 +610,7 @@ int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault) · 30 unmodified lines | ||
| 606 | 610 | fault->callback(fault); |
| 607 | 611 | pgste_set_unlock(fault->ptep, pgste); |
| 608 | 612 | } else { |
| 609 | − rc = gmap_handle_minor_crste_fault(gmap->asce, fault); | |
| 613 | + rc = gmap_handle_minor_crste_fault(gmap, fault); | |
| 610 | 614 | if (!rc && fault->callback) |
| 611 | 615 | fault->callback(fault); |
| 612 | 616 | } |
| @@ -623,10 +627,61 @@ static inline bool gmap_1m_allowed(struct gmap *gmap, gfn_t gfn) · 10 unmodified lines | ||
| 623 | 627 | return test_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &gmap->flags); |
| 624 | 628 | } |
| 625 | 629 | |
| 630 | +static int _gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, int level, | |
| 631 | + struct guest_fault *f) | |
| 632 | +{ | |
| 633 | + union crste oldval, newval; | |
| 634 | + union pte newpte, oldpte; | |
| 635 | + union pgste pgste; | |
| 636 | + int rc = 0; | |
| 637 | + | |
| 638 | + rc = dat_entry_walk(mc, f->gfn, gmap->asce, DAT_WALK_ALLOC_CONTINUE, level, | |
| 639 | + &f->crstep, &f->ptep); | |
| 640 | + if (rc == -ENOMEM) | |
| 641 | + return rc; | |
| 642 | + if (KVM_BUG_ON(rc == -EINVAL, gmap->kvm)) | |
| 643 | + return rc; | |
| 644 | + if (rc) | |
| 645 | + return -EAGAIN; | |
| 646 | + if (KVM_BUG_ON(get_level(f->crstep, f->ptep) > level, gmap->kvm)) | |
| 647 | + return -EINVAL; | |
| 648 | + | |
| 649 | + if (f->ptep) { | |
| 650 | + pgste = pgste_get_lock(f->ptep); | |
| 651 | + oldpte = *f->ptep; | |
| 652 | + newpte = _pte(f->pfn, f->writable, f->write_attempt | oldpte.s.d, !f->page); | |
| 653 | + newpte.s.sd = oldpte.s.sd; | |
| 654 | + oldpte.s.sd = 0; | |
| 655 | + if (oldpte.val == _PTE_EMPTY.val || oldpte.h.pfra == f->pfn) { | |
| 656 | + pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, pgste, f->gfn); | |
| 657 | + if (f->callback) | |
| 658 | + f->callback(f); | |
| 659 | + } else { | |
| 660 | + rc = -EAGAIN; | |
| 661 | + } | |
| 662 | + pgste_set_unlock(f->ptep, pgste); | |
| 663 | + } else { | |
| 664 | + do { | |
| 665 | + oldval = READ_ONCE(*f->crstep); | |
| 666 | + newval = _crste_fc1(f->pfn, oldval.h.tt, f->writable, | |
| 667 | + f->write_attempt | oldval.s.fc1.d); | |
| 668 | + newval.s.fc1.s = !f->page; | |
| 669 | + newval.s.fc1.sd = oldval.s.fc1.sd; | |
| 670 | + if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && | |
| 671 | + crste_origin_large(oldval) != crste_origin_large(newval)) | |
| 672 | + return -EAGAIN; | |
| 673 | + } while (!gmap_crstep_xchg_atomic(gmap, f->crstep, oldval, newval, f->gfn)); | |
| 674 | + if (f->callback) | |
| 675 | + f->callback(f); | |
| 676 | + } | |
| 677 | + | |
| 678 | + return rc; | |
| 679 | +} | |
| 680 | + | |
| 626 | 681 | int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fault *f) |
| 627 | 682 | { |
| 628 | 683 | unsigned int order; |
| 629 | − int rc, level; | |
| 684 | + int level; | |
| 630 | 685 | |
| 631 | 686 | lockdep_assert_held(&gmap->kvm->mmu_lock); |
| 632 | 687 | |
| @@ -638,16 +693,14 @@ int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fau · 5 unmodified lines | ||
| 638 | 693 | else if (order >= get_order(_SEGMENT_SIZE) && gmap_1m_allowed(gmap, f->gfn)) |
| 639 | 694 | level = TABLE_TYPE_SEGMENT; |
| 640 | 695 | } |
| 641 | − rc = dat_link(mc, gmap->asce, level, uses_skeys(gmap), f); | |
| 642 | − KVM_BUG_ON(rc == -EINVAL, gmap->kvm); | |
| 643 | − return rc; | |
| 696 | + return _gmap_link(mc, gmap, level, f); | |
| 644 | 697 | } |
| 645 | 698 | |
| 646 | 699 | static int gmap_ucas_map_one(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, |
| 647 | 700 | gfn_t p_gfn, gfn_t c_gfn, bool force_alloc) |
| 648 | 701 | { |
| 702 | + union crste newcrste, oldcrste; | |
| 649 | 703 | struct page_table *pt; |
| 650 | − union crste newcrste; | |
| 651 | 704 | union crste *crstep; |
| 652 | 705 | union pte *ptep; |
| 653 | 706 | int rc; |
| @@ -673,7 +726,11 @@ static int gmap_ucas_map_one(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, · 19 unmodified lines | ||
| 673 | 726 | &crstep, &ptep); |
| 674 | 727 | if (rc) |
| 675 | 728 | return rc; |
| 676 | − dat_crstep_xchg(crstep, newcrste, c_gfn, gmap->asce); | |
| 729 | + do { | |
| 730 | + oldcrste = READ_ONCE(*crstep); | |
| 731 | + if (oldcrste.val == newcrste.val) | |
| 732 | + break; | |
| 733 | + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, c_gfn, gmap->asce)); | |
| 677 | 734 | return 0; |
| 678 | 735 | } |
| 679 | 736 | |
| @@ -777,8 +834,10 @@ static void gmap_ucas_unmap_one(struct gmap *gmap, gfn_t c_gfn) · 97 unmodified lines | ||
| 777 | 834 | int rc; |
| 778 | 835 | |
| 779 | 836 | rc = dat_entry_walk(NULL, c_gfn, gmap->asce, 0, TABLE_TYPE_SEGMENT, &crstep, &ptep); |
| 780 | − if (!rc) | |
| 781 | − dat_crstep_xchg(crstep, _PMD_EMPTY, c_gfn, gmap->asce); | |
| 837 | + if (rc) | |
| 838 | + return; | |
| 839 | + while (!dat_crstep_xchg_atomic(crstep, READ_ONCE(*crstep), _PMD_EMPTY, c_gfn, gmap->asce)) | |
| 840 | + ; | |
| 782 | 841 | } |
| 783 | 842 | |
| 784 | 843 | void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count) |
| @@ -1017,8 +1076,8 @@ static void gmap_unshadow_level(struct gmap *sg, gfn_t r_gfn, int level) · 232 unmodified lines | ||
| 1017 | 1076 | dat_ptep_xchg(ptep, _PTE_EMPTY, r_gfn, sg->asce, uses_skeys(sg)); |
| 1018 | 1077 | return; |
| 1019 | 1078 | } |
| 1020 | − crste = READ_ONCE(*crstep); | |
| 1021 | − dat_crstep_clear(crstep, r_gfn, sg->asce); | |
| 1079 | + | |
| 1080 | + crste = dat_crstep_clear_atomic(crstep, r_gfn, sg->asce); | |
| 1022 | 1081 | if (crste_leaf(crste) || crste.h.i) |
| 1023 | 1082 | return; |
| 1024 | 1083 | if (is_pmd(crste)) |
| @@ -1101,6 +1160,7 @@ struct gmap_protect_asce_top_level { · 76 unmodified lines | ||
| 1101 | 1160 | static inline int __gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg, |
| 1102 | 1161 | struct gmap_protect_asce_top_level *context) |
| 1103 | 1162 | { |
| 1163 | + struct gmap *parent; | |
| 1104 | 1164 | int rc, i; |
| 1105 | 1165 | |
| 1106 | 1166 | guard(write_lock)(&sg->kvm->mmu_lock); |
| @@ -1108,7 +1168,12 @@ static inline int __gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, s · 1 unmodified line | ||
| 1108 | 1168 | if (kvm_s390_array_needs_retry_safe(sg->kvm, context->seq, context->f)) |
| 1109 | 1169 | return -EAGAIN; |
| 1110 | 1170 | |
| 1111 | − scoped_guard(spinlock, &sg->parent->children_lock) { | |
| 1171 | + parent = READ_ONCE(sg->parent); | |
| 1172 | + if (!parent) | |
| 1173 | + return -EAGAIN; | |
| 1174 | + scoped_guard(spinlock, &parent->children_lock) { | |
| 1175 | + if (READ_ONCE(sg->parent) != parent) | |
| 1176 | + return -EAGAIN; | |
| 1112 | 1177 | for (i = 0; i < CRST_TABLE_PAGES; i++) { |
| 1113 | 1178 | if (!context->f[i].valid) |
| 1114 | 1179 | continue; |
| @@ -1191,6 +1256,9 @@ struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *pare · 76 unmodified lines | ||
| 1191 | 1256 | struct gmap *sg, *new; |
| 1192 | 1257 | int rc; |
| 1193 | 1258 | |
| 1259 | + if (WARN_ON(!parent)) | |
| 1260 | + return ERR_PTR(-EINVAL); | |
| 1261 | + | |
| 1194 | 1262 | scoped_guard(spinlock, &parent->children_lock) { |
| 1195 | 1263 | sg = gmap_find_shadow(parent, asce, edat_level); |
| 1196 | 1264 | if (sg) { |
| @@ -185,6 +185,8 @@ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, un · 184 unmodified lines | ||
| 185 | 185 | else |
| 186 | 186 | _gmap_handle_vsie_unshadow_event(gmap, gfn); |
| 187 | 187 | } |
| 188 | + if (!ptep->s.d && newpte.s.d && !newpte.s.s) | |
| 189 | + SetPageDirty(pfn_to_page(newpte.h.pfra)); | |
| 188 | 190 | return __dat_ptep_xchg(ptep, pgste, newpte, gfn, gmap->asce, uses_skeys(gmap)); |
| 189 | 191 | } |
| 190 | 192 | |
| @@ -194,35 +196,42 @@ static inline union pgste gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, uni · 3 unmodified lines | ||
| 194 | 196 | return _gmap_ptep_xchg(gmap, ptep, newpte, pgste, gfn, true); |
| 195 | 197 | } |
| 196 | 198 | |
| 197 | −static inline void _gmap_crstep_xchg(struct gmap *gmap, union crste *crstep, union crste ne, | |
| 198 | − gfn_t gfn, bool needs_lock) | |
| 199 | +static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, | |
| 200 | + union crste oldcrste, union crste newcrste, | |
| 201 | + gfn_t gfn, bool needs_lock) | |
| 199 | 202 | { |
| 200 | − unsigned long align = 8 + (is_pmd(*crstep) ? 0 : 11); | |
| 203 | + unsigned long align = is_pmd(newcrste) ? _PAGE_ENTRIES : _PAGE_ENTRIES * _CRST_ENTRIES; | |
| 204 | + | |
| 205 | + if (KVM_BUG_ON(crstep->h.tt != oldcrste.h.tt || newcrste.h.tt != oldcrste.h.tt, gmap->kvm)) | |
| 206 | + return true; | |
| 201 | 207 | |
| 202 | 208 | lockdep_assert_held(&gmap->kvm->mmu_lock); |
| 203 | 209 | if (!needs_lock) |
| 204 | 210 | lockdep_assert_held(&gmap->children_lock); |
| 205 | 211 | |
| 206 | 212 | gfn = ALIGN_DOWN(gfn, align); |
| 207 | − if (crste_prefix(*crstep) && (ne.h.p || ne.h.i || !crste_prefix(ne))) { | |
| 208 | − ne.s.fc1.prefix_notif = 0; | |
| 213 | + if (crste_prefix(oldcrste) && (newcrste.h.p || newcrste.h.i || !crste_prefix(newcrste))) { | |
| 214 | + newcrste.s.fc1.prefix_notif = 0; | |
| 209 | 215 | gmap_unmap_prefix(gmap, gfn, gfn + align); |
| 210 | 216 | } |
| 211 | − if (crste_leaf(*crstep) && crstep->s.fc1.vsie_notif && | |
| 212 | − (ne.h.p || ne.h.i || !ne.s.fc1.vsie_notif)) { | |
| 213 | − ne.s.fc1.vsie_notif = 0; | |
| 217 | + if (crste_leaf(oldcrste) && oldcrste.s.fc1.vsie_notif && | |
| 218 | + (newcrste.h.p || newcrste.h.i || !newcrste.s.fc1.vsie_notif)) { | |
| 219 | + newcrste.s.fc1.vsie_notif = 0; | |
| 214 | 220 | if (needs_lock) |
| 215 | 221 | gmap_handle_vsie_unshadow_event(gmap, gfn); |
| 216 | 222 | else |
| 217 | 223 | _gmap_handle_vsie_unshadow_event(gmap, gfn); |
| 218 | 224 | } |
| 219 | − dat_crstep_xchg(crstep, ne, gfn, gmap->asce); | |
| 225 | + if (!oldcrste.s.fc1.d && newcrste.s.fc1.d && !newcrste.s.fc1.s) | |
| 226 | + SetPageDirty(phys_to_page(crste_origin_large(newcrste))); | |
| 227 | + return dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce); | |
| 220 | 228 | } |
| 221 | 229 | |
| 222 | −static inline void gmap_crstep_xchg(struct gmap *gmap, union crste *crstep, union crste ne, | |
| 223 | − gfn_t gfn) | |
| 230 | +static inline bool __must_check gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, | |
| 231 | + union crste oldcrste, union crste newcrste, | |
| 232 | + gfn_t gfn) | |
| 224 | 233 | { |
| 225 | − return _gmap_crstep_xchg(gmap, crstep, ne, gfn, true); | |
| 234 | + return _gmap_crstep_xchg_atomic(gmap, crstep, oldcrste, newcrste, gfn, true); | |
| 226 | 235 | } |
| 227 | 236 | |
| 228 | 237 | /** |
| @@ -2724,6 +2724,9 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) · 2723 unmodified lines | ||
| 2724 | 2724 | |
| 2725 | 2725 | bit = bit_nr + (addr % PAGE_SIZE) * 8; |
| 2726 | 2726 | |
| 2727 | + /* kvm_set_routing_entry() should never allow this to happen */ | |
| 2728 | + WARN_ON_ONCE(bit > (PAGE_SIZE * BITS_PER_BYTE - 1)); | |
| 2729 | + | |
| 2727 | 2730 | return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; |
| 2728 | 2731 | } |
| 2729 | 2732 | |
| @@ -2824,6 +2827,12 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, · 94 unmodified lines | ||
| 2824 | 2827 | int rc; |
| 2825 | 2828 | |
| 2826 | 2829 | mci.val = mcck_info->mcic; |
| 2830 | + | |
| 2831 | + /* log machine checks being reinjected on all debugs */ | |
| 2832 | + VCPU_EVENT(vcpu, 2, "guest machine check %lx", mci.val); | |
| 2833 | + KVM_EVENT(2, "guest machine check %lx", mci.val); | |
| 2834 | + pr_info("guest machine check pid %d: %lx", current->pid, mci.val); | |
| 2835 | + | |
| 2827 | 2836 | if (mci.sr) |
| 2828 | 2837 | cr14 |= CR14_RECOVERY_SUBMASK; |
| 2829 | 2838 | if (mci.dg) |
| @@ -2852,6 +2861,7 @@ int kvm_set_routing_entry(struct kvm *kvm, · 22 unmodified lines | ||
| 2852 | 2861 | struct kvm_kernel_irq_routing_entry *e, |
| 2853 | 2862 | const struct kvm_irq_routing_entry *ue) |
| 2854 | 2863 | { |
| 2864 | + const struct kvm_irq_routing_s390_adapter *adapter; | |
| 2855 | 2865 | u64 uaddr_s, uaddr_i; |
| 2856 | 2866 | int idx; |
| 2857 | 2867 | |
| @@ -2862,6 +2872,14 @@ int kvm_set_routing_entry(struct kvm *kvm, · 4 unmodified lines | ||
| 2862 | 2872 | return -EINVAL; |
| 2863 | 2873 | e->set = set_adapter_int; |
| 2864 | 2874 | |
| 2875 | + adapter = &ue->u.adapter; | |
| 2876 | + if (adapter->summary_addr + (adapter->summary_offset / 8) >= | |
| 2877 | + (adapter->summary_addr & PAGE_MASK) + PAGE_SIZE) | |
| 2878 | + return -EINVAL; | |
| 2879 | + if (adapter->ind_addr + (adapter->ind_offset / 8) >= | |
| 2880 | + (adapter->ind_addr & PAGE_MASK) + PAGE_SIZE) | |
| 2881 | + return -EINVAL; | |
| 2882 | + | |
| 2865 | 2883 | idx = srcu_read_lock(&kvm->srcu); |
| 2866 | 2884 | uaddr_s = gpa_to_hva(kvm, ue->u.adapter.summary_addr); |
| 2867 | 2885 | uaddr_i = gpa_to_hva(kvm, ue->u.adapter.ind_addr); |
| @@ -4617,7 +4617,7 @@ static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu) · 4616 unmodified lines | ||
| 4617 | 4617 | return 0; |
| 4618 | 4618 | } |
| 4619 | 4619 | |
| 4620 | −static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) | |
| 4620 | +static int vcpu_post_run(struct kvm_vcpu *vcpu, int sie_return) | |
| 4621 | 4621 | { |
| 4622 | 4622 | struct mcck_volatile_info *mcck_info; |
| 4623 | 4623 | struct sie_page *sie_page; |
| @@ -4633,14 +4633,14 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) · 9 unmodified lines | ||
| 4633 | 4633 | vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14; |
| 4634 | 4634 | vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; |
| 4635 | 4635 | |
| 4636 | − if (exit_reason == -EINTR) { | |
| 4637 | − VCPU_EVENT(vcpu, 3, "%s", "machine check"); | |
| 4636 | + if (sie_return == SIE64_RETURN_MCCK) { | |
| 4638 | 4637 | sie_page = container_of(vcpu->arch.sie_block, |
| 4639 | 4638 | struct sie_page, sie_block); |
| 4640 | 4639 | mcck_info = &sie_page->mcck_info; |
| 4641 | 4640 | kvm_s390_reinject_machine_check(vcpu, mcck_info); |
| 4642 | 4641 | return 0; |
| 4643 | 4642 | } |
| 4643 | + WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL); | |
| 4644 | 4644 | |
| 4645 | 4645 | if (vcpu->arch.sie_block->icptcode > 0) { |
| 4646 | 4646 | rc = kvm_handle_sie_intercept(vcpu); |
| @@ -4679,7 +4679,7 @@ int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, · 32 unmodified lines | ||
| 4679 | 4679 | #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK) |
| 4680 | 4680 | static int __vcpu_run(struct kvm_vcpu *vcpu) |
| 4681 | 4681 | { |
| 4682 | − int rc, exit_reason; | |
| 4682 | + int rc, sie_return; | |
| 4683 | 4683 | struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block; |
| 4684 | 4684 | |
| 4685 | 4685 | /* |
| @@ -4719,9 +4719,9 @@ xfer_to_guest_mode_check: · 33 unmodified lines | ||
| 4719 | 4719 | guest_timing_enter_irqoff(); |
| 4720 | 4720 | __disable_cpu_timer_accounting(vcpu); |
| 4721 | 4721 | |
| 4722 | − exit_reason = kvm_s390_enter_exit_sie(vcpu->arch.sie_block, | |
| 4723 | − vcpu->run->s.regs.gprs, | |
| 4724 | − vcpu->arch.gmap->asce.val); | |
| 4722 | + sie_return = kvm_s390_enter_exit_sie(vcpu->arch.sie_block, | |
| 4723 | + vcpu->run->s.regs.gprs, | |
| 4724 | + vcpu->arch.gmap->asce.val); | |
| 4725 | 4725 | |
| 4726 | 4726 | __enable_cpu_timer_accounting(vcpu); |
| 4727 | 4727 | guest_timing_exit_irqoff(); |
| @@ -4744,7 +4744,7 @@ xfer_to_guest_mode_check: · 16 unmodified lines | ||
| 4744 | 4744 | } |
| 4745 | 4745 | kvm_vcpu_srcu_read_lock(vcpu); |
| 4746 | 4746 | |
| 4747 | − rc = vcpu_post_run(vcpu, exit_reason); | |
| 4747 | + rc = vcpu_post_run(vcpu, sie_return); | |
| 4748 | 4748 | if (rc || guestdbg_exit_pending(vcpu)) { |
| 4749 | 4749 | kvm_vcpu_srcu_read_unlock(vcpu); |
| 4750 | 4750 | break; |
| @@ -5520,9 +5520,21 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 769 unmodified lines | ||
| 5520 | 5520 | } |
| 5521 | 5521 | #endif |
| 5522 | 5522 | case KVM_S390_VCPU_FAULT: { |
| 5523 | − idx = srcu_read_lock(&vcpu->kvm->srcu); | |
| 5524 | − r = vcpu_dat_fault_handler(vcpu, arg, 0); | |
| 5525 | − srcu_read_unlock(&vcpu->kvm->srcu, idx); | |
| 5523 | + gpa_t gaddr = arg; | |
| 5524 | + | |
| 5525 | + scoped_guard(srcu, &vcpu->kvm->srcu) { | |
| 5526 | + r = vcpu_ucontrol_translate(vcpu, &gaddr); | |
| 5527 | + if (r) | |
| 5528 | + break; | |
| 5529 | + | |
| 5530 | + r = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(gaddr), false); | |
| 5531 | + if (r == PGM_ADDRESSING) | |
| 5532 | + r = -EFAULT; | |
| 5533 | + if (r <= 0) | |
| 5534 | + break; | |
| 5535 | + r = -EIO; | |
| 5536 | + KVM_BUG_ON(r, vcpu->kvm); | |
| 5537 | + } | |
| 5526 | 5538 | break; |
| 5527 | 5539 | } |
| 5528 | 5540 | case KVM_ENABLE_CAP: |
| @@ -1122,6 +1122,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struc · 1121 unmodified lines | ||
| 1122 | 1122 | { |
| 1123 | 1123 | struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; |
| 1124 | 1124 | struct kvm_s390_sie_block *scb_o = vsie_page->scb_o; |
| 1125 | + unsigned long sie_return = SIE64_RETURN_NORMAL; | |
| 1125 | 1126 | int guest_bp_isolation; |
| 1126 | 1127 | int rc = 0; |
| 1127 | 1128 | |
| @@ -1163,7 +1164,7 @@ xfer_to_guest_mode_check: · 35 unmodified lines | ||
| 1163 | 1164 | goto xfer_to_guest_mode_check; |
| 1164 | 1165 | } |
| 1165 | 1166 | guest_timing_enter_irqoff(); |
| 1166 | − rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, sg->asce.val); | |
| 1167 | + sie_return = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, sg->asce.val); | |
| 1167 | 1168 | guest_timing_exit_irqoff(); |
| 1168 | 1169 | local_irq_enable(); |
| 1169 | 1170 | } |
| @@ -1178,12 +1179,13 @@ skip_sie: · 8 unmodified lines | ||
| 1178 | 1179 | |
| 1179 | 1180 | kvm_vcpu_srcu_read_lock(vcpu); |
| 1180 | 1181 | |
| 1181 | − if (rc == -EINTR) { | |
| 1182 | − VCPU_EVENT(vcpu, 3, "%s", "machine check"); | |
| 1182 | + if (sie_return == SIE64_RETURN_MCCK) { | |
| 1183 | 1183 | kvm_s390_reinject_machine_check(vcpu, &vsie_page->mcck_info); |
| 1184 | 1184 | return 0; |
| 1185 | 1185 | } |
| 1186 | 1186 | |
| 1187 | + WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL); | |
| 1188 | + | |
| 1187 | 1189 | if (rc > 0) |
| 1188 | 1190 | rc = 0; /* we could still have an icpt */ |
| 1189 | 1191 | else if (current->thread.gmap_int_code) |
| @@ -1326,7 +1328,7 @@ static void unregister_shadow_scb(struct kvm_vcpu *vcpu) · 136 unmodified lines | ||
| 1326 | 1328 | static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) |
| 1327 | 1329 | { |
| 1328 | 1330 | struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; |
| 1329 | − struct gmap *sg; | |
| 1331 | + struct gmap *sg = NULL; | |
| 1330 | 1332 | int rc = 0; |
| 1331 | 1333 | |
| 1332 | 1334 | while (1) { |
| @@ -1366,6 +1368,8 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) · 33 unmodified lines | ||
| 1366 | 1368 | sg = gmap_put(sg); |
| 1367 | 1369 | cond_resched(); |
| 1368 | 1370 | } |
| 1371 | + if (sg) | |
| 1372 | + sg = gmap_put(sg); | |
| 1369 | 1373 | |
| 1370 | 1374 | if (rc == -EFAULT) { |
| 1371 | 1375 | /* |
| @@ -441,10 +441,17 @@ void do_secure_storage_access(struct pt_regs *regs) · 440 unmodified lines | ||
| 441 | 441 | folio = phys_to_folio(addr); |
| 442 | 442 | if (unlikely(!folio_try_get(folio))) |
| 443 | 443 | return; |
| 444 | − rc = arch_make_folio_accessible(folio); | |
| 444 | + rc = uv_convert_from_secure(folio_to_phys(folio)); | |
| 445 | + if (!rc) | |
| 446 | + clear_bit(PG_arch_1, &folio->flags.f); | |
| 445 | 447 | folio_put(folio); |
| 448 | + /* | |
| 449 | + * There are some valid fixup types for kernel | |
| 450 | + * accesses to donated secure memory. zeropad is one | |
| 451 | + * of them. | |
| 452 | + */ | |
| 446 | 453 | if (rc) |
| 447 | − BUG(); | |
| 454 | + return handle_fault_error_nolock(regs, 0); | |
| 448 | 455 | } else { |
| 449 | 456 | if (faulthandler_disabled()) |
| 450 | 457 | return handle_fault_error_nolock(regs, 0); |
| @@ -121,6 +121,9 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) · 120 unmodified lines | ||
| 121 | 121 | |
| 122 | 122 | WARN_ON(!irqs_disabled()); |
| 123 | 123 | |
| 124 | + if (!sev_cfg.ghcbs_initialized) | |
| 125 | + return boot_ghcb; | |
| 126 | + | |
| 124 | 127 | data = this_cpu_read(runtime_data); |
| 125 | 128 | ghcb = &data->ghcb_page; |
| 126 | 129 | |
| @@ -164,6 +167,9 @@ noinstr void __sev_put_ghcb(struct ghcb_state *state) · 37 unmodified lines | ||
| 164 | 167 | |
| 165 | 168 | WARN_ON(!irqs_disabled()); |
| 166 | 169 | |
| 170 | + if (!sev_cfg.ghcbs_initialized) | |
| 171 | + return; | |
| 172 | + | |
| 167 | 173 | data = this_cpu_read(runtime_data); |
| 168 | 174 | ghcb = &data->ghcb_page; |
| 169 | 175 | |
| @@ -177,6 +177,16 @@ static noinstr void fred_extint(struct pt_regs *regs) · 176 unmodified lines | ||
| 177 | 177 | } |
| 178 | 178 | } |
| 179 | 179 | |
| 180 | +#ifdef CONFIG_AMD_MEM_ENCRYPT | |
| 181 | +noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) | |
| 182 | +{ | |
| 183 | + if (user_mode(regs)) | |
| 184 | + return user_exc_vmm_communication(regs, error_code); | |
| 185 | + else | |
| 186 | + return kernel_exc_vmm_communication(regs, error_code); | |
| 187 | +} | |
| 188 | +#endif | |
| 189 | + | |
| 180 | 190 | static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) |
| 181 | 191 | { |
| 182 | 192 | /* Optimize for #PF. That's the only exception which matters performance wise */ |
| @@ -207,6 +217,10 @@ static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) · 24 unmodified lines | ||
| 207 | 217 | #ifdef CONFIG_X86_CET |
| 208 | 218 | case X86_TRAP_CP: return exc_control_protection(regs, error_code); |
| 209 | 219 | #endif |
| 220 | +#ifdef CONFIG_AMD_MEM_ENCRYPT | |
| 221 | + case X86_TRAP_VC: return exc_vmm_communication(regs, error_code); | |
| 222 | +#endif | |
| 223 | + | |
| 210 | 224 | default: return fred_bad_type(regs, error_code); |
| 211 | 225 | } |
| 212 | 226 | |
| @@ -433,7 +433,20 @@ static __always_inline void setup_lass(struct cpuinfo_x86 *c) · 432 unmodified lines | ||
| 433 | 433 | |
| 434 | 434 | /* These bits should not change their value after CPU init is finished. */ |
| 435 | 435 | static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | |
| 436 | − X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED; | |
| 436 | + X86_CR4_FSGSBASE | X86_CR4_CET; | |
| 437 | + | |
| 438 | +/* | |
| 439 | + * The CR pinning protects against ROP on the 'mov %reg, %CRn' instruction(s). | |
| 440 | + * Since you can ROP directly to these instructions (barring shadow stack), | |
| 441 | + * any protection must follow immediately and unconditionally after that. | |
| 442 | + * | |
| 443 | + * Specifically, the CR[04] write functions below will have the value | |
| 444 | + * validation controlled by the @cr_pinning static_branch which is | |
| 445 | + * __ro_after_init, just like the cr4_pinned_bits value. | |
| 446 | + * | |
| 447 | + * Once set, an attacker will have to defeat page-tables to get around these | |
| 448 | + * restrictions. Which is a much bigger ask than 'simple' ROP. | |
| 449 | + */ | |
| 437 | 450 | static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); |
| 438 | 451 | static unsigned long cr4_pinned_bits __ro_after_init; |
| 439 | 452 | |
| @@ -2050,12 +2063,6 @@ static void identify_cpu(struct cpuinfo_x86 *c) · 1610 unmodified lines | ||
| 2050 | 2063 | setup_umip(c); |
| 2051 | 2064 | setup_lass(c); |
| 2052 | 2065 | |
| 2053 | − /* Enable FSGSBASE instructions if available. */ | |
| 2054 | − if (cpu_has(c, X86_FEATURE_FSGSBASE)) { | |
| 2055 | − cr4_set_bits(X86_CR4_FSGSBASE); | |
| 2056 | − elf_hwcap2 |= HWCAP2_FSGSBASE; | |
| 2057 | − } | |
| 2058 | − | |
| 2059 | 2066 | /* |
| 2060 | 2067 | * The vendor-specific functions might have changed features. |
| 2061 | 2068 | * Now we do "generic changes." |
| @@ -2416,6 +2423,18 @@ void cpu_init_exception_handling(bool boot_cpu) · 354 unmodified lines | ||
| 2416 | 2423 | /* GHCB needs to be setup to handle #VC. */ |
| 2417 | 2424 | setup_ghcb(); |
| 2418 | 2425 | |
| 2426 | + /* | |
| 2427 | + * On CPUs with FSGSBASE support, paranoid_entry() uses | |
| 2428 | + * ALTERNATIVE-patched RDGSBASE/WRGSBASE instructions. Secondary CPUs | |
| 2429 | + * boot after alternatives are patched globally, so early exceptions | |
| 2430 | + * execute patched code that depends on FSGSBASE. Enable the feature | |
| 2431 | + * before any exceptions occur. | |
| 2432 | + */ | |
| 2433 | + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { | |
| 2434 | + cr4_set_bits(X86_CR4_FSGSBASE); | |
| 2435 | + elf_hwcap2 |= HWCAP2_FSGSBASE; | |
| 2436 | + } | |
| 2437 | + | |
| 2419 | 2438 | if (cpu_feature_enabled(X86_FEATURE_FRED)) { |
| 2420 | 2439 | /* The boot CPU has enabled FRED during early boot */ |
| 2421 | 2440 | if (!boot_cpu) |
| @@ -3044,12 +3044,6 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, · 3043 unmodified lines | ||
| 3044 | 3044 | bool prefetch = !fault || fault->prefetch; |
| 3045 | 3045 | bool write_fault = fault && fault->write; |
| 3046 | 3046 | |
| 3047 | − if (unlikely(is_noslot_pfn(pfn))) { | |
| 3048 | − vcpu->stat.pf_mmio_spte_created++; | |
| 3049 | − mark_mmio_spte(vcpu, sptep, gfn, pte_access); | |
| 3050 | − return RET_PF_EMULATE; | |
| 3051 | − } | |
| 3052 | − | |
| 3053 | 3047 | if (is_shadow_present_pte(*sptep)) { |
| 3054 | 3048 | if (prefetch && is_last_spte(*sptep, level) && |
| 3055 | 3049 | pfn == spte_to_pfn(*sptep)) |
| @@ -3066,13 +3060,22 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, · 10 unmodified lines | ||
| 3066 | 3060 | child = spte_to_child_sp(pte); |
| 3067 | 3061 | drop_parent_pte(vcpu->kvm, child, sptep); |
| 3068 | 3062 | flush = true; |
| 3069 | − } else if (WARN_ON_ONCE(pfn != spte_to_pfn(*sptep))) { | |
| 3063 | + } else if (pfn != spte_to_pfn(*sptep)) { | |
| 3064 | + WARN_ON_ONCE(vcpu->arch.mmu->root_role.direct); | |
| 3070 | 3065 | drop_spte(vcpu->kvm, sptep); |
| 3071 | 3066 | flush = true; |
| 3072 | 3067 | } else |
| 3073 | 3068 | was_rmapped = 1; |
| 3074 | 3069 | } |
| 3075 | 3070 | |
| 3071 | + if (unlikely(is_noslot_pfn(pfn))) { | |
| 3072 | + vcpu->stat.pf_mmio_spte_created++; | |
| 3073 | + mark_mmio_spte(vcpu, sptep, gfn, pte_access); | |
| 3074 | + if (flush) | |
| 3075 | + kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level); | |
| 3076 | + return RET_PF_EMULATE; | |
| 3077 | + } | |
| 3078 | + | |
| 3076 | 3079 | wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch, |
| 3077 | 3080 | false, host_writable, &spte); |
| 3078 | 3081 | |
| @@ -424,7 +424,7 @@ void __init efi_unmap_boot_services(void) · 423 unmodified lines | ||
| 424 | 424 | if (efi_enabled(EFI_DBG)) |
| 425 | 425 | return; |
| 426 | 426 | |
| 427 | − sz = sizeof(*ranges_to_free) * efi.memmap.nr_map + 1; | |
| 427 | + sz = sizeof(*ranges_to_free) * (efi.memmap.nr_map + 1); | |
| 428 | 428 | ranges_to_free = kzalloc(sz, GFP_KERNEL); |
| 429 | 429 | if (!ranges_to_free) { |
| 430 | 430 | pr_err("Failed to allocate storage for freeable EFI regions\n"); |
| @@ -35,6 +35,7 @@ · 34 unmodified lines | ||
| 35 | 35 | #define IVPU_HW_IP_60XX 60 |
| 36 | 36 | |
| 37 | 37 | #define IVPU_HW_IP_REV_LNL_B0 4 |
| 38 | +#define IVPU_HW_IP_REV_NVL_A0 0 | |
| 38 | 39 | |
| 39 | 40 | #define IVPU_HW_BTRS_MTL 1 |
| 40 | 41 | #define IVPU_HW_BTRS_LNL 2 |
| @@ -70,8 +70,10 @@ static void wa_init(struct ivpu_device *vdev) · 69 unmodified lines | ||
| 70 | 70 | if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL) |
| 71 | 71 | vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev); |
| 72 | 72 | |
| 73 | − if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL && | |
| 74 | − ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) | |
| 73 | + if ((ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL && | |
| 74 | + ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) || | |
| 75 | + (ivpu_device_id(vdev) == PCI_DEVICE_ID_NVL && | |
| 76 | + ivpu_revision(vdev) == IVPU_HW_IP_REV_NVL_A0)) | |
| 75 | 77 | vdev->wa.disable_clock_relinquish = true; |
| 76 | 78 | |
| 77 | 79 | if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE) |
| @@ -1656,6 +1656,8 @@ static int acpi_ec_setup(struct acpi_ec *ec, struct acpi_device *device, bool ca · 1655 unmodified lines | ||
| 1656 | 1656 | |
| 1657 | 1657 | ret = ec_install_handlers(ec, device, call_reg); |
| 1658 | 1658 | if (ret) { |
| 1659 | + ec_remove_handlers(ec); | |
| 1660 | + | |
| 1659 | 1661 | if (ec == first_ec) |
| 1660 | 1662 | first_ec = NULL; |
| 1661 | 1663 | |
| @@ -1545,6 +1545,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, · 1544 unmodified lines | ||
| 1545 | 1545 | unsigned int val_num) |
| 1546 | 1546 | { |
| 1547 | 1547 | void *orig_work_buf; |
| 1548 | + unsigned int selector_reg; | |
| 1548 | 1549 | unsigned int win_offset; |
| 1549 | 1550 | unsigned int win_page; |
| 1550 | 1551 | bool page_chg; |
| @@ -1563,10 +1564,31 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, · 12 unmodified lines | ||
| 1563 | 1564 | return -EINVAL; |
| 1564 | 1565 | } |
| 1565 | 1566 | |
| 1566 | − /* It is possible to have selector register inside data window. | |
| 1567 | − In that case, selector register is located on every page and | |
| 1568 | − it needs no page switching, when accessed alone. */ | |
| 1567 | + /* | |
| 1568 | + * Calculate the address of the selector register in the corresponding | |
| 1569 | + * data window if it is located on every page. | |
| 1570 | + */ | |
| 1571 | + page_chg = in_range(range->selector_reg, range->window_start, range->window_len); | |
| 1572 | + if (page_chg) | |
| 1573 | + selector_reg = range->range_min + win_page * range->window_len + | |
| 1574 | + range->selector_reg - range->window_start; | |
| 1575 | + | |
| 1576 | + /* | |
| 1577 | + * It is possible to have selector register inside data window. | |
| 1578 | + * In that case, selector register is located on every page and it | |
| 1579 | + * needs no page switching, when accessed alone. | |
| 1580 | + * | |
| 1581 | + * Nevertheless we should synchronize the cache values for it. | |
| 1582 | + * This can't be properly achieved if the selector register is | |
| 1583 | + * the first and the only one to be read inside the data window. | |
| 1584 | + * That's why we update it in that case as well. | |
| 1585 | + * | |
| 1586 | + * However, we specifically avoid updating it for the default page, | |
| 1587 | + * when it's overlapped with the real data window, to prevent from | |
| 1588 | + * infinite looping. | |
| 1589 | + */ | |
| 1569 | 1590 | if (val_num > 1 || |
| 1591 | + (page_chg && selector_reg != range->selector_reg) || | |
| 1570 | 1592 | range->window_start + win_offset != range->selector_reg) { |
| 1571 | 1593 | /* Use separate work_buf during page switching */ |
| 1572 | 1594 | orig_work_buf = map->work_buf; |
| @@ -1575,7 +1597,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, · 2 unmodified lines | ||
| 1575 | 1597 | ret = _regmap_update_bits(map, range->selector_reg, |
| 1576 | 1598 | range->selector_mask, |
| 1577 | 1599 | win_page << range->selector_shift, |
| 1578 | − &page_chg, false); | |
| 1600 | + NULL, false); | |
| 1579 | 1601 | |
| 1580 | 1602 | map->work_buf = orig_work_buf; |
| 1581 | 1603 | |
| @@ -917,9 +917,8 @@ static void zram_account_writeback_submit(struct zram *zram) · 916 unmodified lines | ||
| 917 | 917 | |
| 918 | 918 | static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req) |
| 919 | 919 | { |
| 920 | − u32 size, index = req->pps->index; | |
| 921 | − int err, prio; | |
| 922 | − bool huge; | |
| 920 | + u32 index = req->pps->index; | |
| 921 | + int err; | |
| 923 | 922 | |
| 924 | 923 | err = blk_status_to_errno(req->bio.bi_status); |
| 925 | 924 | if (err) { |
| @@ -946,28 +945,13 @@ static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req) · 20 unmodified lines | ||
| 946 | 945 | goto out; |
| 947 | 946 | } |
| 948 | 947 | |
| 949 | − if (zram->compressed_wb) { | |
| 950 | − /* | |
| 951 | − * ZRAM_WB slots get freed, we need to preserve data required | |
| 952 | − * for read decompression. | |
| 953 | − */ | |
| 954 | − size = get_slot_size(zram, index); | |
| 955 | − prio = get_slot_comp_priority(zram, index); | |
| 956 | − huge = test_slot_flag(zram, index, ZRAM_HUGE); | |
| 957 | − } | |
| 958 | − | |
| 959 | − slot_free(zram, index); | |
| 960 | − set_slot_flag(zram, index, ZRAM_WB); | |
| 948 | + clear_slot_flag(zram, index, ZRAM_IDLE); | |
| 949 | + if (test_slot_flag(zram, index, ZRAM_HUGE)) | |
| 950 | + atomic64_dec(&zram->stats.huge_pages); | |
| 951 | + atomic64_sub(get_slot_size(zram, index), &zram->stats.compr_data_size); | |
| 952 | + zs_free(zram->mem_pool, get_slot_handle(zram, index)); | |
| 961 | 953 | set_slot_handle(zram, index, req->blk_idx); |
| 962 | − | |
| 963 | − if (zram->compressed_wb) { | |
| 964 | − if (huge) | |
| 965 | − set_slot_flag(zram, index, ZRAM_HUGE); | |
| 966 | − set_slot_size(zram, index, size); | |
| 967 | − set_slot_comp_priority(zram, index, prio); | |
| 968 | − } | |
| 969 | − | |
| 970 | − atomic64_inc(&zram->stats.pages_stored); | |
| 954 | + set_slot_flag(zram, index, ZRAM_WB); | |
| 971 | 955 | |
| 972 | 956 | out: |
| 973 | 957 | slot_unlock(zram, index); |
| @@ -2010,8 +1994,13 @@ static void slot_free(struct zram *zram, u32 index) · 1036 unmodified lines | ||
| 2010 | 1994 | set_slot_comp_priority(zram, index, 0); |
| 2011 | 1995 | |
| 2012 | 1996 | if (test_slot_flag(zram, index, ZRAM_HUGE)) { |
| 1997 | + /* | |
| 1998 | + * Writeback completion decrements ->huge_pages but keeps | |
| 1999 | + * ZRAM_HUGE flag for deferred decompression path. | |
| 2000 | + */ | |
| 2001 | + if (!test_slot_flag(zram, index, ZRAM_WB)) | |
| 2002 | + atomic64_dec(&zram->stats.huge_pages); | |
| 2013 | 2003 | clear_slot_flag(zram, index, ZRAM_HUGE); |
| 2014 | − atomic64_dec(&zram->stats.huge_pages); | |
| 2015 | 2004 | } |
| 2016 | 2005 | |
| 2017 | 2006 | if (test_slot_flag(zram, index, ZRAM_WB)) { |
| @@ -251,11 +251,13 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code) · 250 unmodified lines | ||
| 251 | 251 | |
| 252 | 252 | bt_dev_err(hdev, "Hardware error 0x%2.2x", code); |
| 253 | 253 | |
| 254 | + hci_req_sync_lock(hdev); | |
| 255 | + | |
| 254 | 256 | skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT); |
| 255 | 257 | if (IS_ERR(skb)) { |
| 256 | 258 | bt_dev_err(hdev, "Reset after hardware error failed (%ld)", |
| 257 | 259 | PTR_ERR(skb)); |
| 258 | − return; | |
| 260 | + goto unlock; | |
| 259 | 261 | } |
| 260 | 262 | kfree_skb(skb); |
| 261 | 263 | |
| @@ -263,18 +265,21 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code) · 1 unmodified line | ||
| 263 | 265 | if (IS_ERR(skb)) { |
| 264 | 266 | bt_dev_err(hdev, "Retrieving Intel exception info failed (%ld)", |
| 265 | 267 | PTR_ERR(skb)); |
| 266 | − return; | |
| 268 | + goto unlock; | |
| 267 | 269 | } |
| 268 | 270 | |
| 269 | 271 | if (skb->len != 13) { |
| 270 | 272 | bt_dev_err(hdev, "Exception info size mismatch"); |
| 271 | 273 | kfree_skb(skb); |
| 272 | − return; | |
| 274 | + goto unlock; | |
| 273 | 275 | } |
| 274 | 276 | |
| 275 | 277 | bt_dev_err(hdev, "Exception info %s", (char *)(skb->data + 1)); |
| 276 | 278 | |
| 277 | 279 | kfree_skb(skb); |
| 280 | + | |
| 281 | +unlock: | |
| 282 | + hci_req_sync_unlock(hdev); | |
| 278 | 283 | } |
| 279 | 284 | EXPORT_SYMBOL_GPL(btintel_hw_error); |
| 280 | 285 | |
| @@ -2376,8 +2376,11 @@ static void btusb_work(struct work_struct *work) · 2375 unmodified lines | ||
| 2376 | 2376 | if (data->air_mode == HCI_NOTIFY_ENABLE_SCO_CVSD) { |
| 2377 | 2377 | if (hdev->voice_setting & 0x0020) { |
| 2378 | 2378 | static const int alts[3] = { 2, 4, 5 }; |
| 2379 | + unsigned int sco_idx; | |
| 2379 | 2380 | |
| 2380 | − new_alts = alts[data->sco_num - 1]; | |
| 2381 | + sco_idx = min_t(unsigned int, data->sco_num - 1, | |
| 2382 | + ARRAY_SIZE(alts) - 1); | |
| 2383 | + new_alts = alts[sco_idx]; | |
| 2381 | 2384 | } else { |
| 2382 | 2385 | new_alts = data->sco_num; |
| 2383 | 2386 | } |
| @@ -541,6 +541,8 @@ static int download_firmware(struct ll_device *lldev) · 540 unmodified lines | ||
| 541 | 541 | if (err || !fw->data || !fw->size) { |
| 542 | 542 | bt_dev_err(lldev->hu.hdev, "request_firmware failed(errno %d) for %s", |
| 543 | 543 | err, bts_scr_name); |
| 544 | + if (!err) | |
| 545 | + release_firmware(fw); | |
| 544 | 546 | return -EINVAL; |
| 545 | 547 | } |
| 546 | 548 | ptr = (void *)fw->data; |
| @@ -1427,12 +1427,9 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, · 1426 unmodified lines | ||
| 1427 | 1427 | * If there is a problem with its frequency table, take it |
| 1428 | 1428 | * offline and drop it. |
| 1429 | 1429 | */ |
| 1430 | − if (policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_ASCENDING && | |
| 1431 | − policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_DESCENDING) { | |
| 1432 | − ret = cpufreq_table_validate_and_sort(policy); | |
| 1433 | − if (ret) | |
| 1434 | − goto out_offline_policy; | |
| 1435 | − } | |
| 1430 | + ret = cpufreq_table_validate_and_sort(policy); | |
| 1431 | + if (ret) | |
| 1432 | + goto out_offline_policy; | |
| 1436 | 1433 | |
| 1437 | 1434 | /* related_cpus should at least include policy->cpus. */ |
| 1438 | 1435 | cpumask_copy(policy->related_cpus, policy->cpus); |
| @@ -313,6 +313,17 @@ static void cs_start(struct cpufreq_policy *policy) · 312 unmodified lines | ||
| 313 | 313 | dbs_info->requested_freq = policy->cur; |
| 314 | 314 | } |
| 315 | 315 | |
| 316 | +static void cs_limits(struct cpufreq_policy *policy) | |
| 317 | +{ | |
| 318 | + struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); | |
| 319 | + | |
| 320 | + /* | |
| 321 | + * The limits have changed, so may have the current frequency. Reset | |
| 322 | + * requested_freq to avoid any unintended outcomes due to the mismatch. | |
| 323 | + */ | |
| 324 | + dbs_info->requested_freq = policy->cur; | |
| 325 | +} | |
| 326 | + | |
| 316 | 327 | static struct dbs_governor cs_governor = { |
| 317 | 328 | .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), |
| 318 | 329 | .kobj_type = { .default_groups = cs_groups }, |
| @@ -322,6 +333,7 @@ static struct dbs_governor cs_governor = { · 3 unmodified lines | ||
| 322 | 333 | .init = cs_init, |
| 323 | 334 | .exit = cs_exit, |
| 324 | 335 | .start = cs_start, |
| 336 | + .limits = cs_limits, | |
| 325 | 337 | }; |
| 326 | 338 | |
| 327 | 339 | #define CPU_FREQ_GOV_CONSERVATIVE (cs_governor.gov) |
| @@ -563,6 +563,7 @@ EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop); · 562 unmodified lines | ||
| 563 | 563 | |
| 564 | 564 | void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) |
| 565 | 565 | { |
| 566 | + struct dbs_governor *gov = dbs_governor_of(policy); | |
| 566 | 567 | struct policy_dbs_info *policy_dbs; |
| 567 | 568 | |
| 568 | 569 | /* Protect gov->gdbs_data against cpufreq_dbs_governor_exit() */ |
| @@ -574,6 +575,8 @@ void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) · 5 unmodified lines | ||
| 574 | 575 | mutex_lock(&policy_dbs->update_mutex); |
| 575 | 576 | cpufreq_policy_apply_limits(policy); |
| 576 | 577 | gov_update_sample_delay(policy_dbs, 0); |
| 578 | + if (gov->limits) | |
| 579 | + gov->limits(policy); | |
| 577 | 580 | mutex_unlock(&policy_dbs->update_mutex); |
| 578 | 581 | |
| 579 | 582 | out: |
| @@ -138,6 +138,7 @@ struct dbs_governor { · 137 unmodified lines | ||
| 138 | 138 | int (*init)(struct dbs_data *dbs_data); |
| 139 | 139 | void (*exit)(struct dbs_data *dbs_data); |
| 140 | 140 | void (*start)(struct cpufreq_policy *policy); |
| 141 | + void (*limits)(struct cpufreq_policy *policy); | |
| 141 | 142 | }; |
| 142 | 143 | |
| 143 | 144 | static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy) |
| @@ -360,6 +360,10 @@ int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy) · 359 unmodified lines | ||
| 360 | 360 | if (policy_has_boost_freq(policy)) |
| 361 | 361 | policy->boost_supported = true; |
| 362 | 362 | |
| 363 | + if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING || | |
| 364 | + policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_DESCENDING) | |
| 365 | + return 0; | |
| 366 | + | |
| 363 | 367 | return set_freq_table_sorted(policy); |
| 364 | 368 | } |
| 365 | 369 | |
| @@ -59,6 +59,7 @@ config CXL_ACPI · 58 unmodified lines | ||
| 59 | 59 | tristate "CXL ACPI: Platform Support" |
| 60 | 60 | depends on ACPI |
| 61 | 61 | depends on ACPI_NUMA |
| 62 | + depends on CXL_PMEM || !CXL_PMEM | |
| 62 | 63 | default CXL_BUS |
| 63 | 64 | select ACPI_TABLE_LIB |
| 64 | 65 | select ACPI_HMAT |
| @@ -94,7 +94,6 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) · 93 unmodified lines | ||
| 94 | 94 | struct cxl_hdm *cxlhdm; |
| 95 | 95 | void __iomem *hdm; |
| 96 | 96 | u32 ctrl; |
| 97 | − int i; | |
| 98 | 97 | |
| 99 | 98 | if (!info) |
| 100 | 99 | return false; |
| @@ -113,22 +112,16 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) · 12 unmodified lines | ||
| 113 | 112 | return false; |
| 114 | 113 | |
| 115 | 114 | /* |
| 116 | − * If any decoders are committed already, there should not be any | |
| 117 | − * emulated DVSEC decoders. | |
| 115 | + * If HDM decoders are globally enabled, do not fall back to DVSEC | |
| 116 | + * range emulation. Zeroed decoder registers after region teardown | |
| 117 | + * do not imply absence of HDM capability. | |
| 118 | + * | |
| 119 | + * Falling back to DVSEC here would treat the decoder as AUTO and | |
| 120 | + * may incorrectly latch default interleave settings. | |
| 118 | 121 | */ |
| 119 | − for (i = 0; i < cxlhdm->decoder_count; i++) { | |
| 120 | − ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); | |
| 121 | − dev_dbg(&info->port->dev, | |
| 122 | − "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n", | |
| 123 | − info->port->id, i, | |
| 124 | − FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl), | |
| 125 | − readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)), | |
| 126 | − readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)), | |
| 127 | − readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)), | |
| 128 | − readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i))); | |
| 129 | − if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) | |
| 130 | − return false; | |
| 131 | − } | |
| 122 | + ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); | |
| 123 | + if (ctrl & CXL_HDM_DECODER_ENABLE) | |
| 124 | + return false; | |
| 132 | 125 | |
| 133 | 126 | return true; |
| 134 | 127 | } |
| @@ -1301,7 +1301,7 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd) · 1300 unmodified lines | ||
| 1301 | 1301 | * Require an endpoint to be safe otherwise the driver can not |
| 1302 | 1302 | * be sure that the device is unmapped. |
| 1303 | 1303 | */ |
| 1304 | − if (endpoint && cxl_num_decoders_committed(endpoint) == 0) | |
| 1304 | + if (cxlmd->dev.driver && cxl_num_decoders_committed(endpoint) == 0) | |
| 1305 | 1305 | return __cxl_mem_sanitize(mds, cmd); |
| 1306 | 1306 | |
| 1307 | 1307 | return -EBUSY; |
| @@ -552,10 +552,13 @@ static void cxl_port_release(struct device *dev) · 551 unmodified lines | ||
| 552 | 552 | xa_destroy(&port->dports); |
| 553 | 553 | xa_destroy(&port->regions); |
| 554 | 554 | ida_free(&cxl_port_ida, port->id); |
| 555 | − if (is_cxl_root(port)) | |
| 555 | + | |
| 556 | + if (is_cxl_root(port)) { | |
| 556 | 557 | kfree(to_cxl_root(port)); |
| 557 | − else | |
| 558 | + } else { | |
| 559 | + put_device(dev->parent); | |
| 558 | 560 | kfree(port); |
| 561 | + } | |
| 559 | 562 | } |
| 560 | 563 | |
| 561 | 564 | static ssize_t decoders_committed_show(struct device *dev, |
| @@ -707,6 +710,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, · 145 unmodified lines | ||
| 707 | 710 | struct cxl_port *iter; |
| 708 | 711 | |
| 709 | 712 | dev->parent = &parent_port->dev; |
| 713 | + get_device(dev->parent); | |
| 710 | 714 | port->depth = parent_port->depth + 1; |
| 711 | 715 | port->parent_dport = parent_dport; |
| 712 | 716 | |
| @@ -3854,8 +3854,10 @@ static int __construct_region(struct cxl_region *cxlr, · 3853 unmodified lines | ||
| 3854 | 3854 | } |
| 3855 | 3855 | |
| 3856 | 3856 | rc = sysfs_update_group(&cxlr->dev.kobj, &cxl_region_group); |
| 3857 | − if (rc) | |
| 3857 | + if (rc) { | |
| 3858 | + kfree(res); | |
| 3858 | 3859 | return rc; |
| 3860 | + } | |
| 3859 | 3861 | |
| 3860 | 3862 | rc = insert_resource(cxlrd->res, res); |
| 3861 | 3863 | if (rc) { |
| @@ -554,7 +554,7 @@ static __exit void cxl_pmem_exit(void) · 553 unmodified lines | ||
| 554 | 554 | |
| 555 | 555 | MODULE_DESCRIPTION("CXL PMEM: Persistent Memory Support"); |
| 556 | 556 | MODULE_LICENSE("GPL v2"); |
| 557 | −module_init(cxl_pmem_init); | |
| 557 | +subsys_initcall(cxl_pmem_init); | |
| 558 | 558 | module_exit(cxl_pmem_exit); |
| 559 | 559 | MODULE_IMPORT_NS("CXL"); |
| 560 | 560 | MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); |
| @@ -844,6 +844,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, · 843 unmodified lines | ||
| 844 | 844 | { |
| 845 | 845 | struct dw_edma_chip *chip = dw->chip; |
| 846 | 846 | struct device *dev = dw->chip->dev; |
| 847 | + struct msi_desc *msi_desc; | |
| 847 | 848 | u32 wr_mask = 1; |
| 848 | 849 | u32 rd_mask = 1; |
| 849 | 850 | int i, err = 0; |
| @@ -895,9 +896,12 @@ static int dw_edma_irq_request(struct dw_edma *dw, · 45 unmodified lines | ||
| 895 | 896 | &dw->irq[i]); |
| 896 | 897 | if (err) |
| 897 | 898 | goto err_irq_free; |
| 898 | − | |
| 899 | − if (irq_get_msi_desc(irq)) | |
| 899 | + msi_desc = irq_get_msi_desc(irq); | |
| 900 | + if (msi_desc) { | |
| 900 | 901 | get_cached_msi_msg(irq, &dw->irq[i].msi); |
| 902 | + if (!msi_desc->pci.msi_attrib.is_msix) | |
| 903 | + dw->irq[i].msi.data = dw->irq[0].msi.data + i; | |
| 904 | + } | |
| 901 | 905 | } |
| 902 | 906 | |
| 903 | 907 | dw->nr_irqs = i; |
| @@ -252,10 +252,10 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) · 251 unmodified lines | ||
| 252 | 252 | lower_32_bits(chunk->ll_region.paddr)); |
| 253 | 253 | SET_CH_32(dw, chan->dir, chan->id, llp.msb, |
| 254 | 254 | upper_32_bits(chunk->ll_region.paddr)); |
| 255 | + /* Set consumer cycle */ | |
| 256 | + SET_CH_32(dw, chan->dir, chan->id, cycle_sync, | |
| 257 | + HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); | |
| 255 | 258 | } |
| 256 | − /* Set consumer cycle */ | |
| 257 | − SET_CH_32(dw, chan->dir, chan->id, cycle_sync, | |
| 258 | − HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); | |
| 259 | 259 | |
| 260 | 260 | dw_hdma_v0_sync_ll_data(chunk); |
| 261 | 261 | |
| @@ -317,10 +317,8 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, · 316 unmodified lines | ||
| 317 | 317 | return NULL; |
| 318 | 318 | i = fsl_chan - fsl_edma->chans; |
| 319 | 319 | |
| 320 | − fsl_chan->priority = dma_spec->args[1]; | |
| 321 | − fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; | |
| 322 | − fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; | |
| 323 | − fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; | |
| 320 | + if (!b_chmux && i != dma_spec->args[0]) | |
| 321 | + continue; | |
| 324 | 322 | |
| 325 | 323 | if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) |
| 326 | 324 | continue; |
| @@ -328,17 +326,15 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, · 1 unmodified line | ||
| 328 | 326 | if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) |
| 329 | 327 | continue; |
| 330 | 328 | |
| 331 | − if (!b_chmux && i == dma_spec->args[0]) { | |
| 332 | − chan = dma_get_slave_channel(chan); | |
| 333 | − chan->device->privatecnt++; | |
| 334 | − return chan; | |
| 335 | − } else if (b_chmux && !fsl_chan->srcid) { | |
| 336 | − /* if controller support channel mux, choose a free channel */ | |
| 337 | − chan = dma_get_slave_channel(chan); | |
| 338 | − chan->device->privatecnt++; | |
| 339 | − fsl_chan->srcid = dma_spec->args[0]; | |
| 340 | − return chan; | |
| 341 | − } | |
| 329 | + fsl_chan->srcid = dma_spec->args[0]; | |
| 330 | + fsl_chan->priority = dma_spec->args[1]; | |
| 331 | + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; | |
| 332 | + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; | |
| 333 | + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; | |
| 334 | + | |
| 335 | + chan = dma_get_slave_channel(chan); | |
| 336 | + chan->device->privatecnt++; | |
| 337 | + return chan; | |
| 342 | 338 | } |
| 343 | 339 | return NULL; |
| 344 | 340 | } |
| @@ -158,11 +158,7 @@ static const struct device_type idxd_cdev_file_type = { · 157 unmodified lines | ||
| 158 | 158 | static void idxd_cdev_dev_release(struct device *dev) |
| 159 | 159 | { |
| 160 | 160 | struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); |
| 161 | − struct idxd_cdev_context *cdev_ctx; | |
| 162 | − struct idxd_wq *wq = idxd_cdev->wq; | |
| 163 | 161 | |
| 164 | − cdev_ctx = &ictx[wq->idxd->data->type]; | |
| 165 | − ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); | |
| 166 | 162 | kfree(idxd_cdev); |
| 167 | 163 | } |
| 168 | 164 | |
| @@ -582,11 +578,15 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) · 413 unmodified lines | ||
| 582 | 578 | |
| 583 | 579 | void idxd_wq_del_cdev(struct idxd_wq *wq) |
| 584 | 580 | { |
| 581 | + struct idxd_cdev_context *cdev_ctx; | |
| 585 | 582 | struct idxd_cdev *idxd_cdev; |
| 586 | 583 | |
| 587 | 584 | idxd_cdev = wq->idxd_cdev; |
| 588 | 585 | wq->idxd_cdev = NULL; |
| 589 | 586 | cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); |
| 587 | + | |
| 588 | + cdev_ctx = &ictx[wq->idxd->data->type]; | |
| 589 | + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); | |
| 590 | 590 | put_device(cdev_dev(idxd_cdev)); |
| 591 | 591 | } |
| 592 | 592 | |
| @@ -175,6 +175,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq) · 174 unmodified lines | ||
| 175 | 175 | free_descs(wq); |
| 176 | 176 | dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); |
| 177 | 177 | sbitmap_queue_free(&wq->sbq); |
| 178 | + wq->type = IDXD_WQT_NONE; | |
| 178 | 179 | } |
| 179 | 180 | EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, "IDXD"); |
| 180 | 181 | |
| @@ -382,7 +383,6 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) · 201 unmodified lines | ||
| 382 | 383 | lockdep_assert_held(&wq->wq_lock); |
| 383 | 384 | wq->state = IDXD_WQ_DISABLED; |
| 384 | 385 | memset(wq->wqcfg, 0, idxd->wqcfg_size); |
| 385 | − wq->type = IDXD_WQT_NONE; | |
| 386 | 386 | wq->threshold = 0; |
| 387 | 387 | wq->priority = 0; |
| 388 | 388 | wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; |
| @@ -831,8 +831,7 @@ static void idxd_device_evl_free(struct idxd_device *idxd) · 442 unmodified lines | ||
| 831 | 831 | struct device *dev = &idxd->pdev->dev; |
| 832 | 832 | struct idxd_evl *evl = idxd->evl; |
| 833 | 833 | |
| 834 | − gencfg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); | |
| 835 | − if (!gencfg.evl_en) | |
| 834 | + if (!evl) | |
| 836 | 835 | return; |
| 837 | 836 | |
| 838 | 837 | mutex_lock(&evl->lock); |
| @@ -1125,7 +1124,11 @@ int idxd_device_config(struct idxd_device *idxd) · 286 unmodified lines | ||
| 1125 | 1124 | { |
| 1126 | 1125 | int rc; |
| 1127 | 1126 | |
| 1128 | − lockdep_assert_held(&idxd->dev_lock); | |
| 1127 | + guard(spinlock)(&idxd->dev_lock); | |
| 1128 | + | |
| 1129 | + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) | |
| 1130 | + return 0; | |
| 1131 | + | |
| 1129 | 1132 | rc = idxd_wqs_setup(idxd); |
| 1130 | 1133 | if (rc < 0) |
| 1131 | 1134 | return rc; |
| @@ -1332,6 +1335,11 @@ void idxd_wq_free_irq(struct idxd_wq *wq) · 200 unmodified lines | ||
| 1332 | 1335 | |
| 1333 | 1336 | free_irq(ie->vector, ie); |
| 1334 | 1337 | idxd_flush_pending_descs(ie); |
| 1338 | + | |
| 1339 | + /* The interrupt might have been already released by FLR */ | |
| 1340 | + if (ie->int_handle == INVALID_INT_HANDLE) | |
| 1341 | + return; | |
| 1342 | + | |
| 1335 | 1343 | if (idxd->request_int_handles) |
| 1336 | 1344 | idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); |
| 1337 | 1345 | idxd_device_clear_perm_entry(idxd, ie); |
| @@ -1340,6 +1348,23 @@ void idxd_wq_free_irq(struct idxd_wq *wq) · 2 unmodified lines | ||
| 1340 | 1348 | ie->pasid = IOMMU_PASID_INVALID; |
| 1341 | 1349 | } |
| 1342 | 1350 | |
| 1351 | +void idxd_wq_flush_descs(struct idxd_wq *wq) | |
| 1352 | +{ | |
| 1353 | + struct idxd_irq_entry *ie = &wq->ie; | |
| 1354 | + struct idxd_device *idxd = wq->idxd; | |
| 1355 | + | |
| 1356 | + guard(mutex)(&wq->wq_lock); | |
| 1357 | + | |
| 1358 | + if (wq->state != IDXD_WQ_ENABLED || wq->type != IDXD_WQT_KERNEL) | |
| 1359 | + return; | |
| 1360 | + | |
| 1361 | + idxd_flush_pending_descs(ie); | |
| 1362 | + if (idxd->request_int_handles) | |
| 1363 | + idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); | |
| 1364 | + idxd_device_clear_perm_entry(idxd, ie); | |
| 1365 | + ie->int_handle = INVALID_INT_HANDLE; | |
| 1366 | +} | |
| 1367 | + | |
| 1343 | 1368 | int idxd_wq_request_irq(struct idxd_wq *wq) |
| 1344 | 1369 | { |
| 1345 | 1370 | struct idxd_device *idxd = wq->idxd; |
| @@ -1454,11 +1479,7 @@ int idxd_drv_enable_wq(struct idxd_wq *wq) · 108 unmodified lines | ||
| 1454 | 1479 | } |
| 1455 | 1480 | } |
| 1456 | 1481 | |
| 1457 | − rc = 0; | |
| 1458 | − spin_lock(&idxd->dev_lock); | |
| 1459 | − if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) | |
| 1460 | − rc = idxd_device_config(idxd); | |
| 1461 | − spin_unlock(&idxd->dev_lock); | |
| 1482 | + rc = idxd_device_config(idxd); | |
| 1462 | 1483 | if (rc < 0) { |
| 1463 | 1484 | dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); |
| 1464 | 1485 | goto err; |
| @@ -1533,7 +1554,6 @@ void idxd_drv_disable_wq(struct idxd_wq *wq) · 68 unmodified lines | ||
| 1533 | 1554 | idxd_wq_reset(wq); |
| 1534 | 1555 | idxd_wq_free_resources(wq); |
| 1535 | 1556 | percpu_ref_exit(&wq->wq_active); |
| 1536 | − wq->type = IDXD_WQT_NONE; | |
| 1537 | 1557 | wq->client_count = 0; |
| 1538 | 1558 | } |
| 1539 | 1559 | EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, "IDXD"); |
| @@ -1554,10 +1574,7 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) · 14 unmodified lines | ||
| 1554 | 1574 | } |
| 1555 | 1575 | |
| 1556 | 1576 | /* Device configuration */ |
| 1557 | − spin_lock(&idxd->dev_lock); | |
| 1558 | − if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) | |
| 1559 | − rc = idxd_device_config(idxd); | |
| 1560 | − spin_unlock(&idxd->dev_lock); | |
| 1577 | + rc = idxd_device_config(idxd); | |
| 1561 | 1578 | if (rc < 0) |
| 1562 | 1579 | return -ENXIO; |
| 1563 | 1580 | |
| @@ -194,6 +194,22 @@ static void idxd_dma_release(struct dma_device *device) · 193 unmodified lines | ||
| 194 | 194 | kfree(idxd_dma); |
| 195 | 195 | } |
| 196 | 196 | |
| 197 | +static int idxd_dma_terminate_all(struct dma_chan *c) | |
| 198 | +{ | |
| 199 | + struct idxd_wq *wq = to_idxd_wq(c); | |
| 200 | + | |
| 201 | + idxd_wq_flush_descs(wq); | |
| 202 | + | |
| 203 | + return 0; | |
| 204 | +} | |
| 205 | + | |
| 206 | +static void idxd_dma_synchronize(struct dma_chan *c) | |
| 207 | +{ | |
| 208 | + struct idxd_wq *wq = to_idxd_wq(c); | |
| 209 | + | |
| 210 | + idxd_wq_drain(wq); | |
| 211 | +} | |
| 212 | + | |
| 197 | 213 | int idxd_register_dma_device(struct idxd_device *idxd) |
| 198 | 214 | { |
| 199 | 215 | struct idxd_dma_dev *idxd_dma; |
| @@ -224,6 +240,8 @@ int idxd_register_dma_device(struct idxd_device *idxd) · 24 unmodified lines | ||
| 224 | 240 | dma->device_issue_pending = idxd_dma_issue_pending; |
| 225 | 241 | dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources; |
| 226 | 242 | dma->device_free_chan_resources = idxd_dma_free_chan_resources; |
| 243 | + dma->device_terminate_all = idxd_dma_terminate_all; | |
| 244 | + dma->device_synchronize = idxd_dma_synchronize; | |
| 227 | 245 | |
| 228 | 246 | rc = dma_async_device_register(dma); |
| 229 | 247 | if (rc < 0) { |
| @@ -803,6 +803,7 @@ void idxd_wq_quiesce(struct idxd_wq *wq); · 802 unmodified lines | ||
| 803 | 803 | int idxd_wq_init_percpu_ref(struct idxd_wq *wq); |
| 804 | 804 | void idxd_wq_free_irq(struct idxd_wq *wq); |
| 805 | 805 | int idxd_wq_request_irq(struct idxd_wq *wq); |
| 806 | +void idxd_wq_flush_descs(struct idxd_wq *wq); | |
| 806 | 807 | |
| 807 | 808 | /* submission */ |
| 808 | 809 | int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); |
| @@ -973,7 +973,8 @@ static void idxd_device_config_restore(struct idxd_device *idxd, · 972 unmodified lines | ||
| 973 | 973 | |
| 974 | 974 | idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; |
| 975 | 975 | |
| 976 | − idxd->evl->size = saved_evl->size; | |
| 976 | + if (idxd->evl) | |
| 977 | + idxd->evl->size = saved_evl->size; | |
| 977 | 978 | |
| 978 | 979 | for (i = 0; i < idxd->max_groups; i++) { |
| 979 | 980 | struct idxd_group *saved_group, *group; |
| @@ -1104,12 +1105,10 @@ static void idxd_reset_done(struct pci_dev *pdev) · 124 unmodified lines | ||
| 1104 | 1105 | idxd_device_config_restore(idxd, idxd->idxd_saved); |
| 1105 | 1106 | |
| 1106 | 1107 | /* Re-configure IDXD device if allowed. */ |
| 1107 | − if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { | |
| 1108 | − rc = idxd_device_config(idxd); | |
| 1109 | − if (rc < 0) { | |
| 1110 | − dev_err(dev, "HALT: %s config fails\n", idxd_name); | |
| 1111 | − goto out; | |
| 1112 | − } | |
| 1108 | + rc = idxd_device_config(idxd); | |
| 1109 | + if (rc < 0) { | |
| 1110 | + dev_err(dev, "HALT: %s config fails\n", idxd_name); | |
| 1111 | + goto out; | |
| 1113 | 1112 | } |
| 1114 | 1113 | |
| 1115 | 1114 | /* Bind IDXD device to driver. */ |
| @@ -1147,6 +1146,7 @@ static void idxd_reset_done(struct pci_dev *pdev) · 31 unmodified lines | ||
| 1147 | 1146 | } |
| 1148 | 1147 | out: |
| 1149 | 1148 | kfree(idxd->idxd_saved); |
| 1149 | + idxd->idxd_saved = NULL; | |
| 1150 | 1150 | } |
| 1151 | 1151 | |
| 1152 | 1152 | static const struct pci_error_handlers idxd_error_handler = { |
| @@ -397,6 +397,17 @@ static void idxd_device_flr(struct work_struct *work) · 396 unmodified lines | ||
| 397 | 397 | dev_err(&idxd->pdev->dev, "FLR failed\n"); |
| 398 | 398 | } |
| 399 | 399 | |
| 400 | +static void idxd_wqs_flush_descs(struct idxd_device *idxd) | |
| 401 | +{ | |
| 402 | + int i; | |
| 403 | + | |
| 404 | + for (i = 0; i < idxd->max_wqs; i++) { | |
| 405 | + struct idxd_wq *wq = idxd->wqs[i]; | |
| 406 | + | |
| 407 | + idxd_wq_flush_descs(wq); | |
| 408 | + } | |
| 409 | +} | |
| 410 | + | |
| 400 | 411 | static irqreturn_t idxd_halt(struct idxd_device *idxd) |
| 401 | 412 | { |
| 402 | 413 | union gensts_reg gensts; |
| @@ -415,6 +426,11 @@ static irqreturn_t idxd_halt(struct idxd_device *idxd) · 12 unmodified lines | ||
| 415 | 426 | } else if (gensts.reset_type == IDXD_DEVICE_RESET_FLR) { |
| 416 | 427 | idxd->state = IDXD_DEV_HALTED; |
| 417 | 428 | idxd_mask_error_interrupts(idxd); |
| 429 | + /* Flush all pending descriptors, and disable | |
| 430 | + * interrupts, they will be re-enabled when FLR | |
| 431 | + * concludes. | |
| 432 | + */ | |
| 433 | + idxd_wqs_flush_descs(idxd); | |
| 418 | 434 | dev_dbg(&idxd->pdev->dev, |
| 419 | 435 | "idxd halted, doing FLR. After FLR, configs are restored\n"); |
| 420 | 436 | INIT_WORK(&idxd->work, idxd_device_flr); |
| @@ -138,7 +138,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, · 137 unmodified lines | ||
| 138 | 138 | */ |
| 139 | 139 | list_for_each_entry_safe(d, t, &flist, list) { |
| 140 | 140 | list_del_init(&d->list); |
| 141 | − idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true, | |
| 141 | + idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true, | |
| 142 | 142 | NULL, NULL); |
| 143 | 143 | } |
| 144 | 144 | } |
| @@ -1836,6 +1836,7 @@ static void idxd_conf_device_release(struct device *dev) · 1835 unmodified lines | ||
| 1836 | 1836 | { |
| 1837 | 1837 | struct idxd_device *idxd = confdev_to_idxd(dev); |
| 1838 | 1838 | |
| 1839 | + destroy_workqueue(idxd->wq); | |
| 1839 | 1840 | kfree(idxd->groups); |
| 1840 | 1841 | bitmap_free(idxd->wq_enable_map); |
| 1841 | 1842 | kfree(idxd->wqs); |
| @@ -10,6 +10,7 @@ · 9 unmodified lines | ||
| 10 | 10 | */ |
| 11 | 11 | |
| 12 | 12 | #include <linux/bitfield.h> |
| 13 | +#include <linux/cleanup.h> | |
| 13 | 14 | #include <linux/dma-mapping.h> |
| 14 | 15 | #include <linux/dmaengine.h> |
| 15 | 16 | #include <linux/interrupt.h> |
| @@ -296,13 +297,10 @@ static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) · 280 unmodified lines | ||
| 296 | 297 | { |
| 297 | 298 | struct dma_chan *chan = &channel->vc.chan; |
| 298 | 299 | struct rz_dmac *dmac = to_rz_dmac(chan->device); |
| 299 | − unsigned long flags; | |
| 300 | 300 | |
| 301 | 301 | dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); |
| 302 | 302 | |
| 303 | − local_irq_save(flags); | |
| 304 | 303 | rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); |
| 305 | − local_irq_restore(flags); | |
| 306 | 304 | } |
| 307 | 305 | |
| 308 | 306 | static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) |
| @@ -447,6 +445,7 @@ static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) · 138 unmodified lines | ||
| 447 | 445 | if (!desc) |
| 448 | 446 | break; |
| 449 | 447 | |
| 448 | + /* No need to lock. This is called only for the 1st client. */ | |
| 450 | 449 | list_add_tail(&desc->node, &channel->ld_free); |
| 451 | 450 | channel->descs_allocated++; |
| 452 | 451 | } |
| @@ -502,18 +501,21 @@ rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, · 49 unmodified lines | ||
| 502 | 501 | dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", |
| 503 | 502 | __func__, channel->index, &src, &dest, len); |
| 504 | 503 | |
| 505 | − if (list_empty(&channel->ld_free)) | |
| 506 | − return NULL; | |
| 504 | + scoped_guard(spinlock_irqsave, &channel->vc.lock) { | |
| 505 | + if (list_empty(&channel->ld_free)) | |
| 506 | + return NULL; | |
| 507 | 507 | |
| 508 | − desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); | |
| 508 | + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); | |
| 509 | 509 | |
| 510 | − desc->type = RZ_DMAC_DESC_MEMCPY; | |
| 511 | − desc->src = src; | |
| 512 | − desc->dest = dest; | |
| 513 | − desc->len = len; | |
| 514 | − desc->direction = DMA_MEM_TO_MEM; | |
| 510 | + desc->type = RZ_DMAC_DESC_MEMCPY; | |
| 511 | + desc->src = src; | |
| 512 | + desc->dest = dest; | |
| 513 | + desc->len = len; | |
| 514 | + desc->direction = DMA_MEM_TO_MEM; | |
| 515 | + | |
| 516 | + list_move_tail(channel->ld_free.next, &channel->ld_queue); | |
| 517 | + } | |
| 515 | 518 | |
| 516 | − list_move_tail(channel->ld_free.next, &channel->ld_queue); | |
| 517 | 519 | return vchan_tx_prep(&channel->vc, &desc->vd, flags); |
| 518 | 520 | } |
| 519 | 521 | |
| @@ -529,27 +531,29 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, · 9 unmodified lines | ||
| 529 | 531 | int dma_length = 0; |
| 530 | 532 | int i = 0; |
| 531 | 533 | |
| 532 | − if (list_empty(&channel->ld_free)) | |
| 533 | − return NULL; | |
| 534 | + scoped_guard(spinlock_irqsave, &channel->vc.lock) { | |
| 535 | + if (list_empty(&channel->ld_free)) | |
| 536 | + return NULL; | |
| 534 | 537 | |
| 535 | − desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); | |
| 538 | + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); | |
| 536 | 539 | |
| 537 | − for_each_sg(sgl, sg, sg_len, i) { | |
| 538 | − dma_length += sg_dma_len(sg); | |
| 539 | − } | |
| 540 | + for_each_sg(sgl, sg, sg_len, i) | |
| 541 | + dma_length += sg_dma_len(sg); | |
| 540 | 542 | |
| 541 | − desc->type = RZ_DMAC_DESC_SLAVE_SG; | |
| 542 | − desc->sg = sgl; | |
| 543 | − desc->sgcount = sg_len; | |
| 544 | − desc->len = dma_length; | |
| 545 | − desc->direction = direction; | |
| 543 | + desc->type = RZ_DMAC_DESC_SLAVE_SG; | |
| 544 | + desc->sg = sgl; | |
| 545 | + desc->sgcount = sg_len; | |
| 546 | + desc->len = dma_length; | |
| 547 | + desc->direction = direction; | |
| 546 | 548 | |
| 547 | − if (direction == DMA_DEV_TO_MEM) | |
| 548 | − desc->src = channel->src_per_address; | |
| 549 | − else | |
| 550 | − desc->dest = channel->dst_per_address; | |
| 549 | + if (direction == DMA_DEV_TO_MEM) | |
| 550 | + desc->src = channel->src_per_address; | |
| 551 | + else | |
| 552 | + desc->dest = channel->dst_per_address; | |
| 553 | + | |
| 554 | + list_move_tail(channel->ld_free.next, &channel->ld_queue); | |
| 555 | + } | |
| 551 | 556 | |
| 552 | − list_move_tail(channel->ld_free.next, &channel->ld_queue); | |
| 553 | 557 | return vchan_tx_prep(&channel->vc, &desc->vd, flags); |
| 554 | 558 | } |
| 555 | 559 | |
| @@ -561,8 +565,8 @@ static int rz_dmac_terminate_all(struct dma_chan *chan) · 5 unmodified lines | ||
| 561 | 565 | unsigned int i; |
| 562 | 566 | LIST_HEAD(head); |
| 563 | 567 | |
| 564 | − rz_dmac_disable_hw(channel); | |
| 565 | 568 | spin_lock_irqsave(&channel->vc.lock, flags); |
| 569 | + rz_dmac_disable_hw(channel); | |
| 566 | 570 | for (i = 0; i < DMAC_NR_LMDESC; i++) |
| 567 | 571 | lmdesc[i].header = 0; |
| 568 | 572 | |
| @@ -699,7 +703,9 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) · 130 unmodified lines | ||
| 699 | 703 | if (chstat & CHSTAT_ER) { |
| 700 | 704 | dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", |
| 701 | 705 | channel->index, chstat); |
| 702 | − rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); | |
| 706 | + | |
| 707 | + scoped_guard(spinlock_irqsave, &channel->vc.lock) | |
| 708 | + rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); | |
| 703 | 709 | goto done; |
| 704 | 710 | } |
| 705 | 711 | |
| @@ -1234,8 +1234,8 @@ static int xdma_probe(struct platform_device *pdev) · 1233 unmodified lines | ||
| 1234 | 1234 | |
| 1235 | 1235 | xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base, |
| 1236 | 1236 | &xdma_regmap_config); |
| 1237 | − if (!xdev->rmap) { | |
| 1238 | − xdma_err(xdev, "config regmap failed: %d", ret); | |
| 1237 | + if (IS_ERR(xdev->rmap)) { | |
| 1238 | + xdma_err(xdev, "config regmap failed: %pe", xdev->rmap); | |
| 1239 | 1239 | goto failed; |
| 1240 | 1240 | } |
| 1241 | 1241 | INIT_LIST_HEAD(&xdev->dma_dev.channels); |
| @@ -997,16 +997,16 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, · 996 unmodified lines | ||
| 997 | 997 | struct xilinx_cdma_tx_segment, |
| 998 | 998 | node); |
| 999 | 999 | cdma_hw = &cdma_seg->hw; |
| 1000 | − residue += (cdma_hw->control - cdma_hw->status) & | |
| 1001 | − chan->xdev->max_buffer_len; | |
| 1000 | + residue += (cdma_hw->control & chan->xdev->max_buffer_len) - | |
| 1001 | + (cdma_hw->status & chan->xdev->max_buffer_len); | |
| 1002 | 1002 | } else if (chan->xdev->dma_config->dmatype == |
| 1003 | 1003 | XDMA_TYPE_AXIDMA) { |
| 1004 | 1004 | axidma_seg = list_entry(entry, |
| 1005 | 1005 | struct xilinx_axidma_tx_segment, |
| 1006 | 1006 | node); |
| 1007 | 1007 | axidma_hw = &axidma_seg->hw; |
| 1008 | − residue += (axidma_hw->control - axidma_hw->status) & | |
| 1009 | − chan->xdev->max_buffer_len; | |
| 1008 | + residue += (axidma_hw->control & chan->xdev->max_buffer_len) - | |
| 1009 | + (axidma_hw->status & chan->xdev->max_buffer_len); | |
| 1010 | 1010 | } else { |
| 1011 | 1011 | aximcdma_seg = |
| 1012 | 1012 | list_entry(entry, |
| @@ -1014,8 +1014,8 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, · 1 unmodified line | ||
| 1014 | 1014 | node); |
| 1015 | 1015 | aximcdma_hw = &aximcdma_seg->hw; |
| 1016 | 1016 | residue += |
| 1017 | − (aximcdma_hw->control - aximcdma_hw->status) & | |
| 1018 | − chan->xdev->max_buffer_len; | |
| 1017 | + (aximcdma_hw->control & chan->xdev->max_buffer_len) - | |
| 1018 | + (aximcdma_hw->status & chan->xdev->max_buffer_len); | |
| 1019 | 1019 | } |
| 1020 | 1020 | } |
| 1021 | 1021 | |
| @@ -1235,14 +1235,6 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) · 213 unmodified lines | ||
| 1235 | 1235 | |
| 1236 | 1236 | dma_cookie_init(dchan); |
| 1237 | 1237 | |
| 1238 | − if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { | |
| 1239 | − /* For AXI DMA resetting once channel will reset the | |
| 1240 | − * other channel as well so enable the interrupts here. | |
| 1241 | − */ | |
| 1242 | − dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, | |
| 1243 | − XILINX_DMA_DMAXR_ALL_IRQ_MASK); | |
| 1244 | − } | |
| 1245 | − | |
| 1246 | 1238 | if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) |
| 1247 | 1239 | dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, |
| 1248 | 1240 | XILINX_CDMA_CR_SGMODE); |
| @@ -1564,8 +1556,29 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) · 315 unmodified lines | ||
| 1564 | 1556 | if (chan->err) |
| 1565 | 1557 | return; |
| 1566 | 1558 | |
| 1567 | − if (list_empty(&chan->pending_list)) | |
| 1559 | + if (list_empty(&chan->pending_list)) { | |
| 1560 | + if (chan->cyclic) { | |
| 1561 | + struct xilinx_dma_tx_descriptor *desc; | |
| 1562 | + struct list_head *entry; | |
| 1563 | + | |
| 1564 | + desc = list_last_entry(&chan->done_list, | |
| 1565 | + struct xilinx_dma_tx_descriptor, node); | |
| 1566 | + list_for_each(entry, &desc->segments) { | |
| 1567 | + struct xilinx_axidma_tx_segment *axidma_seg; | |
| 1568 | + struct xilinx_axidma_desc_hw *axidma_hw; | |
| 1569 | + axidma_seg = list_entry(entry, | |
| 1570 | + struct xilinx_axidma_tx_segment, | |
| 1571 | + node); | |
| 1572 | + axidma_hw = &axidma_seg->hw; | |
| 1573 | + axidma_hw->status = 0; | |
| 1574 | + } | |
| 1575 | + | |
| 1576 | + list_splice_tail_init(&chan->done_list, &chan->active_list); | |
| 1577 | + chan->desc_pendingcount = 0; | |
| 1578 | + chan->idle = false; | |
| 1579 | + } | |
| 1568 | 1580 | return; |
| 1581 | + } | |
| 1569 | 1582 | |
| 1570 | 1583 | if (!chan->idle) |
| 1571 | 1584 | return; |
| @@ -1591,6 +1604,7 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) · 19 unmodified lines | ||
| 1591 | 1604 | head_desc->async_tx.phys); |
| 1592 | 1605 | reg &= ~XILINX_DMA_CR_DELAY_MAX; |
| 1593 | 1606 | reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; |
| 1607 | + reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK; | |
| 1594 | 1608 | dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); |
| 1595 | 1609 | |
| 1596 | 1610 | xilinx_dma_start(chan); |
| @@ -3024,7 +3038,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, · 1427 unmodified lines | ||
| 3024 | 3038 | return -EINVAL; |
| 3025 | 3039 | } |
| 3026 | 3040 | |
| 3027 | − xdev->common.directions |= chan->direction; | |
| 3041 | + xdev->common.directions |= BIT(chan->direction); | |
| 3028 | 3042 | |
| 3029 | 3043 | /* Request the interrupt */ |
| 3030 | 3044 | chan->irq = of_irq_get(node, chan->tdest); |
| @@ -692,9 +692,9 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev, · 691 unmodified lines | ||
| 692 | 692 | goto err_ib_sched; |
| 693 | 693 | } |
| 694 | 694 | |
| 695 | − /* Drop the initial kref_init count (see drm_sched_main as example) */ | |
| 696 | − dma_fence_put(f); | |
| 697 | 695 | ret = dma_fence_wait(f, false); |
| 696 | + /* Drop the returned fence reference after the wait completes */ | |
| 697 | + dma_fence_put(f); | |
| 698 | 698 | |
| 699 | 699 | err_ib_sched: |
| 700 | 700 | amdgpu_job_free(job); |
| @@ -4207,7 +4207,8 @@ fail: · 4206 unmodified lines | ||
| 4207 | 4207 | |
| 4208 | 4208 | static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) |
| 4209 | 4209 | { |
| 4210 | − char *input = amdgpu_lockup_timeout; | |
| 4210 | + char buf[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH]; | |
| 4211 | + char *input = buf; | |
| 4211 | 4212 | char *timeout_setting = NULL; |
| 4212 | 4213 | int index = 0; |
| 4213 | 4214 | long timeout; |
| @@ -4217,9 +4218,17 @@ static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) · 3 unmodified lines | ||
| 4217 | 4218 | adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = |
| 4218 | 4219 | adev->video_timeout = msecs_to_jiffies(2000); |
| 4219 | 4220 | |
| 4220 | − if (!strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) | |
| 4221 | + if (!strnlen(amdgpu_lockup_timeout, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) | |
| 4221 | 4222 | return 0; |
| 4222 | 4223 | |
| 4224 | + /* | |
| 4225 | + * strsep() destructively modifies its input by replacing delimiters | |
| 4226 | + * with '\0'. Use a stack copy so the global module parameter buffer | |
| 4227 | + * remains intact for multi-GPU systems where this function is called | |
| 4228 | + * once per device. | |
| 4229 | + */ | |
| 4230 | + strscpy(buf, amdgpu_lockup_timeout, sizeof(buf)); | |
| 4231 | + | |
| 4223 | 4232 | while ((timeout_setting = strsep(&input, ",")) && |
| 4224 | 4233 | strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { |
| 4225 | 4234 | ret = kstrtol(timeout_setting, 0, &timeout); |
| @@ -35,10 +35,13 @@ · 34 unmodified lines | ||
| 35 | 35 | * PASIDs are global address space identifiers that can be shared |
| 36 | 36 | * between the GPU, an IOMMU and the driver. VMs on different devices |
| 37 | 37 | * may use the same PASID if they share the same address |
| 38 | − * space. Therefore PASIDs are allocated using a global IDA. VMs are | |
| 39 | − * looked up from the PASID per amdgpu_device. | |
| 38 | + * space. Therefore PASIDs are allocated using IDR cyclic allocator | |
| 39 | + * (similar to kernel PID allocation) which naturally delays reuse. | |
| 40 | + * VMs are looked up from the PASID per amdgpu_device. | |
| 40 | 41 | */ |
| 41 | −static DEFINE_IDA(amdgpu_pasid_ida); | |
| 42 | + | |
| 43 | +static DEFINE_IDR(amdgpu_pasid_idr); | |
| 44 | +static DEFINE_SPINLOCK(amdgpu_pasid_idr_lock); | |
| 42 | 45 | |
| 43 | 46 | /* Helper to free pasid from a fence callback */ |
| 44 | 47 | struct amdgpu_pasid_cb { |
| @@ -50,8 +53,8 @@ struct amdgpu_pasid_cb { · 5 unmodified lines | ||
| 50 | 53 | * amdgpu_pasid_alloc - Allocate a PASID |
| 51 | 54 | * @bits: Maximum width of the PASID in bits, must be at least 1 |
| 52 | 55 | * |
| 53 | − * Allocates a PASID of the given width while keeping smaller PASIDs | |
| 54 | − * available if possible. | |
| 56 | + * Uses kernel's IDR cyclic allocator (same as PID allocation). | |
| 57 | + * Allocates sequentially with automatic wrap-around. | |
| 55 | 58 | * |
| 56 | 59 | * Returns a positive integer on success. Returns %-EINVAL if bits==0. |
| 57 | 60 | * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on |
| @@ -59,14 +62,15 @@ struct amdgpu_pasid_cb { · 1 unmodified line | ||
| 59 | 62 | */ |
| 60 | 63 | int amdgpu_pasid_alloc(unsigned int bits) |
| 61 | 64 | { |
| 62 | − int pasid = -EINVAL; | |
| 65 | + int pasid; | |
| 63 | 66 | |
| 64 | − for (bits = min(bits, 31U); bits > 0; bits--) { | |
| 65 | − pasid = ida_alloc_range(&amdgpu_pasid_ida, 1U << (bits - 1), | |
| 66 | − (1U << bits) - 1, GFP_KERNEL); | |
| 67 | − if (pasid != -ENOSPC) | |
| 68 | − break; | |
| 69 | − } | |
| 67 | + if (bits == 0) | |
| 68 | + return -EINVAL; | |
| 69 | + | |
| 70 | + spin_lock(&amdgpu_pasid_idr_lock); | |
| 71 | + pasid = idr_alloc_cyclic(&amdgpu_pasid_idr, NULL, 1, | |
| 72 | + 1U << bits, GFP_KERNEL); | |
| 73 | + spin_unlock(&amdgpu_pasid_idr_lock); | |
| 70 | 74 | |
| 71 | 75 | if (pasid >= 0) |
| 72 | 76 | trace_amdgpu_pasid_allocated(pasid); |
| @@ -81,7 +85,10 @@ int amdgpu_pasid_alloc(unsigned int bits) · 8 unmodified lines | ||
| 81 | 85 | void amdgpu_pasid_free(u32 pasid) |
| 82 | 86 | { |
| 83 | 87 | trace_amdgpu_pasid_freed(pasid); |
| 84 | − ida_free(&amdgpu_pasid_ida, pasid); | |
| 88 | + | |
| 89 | + spin_lock(&amdgpu_pasid_idr_lock); | |
| 90 | + idr_remove(&amdgpu_pasid_idr, pasid); | |
| 91 | + spin_unlock(&amdgpu_pasid_idr_lock); | |
| 85 | 92 | } |
| 86 | 93 | |
| 87 | 94 | static void amdgpu_pasid_free_cb(struct dma_fence *fence, |
| @@ -616,3 +623,15 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) · 528 unmodified lines | ||
| 616 | 623 | } |
| 617 | 624 | } |
| 618 | 625 | } |
| 626 | + | |
| 627 | +/** | |
| 628 | + * amdgpu_pasid_mgr_cleanup - cleanup PASID manager | |
| 629 | + * | |
| 630 | + * Cleanup the IDR allocator. | |
| 631 | + */ | |
| 632 | +void amdgpu_pasid_mgr_cleanup(void) | |
| 633 | +{ | |
| 634 | + spin_lock(&amdgpu_pasid_idr_lock); | |
| 635 | + idr_destroy(&amdgpu_pasid_idr); | |
| 636 | + spin_unlock(&amdgpu_pasid_idr_lock); | |
| 637 | +} | |
| @@ -74,6 +74,7 @@ int amdgpu_pasid_alloc(unsigned int bits); · 73 unmodified lines | ||
| 74 | 74 | void amdgpu_pasid_free(u32 pasid); |
| 75 | 75 | void amdgpu_pasid_free_delayed(struct dma_resv *resv, |
| 76 | 76 | u32 pasid); |
| 77 | +void amdgpu_pasid_mgr_cleanup(void); | |
| 77 | 78 | |
| 78 | 79 | bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev, |
| 79 | 80 | struct amdgpu_vmid *id); |
| @@ -2898,6 +2898,7 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev) · 2897 unmodified lines | ||
| 2898 | 2898 | xa_destroy(&adev->vm_manager.pasids); |
| 2899 | 2899 | |
| 2900 | 2900 | amdgpu_vmid_mgr_fini(adev); |
| 2901 | + amdgpu_pasid_mgr_cleanup(); | |
| 2901 | 2902 | } |
| 2902 | 2903 | |
| 2903 | 2904 | /** |
| @@ -2973,14 +2974,14 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, · 69 unmodified lines | ||
| 2973 | 2974 | if (!root) |
| 2974 | 2975 | return false; |
| 2975 | 2976 | |
| 2976 | − addr /= AMDGPU_GPU_PAGE_SIZE; | |
| 2977 | − | |
| 2978 | 2977 | if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid, |
| 2979 | − node_id, addr, ts, write_fault)) { | |
| 2978 | + node_id, addr >> PAGE_SHIFT, ts, write_fault)) { | |
| 2980 | 2979 | amdgpu_bo_unref(&root); |
| 2981 | 2980 | return true; |
| 2982 | 2981 | } |
| 2983 | 2982 | |
| 2983 | + addr /= AMDGPU_GPU_PAGE_SIZE; | |
| 2984 | + | |
| 2984 | 2985 | r = amdgpu_bo_reserve(root, true); |
| 2985 | 2986 | if (r) |
| 2986 | 2987 | goto error_unref; |
| @@ -3170,11 +3170,11 @@ static int kfd_ioctl_create_process(struct file *filep, struct kfd_process *p, v · 3169 unmodified lines | ||
| 3170 | 3170 | struct kfd_process *process; |
| 3171 | 3171 | int ret; |
| 3172 | 3172 | |
| 3173 | − /* Each FD owns only one kfd_process */ | |
| 3174 | − if (p->context_id != KFD_CONTEXT_ID_PRIMARY) | |
| 3173 | + if (!filep->private_data || !p) | |
| 3175 | 3174 | return -EINVAL; |
| 3176 | 3175 | |
| 3177 | − if (!filep->private_data || !p) | |
| 3176 | + /* Each FD owns only one kfd_process */ | |
| 3177 | + if (p->context_id != KFD_CONTEXT_ID_PRIMARY) | |
| 3178 | 3178 | return -EINVAL; |
| 3179 | 3179 | |
| 3180 | 3180 | mutex_lock(&kfd_processes_mutex); |
| @@ -3909,8 +3909,9 @@ void amdgpu_dm_update_connector_after_detect( · 3908 unmodified lines | ||
| 3909 | 3909 | |
| 3910 | 3910 | aconnector->dc_sink = sink; |
| 3911 | 3911 | dc_sink_retain(aconnector->dc_sink); |
| 3912 | + drm_edid_free(aconnector->drm_edid); | |
| 3913 | + aconnector->drm_edid = NULL; | |
| 3912 | 3914 | if (sink->dc_edid.length == 0) { |
| 3913 | − aconnector->drm_edid = NULL; | |
| 3914 | 3915 | hdmi_cec_unset_edid(aconnector); |
| 3915 | 3916 | if (aconnector->dc_link->aux_mode) { |
| 3916 | 3917 | drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux); |
| @@ -5422,7 +5423,7 @@ static void setup_backlight_device(struct amdgpu_display_manager *dm, · 1505 unmodified lines | ||
| 5422 | 5423 | caps = &dm->backlight_caps[aconnector->bl_idx]; |
| 5423 | 5424 | |
| 5424 | 5425 | /* Only offer ABM property when non-OLED and user didn't turn off by module parameter */ |
| 5425 | − if (!caps->ext_caps->bits.oled && amdgpu_dm_abm_level < 0) | |
| 5426 | + if (caps->ext_caps && !caps->ext_caps->bits.oled && amdgpu_dm_abm_level < 0) | |
| 5426 | 5427 | drm_object_attach_property(&aconnector->base.base, |
| 5427 | 5428 | dm->adev->mode_info.abm_level_property, |
| 5428 | 5429 | ABM_SYSFS_CONTROL); |
| @@ -12523,6 +12524,11 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, · 7094 unmodified lines | ||
| 12523 | 12524 | } |
| 12524 | 12525 | |
| 12525 | 12526 | if (dc_resource_is_dsc_encoding_supported(dc)) { |
| 12527 | + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { | |
| 12528 | + dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); | |
| 12529 | + dm_new_crtc_state->mode_changed_independent_from_dsc = new_crtc_state->mode_changed; | |
| 12530 | + } | |
| 12531 | + | |
| 12526 | 12532 | for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { |
| 12527 | 12533 | if (drm_atomic_crtc_needs_modeset(new_crtc_state)) { |
| 12528 | 12534 | ret = add_affected_mst_dsc_crtcs(state, crtc); |
| @@ -984,6 +984,7 @@ struct dm_crtc_state { · 983 unmodified lines | ||
| 984 | 984 | |
| 985 | 985 | bool freesync_vrr_info_changed; |
| 986 | 986 | |
| 987 | + bool mode_changed_independent_from_dsc; | |
| 987 | 988 | bool dsc_force_changed; |
| 988 | 989 | bool vrr_supported; |
| 989 | 990 | struct mod_freesync_config freesync_config; |
| @@ -1744,9 +1744,11 @@ int pre_validate_dsc(struct drm_atomic_state *state, · 1743 unmodified lines | ||
| 1744 | 1744 | int ind = find_crtc_index_in_state_by_stream(state, stream); |
| 1745 | 1745 | |
| 1746 | 1746 | if (ind >= 0) { |
| 1747 | + struct dm_crtc_state *dm_new_crtc_state = to_dm_crtc_state(state->crtcs[ind].new_state); | |
| 1748 | + | |
| 1747 | 1749 | DRM_INFO_ONCE("%s:%d MST_DSC no mode changed for stream 0x%p\n", |
| 1748 | 1750 | __func__, __LINE__, stream); |
| 1749 | − state->crtcs[ind].new_state->mode_changed = 0; | |
| 1751 | + dm_new_crtc_state->base.mode_changed = dm_new_crtc_state->mode_changed_independent_from_dsc; | |
| 1750 | 1752 | } |
| 1751 | 1753 | } |
| 1752 | 1754 | } |
| @@ -650,9 +650,6 @@ static struct link_encoder *dce100_link_encoder_create( · 649 unmodified lines | ||
| 650 | 650 | return &enc110->base; |
| 651 | 651 | } |
| 652 | 652 | |
| 653 | − if (enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) | |
| 654 | − return NULL; | |
| 655 | − | |
| 656 | 653 | link_regs_id = |
| 657 | 654 | map_transmitter_id_to_phy_instance(enc_init_data->transmitter); |
| 658 | 655 | |
| @@ -661,7 +658,8 @@ static struct link_encoder *dce100_link_encoder_create( · 2 unmodified lines | ||
| 661 | 658 | &link_enc_feature, |
| 662 | 659 | &link_enc_regs[link_regs_id], |
| 663 | 660 | &link_enc_aux_regs[enc_init_data->channel - 1], |
| 664 | − &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 661 | + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? | |
| 662 | + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 665 | 663 | return &enc110->base; |
| 666 | 664 | } |
| 667 | 665 | |
| @@ -671,7 +671,7 @@ static struct link_encoder *dce110_link_encoder_create( · 670 unmodified lines | ||
| 671 | 671 | kzalloc_obj(struct dce110_link_encoder); |
| 672 | 672 | int link_regs_id; |
| 673 | 673 | |
| 674 | − if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) | |
| 674 | + if (!enc110) | |
| 675 | 675 | return NULL; |
| 676 | 676 | |
| 677 | 677 | link_regs_id = |
| @@ -682,7 +682,8 @@ static struct link_encoder *dce110_link_encoder_create( · 4 unmodified lines | ||
| 682 | 682 | &link_enc_feature, |
| 683 | 683 | &link_enc_regs[link_regs_id], |
| 684 | 684 | &link_enc_aux_regs[enc_init_data->channel - 1], |
| 685 | − &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 685 | + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? | |
| 686 | + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 686 | 687 | return &enc110->base; |
| 687 | 688 | } |
| 688 | 689 | |
| @@ -632,7 +632,7 @@ static struct link_encoder *dce112_link_encoder_create( · 631 unmodified lines | ||
| 632 | 632 | kzalloc_obj(struct dce110_link_encoder); |
| 633 | 633 | int link_regs_id; |
| 634 | 634 | |
| 635 | − if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) | |
| 635 | + if (!enc110) | |
| 636 | 636 | return NULL; |
| 637 | 637 | |
| 638 | 638 | link_regs_id = |
| @@ -643,7 +643,8 @@ static struct link_encoder *dce112_link_encoder_create( · 4 unmodified lines | ||
| 643 | 643 | &link_enc_feature, |
| 644 | 644 | &link_enc_regs[link_regs_id], |
| 645 | 645 | &link_enc_aux_regs[enc_init_data->channel - 1], |
| 646 | − &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 646 | + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? | |
| 647 | + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 647 | 648 | return &enc110->base; |
| 648 | 649 | } |
| 649 | 650 | |
| @@ -716,7 +716,7 @@ static struct link_encoder *dce120_link_encoder_create( · 715 unmodified lines | ||
| 716 | 716 | kzalloc_obj(struct dce110_link_encoder); |
| 717 | 717 | int link_regs_id; |
| 718 | 718 | |
| 719 | − if (!enc110 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) | |
| 719 | + if (!enc110) | |
| 720 | 720 | return NULL; |
| 721 | 721 | |
| 722 | 722 | link_regs_id = |
| @@ -727,7 +727,8 @@ static struct link_encoder *dce120_link_encoder_create( · 4 unmodified lines | ||
| 727 | 727 | &link_enc_feature, |
| 728 | 728 | &link_enc_regs[link_regs_id], |
| 729 | 729 | &link_enc_aux_regs[enc_init_data->channel - 1], |
| 730 | − &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 730 | + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? | |
| 731 | + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 731 | 732 | |
| 732 | 733 | return &enc110->base; |
| 733 | 734 | } |
| @@ -746,18 +746,16 @@ static struct link_encoder *dce60_link_encoder_create( · 745 unmodified lines | ||
| 746 | 746 | return &enc110->base; |
| 747 | 747 | } |
| 748 | 748 | |
| 749 | − if (enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) | |
| 750 | − return NULL; | |
| 751 | − | |
| 752 | 749 | link_regs_id = |
| 753 | 750 | map_transmitter_id_to_phy_instance(enc_init_data->transmitter); |
| 754 | 751 | |
| 755 | 752 | dce60_link_encoder_construct(enc110, |
| 756 | − enc_init_data, | |
| 757 | − &link_enc_feature, | |
| 758 | − &link_enc_regs[link_regs_id], | |
| 759 | − &link_enc_aux_regs[enc_init_data->channel - 1], | |
| 760 | − &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 753 | + enc_init_data, | |
| 754 | + &link_enc_feature, | |
| 755 | + &link_enc_regs[link_regs_id], | |
| 756 | + &link_enc_aux_regs[enc_init_data->channel - 1], | |
| 757 | + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? | |
| 758 | + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 761 | 759 | return &enc110->base; |
| 762 | 760 | } |
| 763 | 761 | |
| @@ -752,9 +752,6 @@ static struct link_encoder *dce80_link_encoder_create( · 751 unmodified lines | ||
| 752 | 752 | return &enc110->base; |
| 753 | 753 | } |
| 754 | 754 | |
| 755 | − if (enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs)) | |
| 756 | − return NULL; | |
| 757 | − | |
| 758 | 755 | link_regs_id = |
| 759 | 756 | map_transmitter_id_to_phy_instance(enc_init_data->transmitter); |
| 760 | 757 | |
| @@ -763,7 +760,8 @@ static struct link_encoder *dce80_link_encoder_create( · 2 unmodified lines | ||
| 763 | 760 | &link_enc_feature, |
| 764 | 761 | &link_enc_regs[link_regs_id], |
| 765 | 762 | &link_enc_aux_regs[enc_init_data->channel - 1], |
| 766 | − &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 763 | + enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? | |
| 764 | + NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); | |
| 767 | 765 | return &enc110->base; |
| 768 | 766 | } |
| 769 | 767 | |
| @@ -59,6 +59,10 @@ · 58 unmodified lines | ||
| 59 | 59 | |
| 60 | 60 | #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) |
| 61 | 61 | |
| 62 | +static void smu_v13_0_0_get_od_setting_limits(struct smu_context *smu, | |
| 63 | + int od_feature_bit, | |
| 64 | + int32_t *min, int32_t *max); | |
| 65 | + | |
| 62 | 66 | static const struct smu_feature_bits smu_v13_0_0_dpm_features = { |
| 63 | 67 | .bits = { |
| 64 | 68 | SMU_FEATURE_BIT_INIT(FEATURE_DPM_GFXCLK_BIT), |
| @@ -1043,8 +1047,35 @@ static bool smu_v13_0_0_is_od_feature_supported(struct smu_context *smu, · 978 unmodified lines | ||
| 1043 | 1047 | PPTable_t *pptable = smu->smu_table.driver_pptable; |
| 1044 | 1048 | const OverDriveLimits_t * const overdrive_upperlimits = |
| 1045 | 1049 | &pptable->SkuTable.OverDriveLimitsBasicMax; |
| 1050 | + int32_t min_value, max_value; | |
| 1051 | + bool feature_enabled; | |
| 1046 | 1052 | |
| 1047 | − return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); | |
| 1053 | + switch (od_feature_bit) { | |
| 1054 | + case PP_OD_FEATURE_FAN_CURVE_BIT: | |
| 1055 | + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); | |
| 1056 | + if (feature_enabled) { | |
| 1057 | + smu_v13_0_0_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_TEMP, | |
| 1058 | + &min_value, &max_value); | |
| 1059 | + if (!min_value && !max_value) { | |
| 1060 | + feature_enabled = false; | |
| 1061 | + goto out; | |
| 1062 | + } | |
| 1063 | + | |
| 1064 | + smu_v13_0_0_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_PWM, | |
| 1065 | + &min_value, &max_value); | |
| 1066 | + if (!min_value && !max_value) { | |
| 1067 | + feature_enabled = false; | |
| 1068 | + goto out; | |
| 1069 | + } | |
| 1070 | + } | |
| 1071 | + break; | |
| 1072 | + default: | |
| 1073 | + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); | |
| 1074 | + break; | |
| 1075 | + } | |
| 1076 | + | |
| 1077 | +out: | |
| 1078 | + return feature_enabled; | |
| 1048 | 1079 | } |
| 1049 | 1080 | |
| 1050 | 1081 | static void smu_v13_0_0_get_od_setting_limits(struct smu_context *smu, |
| @@ -1391,7 +1391,7 @@ static int smu_v13_0_6_emit_clk_levels(struct smu_context *smu, · 1390 unmodified lines | ||
| 1391 | 1391 | break; |
| 1392 | 1392 | case SMU_OD_MCLK: |
| 1393 | 1393 | if (!smu_v13_0_6_cap_supported(smu, SMU_CAP(SET_UCLK_MAX))) |
| 1394 | − return 0; | |
| 1394 | + return -EOPNOTSUPP; | |
| 1395 | 1395 | |
| 1396 | 1396 | size += sysfs_emit_at(buf, size, "%s:\n", "OD_MCLK"); |
| 1397 | 1397 | size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n", |
| @@ -2122,6 +2122,7 @@ static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu, · 724 unmodified lines | ||
| 2122 | 2122 | { |
| 2123 | 2123 | struct smu_dpm_context *smu_dpm = &(smu->smu_dpm); |
| 2124 | 2124 | struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context; |
| 2125 | + struct smu_dpm_table *uclk_table = &dpm_context->dpm_tables.uclk_table; | |
| 2125 | 2126 | struct smu_umd_pstate_table *pstate_table = &smu->pstate_table; |
| 2126 | 2127 | uint32_t min_clk; |
| 2127 | 2128 | uint32_t max_clk; |
| @@ -2221,14 +2222,16 @@ static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu, · 93 unmodified lines | ||
| 2221 | 2222 | if (ret) |
| 2222 | 2223 | return ret; |
| 2223 | 2224 | |
| 2224 | − min_clk = SMU_DPM_TABLE_MIN( | |
| 2225 | − &dpm_context->dpm_tables.uclk_table); | |
| 2226 | − max_clk = SMU_DPM_TABLE_MAX( | |
| 2227 | − &dpm_context->dpm_tables.uclk_table); | |
| 2228 | − ret = smu_v13_0_6_set_soft_freq_limited_range( | |
| 2229 | − smu, SMU_UCLK, min_clk, max_clk, false); | |
| 2230 | − if (ret) | |
| 2231 | − return ret; | |
| 2225 | + if (SMU_DPM_TABLE_MAX(uclk_table) != | |
| 2226 | + pstate_table->uclk_pstate.curr.max) { | |
| 2227 | + min_clk = SMU_DPM_TABLE_MIN(&dpm_context->dpm_tables.uclk_table); | |
| 2228 | + max_clk = SMU_DPM_TABLE_MAX(&dpm_context->dpm_tables.uclk_table); | |
| 2229 | + ret = smu_v13_0_6_set_soft_freq_limited_range(smu, | |
| 2230 | + SMU_UCLK, min_clk, | |
| 2231 | + max_clk, false); | |
| 2232 | + if (ret) | |
| 2233 | + return ret; | |
| 2234 | + } | |
| 2232 | 2235 | smu_v13_0_reset_custom_level(smu); |
| 2233 | 2236 | } |
| 2234 | 2237 | break; |
| @@ -59,6 +59,10 @@ · 58 unmodified lines | ||
| 59 | 59 | |
| 60 | 60 | #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) |
| 61 | 61 | |
| 62 | +static void smu_v13_0_7_get_od_setting_limits(struct smu_context *smu, | |
| 63 | + int od_feature_bit, | |
| 64 | + int32_t *min, int32_t *max); | |
| 65 | + | |
| 62 | 66 | static const struct smu_feature_bits smu_v13_0_7_dpm_features = { |
| 63 | 67 | .bits = { |
| 64 | 68 | SMU_FEATURE_BIT_INIT(FEATURE_DPM_GFXCLK_BIT), |
| @@ -1053,8 +1057,35 @@ static bool smu_v13_0_7_is_od_feature_supported(struct smu_context *smu, · 988 unmodified lines | ||
| 1053 | 1057 | PPTable_t *pptable = smu->smu_table.driver_pptable; |
| 1054 | 1058 | const OverDriveLimits_t * const overdrive_upperlimits = |
| 1055 | 1059 | &pptable->SkuTable.OverDriveLimitsBasicMax; |
| 1060 | + int32_t min_value, max_value; | |
| 1061 | + bool feature_enabled; | |
| 1056 | 1062 | |
| 1057 | − return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); | |
| 1063 | + switch (od_feature_bit) { | |
| 1064 | + case PP_OD_FEATURE_FAN_CURVE_BIT: | |
| 1065 | + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); | |
| 1066 | + if (feature_enabled) { | |
| 1067 | + smu_v13_0_7_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_TEMP, | |
| 1068 | + &min_value, &max_value); | |
| 1069 | + if (!min_value && !max_value) { | |
| 1070 | + feature_enabled = false; | |
| 1071 | + goto out; | |
| 1072 | + } | |
| 1073 | + | |
| 1074 | + smu_v13_0_7_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_PWM, | |
| 1075 | + &min_value, &max_value); | |
| 1076 | + if (!min_value && !max_value) { | |
| 1077 | + feature_enabled = false; | |
| 1078 | + goto out; | |
| 1079 | + } | |
| 1080 | + } | |
| 1081 | + break; | |
| 1082 | + default: | |
| 1083 | + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); | |
| 1084 | + break; | |
| 1085 | + } | |
| 1086 | + | |
| 1087 | +out: | |
| 1088 | + return feature_enabled; | |
| 1058 | 1089 | } |
| 1059 | 1090 | |
| 1060 | 1091 | static void smu_v13_0_7_get_od_setting_limits(struct smu_context *smu, |
| @@ -56,6 +56,10 @@ · 55 unmodified lines | ||
| 56 | 56 | |
| 57 | 57 | #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c)) |
| 58 | 58 | |
| 59 | +static void smu_v14_0_2_get_od_setting_limits(struct smu_context *smu, | |
| 60 | + int od_feature_bit, | |
| 61 | + int32_t *min, int32_t *max); | |
| 62 | + | |
| 59 | 63 | static const struct smu_feature_bits smu_v14_0_2_dpm_features = { |
| 60 | 64 | .bits = { SMU_FEATURE_BIT_INIT(FEATURE_DPM_GFXCLK_BIT), |
| 61 | 65 | SMU_FEATURE_BIT_INIT(FEATURE_DPM_UCLK_BIT), |
| @@ -922,8 +926,35 @@ static bool smu_v14_0_2_is_od_feature_supported(struct smu_context *smu, · 860 unmodified lines | ||
| 922 | 926 | PPTable_t *pptable = smu->smu_table.driver_pptable; |
| 923 | 927 | const OverDriveLimits_t * const overdrive_upperlimits = |
| 924 | 928 | &pptable->SkuTable.OverDriveLimitsBasicMax; |
| 929 | + int32_t min_value, max_value; | |
| 930 | + bool feature_enabled; | |
| 925 | 931 | |
| 926 | − return overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit); | |
| 932 | + switch (od_feature_bit) { | |
| 933 | + case PP_OD_FEATURE_FAN_CURVE_BIT: | |
| 934 | + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); | |
| 935 | + if (feature_enabled) { | |
| 936 | + smu_v14_0_2_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_TEMP, | |
| 937 | + &min_value, &max_value); | |
| 938 | + if (!min_value && !max_value) { | |
| 939 | + feature_enabled = false; | |
| 940 | + goto out; | |
| 941 | + } | |
| 942 | + | |
| 943 | + smu_v14_0_2_get_od_setting_limits(smu, PP_OD_FEATURE_FAN_CURVE_PWM, | |
| 944 | + &min_value, &max_value); | |
| 945 | + if (!min_value && !max_value) { | |
| 946 | + feature_enabled = false; | |
| 947 | + goto out; | |
| 948 | + } | |
| 949 | + } | |
| 950 | + break; | |
| 951 | + default: | |
| 952 | + feature_enabled = !!(overdrive_upperlimits->FeatureCtrlMask & (1U << od_feature_bit)); | |
| 953 | + break; | |
| 954 | + } | |
| 955 | + | |
| 956 | +out: | |
| 957 | + return feature_enabled; | |
| 927 | 958 | } |
| 928 | 959 | |
| 929 | 960 | static void smu_v14_0_2_get_od_setting_limits(struct smu_context *smu, |
| @@ -550,27 +550,27 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, · 549 unmodified lines | ||
| 550 | 550 | } |
| 551 | 551 | EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); |
| 552 | 552 | |
| 553 | −static bool drm_gem_shmem_try_map_pmd(struct vm_fault *vmf, unsigned long addr, | |
| 554 | − struct page *page) | |
| 553 | +static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, | |
| 554 | + unsigned long pfn) | |
| 555 | 555 | { |
| 556 | + if (!order) { | |
| 557 | + return vmf_insert_pfn(vmf->vma, vmf->address, pfn); | |
| 556 | 558 | #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP |
| 557 | − unsigned long pfn = page_to_pfn(page); | |
| 558 | − unsigned long paddr = pfn << PAGE_SHIFT; | |
| 559 | − bool aligned = (addr & ~PMD_MASK) == (paddr & ~PMD_MASK); | |
| 560 | − | |
| 561 | − if (aligned && | |
| 562 | − pmd_none(*vmf->pmd) && | |
| 563 | − folio_test_pmd_mappable(page_folio(page))) { | |
| 564 | − pfn &= PMD_MASK >> PAGE_SHIFT; | |
| 565 | − if (vmf_insert_pfn_pmd(vmf, pfn, false) == VM_FAULT_NOPAGE) | |
| 566 | − return true; | |
| 567 | − } | |
| 559 | + } else if (order == PMD_ORDER) { | |
| 560 | + unsigned long paddr = pfn << PAGE_SHIFT; | |
| 561 | + bool aligned = (vmf->address & ~PMD_MASK) == (paddr & ~PMD_MASK); | |
| 562 | + | |
| 563 | + if (aligned && | |
| 564 | + folio_test_pmd_mappable(page_folio(pfn_to_page(pfn)))) { | |
| 565 | + pfn &= PMD_MASK >> PAGE_SHIFT; | |
| 566 | + return vmf_insert_pfn_pmd(vmf, pfn, false); | |
| 567 | + } | |
| 568 | 568 | #endif |
| 569 | − | |
| 570 | − return false; | |
| 569 | + } | |
| 570 | + return VM_FAULT_FALLBACK; | |
| 571 | 571 | } |
| 572 | 572 | |
| 573 | −static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) | |
| 573 | +static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int order) | |
| 574 | 574 | { |
| 575 | 575 | struct vm_area_struct *vma = vmf->vma; |
| 576 | 576 | struct drm_gem_object *obj = vma->vm_private_data; |
| @@ -581,6 +581,9 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) · 4 unmodified lines | ||
| 581 | 581 | pgoff_t page_offset; |
| 582 | 582 | unsigned long pfn; |
| 583 | 583 | |
| 584 | + if (order && order != PMD_ORDER) | |
| 585 | + return VM_FAULT_FALLBACK; | |
| 586 | + | |
| 584 | 587 | /* Offset to faulty address in the VMA. */ |
| 585 | 588 | page_offset = vmf->pgoff - vma->vm_pgoff; |
| 586 | 589 | |
| @@ -593,13 +596,8 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) · 6 unmodified lines | ||
| 593 | 596 | goto out; |
| 594 | 597 | } |
| 595 | 598 | |
| 596 | − if (drm_gem_shmem_try_map_pmd(vmf, vmf->address, pages[page_offset])) { | |
| 597 | − ret = VM_FAULT_NOPAGE; | |
| 598 | − goto out; | |
| 599 | − } | |
| 600 | − | |
| 601 | 599 | pfn = page_to_pfn(pages[page_offset]); |
| 602 | − ret = vmf_insert_pfn(vma, vmf->address, pfn); | |
| 600 | + ret = try_insert_pfn(vmf, order, pfn); | |
| 603 | 601 | |
| 604 | 602 | out: |
| 605 | 603 | dma_resv_unlock(shmem->base.resv); |
| @@ -607,6 +605,11 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) · 1 unmodified line | ||
| 607 | 605 | return ret; |
| 608 | 606 | } |
| 609 | 607 | |
| 608 | +static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) | |
| 609 | +{ | |
| 610 | + return drm_gem_shmem_any_fault(vmf, 0); | |
| 611 | +} | |
| 612 | + | |
| 610 | 613 | static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) |
| 611 | 614 | { |
| 612 | 615 | struct drm_gem_object *obj = vma->vm_private_data; |
| @@ -643,6 +646,9 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) · 30 unmodified lines | ||
| 643 | 646 | |
| 644 | 647 | const struct vm_operations_struct drm_gem_shmem_vm_ops = { |
| 645 | 648 | .fault = drm_gem_shmem_fault, |
| 649 | +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP | |
| 650 | + .huge_fault = drm_gem_shmem_any_fault, | |
| 651 | +#endif | |
| 646 | 652 | .open = drm_gem_shmem_vm_open, |
| 647 | 653 | .close = drm_gem_shmem_vm_close, |
| 648 | 654 | }; |
| @@ -602,7 +602,7 @@ int drm_syncobj_get_handle(struct drm_file *file_private, · 601 unmodified lines | ||
| 602 | 602 | drm_syncobj_get(syncobj); |
| 603 | 603 | |
| 604 | 604 | ret = xa_alloc(&file_private->syncobj_xa, handle, syncobj, xa_limit_32b, |
| 605 | − GFP_NOWAIT); | |
| 605 | + GFP_KERNEL); | |
| 606 | 606 | if (ret) |
| 607 | 607 | drm_syncobj_put(syncobj); |
| 608 | 608 | |
| @@ -716,7 +716,7 @@ static int drm_syncobj_fd_to_handle(struct drm_file *file_private, · 107 unmodified lines | ||
| 716 | 716 | drm_syncobj_get(syncobj); |
| 717 | 717 | |
| 718 | 718 | ret = xa_alloc(&file_private->syncobj_xa, handle, syncobj, xa_limit_32b, |
| 719 | − GFP_NOWAIT); | |
| 719 | + GFP_KERNEL); | |
| 720 | 720 | if (ret) |
| 721 | 721 | drm_syncobj_put(syncobj); |
| 722 | 722 | |
| @@ -4602,6 +4602,7 @@ intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, · 4601 unmodified lines | ||
| 4602 | 4602 | struct intel_crtc_state *crtc_state = |
| 4603 | 4603 | intel_atomic_get_new_crtc_state(state, crtc); |
| 4604 | 4604 | struct intel_crtc_state *saved_state; |
| 4605 | + int err; | |
| 4605 | 4606 | |
| 4606 | 4607 | saved_state = intel_crtc_state_alloc(crtc); |
| 4607 | 4608 | if (!saved_state) |
| @@ -4610,7 +4611,12 @@ intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, · 2 unmodified lines | ||
| 4610 | 4611 | /* free the old crtc_state->hw members */ |
| 4611 | 4612 | intel_crtc_free_hw_state(crtc_state); |
| 4612 | 4613 | |
| 4613 | − intel_dp_tunnel_atomic_clear_stream_bw(state, crtc_state); | |
| 4614 | + err = intel_dp_tunnel_atomic_clear_stream_bw(state, crtc_state); | |
| 4615 | + if (err) { | |
| 4616 | + kfree(saved_state); | |
| 4617 | + | |
| 4618 | + return err; | |
| 4619 | + } | |
| 4614 | 4620 | |
| 4615 | 4621 | /* FIXME: before the switch to atomic started, a new pipe_config was |
| 4616 | 4622 | * kzalloc'd. Code that depends on any field being zero should be |
| @@ -621,19 +621,27 @@ int intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, · 620 unmodified lines | ||
| 621 | 621 | * |
| 622 | 622 | * Clear any DP tunnel stream BW requirement set by |
| 623 | 623 | * intel_dp_tunnel_atomic_compute_stream_bw(). |
| 624 | + * | |
| 625 | + * Returns 0 in case of success, a negative error code otherwise. | |
| 624 | 626 | */ |
| 625 | −void intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, | |
| 626 | − struct intel_crtc_state *crtc_state) | |
| 627 | +int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, | |
| 628 | + struct intel_crtc_state *crtc_state) | |
| 627 | 629 | { |
| 628 | 630 | struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); |
| 631 | + int err; | |
| 629 | 632 | |
| 630 | 633 | if (!crtc_state->dp_tunnel_ref.tunnel) |
| 631 | − return; | |
| 634 | + return 0; | |
| 635 | + | |
| 636 | + err = drm_dp_tunnel_atomic_set_stream_bw(&state->base, | |
| 637 | + crtc_state->dp_tunnel_ref.tunnel, | |
| 638 | + crtc->pipe, 0); | |
| 639 | + if (err) | |
| 640 | + return err; | |
| 632 | 641 | |
| 633 | − drm_dp_tunnel_atomic_set_stream_bw(&state->base, | |
| 634 | − crtc_state->dp_tunnel_ref.tunnel, | |
| 635 | − crtc->pipe, 0); | |
| 636 | 642 | drm_dp_tunnel_ref_put(&crtc_state->dp_tunnel_ref); |
| 643 | + | |
| 644 | + return 0; | |
| 637 | 645 | } |
| 638 | 646 | |
| 639 | 647 | /** |
| @@ -40,8 +40,8 @@ int intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, · 39 unmodified lines | ||
| 40 | 40 | struct intel_dp *intel_dp, |
| 41 | 41 | const struct intel_connector *connector, |
| 42 | 42 | struct intel_crtc_state *crtc_state); |
| 43 | −void intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, | |
| 44 | − struct intel_crtc_state *crtc_state); | |
| 43 | +int intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, | |
| 44 | + struct intel_crtc_state *crtc_state); | |
| 45 | 45 | |
| 46 | 46 | int intel_dp_tunnel_atomic_add_state_for_crtc(struct intel_atomic_state *state, |
| 47 | 47 | struct intel_crtc *crtc); |
| @@ -88,9 +88,12 @@ intel_dp_tunnel_atomic_compute_stream_bw(struct intel_atomic_state *state, · 40 unmodified lines | ||
| 88 | 88 | return 0; |
| 89 | 89 | } |
| 90 | 90 | |
| 91 | −static inline void | |
| 91 | +static inline int | |
| 92 | 92 | intel_dp_tunnel_atomic_clear_stream_bw(struct intel_atomic_state *state, |
| 93 | − struct intel_crtc_state *crtc_state) {} | |
| 93 | + struct intel_crtc_state *crtc_state) | |
| 94 | +{ | |
| 95 | + return 0; | |
| 96 | +} | |
| 94 | 97 | |
| 95 | 98 | static inline int |
| 96 | 99 | intel_dp_tunnel_atomic_add_state_for_crtc(struct intel_atomic_state *state, |
| @@ -496,8 +496,10 @@ gmbus_xfer_read_chunk(struct intel_display *display, · 495 unmodified lines | ||
| 496 | 496 | |
| 497 | 497 | val = intel_de_read_fw(display, GMBUS3(display)); |
| 498 | 498 | do { |
| 499 | − if (extra_byte_added && len == 1) | |
| 499 | + if (extra_byte_added && len == 1) { | |
| 500 | + len--; | |
| 500 | 501 | break; |
| 502 | + } | |
| 501 | 503 | |
| 502 | 504 | *buf++ = val & 0xff; |
| 503 | 505 | val >>= 8; |
| @@ -436,11 +436,16 @@ void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, · 435 unmodified lines | ||
| 436 | 436 | drm_framebuffer_get(plane_state->hw.fb); |
| 437 | 437 | } |
| 438 | 438 | |
| 439 | +static void unlink_nv12_plane(struct intel_crtc_state *crtc_state, | |
| 440 | + struct intel_plane_state *plane_state); | |
| 441 | + | |
| 439 | 442 | void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, |
| 440 | 443 | struct intel_plane_state *plane_state) |
| 441 | 444 | { |
| 442 | 445 | struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); |
| 443 | 446 | |
| 447 | + unlink_nv12_plane(crtc_state, plane_state); | |
| 448 | + | |
| 444 | 449 | crtc_state->active_planes &= ~BIT(plane->id); |
| 445 | 450 | crtc_state->scaled_planes &= ~BIT(plane->id); |
| 446 | 451 | crtc_state->nv12_planes &= ~BIT(plane->id); |
| @@ -1513,6 +1518,9 @@ static void unlink_nv12_plane(struct intel_crtc_state *crtc_state, · 1066 unmodified lines | ||
| 1513 | 1518 | struct intel_display *display = to_intel_display(plane_state); |
| 1514 | 1519 | struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); |
| 1515 | 1520 | |
| 1521 | + if (!plane_state->planar_linked_plane) | |
| 1522 | + return; | |
| 1523 | + | |
| 1516 | 1524 | plane_state->planar_linked_plane = NULL; |
| 1517 | 1525 | |
| 1518 | 1526 | if (!plane_state->is_y_plane) |
| @@ -1550,8 +1558,7 @@ static int icl_check_nv12_planes(struct intel_atomic_state *state, · 31 unmodified lines | ||
| 1550 | 1558 | if (plane->pipe != crtc->pipe) |
| 1551 | 1559 | continue; |
| 1552 | 1560 | |
| 1553 | − if (plane_state->planar_linked_plane) | |
| 1554 | − unlink_nv12_plane(crtc_state, plane_state); | |
| 1561 | + unlink_nv12_plane(crtc_state, plane_state); | |
| 1555 | 1562 | } |
| 1556 | 1563 | |
| 1557 | 1564 | if (!crtc_state->nv12_planes) |
| @@ -25,9 +25,9 @@ · 24 unmodified lines | ||
| 25 | 25 | might_sleep(); \ |
| 26 | 26 | for (;;) { \ |
| 27 | 27 | const bool expired__ = ktime_after(ktime_get_raw(), end__); \ |
| 28 | − OP; \ | |
| 29 | 28 | /* Guarantee COND check prior to timeout */ \ |
| 30 | 29 | barrier(); \ |
| 30 | + OP; \ | |
| 31 | 31 | if (COND) { \ |
| 32 | 32 | ret__ = 0; \ |
| 33 | 33 | break; \ |
| @@ -1236,6 +1236,11 @@ static int mtk_dsi_probe(struct platform_device *pdev) · 1235 unmodified lines | ||
| 1236 | 1236 | |
| 1237 | 1237 | dsi->host.ops = &mtk_dsi_ops; |
| 1238 | 1238 | dsi->host.dev = dev; |
| 1239 | + | |
| 1240 | + init_waitqueue_head(&dsi->irq_wait_queue); | |
| 1241 | + | |
| 1242 | + platform_set_drvdata(pdev, dsi); | |
| 1243 | + | |
| 1239 | 1244 | ret = mipi_dsi_host_register(&dsi->host); |
| 1240 | 1245 | if (ret < 0) |
| 1241 | 1246 | return dev_err_probe(dev, ret, "Failed to register DSI host\n"); |
| @@ -1247,10 +1252,6 @@ static int mtk_dsi_probe(struct platform_device *pdev) · 5 unmodified lines | ||
| 1247 | 1252 | return dev_err_probe(&pdev->dev, ret, "Failed to request DSI irq\n"); |
| 1248 | 1253 | } |
| 1249 | 1254 | |
| 1250 | − init_waitqueue_head(&dsi->irq_wait_queue); | |
| 1251 | − | |
| 1252 | − platform_set_drvdata(pdev, dsi); | |
| 1253 | − | |
| 1254 | 1255 | dsi->bridge.of_node = dev->of_node; |
| 1255 | 1256 | dsi->bridge.type = DRM_MODE_CONNECTOR_DSI; |
| 1256 | 1257 | |
| @@ -553,6 +553,7 @@ · 552 unmodified lines | ||
| 553 | 553 | #define ENABLE_SMP_LD_RENDER_SURFACE_CONTROL REG_BIT(44 - 32) |
| 554 | 554 | #define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32) |
| 555 | 555 | #define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32) |
| 556 | +#define L3_128B_256B_WRT_DIS REG_BIT(40 - 32) | |
| 556 | 557 | #define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32) |
| 557 | 558 | #define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32) |
| 558 | 559 | |
| @@ -1442,9 +1442,9 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, · 1441 unmodified lines | ||
| 1442 | 1442 | err = vma_check_userptr(vm, op->map.vma, pt_update); |
| 1443 | 1443 | break; |
| 1444 | 1444 | case DRM_GPUVA_OP_REMAP: |
| 1445 | − if (op->remap.prev) | |
| 1445 | + if (op->remap.prev && !op->remap.skip_prev) | |
| 1446 | 1446 | err = vma_check_userptr(vm, op->remap.prev, pt_update); |
| 1447 | − if (!err && op->remap.next) | |
| 1447 | + if (!err && op->remap.next && !op->remap.skip_next) | |
| 1448 | 1448 | err = vma_check_userptr(vm, op->remap.next, pt_update); |
| 1449 | 1449 | break; |
| 1450 | 1450 | case DRM_GPUVA_OP_UNMAP: |
| @@ -2198,12 +2198,12 @@ static int op_prepare(struct xe_vm *vm, · 747 unmodified lines | ||
| 2198 | 2198 | |
| 2199 | 2199 | err = unbind_op_prepare(tile, pt_update_ops, old); |
| 2200 | 2200 | |
| 2201 | − if (!err && op->remap.prev) { | |
| 2201 | + if (!err && op->remap.prev && !op->remap.skip_prev) { | |
| 2202 | 2202 | err = bind_op_prepare(vm, tile, pt_update_ops, |
| 2203 | 2203 | op->remap.prev, false); |
| 2204 | 2204 | pt_update_ops->wait_vm_bookkeep = true; |
| 2205 | 2205 | } |
| 2206 | − if (!err && op->remap.next) { | |
| 2206 | + if (!err && op->remap.next && !op->remap.skip_next) { | |
| 2207 | 2207 | err = bind_op_prepare(vm, tile, pt_update_ops, |
| 2208 | 2208 | op->remap.next, false); |
| 2209 | 2209 | pt_update_ops->wait_vm_bookkeep = true; |
| @@ -2428,10 +2428,10 @@ static void op_commit(struct xe_vm *vm, · 218 unmodified lines | ||
| 2428 | 2428 | |
| 2429 | 2429 | unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2); |
| 2430 | 2430 | |
| 2431 | − if (op->remap.prev) | |
| 2431 | + if (op->remap.prev && !op->remap.skip_prev) | |
| 2432 | 2432 | bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, |
| 2433 | 2433 | fence, fence2, false); |
| 2434 | − if (op->remap.next) | |
| 2434 | + if (op->remap.next && !op->remap.skip_next) | |
| 2435 | 2435 | bind_op_commit(vm, tile, pt_update_ops, op->remap.next, |
| 2436 | 2436 | fence, fence2, false); |
| 2437 | 2437 | break; |
| @@ -341,6 +341,8 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, · 340 unmodified lines | ||
| 341 | 341 | ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data); |
| 342 | 342 | if (ret) { |
| 343 | 343 | xe_sriov_packet_free(*data); |
| 344 | + *data = NULL; | |
| 345 | + | |
| 344 | 346 | return ret; |
| 345 | 347 | } |
| 346 | 348 | |
| @@ -2554,7 +2554,6 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) · 2553 unmodified lines | ||
| 2554 | 2554 | if (!err && op->remap.skip_prev) { |
| 2555 | 2555 | op->remap.prev->tile_present = |
| 2556 | 2556 | tile_present; |
| 2557 | − op->remap.prev = NULL; | |
| 2558 | 2557 | } |
| 2559 | 2558 | } |
| 2560 | 2559 | if (op->remap.next) { |
| @@ -2564,11 +2563,13 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) · 3 unmodified lines | ||
| 2564 | 2563 | if (!err && op->remap.skip_next) { |
| 2565 | 2564 | op->remap.next->tile_present = |
| 2566 | 2565 | tile_present; |
| 2567 | − op->remap.next = NULL; | |
| 2568 | 2566 | } |
| 2569 | 2567 | } |
| 2570 | 2568 | |
| 2571 | − /* Adjust for partial unbind after removing VMA from VM */ | |
| 2569 | + /* | |
| 2570 | + * Adjust for partial unbind after removing VMA from VM. In case | |
| 2571 | + * of unwind we might need to undo this later. | |
| 2572 | + */ | |
| 2572 | 2573 | if (!err) { |
| 2573 | 2574 | op->base.remap.unmap->va->va.addr = op->remap.start; |
| 2574 | 2575 | op->base.remap.unmap->va->va.range = op->remap.range; |
| @@ -2687,6 +2688,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, · 112 unmodified lines | ||
| 2687 | 2688 | |
| 2688 | 2689 | op->remap.start = xe_vma_start(old); |
| 2689 | 2690 | op->remap.range = xe_vma_size(old); |
| 2691 | + op->remap.old_start = op->remap.start; | |
| 2692 | + op->remap.old_range = op->remap.range; | |
| 2690 | 2693 | |
| 2691 | 2694 | flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK; |
| 2692 | 2695 | if (op->base.remap.prev) { |
| @@ -2835,8 +2838,19 @@ static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, · 142 unmodified lines | ||
| 2835 | 2838 | xe_svm_notifier_lock(vm); |
| 2836 | 2839 | vma->gpuva.flags &= ~XE_VMA_DESTROYED; |
| 2837 | 2840 | xe_svm_notifier_unlock(vm); |
| 2838 | − if (post_commit) | |
| 2841 | + if (post_commit) { | |
| 2842 | + /* | |
| 2843 | + * Restore the old va range, in case of the | |
| 2844 | + * prev/next skip optimisation. Otherwise what | |
| 2845 | + * we re-insert here could be smaller than the | |
| 2846 | + * original range. | |
| 2847 | + */ | |
| 2848 | + op->base.remap.unmap->va->va.addr = | |
| 2849 | + op->remap.old_start; | |
| 2850 | + op->base.remap.unmap->va->va.range = | |
| 2851 | + op->remap.old_range; | |
| 2839 | 2852 | xe_vm_insert_vma(vm, vma); |
| 2853 | + } | |
| 2840 | 2854 | } |
| 2841 | 2855 | break; |
| 2842 | 2856 | } |
| @@ -373,6 +373,10 @@ struct xe_vma_op_remap { · 372 unmodified lines | ||
| 373 | 373 | u64 start; |
| 374 | 374 | /** @range: range of the VMA unmap */ |
| 375 | 375 | u64 range; |
| 376 | + /** @old_start: Original start of the VMA we unmap */ | |
| 377 | + u64 old_start; | |
| 378 | + /** @old_range: Original range of the VMA we unmap */ | |
| 379 | + u64 old_range; | |
| 376 | 380 | /** @skip_prev: skip prev rebind */ |
| 377 | 381 | bool skip_prev; |
| 378 | 382 | /** @skip_next: skip next rebind */ |
| @@ -247,7 +247,8 @@ static const struct xe_rtp_entry_sr gt_was[] = { · 246 unmodified lines | ||
| 247 | 247 | LSN_DIM_Z_WGT_MASK, |
| 248 | 248 | LSN_LNI_WGT(1) | LSN_LNE_WGT(1) | |
| 249 | 249 | LSN_DIM_X_WGT(1) | LSN_DIM_Y_WGT(1) | |
| 250 | − LSN_DIM_Z_WGT(1))) | |
| 250 | + LSN_DIM_Z_WGT(1)), | |
| 251 | + SET(LSC_CHICKEN_BIT_0_UDW, L3_128B_256B_WRT_DIS)) | |
| 251 | 252 | }, |
| 252 | 253 | |
| 253 | 254 | /* Xe2_HPM */ |
| @@ -10,6 +10,8 @@ · 9 unmodified lines | ||
| 10 | 10 | #include <linux/hwmon.h> |
| 11 | 11 | #include <linux/i2c.h> |
| 12 | 12 | #include <linux/init.h> |
| 13 | +#include <linux/math64.h> | |
| 14 | +#include <linux/minmax.h> | |
| 13 | 15 | #include <linux/module.h> |
| 14 | 16 | #include <linux/regulator/consumer.h> |
| 15 | 17 | |
| @@ -33,7 +35,7 @@ · 17 unmodified lines | ||
| 33 | 35 | struct adm1177_state { |
| 34 | 36 | struct i2c_client *client; |
| 35 | 37 | u32 r_sense_uohm; |
| 36 | − u32 alert_threshold_ua; | |
| 38 | + u64 alert_threshold_ua; | |
| 37 | 39 | bool vrange_high; |
| 38 | 40 | }; |
| 39 | 41 | |
| @@ -48,7 +50,7 @@ static int adm1177_write_cmd(struct adm1177_state *st, u8 cmd) · 8 unmodified lines | ||
| 48 | 50 | } |
| 49 | 51 | |
| 50 | 52 | static int adm1177_write_alert_thr(struct adm1177_state *st, |
| 51 | − u32 alert_threshold_ua) | |
| 53 | + u64 alert_threshold_ua) | |
| 52 | 54 | { |
| 53 | 55 | u64 val; |
| 54 | 56 | int ret; |
| @@ -91,8 +93,8 @@ static int adm1177_read(struct device *dev, enum hwmon_sensor_types type, · 36 unmodified lines | ||
| 91 | 93 | *val = div_u64((105840000ull * dummy), |
| 92 | 94 | 4096 * st->r_sense_uohm); |
| 93 | 95 | return 0; |
| 94 | − case hwmon_curr_max_alarm: | |
| 95 | − *val = st->alert_threshold_ua; | |
| 96 | + case hwmon_curr_max: | |
| 97 | + *val = div_u64(st->alert_threshold_ua, 1000); | |
| 96 | 98 | return 0; |
| 97 | 99 | default: |
| 98 | 100 | return -EOPNOTSUPP; |
| @@ -126,9 +128,10 @@ static int adm1177_write(struct device *dev, enum hwmon_sensor_types type, · 27 unmodified lines | ||
| 126 | 128 | switch (type) { |
| 127 | 129 | case hwmon_curr: |
| 128 | 130 | switch (attr) { |
| 129 | − case hwmon_curr_max_alarm: | |
| 130 | − adm1177_write_alert_thr(st, val); | |
| 131 | − return 0; | |
| 131 | + case hwmon_curr_max: | |
| 132 | + val = clamp_val(val, 0, | |
| 133 | + div_u64(105840000ULL, st->r_sense_uohm)); | |
| 134 | + return adm1177_write_alert_thr(st, (u64)val * 1000); | |
| 132 | 135 | default: |
| 133 | 136 | return -EOPNOTSUPP; |
| 134 | 137 | } |
| @@ -156,7 +159,7 @@ static umode_t adm1177_is_visible(const void *data, · 21 unmodified lines | ||
| 156 | 159 | if (st->r_sense_uohm) |
| 157 | 160 | return 0444; |
| 158 | 161 | return 0; |
| 159 | − case hwmon_curr_max_alarm: | |
| 162 | + case hwmon_curr_max: | |
| 160 | 163 | if (st->r_sense_uohm) |
| 161 | 164 | return 0644; |
| 162 | 165 | return 0; |
| @@ -170,7 +173,7 @@ static umode_t adm1177_is_visible(const void *data, · 7 unmodified lines | ||
| 170 | 173 | |
| 171 | 174 | static const struct hwmon_channel_info * const adm1177_info[] = { |
| 172 | 175 | HWMON_CHANNEL_INFO(curr, |
| 173 | − HWMON_C_INPUT | HWMON_C_MAX_ALARM), | |
| 176 | + HWMON_C_INPUT | HWMON_C_MAX), | |
| 174 | 177 | HWMON_CHANNEL_INFO(in, |
| 175 | 178 | HWMON_I_INPUT), |
| 176 | 179 | NULL |
| @@ -192,7 +195,8 @@ static int adm1177_probe(struct i2c_client *client) · 15 unmodified lines | ||
| 192 | 195 | struct device *dev = &client->dev; |
| 193 | 196 | struct device *hwmon_dev; |
| 194 | 197 | struct adm1177_state *st; |
| 195 | − u32 alert_threshold_ua; | |
| 198 | + u64 alert_threshold_ua; | |
| 199 | + u32 prop; | |
| 196 | 200 | int ret; |
| 197 | 201 | |
| 198 | 202 | st = devm_kzalloc(dev, sizeof(*st), GFP_KERNEL); |
| @@ -208,22 +212,26 @@ static int adm1177_probe(struct i2c_client *client) · 9 unmodified lines | ||
| 208 | 212 | if (device_property_read_u32(dev, "shunt-resistor-micro-ohms", |
| 209 | 213 | &st->r_sense_uohm)) |
| 210 | 214 | st->r_sense_uohm = 0; |
| 211 | − if (device_property_read_u32(dev, "adi,shutdown-threshold-microamp", | |
| 212 | − &alert_threshold_ua)) { | |
| 213 | − if (st->r_sense_uohm) | |
| 214 | − /* | |
| 215 | − * set maximum default value from datasheet based on | |
| 216 | − * shunt-resistor | |
| 217 | − */ | |
| 218 | − alert_threshold_ua = div_u64(105840000000, | |
| 219 | − st->r_sense_uohm); | |
| 220 | − else | |
| 221 | − alert_threshold_ua = 0; | |
| 215 | + if (!device_property_read_u32(dev, "adi,shutdown-threshold-microamp", | |
| 216 | + &prop)) { | |
| 217 | + alert_threshold_ua = prop; | |
| 218 | + } else if (st->r_sense_uohm) { | |
| 219 | + /* | |
| 220 | + * set maximum default value from datasheet based on | |
| 221 | + * shunt-resistor | |
| 222 | + */ | |
| 223 | + alert_threshold_ua = div_u64(105840000000ULL, | |
| 224 | + st->r_sense_uohm); | |
| 225 | + } else { | |
| 226 | + alert_threshold_ua = 0; | |
| 222 | 227 | } |
| 223 | 228 | st->vrange_high = device_property_read_bool(dev, |
| 224 | 229 | "adi,vrange-high-enable"); |
| 225 | − if (alert_threshold_ua && st->r_sense_uohm) | |
| 226 | − adm1177_write_alert_thr(st, alert_threshold_ua); | |
| 230 | + if (alert_threshold_ua && st->r_sense_uohm) { | |
| 231 | + ret = adm1177_write_alert_thr(st, alert_threshold_ua); | |
| 232 | + if (ret) | |
| 233 | + return ret; | |
| 234 | + } | |
| 227 | 235 | |
| 228 | 236 | ret = adm1177_write_cmd(st, ADM1177_CMD_V_CONT | |
| 229 | 237 | ADM1177_CMD_I_CONT | |
| @@ -131,7 +131,7 @@ static int get_temp_target(struct peci_cputemp *priv, enum peci_temp_target_type · 130 unmodified lines | ||
| 131 | 131 | *val = priv->temp.target.tjmax; |
| 132 | 132 | break; |
| 133 | 133 | case crit_hyst_type: |
| 134 | − *val = priv->temp.target.tjmax - priv->temp.target.tcontrol; | |
| 134 | + *val = priv->temp.target.tcontrol; | |
| 135 | 135 | break; |
| 136 | 136 | default: |
| 137 | 137 | ret = -EOPNOTSUPP; |
| @@ -319,7 +319,7 @@ static umode_t cputemp_is_visible(const void *data, enum hwmon_sensor_types type · 181 unmodified lines | ||
| 319 | 319 | { |
| 320 | 320 | const struct peci_cputemp *priv = data; |
| 321 | 321 | |
| 322 | − if (channel > CPUTEMP_CHANNEL_NUMS) | |
| 322 | + if (channel >= CPUTEMP_CHANNEL_NUMS) | |
| 323 | 323 | return 0; |
| 324 | 324 | |
| 325 | 325 | if (channel < channel_core) |
| @@ -72,7 +72,8 @@ static int ina233_read_word_data(struct i2c_client *client, int page, · 71 unmodified lines | ||
| 72 | 72 | |
| 73 | 73 | /* Adjust returned value to match VIN coefficients */ |
| 74 | 74 | /* VIN: 1.25 mV VSHUNT: 2.5 uV LSB */ |
| 75 | − ret = DIV_ROUND_CLOSEST(ret * 25, 12500); | |
| 75 | + ret = clamp_val(DIV_ROUND_CLOSEST((s16)ret * 25, 12500), | |
| 76 | + S16_MIN, S16_MAX) & 0xffff; | |
| 76 | 77 | break; |
| 77 | 78 | default: |
| 78 | 79 | ret = -ENODATA; |
| @@ -96,7 +96,15 @@ static ssize_t isl68137_avs_enable_show_page(struct i2c_client *client, · 95 unmodified lines | ||
| 96 | 96 | int page, |
| 97 | 97 | char *buf) |
| 98 | 98 | { |
| 99 | − int val = pmbus_read_byte_data(client, page, PMBUS_OPERATION); | |
| 99 | + int val; | |
| 100 | + | |
| 101 | + val = pmbus_lock_interruptible(client); | |
| 102 | + if (val) | |
| 103 | + return val; | |
| 104 | + | |
| 105 | + val = pmbus_read_byte_data(client, page, PMBUS_OPERATION); | |
| 106 | + | |
| 107 | + pmbus_unlock(client); | |
| 100 | 108 | |
| 101 | 109 | if (val < 0) |
| 102 | 110 | return val; |
| @@ -118,6 +126,10 @@ static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, · 15 unmodified lines | ||
| 118 | 126 | |
| 119 | 127 | op_val = result ? ISL68137_VOUT_AVS : 0; |
| 120 | 128 | |
| 129 | + rc = pmbus_lock_interruptible(client); | |
| 130 | + if (rc) | |
| 131 | + return rc; | |
| 132 | + | |
| 121 | 133 | /* |
| 122 | 134 | * Writes to VOUT setpoint over AVSBus will persist after the VRM is |
| 123 | 135 | * switched to PMBus control. Switching back to AVSBus control |
| @@ -129,17 +141,20 @@ static ssize_t isl68137_avs_enable_store_page(struct i2c_client *client, · 5 unmodified lines | ||
| 129 | 141 | rc = pmbus_read_word_data(client, page, 0xff, |
| 130 | 142 | PMBUS_VOUT_COMMAND); |
| 131 | 143 | if (rc < 0) |
| 132 | − return rc; | |
| 144 | + goto unlock; | |
| 133 | 145 | |
| 134 | 146 | rc = pmbus_write_word_data(client, page, PMBUS_VOUT_COMMAND, |
| 135 | 147 | rc); |
| 136 | 148 | if (rc < 0) |
| 137 | − return rc; | |
| 149 | + goto unlock; | |
| 138 | 150 | } |
| 139 | 151 | |
| 140 | 152 | rc = pmbus_update_byte_data(client, page, PMBUS_OPERATION, |
| 141 | 153 | ISL68137_VOUT_AVS, op_val); |
| 142 | 154 | |
| 155 | +unlock: | |
| 156 | + pmbus_unlock(client); | |
| 157 | + | |
| 143 | 158 | return (rc < 0) ? rc : count; |
| 144 | 159 | } |
| 145 | 160 | |
| @@ -6,6 +6,7 @@ · 5 unmodified lines | ||
| 6 | 6 | * Copyright (c) 2012 Guenter Roeck |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | +#include <linux/atomic.h> | |
| 9 | 10 | #include <linux/debugfs.h> |
| 10 | 11 | #include <linux/delay.h> |
| 11 | 12 | #include <linux/dcache.h> |
| @@ -21,8 +22,8 @@ · 9 unmodified lines | ||
| 21 | 22 | #include <linux/pmbus.h> |
| 22 | 23 | #include <linux/regulator/driver.h> |
| 23 | 24 | #include <linux/regulator/machine.h> |
| 24 | −#include <linux/of.h> | |
| 25 | 25 | #include <linux/thermal.h> |
| 26 | +#include <linux/workqueue.h> | |
| 26 | 27 | #include "pmbus.h" |
| 27 | 28 | |
| 28 | 29 | /* |
| @@ -112,6 +113,11 @@ struct pmbus_data { · 83 unmodified lines | ||
| 112 | 113 | |
| 113 | 114 | struct mutex update_lock; |
| 114 | 115 | |
| 116 | +#if IS_ENABLED(CONFIG_REGULATOR) | |
| 117 | + atomic_t regulator_events[PMBUS_PAGES]; | |
| 118 | + struct work_struct regulator_notify_work; | |
| 119 | +#endif | |
| 120 | + | |
| 115 | 121 | bool has_status_word; /* device uses STATUS_WORD register */ |
| 116 | 122 | int (*read_status)(struct i2c_client *client, int page); |
| 117 | 123 | |
| @@ -1209,6 +1215,12 @@ static ssize_t pmbus_show_boolean(struct device *dev, · 1091 unmodified lines | ||
| 1209 | 1215 | return sysfs_emit(buf, "%d\n", val); |
| 1210 | 1216 | } |
| 1211 | 1217 | |
| 1218 | +static ssize_t pmbus_show_zero(struct device *dev, | |
| 1219 | + struct device_attribute *devattr, char *buf) | |
| 1220 | +{ | |
| 1221 | + return sysfs_emit(buf, "0\n"); | |
| 1222 | +} | |
| 1223 | + | |
| 1212 | 1224 | static ssize_t pmbus_show_sensor(struct device *dev, |
| 1213 | 1225 | struct device_attribute *devattr, char *buf) |
| 1214 | 1226 | { |
| @@ -1407,7 +1419,7 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, · 192 unmodified lines | ||
| 1407 | 1419 | int reg, |
| 1408 | 1420 | enum pmbus_sensor_classes class, |
| 1409 | 1421 | bool update, bool readonly, |
| 1410 | − bool convert) | |
| 1422 | + bool writeonly, bool convert) | |
| 1411 | 1423 | { |
| 1412 | 1424 | struct pmbus_sensor *sensor; |
| 1413 | 1425 | struct device_attribute *a; |
| @@ -1436,7 +1448,8 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, · 22 unmodified lines | ||
| 1436 | 1448 | sensor->data = -ENODATA; |
| 1437 | 1449 | pmbus_dev_attr_init(a, sensor->name, |
| 1438 | 1450 | readonly ? 0444 : 0644, |
| 1439 | − pmbus_show_sensor, pmbus_set_sensor); | |
| 1451 | + writeonly ? pmbus_show_zero : pmbus_show_sensor, | |
| 1452 | + pmbus_set_sensor); | |
| 1440 | 1453 | |
| 1441 | 1454 | if (pmbus_add_attribute(data, &a->attr)) |
| 1442 | 1455 | return NULL; |
| @@ -1495,8 +1508,10 @@ static int pmbus_add_label(struct pmbus_data *data, · 52 unmodified lines | ||
| 1495 | 1508 | struct pmbus_limit_attr { |
| 1496 | 1509 | u16 reg; /* Limit register */ |
| 1497 | 1510 | u16 sbit; /* Alarm attribute status bit */ |
| 1498 | − bool update; /* True if register needs updates */ | |
| 1499 | − bool low; /* True if low limit; for limits with compare functions only */ | |
| 1511 | + bool readonly:1; /* True if the attribute is read-only */ | |
| 1512 | + bool writeonly:1; /* True if the attribute is write-only */ | |
| 1513 | + bool update:1; /* True if register needs updates */ | |
| 1514 | + bool low:1; /* True if low limit; for limits with compare functions only */ | |
| 1500 | 1515 | const char *attr; /* Attribute name */ |
| 1501 | 1516 | const char *alarm; /* Alarm attribute name */ |
| 1502 | 1517 | }; |
| @@ -1511,9 +1526,9 @@ struct pmbus_sensor_attr { · 8 unmodified lines | ||
| 1511 | 1526 | u8 nlimit; /* # of limit registers */ |
| 1512 | 1527 | enum pmbus_sensor_classes class;/* sensor class */ |
| 1513 | 1528 | const char *label; /* sensor label */ |
| 1514 | − bool paged; /* true if paged sensor */ | |
| 1515 | − bool update; /* true if update needed */ | |
| 1516 | − bool compare; /* true if compare function needed */ | |
| 1529 | + bool paged:1; /* true if paged sensor */ | |
| 1530 | + bool update:1; /* true if update needed */ | |
| 1531 | + bool compare:1; /* true if compare function needed */ | |
| 1517 | 1532 | u32 func; /* sensor mask */ |
| 1518 | 1533 | u32 sfunc; /* sensor status mask */ |
| 1519 | 1534 | int sreg; /* status register */ |
| @@ -1544,7 +1559,7 @@ static int pmbus_add_limit_attrs(struct i2c_client *client, · 24 unmodified lines | ||
| 1544 | 1559 | curr = pmbus_add_sensor(data, name, l->attr, index, |
| 1545 | 1560 | page, 0xff, l->reg, attr->class, |
| 1546 | 1561 | attr->update || l->update, |
| 1547 | − false, true); | |
| 1562 | + l->readonly, l->writeonly, true); | |
| 1548 | 1563 | if (!curr) |
| 1549 | 1564 | return -ENOMEM; |
| 1550 | 1565 | if (l->sbit && (info->func[page] & attr->sfunc)) { |
| @@ -1584,7 +1599,7 @@ static int pmbus_add_sensor_attrs_one(struct i2c_client *client, · 33 unmodified lines | ||
| 1584 | 1599 | return ret; |
| 1585 | 1600 | } |
| 1586 | 1601 | base = pmbus_add_sensor(data, name, "input", index, page, phase, |
| 1587 | − attr->reg, attr->class, true, true, true); | |
| 1602 | + attr->reg, attr->class, true, true, false, true); | |
| 1588 | 1603 | if (!base) |
| 1589 | 1604 | return -ENOMEM; |
| 1590 | 1605 | /* No limit and alarm attributes for phase specific sensors */ |
| @@ -1707,23 +1722,29 @@ static const struct pmbus_limit_attr vin_limit_attrs[] = { · 116 unmodified lines | ||
| 1707 | 1722 | }, { |
| 1708 | 1723 | .reg = PMBUS_VIRT_READ_VIN_AVG, |
| 1709 | 1724 | .update = true, |
| 1725 | + .readonly = true, | |
| 1710 | 1726 | .attr = "average", |
| 1711 | 1727 | }, { |
| 1712 | 1728 | .reg = PMBUS_VIRT_READ_VIN_MIN, |
| 1713 | 1729 | .update = true, |
| 1730 | + .readonly = true, | |
| 1714 | 1731 | .attr = "lowest", |
| 1715 | 1732 | }, { |
| 1716 | 1733 | .reg = PMBUS_VIRT_READ_VIN_MAX, |
| 1717 | 1734 | .update = true, |
| 1735 | + .readonly = true, | |
| 1718 | 1736 | .attr = "highest", |
| 1719 | 1737 | }, { |
| 1720 | 1738 | .reg = PMBUS_VIRT_RESET_VIN_HISTORY, |
| 1739 | + .writeonly = true, | |
| 1721 | 1740 | .attr = "reset_history", |
| 1722 | 1741 | }, { |
| 1723 | 1742 | .reg = PMBUS_MFR_VIN_MIN, |
| 1743 | + .readonly = true, | |
| 1724 | 1744 | .attr = "rated_min", |
| 1725 | 1745 | }, { |
| 1726 | 1746 | .reg = PMBUS_MFR_VIN_MAX, |
| 1747 | + .readonly = true, | |
| 1727 | 1748 | .attr = "rated_max", |
| 1728 | 1749 | }, |
| 1729 | 1750 | }; |
| @@ -1776,23 +1797,29 @@ static const struct pmbus_limit_attr vout_limit_attrs[] = { · 46 unmodified lines | ||
| 1776 | 1797 | }, { |
| 1777 | 1798 | .reg = PMBUS_VIRT_READ_VOUT_AVG, |
| 1778 | 1799 | .update = true, |
| 1800 | + .readonly = true, | |
| 1779 | 1801 | .attr = "average", |
| 1780 | 1802 | }, { |
| 1781 | 1803 | .reg = PMBUS_VIRT_READ_VOUT_MIN, |
| 1782 | 1804 | .update = true, |
| 1805 | + .readonly = true, | |
| 1783 | 1806 | .attr = "lowest", |
| 1784 | 1807 | }, { |
| 1785 | 1808 | .reg = PMBUS_VIRT_READ_VOUT_MAX, |
| 1786 | 1809 | .update = true, |
| 1810 | + .readonly = true, | |
| 1787 | 1811 | .attr = "highest", |
| 1788 | 1812 | }, { |
| 1789 | 1813 | .reg = PMBUS_VIRT_RESET_VOUT_HISTORY, |
| 1814 | + .writeonly = true, | |
| 1790 | 1815 | .attr = "reset_history", |
| 1791 | 1816 | }, { |
| 1792 | 1817 | .reg = PMBUS_MFR_VOUT_MIN, |
| 1818 | + .readonly = true, | |
| 1793 | 1819 | .attr = "rated_min", |
| 1794 | 1820 | }, { |
| 1795 | 1821 | .reg = PMBUS_MFR_VOUT_MAX, |
| 1822 | + .readonly = true, | |
| 1796 | 1823 | .attr = "rated_max", |
| 1797 | 1824 | }, |
| 1798 | 1825 | }; |
| @@ -1852,20 +1879,25 @@ static const struct pmbus_limit_attr iin_limit_attrs[] = { · 53 unmodified lines | ||
| 1852 | 1879 | }, { |
| 1853 | 1880 | .reg = PMBUS_VIRT_READ_IIN_AVG, |
| 1854 | 1881 | .update = true, |
| 1882 | + .readonly = true, | |
| 1855 | 1883 | .attr = "average", |
| 1856 | 1884 | }, { |
| 1857 | 1885 | .reg = PMBUS_VIRT_READ_IIN_MIN, |
| 1858 | 1886 | .update = true, |
| 1887 | + .readonly = true, | |
| 1859 | 1888 | .attr = "lowest", |
| 1860 | 1889 | }, { |
| 1861 | 1890 | .reg = PMBUS_VIRT_READ_IIN_MAX, |
| 1862 | 1891 | .update = true, |
| 1892 | + .readonly = true, | |
| 1863 | 1893 | .attr = "highest", |
| 1864 | 1894 | }, { |
| 1865 | 1895 | .reg = PMBUS_VIRT_RESET_IIN_HISTORY, |
| 1896 | + .writeonly = true, | |
| 1866 | 1897 | .attr = "reset_history", |
| 1867 | 1898 | }, { |
| 1868 | 1899 | .reg = PMBUS_MFR_IIN_MAX, |
| 1900 | + .readonly = true, | |
| 1869 | 1901 | .attr = "rated_max", |
| 1870 | 1902 | }, |
| 1871 | 1903 | }; |
| @@ -1889,20 +1921,25 @@ static const struct pmbus_limit_attr iout_limit_attrs[] = { · 17 unmodified lines | ||
| 1889 | 1921 | }, { |
| 1890 | 1922 | .reg = PMBUS_VIRT_READ_IOUT_AVG, |
| 1891 | 1923 | .update = true, |
| 1924 | + .readonly = true, | |
| 1892 | 1925 | .attr = "average", |
| 1893 | 1926 | }, { |
| 1894 | 1927 | .reg = PMBUS_VIRT_READ_IOUT_MIN, |
| 1895 | 1928 | .update = true, |
| 1929 | + .readonly = true, | |
| 1896 | 1930 | .attr = "lowest", |
| 1897 | 1931 | }, { |
| 1898 | 1932 | .reg = PMBUS_VIRT_READ_IOUT_MAX, |
| 1899 | 1933 | .update = true, |
| 1934 | + .readonly = true, | |
| 1900 | 1935 | .attr = "highest", |
| 1901 | 1936 | }, { |
| 1902 | 1937 | .reg = PMBUS_VIRT_RESET_IOUT_HISTORY, |
| 1938 | + .writeonly = true, | |
| 1903 | 1939 | .attr = "reset_history", |
| 1904 | 1940 | }, { |
| 1905 | 1941 | .reg = PMBUS_MFR_IOUT_MAX, |
| 1942 | + .readonly = true, | |
| 1906 | 1943 | .attr = "rated_max", |
| 1907 | 1944 | }, |
| 1908 | 1945 | }; |
| @@ -1943,20 +1980,25 @@ static const struct pmbus_limit_attr pin_limit_attrs[] = { · 34 unmodified lines | ||
| 1943 | 1980 | }, { |
| 1944 | 1981 | .reg = PMBUS_VIRT_READ_PIN_AVG, |
| 1945 | 1982 | .update = true, |
| 1983 | + .readonly = true, | |
| 1946 | 1984 | .attr = "average", |
| 1947 | 1985 | }, { |
| 1948 | 1986 | .reg = PMBUS_VIRT_READ_PIN_MIN, |
| 1949 | 1987 | .update = true, |
| 1988 | + .readonly = true, | |
| 1950 | 1989 | .attr = "input_lowest", |
| 1951 | 1990 | }, { |
| 1952 | 1991 | .reg = PMBUS_VIRT_READ_PIN_MAX, |
| 1953 | 1992 | .update = true, |
| 1993 | + .readonly = true, | |
| 1954 | 1994 | .attr = "input_highest", |
| 1955 | 1995 | }, { |
| 1956 | 1996 | .reg = PMBUS_VIRT_RESET_PIN_HISTORY, |
| 1997 | + .writeonly = true, | |
| 1957 | 1998 | .attr = "reset_history", |
| 1958 | 1999 | }, { |
| 1959 | 2000 | .reg = PMBUS_MFR_PIN_MAX, |
| 2001 | + .readonly = true, | |
| 1960 | 2002 | .attr = "rated_max", |
| 1961 | 2003 | }, |
| 1962 | 2004 | }; |
| @@ -1980,20 +2022,25 @@ static const struct pmbus_limit_attr pout_limit_attrs[] = { · 17 unmodified lines | ||
| 1980 | 2022 | }, { |
| 1981 | 2023 | .reg = PMBUS_VIRT_READ_POUT_AVG, |
| 1982 | 2024 | .update = true, |
| 2025 | + .readonly = true, | |
| 1983 | 2026 | .attr = "average", |
| 1984 | 2027 | }, { |
| 1985 | 2028 | .reg = PMBUS_VIRT_READ_POUT_MIN, |
| 1986 | 2029 | .update = true, |
| 2030 | + .readonly = true, | |
| 1987 | 2031 | .attr = "input_lowest", |
| 1988 | 2032 | }, { |
| 1989 | 2033 | .reg = PMBUS_VIRT_READ_POUT_MAX, |
| 1990 | 2034 | .update = true, |
| 2035 | + .readonly = true, | |
| 1991 | 2036 | .attr = "input_highest", |
| 1992 | 2037 | }, { |
| 1993 | 2038 | .reg = PMBUS_VIRT_RESET_POUT_HISTORY, |
| 2039 | + .writeonly = true, | |
| 1994 | 2040 | .attr = "reset_history", |
| 1995 | 2041 | }, { |
| 1996 | 2042 | .reg = PMBUS_MFR_POUT_MAX, |
| 2043 | + .readonly = true, | |
| 1997 | 2044 | .attr = "rated_max", |
| 1998 | 2045 | }, |
| 1999 | 2046 | }; |
| @@ -2049,18 +2096,23 @@ static const struct pmbus_limit_attr temp_limit_attrs[] = { · 49 unmodified lines | ||
| 2049 | 2096 | .sbit = PB_TEMP_OT_FAULT, |
| 2050 | 2097 | }, { |
| 2051 | 2098 | .reg = PMBUS_VIRT_READ_TEMP_MIN, |
| 2099 | + .readonly = true, | |
| 2052 | 2100 | .attr = "lowest", |
| 2053 | 2101 | }, { |
| 2054 | 2102 | .reg = PMBUS_VIRT_READ_TEMP_AVG, |
| 2103 | + .readonly = true, | |
| 2055 | 2104 | .attr = "average", |
| 2056 | 2105 | }, { |
| 2057 | 2106 | .reg = PMBUS_VIRT_READ_TEMP_MAX, |
| 2107 | + .readonly = true, | |
| 2058 | 2108 | .attr = "highest", |
| 2059 | 2109 | }, { |
| 2060 | 2110 | .reg = PMBUS_VIRT_RESET_TEMP_HISTORY, |
| 2111 | + .writeonly = true, | |
| 2061 | 2112 | .attr = "reset_history", |
| 2062 | 2113 | }, { |
| 2063 | 2114 | .reg = PMBUS_MFR_MAX_TEMP_1, |
| 2115 | + .readonly = true, | |
| 2064 | 2116 | .attr = "rated_max", |
| 2065 | 2117 | }, |
| 2066 | 2118 | }; |
| @@ -2090,18 +2142,23 @@ static const struct pmbus_limit_attr temp_limit_attrs2[] = { · 23 unmodified lines | ||
| 2090 | 2142 | .sbit = PB_TEMP_OT_FAULT, |
| 2091 | 2143 | }, { |
| 2092 | 2144 | .reg = PMBUS_VIRT_READ_TEMP2_MIN, |
| 2145 | + .readonly = true, | |
| 2093 | 2146 | .attr = "lowest", |
| 2094 | 2147 | }, { |
| 2095 | 2148 | .reg = PMBUS_VIRT_READ_TEMP2_AVG, |
| 2149 | + .readonly = true, | |
| 2096 | 2150 | .attr = "average", |
| 2097 | 2151 | }, { |
| 2098 | 2152 | .reg = PMBUS_VIRT_READ_TEMP2_MAX, |
| 2153 | + .readonly = true, | |
| 2099 | 2154 | .attr = "highest", |
| 2100 | 2155 | }, { |
| 2101 | 2156 | .reg = PMBUS_VIRT_RESET_TEMP2_HISTORY, |
| 2157 | + .writeonly = true, | |
| 2102 | 2158 | .attr = "reset_history", |
| 2103 | 2159 | }, { |
| 2104 | 2160 | .reg = PMBUS_MFR_MAX_TEMP_2, |
| 2161 | + .readonly = true, | |
| 2105 | 2162 | .attr = "rated_max", |
| 2106 | 2163 | }, |
| 2107 | 2164 | }; |
| @@ -2131,6 +2188,7 @@ static const struct pmbus_limit_attr temp_limit_attrs3[] = { · 23 unmodified lines | ||
| 2131 | 2188 | .sbit = PB_TEMP_OT_FAULT, |
| 2132 | 2189 | }, { |
| 2133 | 2190 | .reg = PMBUS_MFR_MAX_TEMP_3, |
| 2191 | + .readonly = true, | |
| 2134 | 2192 | .attr = "rated_max", |
| 2135 | 2193 | }, |
| 2136 | 2194 | }; |
| @@ -2214,7 +2272,7 @@ static int pmbus_add_fan_ctrl(struct i2c_client *client, · 77 unmodified lines | ||
| 2214 | 2272 | |
| 2215 | 2273 | sensor = pmbus_add_sensor(data, "fan", "target", index, page, |
| 2216 | 2274 | 0xff, PMBUS_VIRT_FAN_TARGET_1 + id, PSC_FAN, |
| 2217 | − false, false, true); | |
| 2275 | + false, false, false, true); | |
| 2218 | 2276 | |
| 2219 | 2277 | if (!sensor) |
| 2220 | 2278 | return -ENOMEM; |
| @@ -2225,14 +2283,14 @@ static int pmbus_add_fan_ctrl(struct i2c_client *client, · 4 unmodified lines | ||
| 2225 | 2283 | |
| 2226 | 2284 | sensor = pmbus_add_sensor(data, "pwm", NULL, index, page, |
| 2227 | 2285 | 0xff, PMBUS_VIRT_PWM_1 + id, PSC_PWM, |
| 2228 | − false, false, true); | |
| 2286 | + false, false, false, true); | |
| 2229 | 2287 | |
| 2230 | 2288 | if (!sensor) |
| 2231 | 2289 | return -ENOMEM; |
| 2232 | 2290 | |
| 2233 | 2291 | sensor = pmbus_add_sensor(data, "pwm", "enable", index, page, |
| 2234 | 2292 | 0xff, PMBUS_VIRT_PWM_ENABLE_1 + id, PSC_PWM, |
| 2235 | − true, false, false); | |
| 2293 | + true, false, false, false); | |
| 2236 | 2294 | |
| 2237 | 2295 | if (!sensor) |
| 2238 | 2296 | return -ENOMEM; |
| @@ -2274,7 +2332,7 @@ static int pmbus_add_fan_attributes(struct i2c_client *client, · 35 unmodified lines | ||
| 2274 | 2332 | |
| 2275 | 2333 | if (pmbus_add_sensor(data, "fan", "input", index, |
| 2276 | 2334 | page, 0xff, pmbus_fan_registers[f], |
| 2277 | − PSC_FAN, true, true, true) == NULL) | |
| 2335 | + PSC_FAN, true, true, false, true) == NULL) | |
| 2278 | 2336 | return -ENOMEM; |
| 2279 | 2337 | |
| 2280 | 2338 | /* Fan control */ |
| @@ -3176,12 +3234,19 @@ static int pmbus_regulator_get_voltage(struct regulator_dev *rdev) · 895 unmodified lines | ||
| 3176 | 3234 | .class = PSC_VOLTAGE_OUT, |
| 3177 | 3235 | .convert = true, |
| 3178 | 3236 | }; |
| 3237 | + int ret; | |
| 3179 | 3238 | |
| 3239 | + mutex_lock(&data->update_lock); | |
| 3180 | 3240 | s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_READ_VOUT); |
| 3181 | − if (s.data < 0) | |
| 3182 | − return s.data; | |
| 3241 | + if (s.data < 0) { | |
| 3242 | + ret = s.data; | |
| 3243 | + goto unlock; | |
| 3244 | + } | |
| 3183 | 3245 | |
| 3184 | − return (int)pmbus_reg2data(data, &s) * 1000; /* unit is uV */ | |
| 3246 | + ret = (int)pmbus_reg2data(data, &s) * 1000; /* unit is uV */ | |
| 3247 | +unlock: | |
| 3248 | + mutex_unlock(&data->update_lock); | |
| 3249 | + return ret; | |
| 3185 | 3250 | } |
| 3186 | 3251 | |
| 3187 | 3252 | static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, |
| @@ -3198,16 +3263,22 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, · 10 unmodified lines | ||
| 3198 | 3263 | }; |
| 3199 | 3264 | int val = DIV_ROUND_CLOSEST(min_uv, 1000); /* convert to mV */ |
| 3200 | 3265 | int low, high; |
| 3266 | + int ret; | |
| 3201 | 3267 | |
| 3202 | 3268 | *selector = 0; |
| 3203 | 3269 | |
| 3270 | + mutex_lock(&data->update_lock); | |
| 3204 | 3271 | low = pmbus_regulator_get_low_margin(client, s.page); |
| 3205 | − if (low < 0) | |
| 3206 | − return low; | |
| 3272 | + if (low < 0) { | |
| 3273 | + ret = low; | |
| 3274 | + goto unlock; | |
| 3275 | + } | |
| 3207 | 3276 | |
| 3208 | 3277 | high = pmbus_regulator_get_high_margin(client, s.page); |
| 3209 | − if (high < 0) | |
| 3210 | − return high; | |
| 3278 | + if (high < 0) { | |
| 3279 | + ret = high; | |
| 3280 | + goto unlock; | |
| 3281 | + } | |
| 3211 | 3282 | |
| 3212 | 3283 | /* Make sure we are within margins */ |
| 3213 | 3284 | if (low > val) |
| @@ -3217,7 +3288,10 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, · 3 unmodified lines | ||
| 3217 | 3288 | |
| 3218 | 3289 | val = pmbus_data2reg(data, &s, val); |
| 3219 | 3290 | |
| 3220 | − return _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); | |
| 3291 | + ret = _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); | |
| 3292 | +unlock: | |
| 3293 | + mutex_unlock(&data->update_lock); | |
| 3294 | + return ret; | |
| 3221 | 3295 | } |
| 3222 | 3296 | |
| 3223 | 3297 | static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, |
| @@ -3227,6 +3301,7 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, · 3 unmodified lines | ||
| 3227 | 3301 | struct i2c_client *client = to_i2c_client(dev->parent); |
| 3228 | 3302 | struct pmbus_data *data = i2c_get_clientdata(client); |
| 3229 | 3303 | int val, low, high; |
| 3304 | + int ret; | |
| 3230 | 3305 | |
| 3231 | 3306 | if (data->flags & PMBUS_VOUT_PROTECTED) |
| 3232 | 3307 | return 0; |
| @@ -3239,18 +3314,29 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, · 6 unmodified lines | ||
| 3239 | 3314 | val = DIV_ROUND_CLOSEST(rdev->desc->min_uV + |
| 3240 | 3315 | (rdev->desc->uV_step * selector), 1000); /* convert to mV */ |
| 3241 | 3316 | |
| 3317 | + mutex_lock(&data->update_lock); | |
| 3318 | + | |
| 3242 | 3319 | low = pmbus_regulator_get_low_margin(client, rdev_get_id(rdev)); |
| 3243 | − if (low < 0) | |
| 3244 | − return low; | |
| 3320 | + if (low < 0) { | |
| 3321 | + ret = low; | |
| 3322 | + goto unlock; | |
| 3323 | + } | |
| 3245 | 3324 | |
| 3246 | 3325 | high = pmbus_regulator_get_high_margin(client, rdev_get_id(rdev)); |
| 3247 | − if (high < 0) | |
| 3248 | − return high; | |
| 3326 | + if (high < 0) { | |
| 3327 | + ret = high; | |
| 3328 | + goto unlock; | |
| 3329 | + } | |
| 3249 | 3330 | |
| 3250 | − if (val >= low && val <= high) | |
| 3251 | − return val * 1000; /* unit is uV */ | |
| 3331 | + if (val >= low && val <= high) { | |
| 3332 | + ret = val * 1000; /* unit is uV */ | |
| 3333 | + goto unlock; | |
| 3334 | + } | |
| 3252 | 3335 | |
| 3253 | − return 0; | |
| 3336 | + ret = 0; | |
| 3337 | +unlock: | |
| 3338 | + mutex_unlock(&data->update_lock); | |
| 3339 | + return ret; | |
| 3254 | 3340 | } |
| 3255 | 3341 | |
| 3256 | 3342 | const struct regulator_ops pmbus_regulator_ops = { |
| @@ -3281,12 +3367,42 @@ int pmbus_regulator_init_cb(struct regulator_dev *rdev, · 24 unmodified lines | ||
| 3281 | 3367 | } |
| 3282 | 3368 | EXPORT_SYMBOL_NS_GPL(pmbus_regulator_init_cb, "PMBUS"); |
| 3283 | 3369 | |
| 3370 | +static void pmbus_regulator_notify_work_cancel(void *data) | |
| 3371 | +{ | |
| 3372 | + struct pmbus_data *pdata = data; | |
| 3373 | + | |
| 3374 | + cancel_work_sync(&pdata->regulator_notify_work); | |
| 3375 | +} | |
| 3376 | + | |
| 3377 | +static void pmbus_regulator_notify_worker(struct work_struct *work) | |
| 3378 | +{ | |
| 3379 | + struct pmbus_data *data = | |
| 3380 | + container_of(work, struct pmbus_data, regulator_notify_work); | |
| 3381 | + int i, j; | |
| 3382 | + | |
| 3383 | + for (i = 0; i < data->info->pages; i++) { | |
| 3384 | + int event; | |
| 3385 | + | |
| 3386 | + event = atomic_xchg(&data->regulator_events[i], 0); | |
| 3387 | + if (!event) | |
| 3388 | + continue; | |
| 3389 | + | |
| 3390 | + for (j = 0; j < data->info->num_regulators; j++) { | |
| 3391 | + if (i == rdev_get_id(data->rdevs[j])) { | |
| 3392 | + regulator_notifier_call_chain(data->rdevs[j], | |
| 3393 | + event, NULL); | |
| 3394 | + break; | |
| 3395 | + } | |
| 3396 | + } | |
| 3397 | + } | |
| 3398 | +} | |
| 3399 | + | |
| 3284 | 3400 | static int pmbus_regulator_register(struct pmbus_data *data) |
| 3285 | 3401 | { |
| 3286 | 3402 | struct device *dev = data->dev; |
| 3287 | 3403 | const struct pmbus_driver_info *info = data->info; |
| 3288 | 3404 | const struct pmbus_platform_data *pdata = dev_get_platdata(dev); |
| 3289 | − int i; | |
| 3405 | + int i, ret; | |
| 3290 | 3406 | |
| 3291 | 3407 | data->rdevs = devm_kzalloc(dev, sizeof(struct regulator_dev *) * info->num_regulators, |
| 3292 | 3408 | GFP_KERNEL); |
| @@ -3310,19 +3426,19 @@ static int pmbus_regulator_register(struct pmbus_data *data) · 17 unmodified lines | ||
| 3310 | 3426 | info->reg_desc[i].name); |
| 3311 | 3427 | } |
| 3312 | 3428 | |
| 3429 | + INIT_WORK(&data->regulator_notify_work, pmbus_regulator_notify_worker); | |
| 3430 | + | |
| 3431 | + ret = devm_add_action_or_reset(dev, pmbus_regulator_notify_work_cancel, data); | |
| 3432 | + if (ret) | |
| 3433 | + return ret; | |
| 3434 | + | |
| 3313 | 3435 | return 0; |
| 3314 | 3436 | } |
| 3315 | 3437 | |
| 3316 | 3438 | static void pmbus_regulator_notify(struct pmbus_data *data, int page, int event) |
| 3317 | 3439 | { |
| 3318 | − int j; | |
| 3319 | − | |
| 3320 | − for (j = 0; j < data->info->num_regulators; j++) { | |
| 3321 | − if (page == rdev_get_id(data->rdevs[j])) { | |
| 3322 | − regulator_notifier_call_chain(data->rdevs[j], event, NULL); | |
| 3323 | − break; | |
| 3324 | − } | |
| 3325 | − } | |
| 3440 | + atomic_or(event, &data->regulator_events[page]); | |
| 3441 | + schedule_work(&data->regulator_notify_work); | |
| 3326 | 3442 | } |
| 3327 | 3443 | #else |
| 3328 | 3444 | static int pmbus_regulator_register(struct pmbus_data *data) |
| @@ -7,6 +7,7 @@ · 6 unmodified lines | ||
| 7 | 7 | |
| 8 | 8 | #include <linux/module.h> |
| 9 | 9 | #include <linux/platform_device.h> |
| 10 | +#include <linux/pm_domain.h> | |
| 10 | 11 | #include <linux/pm_runtime.h> |
| 11 | 12 | #include <linux/soc/amd/isp4_misc.h> |
| 12 | 13 | |
| @@ -76,22 +77,20 @@ static int amd_isp_dw_i2c_plat_probe(struct platform_device *pdev) · 63 unmodified lines | ||
| 76 | 77 | |
| 77 | 78 | device_enable_async_suspend(&pdev->dev); |
| 78 | 79 | |
| 79 | − pm_runtime_enable(&pdev->dev); | |
| 80 | − pm_runtime_get_sync(&pdev->dev); | |
| 81 | − | |
| 80 | + dev_pm_genpd_resume(&pdev->dev); | |
| 82 | 81 | ret = i2c_dw_probe(isp_i2c_dev); |
| 83 | 82 | if (ret) { |
| 84 | 83 | dev_err_probe(&pdev->dev, ret, "i2c_dw_probe failed\n"); |
| 85 | 84 | goto error_release_rpm; |
| 86 | 85 | } |
| 87 | − | |
| 88 | − pm_runtime_put_sync(&pdev->dev); | |
| 86 | + dev_pm_genpd_suspend(&pdev->dev); | |
| 87 | + pm_runtime_set_suspended(&pdev->dev); | |
| 88 | + pm_runtime_enable(&pdev->dev); | |
| 89 | 89 | |
| 90 | 90 | return 0; |
| 91 | 91 | |
| 92 | 92 | error_release_rpm: |
| 93 | 93 | amd_isp_dw_i2c_plat_pm_cleanup(isp_i2c_dev); |
| 94 | − pm_runtime_put_sync(&pdev->dev); | |
| 95 | 94 | return ret; |
| 96 | 95 | } |
| 97 | 96 | |
| @@ -1018,8 +1018,9 @@ static inline int i2c_imx_isr_read(struct imx_i2c_struct *i2c_imx) · 1017 unmodified lines | ||
| 1018 | 1018 | return 0; |
| 1019 | 1019 | } |
| 1020 | 1020 | |
| 1021 | −static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) | |
| 1021 | +static inline enum imx_i2c_state i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) | |
| 1022 | 1022 | { |
| 1023 | + enum imx_i2c_state next_state = IMX_I2C_STATE_READ_CONTINUE; | |
| 1023 | 1024 | unsigned int temp; |
| 1024 | 1025 | |
| 1025 | 1026 | if ((i2c_imx->msg->len - 1) == i2c_imx->msg_buf_idx) { |
| @@ -1033,18 +1034,20 @@ static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) · 7 unmodified lines | ||
| 1033 | 1034 | i2c_imx->stopped = 1; |
| 1034 | 1035 | temp &= ~(I2CR_MSTA | I2CR_MTX); |
| 1035 | 1036 | imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); |
| 1036 | − } else { | |
| 1037 | − /* | |
| 1038 | − * For i2c master receiver repeat restart operation like: | |
| 1039 | − * read -> repeat MSTA -> read/write | |
| 1040 | − * The controller must set MTX before read the last byte in | |
| 1041 | − * the first read operation, otherwise the first read cost | |
| 1042 | − * one extra clock cycle. | |
| 1043 | − */ | |
| 1044 | − temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); | |
| 1045 | − temp |= I2CR_MTX; | |
| 1046 | − imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); | |
| 1037 | + | |
| 1038 | + return IMX_I2C_STATE_DONE; | |
| 1047 | 1039 | } |
| 1040 | + /* | |
| 1041 | + * For i2c master receiver repeat restart operation like: | |
| 1042 | + * read -> repeat MSTA -> read/write | |
| 1043 | + * The controller must set MTX before read the last byte in | |
| 1044 | + * the first read operation, otherwise the first read cost | |
| 1045 | + * one extra clock cycle. | |
| 1046 | + */ | |
| 1047 | + temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); | |
| 1048 | + temp |= I2CR_MTX; | |
| 1049 | + imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR); | |
| 1050 | + next_state = IMX_I2C_STATE_DONE; | |
| 1048 | 1051 | } else if (i2c_imx->msg_buf_idx == (i2c_imx->msg->len - 2)) { |
| 1049 | 1052 | temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR); |
| 1050 | 1053 | temp |= I2CR_TXAK; |
| @@ -1052,6 +1055,7 @@ static inline void i2c_imx_isr_read_continue(struct imx_i2c_struct *i2c_imx) · 1 unmodified line | ||
| 1052 | 1055 | } |
| 1053 | 1056 | |
| 1054 | 1057 | i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR); |
| 1058 | + return next_state; | |
| 1055 | 1059 | } |
| 1056 | 1060 | |
| 1057 | 1061 | static inline void i2c_imx_isr_read_block_data_len(struct imx_i2c_struct *i2c_imx) |
| @@ -1088,11 +1092,9 @@ static irqreturn_t i2c_imx_master_isr(struct imx_i2c_struct *i2c_imx, unsigned i · 30 unmodified lines | ||
| 1088 | 1092 | break; |
| 1089 | 1093 | |
| 1090 | 1094 | case IMX_I2C_STATE_READ_CONTINUE: |
| 1091 | − i2c_imx_isr_read_continue(i2c_imx); | |
| 1092 | − if (i2c_imx->msg_buf_idx == i2c_imx->msg->len) { | |
| 1093 | − i2c_imx->state = IMX_I2C_STATE_DONE; | |
| 1095 | + i2c_imx->state = i2c_imx_isr_read_continue(i2c_imx); | |
| 1096 | + if (i2c_imx->state == IMX_I2C_STATE_DONE) | |
| 1094 | 1097 | wake_up(&i2c_imx->queue); |
| 1095 | − } | |
| 1096 | 1098 | break; |
| 1097 | 1099 | |
| 1098 | 1100 | case IMX_I2C_STATE_READ_BLOCK_DATA: |
| @@ -1490,6 +1492,7 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, · 391 unmodified lines | ||
| 1490 | 1492 | bool is_lastmsg) |
| 1491 | 1493 | { |
| 1492 | 1494 | int block_data = msgs->flags & I2C_M_RECV_LEN; |
| 1495 | + int ret = 0; | |
| 1493 | 1496 | |
| 1494 | 1497 | dev_dbg(&i2c_imx->adapter.dev, |
| 1495 | 1498 | "<%s> write slave address: addr=0x%x\n", |
| @@ -1522,10 +1525,20 @@ static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs, · 26 unmodified lines | ||
| 1522 | 1525 | dev_err(&i2c_imx->adapter.dev, "<%s> read timedout\n", __func__); |
| 1523 | 1526 | return -ETIMEDOUT; |
| 1524 | 1527 | } |
| 1525 | − if (!i2c_imx->stopped) | |
| 1526 | − return i2c_imx_bus_busy(i2c_imx, 0, false); | |
| 1528 | + if (i2c_imx->is_lastmsg) { | |
| 1529 | + if (!i2c_imx->stopped) | |
| 1530 | + ret = i2c_imx_bus_busy(i2c_imx, 0, false); | |
| 1531 | + /* | |
| 1532 | + * Only read the last byte of the last message after the bus is | |
| 1533 | + * not busy. Else the controller generates another clock which | |
| 1534 | + * might confuse devices. | |
| 1535 | + */ | |
| 1536 | + if (!ret) | |
| 1537 | + i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = imx_i2c_read_reg(i2c_imx, | |
| 1538 | + IMX_I2C_I2DR); | |
| 1539 | + } | |
| 1527 | 1540 | |
| 1528 | − return 0; | |
| 1541 | + return ret; | |
| 1529 | 1542 | } |
| 1530 | 1543 | |
| 1531 | 1544 | static int i2c_imx_xfer_common(struct i2c_adapter *adapter, |
| @@ -608,14 +608,29 @@ int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, · 607 unmodified lines | ||
| 608 | 608 | if (rdma_rw_io_needs_mr(qp->device, port_num, dir, sg_cnt)) { |
| 609 | 609 | ret = rdma_rw_init_mr_wrs(ctx, qp, port_num, sg, sg_cnt, |
| 610 | 610 | sg_offset, remote_addr, rkey, dir); |
| 611 | − } else if (sg_cnt > 1) { | |
| 611 | + /* | |
| 612 | + * If MR init succeeded or failed for a reason other | |
| 613 | + * than pool exhaustion, that result is final. | |
| 614 | + * | |
| 615 | + * Pool exhaustion (-EAGAIN) from the max_sgl_rd | |
| 616 | + * optimization is recoverable: fall back to | |
| 617 | + * direct SGE posting. iWARP and force_mr require | |
| 618 | + * MRs unconditionally, so -EAGAIN is terminal. | |
| 619 | + */ | |
| 620 | + if (ret != -EAGAIN || | |
| 621 | + rdma_protocol_iwarp(qp->device, port_num) || | |
| 622 | + unlikely(rdma_rw_force_mr)) | |
| 623 | + goto out; | |
| 624 | + } | |
| 625 | + | |
| 626 | + if (sg_cnt > 1) | |
| 612 | 627 | ret = rdma_rw_init_map_wrs(ctx, qp, sg, sg_cnt, sg_offset, |
| 613 | 628 | remote_addr, rkey, dir); |
| 614 | − } else { | |
| 629 | + else | |
| 615 | 630 | ret = rdma_rw_init_single_wr(ctx, qp, sg, sg_offset, |
| 616 | 631 | remote_addr, rkey, dir); |
| 617 | − } | |
| 618 | 632 | |
| 633 | +out: | |
| 619 | 634 | if (ret < 0) |
| 620 | 635 | goto out_unmap_sg; |
| 621 | 636 | return ret; |
| @@ -686,14 +701,16 @@ int rdma_rw_ctx_init_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp, · 64 unmodified lines | ||
| 686 | 701 | return ret; |
| 687 | 702 | |
| 688 | 703 | /* |
| 689 | − * IOVA mapping not available. Check if MR registration provides | |
| 690 | − * better performance than multiple SGE entries. | |
| 704 | + * IOVA not available; fall back to the map_wrs path, which maps | |
| 705 | + * each bvec as a direct SGE. This is always correct: the MR path | |
| 706 | + * is a throughput optimization, not a correctness requirement. | |
| 707 | + * (iWARP, which does require MRs, is handled by the check above.) | |
| 708 | + * | |
| 709 | + * The rdma_rw_io_needs_mr() gate is not used here because nr_bvec | |
| 710 | + * is a raw page count that overstates DMA entry demand -- the bvec | |
| 711 | + * caller has no post-DMA-coalescing segment count, and feeding the | |
| 712 | + * inflated count into the MR path exhausts the pool on RDMA READs. | |
| 691 | 713 | */ |
| 692 | − if (rdma_rw_io_needs_mr(dev, port_num, dir, nr_bvec)) | |
| 693 | − return rdma_rw_init_mr_wrs_bvec(ctx, qp, port_num, bvecs, | |
| 694 | − nr_bvec, &iter, remote_addr, | |
| 695 | − rkey, dir); | |
| 696 | − | |
| 697 | 714 | return rdma_rw_init_map_wrs_bvec(ctx, qp, bvecs, nr_bvec, &iter, |
| 698 | 715 | remote_addr, rkey, dir); |
| 699 | 716 | } |
| @@ -55,7 +55,8 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d · 54 unmodified lines | ||
| 55 | 55 | |
| 56 | 56 | if (dirty) |
| 57 | 57 | ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt, |
| 58 | − DMA_BIDIRECTIONAL, 0); | |
| 58 | + DMA_BIDIRECTIONAL, | |
| 59 | + DMA_ATTR_REQUIRE_COHERENT); | |
| 59 | 60 | |
| 60 | 61 | for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) { |
| 61 | 62 | unpin_user_page_range_dirty_lock(sg_page(sg), |
| @@ -169,7 +170,7 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, · 107 unmodified lines | ||
| 169 | 170 | unsigned long lock_limit; |
| 170 | 171 | unsigned long new_pinned; |
| 171 | 172 | unsigned long cur_base; |
| 172 | − unsigned long dma_attr = 0; | |
| 173 | + unsigned long dma_attr = DMA_ATTR_REQUIRE_COHERENT; | |
| 173 | 174 | struct mm_struct *mm; |
| 174 | 175 | unsigned long npages; |
| 175 | 176 | int pinned, ret; |
| @@ -210,7 +210,7 @@ static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev) · 209 unmodified lines | ||
| 210 | 210 | return rc; |
| 211 | 211 | } |
| 212 | 212 | |
| 213 | −static void bng_re_query_hwrm_version(struct bng_re_dev *rdev) | |
| 213 | +static int bng_re_query_hwrm_version(struct bng_re_dev *rdev) | |
| 214 | 214 | { |
| 215 | 215 | struct bnge_auxr_dev *aux_dev = rdev->aux_dev; |
| 216 | 216 | struct hwrm_ver_get_output ver_get_resp = {}; |
| @@ -230,7 +230,7 @@ static void bng_re_query_hwrm_version(struct bng_re_dev *rdev) · 13 unmodified lines | ||
| 230 | 230 | if (rc) { |
| 231 | 231 | ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x", |
| 232 | 232 | rc); |
| 233 | − return; | |
| 233 | + return rc; | |
| 234 | 234 | } |
| 235 | 235 | |
| 236 | 236 | cctx = rdev->chip_ctx; |
| @@ -244,6 +244,8 @@ static void bng_re_query_hwrm_version(struct bng_re_dev *rdev) · 7 unmodified lines | ||
| 244 | 244 | |
| 245 | 245 | if (!cctx->hwrm_cmd_max_timeout) |
| 246 | 246 | cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT; |
| 247 | + | |
| 248 | + return 0; | |
| 247 | 249 | } |
| 248 | 250 | |
| 249 | 251 | static void bng_re_dev_uninit(struct bng_re_dev *rdev) |
| @@ -306,13 +308,15 @@ static int bng_re_dev_init(struct bng_re_dev *rdev) · 56 unmodified lines | ||
| 306 | 308 | goto msix_ctx_fail; |
| 307 | 309 | } |
| 308 | 310 | |
| 309 | − bng_re_query_hwrm_version(rdev); | |
| 311 | + rc = bng_re_query_hwrm_version(rdev); | |
| 312 | + if (rc) | |
| 313 | + goto destroy_chip_ctx; | |
| 310 | 314 | |
| 311 | 315 | rc = bng_re_alloc_fw_channel(&rdev->bng_res, &rdev->rcfw); |
| 312 | 316 | if (rc) { |
| 313 | 317 | ibdev_err(&rdev->ibdev, |
| 314 | 318 | "Failed to allocate RCFW Channel: %#x\n", rc); |
| 315 | − goto alloc_fw_chl_fail; | |
| 319 | + goto destroy_chip_ctx; | |
| 316 | 320 | } |
| 317 | 321 | |
| 318 | 322 | /* Allocate nq record memory */ |
| @@ -391,7 +395,7 @@ free_rcfw: · 72 unmodified lines | ||
| 391 | 395 | kfree(rdev->nqr); |
| 392 | 396 | nq_alloc_fail: |
| 393 | 397 | bng_re_free_rcfw_channel(&rdev->rcfw); |
| 394 | −alloc_fw_chl_fail: | |
| 398 | +destroy_chip_ctx: | |
| 395 | 399 | bng_re_destroy_chip_ctx(rdev); |
| 396 | 400 | msix_ctx_fail: |
| 397 | 401 | bnge_unregister_dev(rdev->aux_dev); |
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause |
| 2 | 2 | /* |
| 3 | − * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. | |
| 3 | + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. | |
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | #include <linux/log2.h> |
| @@ -310,23 +310,19 @@ static inline struct efa_comp_ctx *efa_com_get_comp_ctx_by_cmd_id(struct efa_com · 303 unmodified lines | ||
| 310 | 310 | return &aq->comp_ctx[ctx_id]; |
| 311 | 311 | } |
| 312 | 312 | |
| 313 | −static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, | |
| 314 | − struct efa_admin_aq_entry *cmd, | |
| 315 | − size_t cmd_size_in_bytes, | |
| 316 | − struct efa_admin_acq_entry *comp, | |
| 317 | − size_t comp_size_in_bytes) | |
| 313 | +static void __efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, | |
| 314 | + struct efa_comp_ctx *comp_ctx, | |
| 315 | + struct efa_admin_aq_entry *cmd, | |
| 316 | + size_t cmd_size_in_bytes, | |
| 317 | + struct efa_admin_acq_entry *comp, | |
| 318 | + size_t comp_size_in_bytes) | |
| 318 | 319 | { |
| 319 | 320 | struct efa_admin_aq_entry *aqe; |
| 320 | − struct efa_comp_ctx *comp_ctx; | |
| 321 | 321 | u16 queue_size_mask; |
| 322 | 322 | u16 cmd_id; |
| 323 | 323 | u16 ctx_id; |
| 324 | 324 | u16 pi; |
| 325 | 325 | |
| 326 | − comp_ctx = efa_com_alloc_comp_ctx(aq); | |
| 327 | − if (!comp_ctx) | |
| 328 | − return ERR_PTR(-EINVAL); | |
| 329 | − | |
| 330 | 326 | queue_size_mask = aq->depth - 1; |
| 331 | 327 | pi = aq->sq.pc & queue_size_mask; |
| 332 | 328 | ctx_id = efa_com_get_comp_ctx_id(aq, comp_ctx); |
| @@ -360,8 +356,6 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu · 27 unmodified lines | ||
| 360 | 356 | |
| 361 | 357 | /* barrier not needed in case of writel */ |
| 362 | 358 | writel(aq->sq.pc, aq->sq.db_addr); |
| 363 | − | |
| 364 | − return comp_ctx; | |
| 365 | 359 | } |
| 366 | 360 | |
| 367 | 361 | static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) |
| @@ -394,28 +388,25 @@ static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) · 26 unmodified lines | ||
| 394 | 388 | return 0; |
| 395 | 389 | } |
| 396 | 390 | |
| 397 | −static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, | |
| 398 | − struct efa_admin_aq_entry *cmd, | |
| 399 | − size_t cmd_size_in_bytes, | |
| 400 | − struct efa_admin_acq_entry *comp, | |
| 401 | − size_t comp_size_in_bytes) | |
| 391 | +static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, | |
| 392 | + struct efa_comp_ctx *comp_ctx, | |
| 393 | + struct efa_admin_aq_entry *cmd, | |
| 394 | + size_t cmd_size_in_bytes, | |
| 395 | + struct efa_admin_acq_entry *comp, | |
| 396 | + size_t comp_size_in_bytes) | |
| 402 | 397 | { |
| 403 | − struct efa_comp_ctx *comp_ctx; | |
| 404 | − | |
| 405 | 398 | spin_lock(&aq->sq.lock); |
| 406 | 399 | if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { |
| 407 | 400 | ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); |
| 408 | 401 | spin_unlock(&aq->sq.lock); |
| 409 | − return ERR_PTR(-ENODEV); | |
| 402 | + return -ENODEV; | |
| 410 | 403 | } |
| 411 | 404 | |
| 412 | − comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp, | |
| 413 | − comp_size_in_bytes); | |
| 405 | + __efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size_in_bytes, comp, | |
| 406 | + comp_size_in_bytes); | |
| 414 | 407 | spin_unlock(&aq->sq.lock); |
| 415 | − if (IS_ERR(comp_ctx)) | |
| 416 | − clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); | |
| 417 | 408 | |
| 418 | − return comp_ctx; | |
| 409 | + return 0; | |
| 419 | 410 | } |
| 420 | 411 | |
| 421 | 412 | static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, |
| @@ -512,7 +503,6 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c · 90 unmodified lines | ||
| 512 | 503 | { |
| 513 | 504 | unsigned long timeout; |
| 514 | 505 | unsigned long flags; |
| 515 | − int err; | |
| 516 | 506 | |
| 517 | 507 | timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); |
| 518 | 508 | |
| @@ -532,24 +522,20 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c · 13 unmodified lines | ||
| 532 | 522 | atomic64_inc(&aq->stats.no_completion); |
| 533 | 523 | |
| 534 | 524 | clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); |
| 535 | − err = -ETIME; | |
| 536 | − goto out; | |
| 525 | + return -ETIME; | |
| 537 | 526 | } |
| 538 | 527 | |
| 539 | 528 | msleep(aq->poll_interval); |
| 540 | 529 | } |
| 541 | 530 | |
| 542 | − err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); | |
| 543 | −out: | |
| 544 | − efa_com_dealloc_comp_ctx(aq, comp_ctx); | |
| 545 | − return err; | |
| 531 | + return efa_com_comp_status_to_errno( | |
| 532 | + comp_ctx->user_cqe->acq_common_descriptor.status); | |
| 546 | 533 | } |
| 547 | 534 | |
| 548 | 535 | static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, |
| 549 | 536 | struct efa_com_admin_queue *aq) |
| 550 | 537 | { |
| 551 | 538 | unsigned long flags; |
| 552 | − int err; | |
| 553 | 539 | |
| 554 | 540 | wait_for_completion_timeout(&comp_ctx->wait_event, |
| 555 | 541 | usecs_to_jiffies(aq->completion_timeout)); |
| @@ -585,14 +571,11 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com · 29 unmodified lines | ||
| 585 | 571 | aq->cq.cc); |
| 586 | 572 | |
| 587 | 573 | clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); |
| 588 | − err = -ETIME; | |
| 589 | − goto out; | |
| 574 | + return -ETIME; | |
| 590 | 575 | } |
| 591 | 576 | |
| 592 | − err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status); | |
| 593 | −out: | |
| 594 | − efa_com_dealloc_comp_ctx(aq, comp_ctx); | |
| 595 | − return err; | |
| 577 | + return efa_com_comp_status_to_errno( | |
| 578 | + comp_ctx->user_cqe->acq_common_descriptor.status); | |
| 596 | 579 | } |
| 597 | 580 | |
| 598 | 581 | /* |
| @@ -642,30 +625,39 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq, · 43 unmodified lines | ||
| 642 | 625 | ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", |
| 643 | 626 | efa_com_cmd_str(cmd->aq_common_descriptor.opcode), |
| 644 | 627 | cmd->aq_common_descriptor.opcode); |
| 645 | − comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size); | |
| 646 | − if (IS_ERR(comp_ctx)) { | |
| 628 | + | |
| 629 | + comp_ctx = efa_com_alloc_comp_ctx(aq); | |
| 630 | + if (!comp_ctx) { | |
| 631 | + clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); | |
| 632 | + up(&aq->avail_cmds); | |
| 633 | + return -EINVAL; | |
| 634 | + } | |
| 635 | + | |
| 636 | + err = efa_com_submit_admin_cmd(aq, comp_ctx, cmd, cmd_size, comp, comp_size); | |
| 637 | + if (err) { | |
| 647 | 638 | ibdev_err_ratelimited( |
| 648 | 639 | aq->efa_dev, |
| 649 | − "Failed to submit command %s (opcode %u) err %pe\n", | |
| 640 | + "Failed to submit command %s (opcode %u) err %d\n", | |
| 650 | 641 | efa_com_cmd_str(cmd->aq_common_descriptor.opcode), |
| 651 | − cmd->aq_common_descriptor.opcode, comp_ctx); | |
| 642 | + cmd->aq_common_descriptor.opcode, err); | |
| 652 | 643 | |
| 644 | + efa_com_dealloc_comp_ctx(aq, comp_ctx); | |
| 653 | 645 | up(&aq->avail_cmds); |
| 654 | 646 | atomic64_inc(&aq->stats.cmd_err); |
| 655 | − return PTR_ERR(comp_ctx); | |
| 647 | + return err; | |
| 656 | 648 | } |
| 657 | 649 | |
| 658 | 650 | err = efa_com_wait_and_process_admin_cq(comp_ctx, aq); |
| 659 | 651 | if (err) { |
| 660 | 652 | ibdev_err_ratelimited( |
| 661 | 653 | aq->efa_dev, |
| 662 | − "Failed to process command %s (opcode %u) comp_status %d err %d\n", | |
| 654 | + "Failed to process command %s (opcode %u) err %d\n", | |
| 663 | 655 | efa_com_cmd_str(cmd->aq_common_descriptor.opcode), |
| 664 | − cmd->aq_common_descriptor.opcode, | |
| 665 | − comp_ctx->user_cqe->acq_common_descriptor.status, err); | |
| 656 | + cmd->aq_common_descriptor.opcode, err); | |
| 666 | 657 | atomic64_inc(&aq->stats.cmd_err); |
| 667 | 658 | } |
| 668 | 659 | |
| 660 | + efa_com_dealloc_comp_ctx(aq, comp_ctx); | |
| 669 | 661 | up(&aq->avail_cmds); |
| 670 | 662 | |
| 671 | 663 | return err; |
| @@ -508,6 +508,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, · 507 unmodified lines | ||
| 508 | 508 | { |
| 509 | 509 | const struct ib_global_route *grh; |
| 510 | 510 | enum rdma_network_type net; |
| 511 | + u8 smac[ETH_ALEN]; | |
| 511 | 512 | u16 vlan; |
| 512 | 513 | int rc; |
| 513 | 514 | |
| @@ -518,7 +519,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, · 4 unmodified lines | ||
| 518 | 519 | |
| 519 | 520 | grh = rdma_ah_read_grh(attr); |
| 520 | 521 | |
| 521 | − rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, &hdr->eth.smac_h[0]); | |
| 522 | + rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac); | |
| 522 | 523 | if (rc) |
| 523 | 524 | return rc; |
| 524 | 525 | |
| @@ -536,6 +537,7 @@ static int ionic_build_hdr(struct ionic_ibdev *dev, · 11 unmodified lines | ||
| 536 | 537 | if (rc) |
| 537 | 538 | return rc; |
| 538 | 539 | |
| 540 | + ether_addr_copy(hdr->eth.smac_h, smac); | |
| 539 | 541 | ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac); |
| 540 | 542 | |
| 541 | 543 | if (net == RDMA_NETWORK_IPV4) { |
| @@ -2241,11 +2241,12 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, · 2240 unmodified lines | ||
| 2241 | 2241 | int oldarpindex; |
| 2242 | 2242 | int arpindex; |
| 2243 | 2243 | struct net_device *netdev = iwdev->netdev; |
| 2244 | + int ret; | |
| 2244 | 2245 | |
| 2245 | 2246 | /* create an hte and cm_node for this instance */ |
| 2246 | 2247 | cm_node = kzalloc_obj(*cm_node, GFP_ATOMIC); |
| 2247 | 2248 | if (!cm_node) |
| 2248 | − return NULL; | |
| 2249 | + return ERR_PTR(-ENOMEM); | |
| 2249 | 2250 | |
| 2250 | 2251 | /* set our node specific transport info */ |
| 2251 | 2252 | cm_node->ipv4 = cm_info->ipv4; |
| @@ -2348,8 +2349,10 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, · 96 unmodified lines | ||
| 2348 | 2349 | arpindex = -EINVAL; |
| 2349 | 2350 | } |
| 2350 | 2351 | |
| 2351 | − if (arpindex < 0) | |
| 2352 | + if (arpindex < 0) { | |
| 2353 | + ret = -EINVAL; | |
| 2352 | 2354 | goto err; |
| 2355 | + } | |
| 2353 | 2356 | |
| 2354 | 2357 | ether_addr_copy(cm_node->rem_mac, |
| 2355 | 2358 | iwdev->rf->arp_table[arpindex].mac_addr); |
| @@ -2360,7 +2363,7 @@ irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, · 4 unmodified lines | ||
| 2360 | 2363 | err: |
| 2361 | 2364 | kfree(cm_node); |
| 2362 | 2365 | |
| 2363 | − return NULL; | |
| 2366 | + return ERR_PTR(ret); | |
| 2364 | 2367 | } |
| 2365 | 2368 | |
| 2366 | 2369 | static void irdma_destroy_connection(struct irdma_cm_node *cm_node) |
| @@ -3021,8 +3024,8 @@ static int irdma_create_cm_node(struct irdma_cm_core *cm_core, · 654 unmodified lines | ||
| 3021 | 3024 | |
| 3022 | 3025 | /* create a CM connection node */ |
| 3023 | 3026 | cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL); |
| 3024 | − if (!cm_node) | |
| 3025 | − return -ENOMEM; | |
| 3027 | + if (IS_ERR(cm_node)) | |
| 3028 | + return PTR_ERR(cm_node); | |
| 3026 | 3029 | |
| 3027 | 3030 | /* set our node side to client (active) side */ |
| 3028 | 3031 | cm_node->tcp_cntxt.client = 1; |
| @@ -3219,9 +3222,9 @@ void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf) · 190 unmodified lines | ||
| 3219 | 3222 | cm_info.cm_id = listener->cm_id; |
| 3220 | 3223 | cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, |
| 3221 | 3224 | listener); |
| 3222 | − if (!cm_node) { | |
| 3225 | + if (IS_ERR(cm_node)) { | |
| 3223 | 3226 | ibdev_dbg(&cm_core->iwdev->ibdev, |
| 3224 | − "CM: allocate node failed\n"); | |
| 3227 | + "CM: allocate node failed ret=%ld\n", PTR_ERR(cm_node)); | |
| 3225 | 3228 | refcount_dec(&listener->refcnt); |
| 3226 | 3229 | return; |
| 3227 | 3230 | } |
| @@ -4239,21 +4242,21 @@ static void irdma_cm_event_handler(struct work_struct *work) · 1011 unmodified lines | ||
| 4239 | 4242 | irdma_cm_event_reset(event); |
| 4240 | 4243 | break; |
| 4241 | 4244 | case IRDMA_CM_EVENT_CONNECTED: |
| 4242 | − if (!event->cm_node->cm_id || | |
| 4243 | − event->cm_node->state != IRDMA_CM_STATE_OFFLOADED) | |
| 4245 | + if (!cm_node->cm_id || | |
| 4246 | + cm_node->state != IRDMA_CM_STATE_OFFLOADED) | |
| 4244 | 4247 | break; |
| 4245 | 4248 | irdma_cm_event_connected(event); |
| 4246 | 4249 | break; |
| 4247 | 4250 | case IRDMA_CM_EVENT_MPA_REJECT: |
| 4248 | − if (!event->cm_node->cm_id || | |
| 4251 | + if (!cm_node->cm_id || | |
| 4249 | 4252 | cm_node->state == IRDMA_CM_STATE_OFFLOADED) |
| 4250 | 4253 | break; |
| 4251 | 4254 | irdma_send_cm_event(cm_node, cm_node->cm_id, |
| 4252 | 4255 | IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); |
| 4253 | 4256 | break; |
| 4254 | 4257 | case IRDMA_CM_EVENT_ABORTED: |
| 4255 | − if (!event->cm_node->cm_id || | |
| 4256 | − event->cm_node->state == IRDMA_CM_STATE_OFFLOADED) | |
| 4258 | + if (!cm_node->cm_id || | |
| 4259 | + cm_node->state == IRDMA_CM_STATE_OFFLOADED) | |
| 4257 | 4260 | break; |
| 4258 | 4261 | irdma_event_connect_error(event); |
| 4259 | 4262 | break; |
| @@ -4263,7 +4266,7 @@ static void irdma_cm_event_handler(struct work_struct *work) · 3 unmodified lines | ||
| 4263 | 4266 | break; |
| 4264 | 4267 | } |
| 4265 | 4268 | |
| 4266 | − irdma_rem_ref_cm_node(event->cm_node); | |
| 4269 | + irdma_rem_ref_cm_node(cm_node); | |
| 4267 | 4270 | kfree(event); |
| 4268 | 4271 | } |
| 4269 | 4272 | |
| @@ -1438,7 +1438,7 @@ exit: · 1437 unmodified lines | ||
| 1438 | 1438 | * irdma_round_up_wq - return round up qp wq depth |
| 1439 | 1439 | * @wqdepth: wq depth in quanta to round up |
| 1440 | 1440 | */ |
| 1441 | −static int irdma_round_up_wq(u32 wqdepth) | |
| 1441 | +static u64 irdma_round_up_wq(u64 wqdepth) | |
| 1442 | 1442 | { |
| 1443 | 1443 | int scount = 1; |
| 1444 | 1444 | |
| @@ -1491,15 +1491,16 @@ void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, · 46 unmodified lines | ||
| 1491 | 1491 | int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, |
| 1492 | 1492 | u32 *sqdepth) |
| 1493 | 1493 | { |
| 1494 | − u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; | |
| 1494 | + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; | |
| 1495 | + u64 hw_quanta = | |
| 1496 | + irdma_round_up_wq(((u64)sq_size << shift) + IRDMA_SQ_RSVD); | |
| 1495 | 1497 | |
| 1496 | − *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD); | |
| 1497 | − | |
| 1498 | − if (*sqdepth < min_size) | |
| 1499 | − *sqdepth = min_size; | |
| 1500 | − else if (*sqdepth > uk_attrs->max_hw_wq_quanta) | |
| 1498 | + if (hw_quanta < min_hw_quanta) | |
| 1499 | + hw_quanta = min_hw_quanta; | |
| 1500 | + else if (hw_quanta > uk_attrs->max_hw_wq_quanta) | |
| 1501 | 1501 | return -EINVAL; |
| 1502 | 1502 | |
| 1503 | + *sqdepth = hw_quanta; | |
| 1503 | 1504 | return 0; |
| 1504 | 1505 | } |
| 1505 | 1506 | |
| @@ -1513,15 +1514,16 @@ int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, · 7 unmodified lines | ||
| 1513 | 1514 | int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, |
| 1514 | 1515 | u32 *rqdepth) |
| 1515 | 1516 | { |
| 1516 | − u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; | |
| 1517 | − | |
| 1518 | − *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD); | |
| 1517 | + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; | |
| 1518 | + u64 hw_quanta = | |
| 1519 | + irdma_round_up_wq(((u64)rq_size << shift) + IRDMA_RQ_RSVD); | |
| 1519 | 1520 | |
| 1520 | − if (*rqdepth < min_size) | |
| 1521 | − *rqdepth = min_size; | |
| 1522 | − else if (*rqdepth > uk_attrs->max_hw_rq_quanta) | |
| 1521 | + if (hw_quanta < min_hw_quanta) | |
| 1522 | + hw_quanta = min_hw_quanta; | |
| 1523 | + else if (hw_quanta > uk_attrs->max_hw_rq_quanta) | |
| 1523 | 1524 | return -EINVAL; |
| 1524 | 1525 | |
| 1526 | + *rqdepth = hw_quanta; | |
| 1525 | 1527 | return 0; |
| 1526 | 1528 | } |
| 1527 | 1529 | |
| @@ -1535,13 +1537,16 @@ int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, · 7 unmodified lines | ||
| 1535 | 1537 | int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, |
| 1536 | 1538 | u32 *srqdepth) |
| 1537 | 1539 | { |
| 1538 | − *srqdepth = irdma_round_up_wq((srq_size << shift) + IRDMA_RQ_RSVD); | |
| 1540 | + u32 min_hw_quanta = (u32)uk_attrs->min_hw_wq_size << shift; | |
| 1541 | + u64 hw_quanta = | |
| 1542 | + irdma_round_up_wq(((u64)srq_size << shift) + IRDMA_RQ_RSVD); | |
| 1539 | 1543 | |
| 1540 | − if (*srqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) | |
| 1541 | − *srqdepth = uk_attrs->min_hw_wq_size << shift; | |
| 1542 | − else if (*srqdepth > uk_attrs->max_hw_srq_quanta) | |
| 1544 | + if (hw_quanta < min_hw_quanta) | |
| 1545 | + hw_quanta = min_hw_quanta; | |
| 1546 | + else if (hw_quanta > uk_attrs->max_hw_srq_quanta) | |
| 1543 | 1547 | return -EINVAL; |
| 1544 | 1548 | |
| 1549 | + *srqdepth = hw_quanta; | |
| 1545 | 1550 | return 0; |
| 1546 | 1551 | } |
| 1547 | 1552 | |
| @@ -2322,8 +2322,6 @@ void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) · 2321 unmodified lines | ||
| 2322 | 2322 | struct irdma_qp *qp = sc_qp->qp_uk.back_qp; |
| 2323 | 2323 | struct ib_qp_attr attr; |
| 2324 | 2324 | |
| 2325 | − if (qp->iwdev->rf->reset) | |
| 2326 | − return; | |
| 2327 | 2325 | attr.qp_state = IB_QPS_ERR; |
| 2328 | 2326 | |
| 2329 | 2327 | if (rdma_protocol_roce(qp->ibqp.device, 1)) |
| @@ -558,7 +558,8 @@ static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) · 557 unmodified lines | ||
| 558 | 558 | } |
| 559 | 559 | |
| 560 | 560 | irdma_qp_rem_ref(&iwqp->ibqp); |
| 561 | − wait_for_completion(&iwqp->free_qp); | |
| 561 | + if (!iwdev->rf->reset) | |
| 562 | + wait_for_completion(&iwqp->free_qp); | |
| 562 | 563 | irdma_free_lsmm_rsrc(iwqp); |
| 563 | 564 | irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); |
| 564 | 565 | |
| @@ -1105,6 +1106,7 @@ static int irdma_create_qp(struct ib_qp *ibqp, · 540 unmodified lines | ||
| 1105 | 1106 | spin_lock_init(&iwqp->sc_qp.pfpdu.lock); |
| 1106 | 1107 | iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; |
| 1107 | 1108 | rf->qp_table[qp_num] = iwqp; |
| 1109 | + init_completion(&iwqp->free_qp); | |
| 1108 | 1110 | |
| 1109 | 1111 | if (udata) { |
| 1110 | 1112 | /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ |
| @@ -1129,7 +1131,6 @@ static int irdma_create_qp(struct ib_qp *ibqp, · 18 unmodified lines | ||
| 1129 | 1131 | } |
| 1130 | 1132 | } |
| 1131 | 1133 | |
| 1132 | − init_completion(&iwqp->free_qp); | |
| 1133 | 1134 | return 0; |
| 1134 | 1135 | |
| 1135 | 1136 | error: |
| @@ -1462,8 +1463,6 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 326 unmodified lines | ||
| 1462 | 1463 | ctx_info->remote_atomics_en = true; |
| 1463 | 1464 | } |
| 1464 | 1465 | |
| 1465 | − wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); | |
| 1466 | − | |
| 1467 | 1466 | ibdev_dbg(&iwdev->ibdev, |
| 1468 | 1467 | "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", |
| 1469 | 1468 | __builtin_return_address(0), ibqp->qp_num, attr->qp_state, |
| @@ -1540,6 +1539,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, · 70 unmodified lines | ||
| 1540 | 1539 | case IB_QPS_ERR: |
| 1541 | 1540 | case IB_QPS_RESET: |
| 1542 | 1541 | if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { |
| 1542 | + iwqp->ibqp_state = attr->qp_state; | |
| 1543 | 1543 | spin_unlock_irqrestore(&iwqp->lock, flags); |
| 1544 | 1544 | if (udata && udata->inlen) { |
| 1545 | 1545 | if (ib_copy_from_udata(&ureq, udata, |
| @@ -1745,6 +1745,7 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, · 199 unmodified lines | ||
| 1745 | 1745 | case IB_QPS_ERR: |
| 1746 | 1746 | case IB_QPS_RESET: |
| 1747 | 1747 | if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { |
| 1748 | + iwqp->ibqp_state = attr->qp_state; | |
| 1748 | 1749 | spin_unlock_irqrestore(&iwqp->lock, flags); |
| 1749 | 1750 | if (udata && udata->inlen) { |
| 1750 | 1751 | if (ib_copy_from_udata(&ureq, udata, |
| @@ -3723,6 +3724,7 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, · 1972 unmodified lines | ||
| 3723 | 3724 | |
| 3724 | 3725 | err: |
| 3725 | 3726 | ib_umem_release(region); |
| 3727 | + iwmr->region = NULL; | |
| 3726 | 3728 | return err; |
| 3727 | 3729 | } |
| 3728 | 3730 | |
| @@ -1211,7 +1211,7 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, · 1210 unmodified lines | ||
| 1211 | 1211 | */ |
| 1212 | 1212 | if (dev_use_swiotlb(dev, size, dir) && |
| 1213 | 1213 | iova_unaligned(iovad, phys, size)) { |
| 1214 | − if (attrs & DMA_ATTR_MMIO) | |
| 1214 | + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) | |
| 1215 | 1215 | return DMA_MAPPING_ERROR; |
| 1216 | 1216 | |
| 1217 | 1217 | phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs); |
| @@ -1223,7 +1223,8 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, · 5 unmodified lines | ||
| 1223 | 1223 | arch_sync_dma_for_device(phys, size, dir); |
| 1224 | 1224 | |
| 1225 | 1225 | iova = __iommu_dma_map(dev, phys, size, prot, dma_mask); |
| 1226 | − if (iova == DMA_MAPPING_ERROR && !(attrs & DMA_ATTR_MMIO)) | |
| 1226 | + if (iova == DMA_MAPPING_ERROR && | |
| 1227 | + !(attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))) | |
| 1227 | 1228 | swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); |
| 1228 | 1229 | return iova; |
| 1229 | 1230 | } |
| @@ -1233,7 +1234,7 @@ void iommu_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle, · 3 unmodified lines | ||
| 1233 | 1234 | { |
| 1234 | 1235 | phys_addr_t phys; |
| 1235 | 1236 | |
| 1236 | − if (attrs & DMA_ATTR_MMIO) { | |
| 1237 | + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) { | |
| 1237 | 1238 | __iommu_dma_unmap(dev, dma_handle, size); |
| 1238 | 1239 | return; |
| 1239 | 1240 | } |
| @@ -1945,9 +1946,21 @@ int dma_iova_link(struct device *dev, struct dma_iova_state *state, · 705 unmodified lines | ||
| 1945 | 1946 | if (WARN_ON_ONCE(iova_start_pad && offset > 0)) |
| 1946 | 1947 | return -EIO; |
| 1947 | 1948 | |
| 1949 | + /* | |
| 1950 | + * DMA_IOVA_USE_SWIOTLB is set on state after some entry | |
| 1951 | + * took SWIOTLB path, which we were supposed to prevent | |
| 1952 | + * for DMA_ATTR_REQUIRE_COHERENT attribute. | |
| 1953 | + */ | |
| 1954 | + if (WARN_ON_ONCE((state->__size & DMA_IOVA_USE_SWIOTLB) && | |
| 1955 | + (attrs & DMA_ATTR_REQUIRE_COHERENT))) | |
| 1956 | + return -EOPNOTSUPP; | |
| 1957 | + | |
| 1958 | + if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT)) | |
| 1959 | + return -EOPNOTSUPP; | |
| 1960 | + | |
| 1948 | 1961 | if (dev_use_swiotlb(dev, size, dir) && |
| 1949 | 1962 | iova_unaligned(iovad, phys, size)) { |
| 1950 | − if (attrs & DMA_ATTR_MMIO) | |
| 1963 | + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) | |
| 1951 | 1964 | return -EPERM; |
| 1952 | 1965 | |
| 1953 | 1966 | return iommu_dma_iova_link_swiotlb(dev, state, phys, offset, |
| @@ -306,6 +306,8 @@ static int mpm_pd_power_off(struct generic_pm_domain *genpd) · 305 unmodified lines | ||
| 306 | 306 | if (ret < 0) |
| 307 | 307 | return ret; |
| 308 | 308 | |
| 309 | + mbox_client_txdone(priv->mbox_chan, 0); | |
| 310 | + | |
| 309 | 311 | return 0; |
| 310 | 312 | } |
| 311 | 313 | |
| @@ -434,6 +436,7 @@ static int qcom_mpm_probe(struct platform_device *pdev, struct device_node *pare · 122 unmodified lines | ||
| 434 | 436 | } |
| 435 | 437 | |
| 436 | 438 | priv->mbox_client.dev = dev; |
| 439 | + priv->mbox_client.knows_txdone = true; | |
| 437 | 440 | priv->mbox_chan = mbox_request_channel(&priv->mbox_client, 0); |
| 438 | 441 | if (IS_ERR(priv->mbox_chan)) { |
| 439 | 442 | ret = PTR_ERR(priv->mbox_chan); |
| @@ -621,7 +621,7 @@ static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_no · 620 unmodified lines | ||
| 621 | 621 | return 0; |
| 622 | 622 | |
| 623 | 623 | pm_put: |
| 624 | − pm_runtime_put(&pdev->dev); | |
| 624 | + pm_runtime_put_sync(&pdev->dev); | |
| 625 | 625 | |
| 626 | 626 | return ret; |
| 627 | 627 | } |
| @@ -3080,8 +3080,6 @@ static int ccs_init_state(struct v4l2_subdev *sd, · 3079 unmodified lines | ||
| 3080 | 3080 | struct v4l2_rect *crop = |
| 3081 | 3081 | v4l2_subdev_state_get_crop(sd_state, pad); |
| 3082 | 3082 | |
| 3083 | − guard(mutex)(&sensor->mutex); | |
| 3084 | − | |
| 3085 | 3083 | ccs_get_native_size(ssd, crop); |
| 3086 | 3084 | |
| 3087 | 3085 | fmt->width = crop->width; |
| @@ -192,6 +192,8 @@ static long media_request_ioctl_reinit(struct media_request *req) · 191 unmodified lines | ||
| 192 | 192 | struct media_device *mdev = req->mdev; |
| 193 | 193 | unsigned long flags; |
| 194 | 194 | |
| 195 | + mutex_lock(&mdev->req_queue_mutex); | |
| 196 | + | |
| 195 | 197 | spin_lock_irqsave(&req->lock, flags); |
| 196 | 198 | if (req->state != MEDIA_REQUEST_STATE_IDLE && |
| 197 | 199 | req->state != MEDIA_REQUEST_STATE_COMPLETE) { |
| @@ -199,6 +201,7 @@ static long media_request_ioctl_reinit(struct media_request *req) · 1 unmodified line | ||
| 199 | 201 | "request: %s not in idle or complete state, cannot reinit\n", |
| 200 | 202 | req->debug_str); |
| 201 | 203 | spin_unlock_irqrestore(&req->lock, flags); |
| 204 | + mutex_unlock(&mdev->req_queue_mutex); | |
| 202 | 205 | return -EBUSY; |
| 203 | 206 | } |
| 204 | 207 | if (req->access_count) { |
| @@ -206,6 +209,7 @@ static long media_request_ioctl_reinit(struct media_request *req) · 1 unmodified line | ||
| 206 | 209 | "request: %s is being accessed, cannot reinit\n", |
| 207 | 210 | req->debug_str); |
| 208 | 211 | spin_unlock_irqrestore(&req->lock, flags); |
| 212 | + mutex_unlock(&mdev->req_queue_mutex); | |
| 209 | 213 | return -EBUSY; |
| 210 | 214 | } |
| 211 | 215 | req->state = MEDIA_REQUEST_STATE_CLEANING; |
| @@ -216,6 +220,7 @@ static long media_request_ioctl_reinit(struct media_request *req) · 4 unmodified lines | ||
| 216 | 220 | spin_lock_irqsave(&req->lock, flags); |
| 217 | 221 | req->state = MEDIA_REQUEST_STATE_IDLE; |
| 218 | 222 | spin_unlock_irqrestore(&req->lock, flags); |
| 223 | + mutex_unlock(&mdev->req_queue_mutex); | |
| 219 | 224 | |
| 220 | 225 | return 0; |
| 221 | 226 | } |
| @@ -500,11 +500,15 @@ void rkvdec_hevc_run_preamble(struct rkvdec_ctx *ctx, · 499 unmodified lines | ||
| 500 | 500 | ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, |
| 501 | 501 | V4L2_CID_STATELESS_HEVC_EXT_SPS_ST_RPS); |
| 502 | 502 | run->ext_sps_st_rps = ctrl ? ctrl->p_cur.p : NULL; |
| 503 | + } else { | |
| 504 | + run->ext_sps_st_rps = NULL; | |
| 503 | 505 | } |
| 504 | 506 | if (ctx->has_sps_lt_rps) { |
| 505 | 507 | ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, |
| 506 | 508 | V4L2_CID_STATELESS_HEVC_EXT_SPS_LT_RPS); |
| 507 | 509 | run->ext_sps_lt_rps = ctrl ? ctrl->p_cur.p : NULL; |
| 510 | + } else { | |
| 511 | + run->ext_sps_lt_rps = NULL; | |
| 508 | 512 | } |
| 509 | 513 | |
| 510 | 514 | rkvdec_run_preamble(ctx, &run->base); |
| @@ -130,7 +130,7 @@ struct rkvdec_h264_ctx { · 129 unmodified lines | ||
| 130 | 130 | struct vdpu383_regs_h26x regs; |
| 131 | 131 | }; |
| 132 | 132 | |
| 133 | −static void set_field_order_cnt(struct rkvdec_pps *pps, const struct v4l2_h264_dpb_entry *dpb) | |
| 133 | +static noinline_for_stack void set_field_order_cnt(struct rkvdec_pps *pps, const struct v4l2_h264_dpb_entry *dpb) | |
| 134 | 134 | { |
| 135 | 135 | pps->top_field_order_cnt0 = dpb[0].top_field_order_cnt; |
| 136 | 136 | pps->bot_field_order_cnt0 = dpb[0].bottom_field_order_cnt; |
| @@ -166,6 +166,31 @@ static void set_field_order_cnt(struct rkvdec_pps *pps, const struct v4l2_h264_d · 29 unmodified lines | ||
| 166 | 166 | pps->bot_field_order_cnt15 = dpb[15].bottom_field_order_cnt; |
| 167 | 167 | } |
| 168 | 168 | |
| 169 | +static noinline_for_stack void set_dec_params(struct rkvdec_pps *pps, const struct v4l2_ctrl_h264_decode_params *dec_params) | |
| 170 | +{ | |
| 171 | + const struct v4l2_h264_dpb_entry *dpb = dec_params->dpb; | |
| 172 | + | |
| 173 | + for (int i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) { | |
| 174 | + if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM) | |
| 175 | + pps->is_longterm |= (1 << i); | |
| 176 | + pps->ref_field_flags |= | |
| 177 | + (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD)) << i; | |
| 178 | + pps->ref_colmv_use_flag |= | |
| 179 | + (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) << i; | |
| 180 | + pps->ref_topfield_used |= | |
| 181 | + (!!(dpb[i].fields & V4L2_H264_TOP_FIELD_REF)) << i; | |
| 182 | + pps->ref_botfield_used |= | |
| 183 | + (!!(dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF)) << i; | |
| 184 | + } | |
| 185 | + pps->pic_field_flag = | |
| 186 | + !!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC); | |
| 187 | + pps->pic_associated_flag = | |
| 188 | + !!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD); | |
| 189 | + | |
| 190 | + pps->cur_top_field = dec_params->top_field_order_cnt; | |
| 191 | + pps->cur_bot_field = dec_params->bottom_field_order_cnt; | |
| 192 | +} | |
| 193 | + | |
| 169 | 194 | static void assemble_hw_pps(struct rkvdec_ctx *ctx, |
| 170 | 195 | struct rkvdec_h264_run *run) |
| 171 | 196 | { |
| @@ -177,7 +202,6 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx, · 5 unmodified lines | ||
| 177 | 202 | struct rkvdec_h264_priv_tbl *priv_tbl = h264_ctx->priv_tbl.cpu; |
| 178 | 203 | struct rkvdec_sps_pps *hw_ps; |
| 179 | 204 | u32 pic_width, pic_height; |
| 180 | − u32 i; | |
| 181 | 205 | |
| 182 | 206 | /* |
| 183 | 207 | * HW read the SPS/PPS information from PPS packet index by PPS id. |
| @@ -261,28 +285,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx, · 77 unmodified lines | ||
| 261 | 285 | !!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT); |
| 262 | 286 | |
| 263 | 287 | set_field_order_cnt(&hw_ps->pps, dpb); |
| 288 | + set_dec_params(&hw_ps->pps, dec_params); | |
| 264 | 289 | |
| 265 | − for (i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) { | |
| 266 | − if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM) | |
| 267 | − hw_ps->pps.is_longterm |= (1 << i); | |
| 268 | − | |
| 269 | − hw_ps->pps.ref_field_flags |= | |
| 270 | − (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD)) << i; | |
| 271 | − hw_ps->pps.ref_colmv_use_flag |= | |
| 272 | − (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) << i; | |
| 273 | − hw_ps->pps.ref_topfield_used |= | |
| 274 | − (!!(dpb[i].fields & V4L2_H264_TOP_FIELD_REF)) << i; | |
| 275 | − hw_ps->pps.ref_botfield_used |= | |
| 276 | − (!!(dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF)) << i; | |
| 277 | − } | |
| 278 | − | |
| 279 | − hw_ps->pps.pic_field_flag = | |
| 280 | − !!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC); | |
| 281 | − hw_ps->pps.pic_associated_flag = | |
| 282 | − !!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD); | |
| 283 | − | |
| 284 | − hw_ps->pps.cur_top_field = dec_params->top_field_order_cnt; | |
| 285 | − hw_ps->pps.cur_bot_field = dec_params->bottom_field_order_cnt; | |
| 286 | 290 | } |
| 287 | 291 | |
| 288 | 292 | static void rkvdec_write_regs(struct rkvdec_ctx *ctx) |
| @@ -893,7 +893,8 @@ out_update_last: · 892 unmodified lines | ||
| 893 | 893 | update_ctx_last_info(vp9_ctx); |
| 894 | 894 | } |
| 895 | 895 | |
| 896 | −static void rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx) | |
| 896 | +static noinline_for_stack void | |
| 897 | +rkvdec_init_v4l2_vp9_count_tbl(struct rkvdec_ctx *ctx) | |
| 897 | 898 | { |
| 898 | 899 | struct rkvdec_vp9_ctx *vp9_ctx = ctx->priv; |
| 899 | 900 | struct rkvdec_vp9_intra_frame_symbol_counts *intra_cnts = vp9_ctx->count_tbl.cpu; |
| @@ -7,6 +7,7 @@ config VIDEO_DW_MIPI_CSI2RX · 6 unmodified lines | ||
| 7 | 7 | depends on VIDEO_DEV |
| 8 | 8 | depends on V4L_PLATFORM_DRIVERS |
| 9 | 9 | depends on PM && COMMON_CLK |
| 10 | + select GENERIC_PHY_MIPI_DPHY | |
| 10 | 11 | select MEDIA_CONTROLLER |
| 11 | 12 | select V4L2_FWNODE |
| 12 | 13 | select VIDEO_V4L2_SUBDEV_API |
| @@ -301,7 +301,7 @@ dw_mipi_csi2rx_enum_mbus_code(struct v4l2_subdev *sd, · 300 unmodified lines | ||
| 301 | 301 | |
| 302 | 302 | return 0; |
| 303 | 303 | case DW_MIPI_CSI2RX_PAD_SINK: |
| 304 | − if (code->index > csi2->formats_num) | |
| 304 | + if (code->index >= csi2->formats_num) | |
| 305 | 305 | return -EINVAL; |
| 306 | 306 | |
| 307 | 307 | code->code = csi2->formats[code->index].code; |
| @@ -343,7 +343,7 @@ const struct hantro_variant imx8mq_vpu_variant = { · 342 unmodified lines | ||
| 343 | 343 | .num_regs = ARRAY_SIZE(imx8mq_reg_names) |
| 344 | 344 | }; |
| 345 | 345 | |
| 346 | −static const struct of_device_id imx8mq_vpu_shared_resources[] __initconst = { | |
| 346 | +static const struct of_device_id imx8mq_vpu_shared_resources[] = { | |
| 347 | 347 | { .compatible = "nxp,imx8mq-vpu-g1", }, |
| 348 | 348 | { .compatible = "nxp,imx8mq-vpu-g2", }, |
| 349 | 349 | { /* sentinel */ } |
| @@ -1751,7 +1751,8 @@ static void uvc_video_complete(struct urb *urb) · 1750 unmodified lines | ||
| 1751 | 1751 | /* |
| 1752 | 1752 | * Free transfer buffers. |
| 1753 | 1753 | */ |
| 1754 | −static void uvc_free_urb_buffers(struct uvc_streaming *stream) | |
| 1754 | +static void uvc_free_urb_buffers(struct uvc_streaming *stream, | |
| 1755 | + unsigned int size) | |
| 1755 | 1756 | { |
| 1756 | 1757 | struct usb_device *udev = stream->dev->udev; |
| 1757 | 1758 | struct uvc_urb *uvc_urb; |
| @@ -1760,7 +1761,7 @@ static void uvc_free_urb_buffers(struct uvc_streaming *stream) · 2 unmodified lines | ||
| 1760 | 1761 | if (!uvc_urb->buffer) |
| 1761 | 1762 | continue; |
| 1762 | 1763 | |
| 1763 | − usb_free_noncoherent(udev, stream->urb_size, uvc_urb->buffer, | |
| 1764 | + usb_free_noncoherent(udev, size, uvc_urb->buffer, | |
| 1764 | 1765 | uvc_stream_dir(stream), uvc_urb->sgt); |
| 1765 | 1766 | uvc_urb->buffer = NULL; |
| 1766 | 1767 | uvc_urb->sgt = NULL; |
| @@ -1820,7 +1821,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, · 53 unmodified lines | ||
| 1820 | 1821 | |
| 1821 | 1822 | if (!uvc_alloc_urb_buffer(stream, uvc_urb, urb_size, |
| 1822 | 1823 | gfp_flags)) { |
| 1823 | − uvc_free_urb_buffers(stream); | |
| 1824 | + uvc_free_urb_buffers(stream, urb_size); | |
| 1824 | 1825 | break; |
| 1825 | 1826 | } |
| 1826 | 1827 | |
| @@ -1868,7 +1869,7 @@ static void uvc_video_stop_transfer(struct uvc_streaming *stream, · 41 unmodified lines | ||
| 1868 | 1869 | } |
| 1869 | 1870 | |
| 1870 | 1871 | if (free_buffers) |
| 1871 | − uvc_free_urb_buffers(stream); | |
| 1872 | + uvc_free_urb_buffers(stream, stream->urb_size); | |
| 1872 | 1873 | } |
| 1873 | 1874 | |
| 1874 | 1875 | /* |
| @@ -3082,13 +3082,14 @@ static long __video_do_ioctl(struct file *file, · 3081 unmodified lines | ||
| 3082 | 3082 | } |
| 3083 | 3083 | |
| 3084 | 3084 | /* |
| 3085 | − * We need to serialize streamon/off with queueing new requests. | |
| 3085 | + * We need to serialize streamon/off/reqbufs with queueing new requests. | |
| 3086 | 3086 | * These ioctls may trigger the cancellation of a streaming |
| 3087 | 3087 | * operation, and that should not be mixed with queueing a new |
| 3088 | 3088 | * request at the same time. |
| 3089 | 3089 | */ |
| 3090 | 3090 | if (v4l2_device_supports_requests(vfd->v4l2_dev) && |
| 3091 | − (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF)) { | |
| 3091 | + (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF || | |
| 3092 | + cmd == VIDIOC_REQBUFS)) { | |
| 3092 | 3093 | req_queue_lock = &vfd->v4l2_dev->mdev->req_queue_mutex; |
| 3093 | 3094 | |
| 3094 | 3095 | if (mutex_lock_interruptible(req_queue_lock)) |
| @@ -601,7 +601,9 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[], · 600 unmodified lines | ||
| 601 | 601 | /* We need synchronization with dev->stop() */ |
| 602 | 602 | ASSERT_RTNL(); |
| 603 | 603 | |
| 604 | − can_ctrlmode_changelink(dev, data, extack); | |
| 604 | + err = can_ctrlmode_changelink(dev, data, extack); | |
| 605 | + if (err) | |
| 606 | + return err; | |
| 605 | 607 | |
| 606 | 608 | if (data[IFLA_CAN_BITTIMING]) { |
| 607 | 609 | struct can_bittiming bt; |
| @@ -1225,7 +1225,11 @@ static int mcp251x_open(struct net_device *net) · 1224 unmodified lines | ||
| 1225 | 1225 | } |
| 1226 | 1226 | |
| 1227 | 1227 | mutex_lock(&priv->mcp_lock); |
| 1228 | − mcp251x_power_enable(priv->transceiver, 1); | |
| 1228 | + ret = mcp251x_power_enable(priv->transceiver, 1); | |
| 1229 | + if (ret) { | |
| 1230 | + dev_err(&spi->dev, "failed to enable transceiver power: %pe\n", ERR_PTR(ret)); | |
| 1231 | + goto out_close_candev; | |
| 1232 | + } | |
| 1229 | 1233 | |
| 1230 | 1234 | priv->force_quit = 0; |
| 1231 | 1235 | priv->tx_skb = NULL; |
| @@ -1272,6 +1276,7 @@ out_free_irq: · 40 unmodified lines | ||
| 1272 | 1276 | mcp251x_hw_sleep(spi); |
| 1273 | 1277 | out_close: |
| 1274 | 1278 | mcp251x_power_enable(priv->transceiver, 0); |
| 1279 | +out_close_candev: | |
| 1275 | 1280 | close_candev(net); |
| 1276 | 1281 | mutex_unlock(&priv->mcp_lock); |
| 1277 | 1282 | if (release_irq) |
| @@ -1516,11 +1521,25 @@ static int __maybe_unused mcp251x_can_resume(struct device *dev) · 238 unmodified lines | ||
| 1516 | 1521 | { |
| 1517 | 1522 | struct spi_device *spi = to_spi_device(dev); |
| 1518 | 1523 | struct mcp251x_priv *priv = spi_get_drvdata(spi); |
| 1524 | + int ret = 0; | |
| 1519 | 1525 | |
| 1520 | − if (priv->after_suspend & AFTER_SUSPEND_POWER) | |
| 1521 | − mcp251x_power_enable(priv->power, 1); | |
| 1522 | − if (priv->after_suspend & AFTER_SUSPEND_UP) | |
| 1523 | − mcp251x_power_enable(priv->transceiver, 1); | |
| 1526 | + if (priv->after_suspend & AFTER_SUSPEND_POWER) { | |
| 1527 | + ret = mcp251x_power_enable(priv->power, 1); | |
| 1528 | + if (ret) { | |
| 1529 | + dev_err(dev, "failed to restore power: %pe\n", ERR_PTR(ret)); | |
| 1530 | + return ret; | |
| 1531 | + } | |
| 1532 | + } | |
| 1533 | + | |
| 1534 | + if (priv->after_suspend & AFTER_SUSPEND_UP) { | |
| 1535 | + ret = mcp251x_power_enable(priv->transceiver, 1); | |
| 1536 | + if (ret) { | |
| 1537 | + dev_err(dev, "failed to restore transceiver power: %pe\n", ERR_PTR(ret)); | |
| 1538 | + if (priv->after_suspend & AFTER_SUSPEND_POWER) | |
| 1539 | + mcp251x_power_enable(priv->power, 0); | |
| 1540 | + return ret; | |
| 1541 | + } | |
| 1542 | + } | |
| 1524 | 1543 | |
| 1525 | 1544 | if (priv->after_suspend & (AFTER_SUSPEND_POWER | AFTER_SUSPEND_UP)) |
| 1526 | 1545 | queue_work(priv->wq, &priv->restart_work); |
| @@ -227,7 +227,9 @@ static int airoha_ppe_get_wdma_info(struct net_device *dev, const u8 *addr, · 226 unmodified lines | ||
| 227 | 227 | if (!dev) |
| 228 | 228 | return -ENODEV; |
| 229 | 229 | |
| 230 | + rcu_read_lock(); | |
| 230 | 231 | err = dev_fill_forward_path(dev, addr, &stack); |
| 232 | + rcu_read_unlock(); | |
| 231 | 233 | if (err) |
| 232 | 234 | return err; |
| 233 | 235 | |
| @@ -25,7 +25,7 @@ config B44 · 24 unmodified lines | ||
| 25 | 25 | select SSB |
| 26 | 26 | select MII |
| 27 | 27 | select PHYLIB |
| 28 | − select FIXED_PHY if BCM47XX | |
| 28 | + select FIXED_PHY | |
| 29 | 29 | help |
| 30 | 30 | If you have a network (Ethernet) controller of this type, say Y |
| 31 | 31 | or M here. |
| @@ -1152,12 +1152,6 @@ void bcmasp_enable_wol(struct bcmasp_intf *intf, bool en) · 1151 unmodified lines | ||
| 1152 | 1152 | } |
| 1153 | 1153 | } |
| 1154 | 1154 | |
| 1155 | −static void bcmasp_wol_irq_destroy(struct bcmasp_priv *priv) | |
| 1156 | −{ | |
| 1157 | − if (priv->wol_irq > 0) | |
| 1158 | − free_irq(priv->wol_irq, priv); | |
| 1159 | −} | |
| 1160 | − | |
| 1161 | 1155 | static void bcmasp_eee_fixup(struct bcmasp_intf *intf, bool en) |
| 1162 | 1156 | { |
| 1163 | 1157 | u32 reg, phy_lpi_overwrite; |
| @@ -1255,7 +1249,7 @@ static int bcmasp_probe(struct platform_device *pdev) · 91 unmodified lines | ||
| 1255 | 1249 | if (priv->irq <= 0) |
| 1256 | 1250 | return -EINVAL; |
| 1257 | 1251 | |
| 1258 | − priv->clk = devm_clk_get_optional_enabled(dev, "sw_asp"); | |
| 1252 | + priv->clk = devm_clk_get_optional(dev, "sw_asp"); | |
| 1259 | 1253 | if (IS_ERR(priv->clk)) |
| 1260 | 1254 | return dev_err_probe(dev, PTR_ERR(priv->clk), |
| 1261 | 1255 | "failed to request clock\n"); |
| @@ -1283,6 +1277,10 @@ static int bcmasp_probe(struct platform_device *pdev) · 21 unmodified lines | ||
| 1283 | 1277 | |
| 1284 | 1278 | bcmasp_set_pdata(priv, pdata); |
| 1285 | 1279 | |
| 1280 | + ret = clk_prepare_enable(priv->clk); | |
| 1281 | + if (ret) | |
| 1282 | + return dev_err_probe(dev, ret, "failed to start clock\n"); | |
| 1283 | + | |
| 1286 | 1284 | /* Enable all clocks to ensure successful probing */ |
| 1287 | 1285 | bcmasp_core_clock_set(priv, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE, 0); |
| 1288 | 1286 | |
| @@ -1294,8 +1292,10 @@ static int bcmasp_probe(struct platform_device *pdev) · 5 unmodified lines | ||
| 1294 | 1292 | |
| 1295 | 1293 | ret = devm_request_irq(&pdev->dev, priv->irq, bcmasp_isr, 0, |
| 1296 | 1294 | pdev->name, priv); |
| 1297 | − if (ret) | |
| 1298 | − return dev_err_probe(dev, ret, "failed to request ASP interrupt: %d", ret); | |
| 1295 | + if (ret) { | |
| 1296 | + dev_err(dev, "Failed to request ASP interrupt: %d", ret); | |
| 1297 | + goto err_clock_disable; | |
| 1298 | + } | |
| 1299 | 1299 | |
| 1300 | 1300 | /* Register mdio child nodes */ |
| 1301 | 1301 | of_platform_populate(dev->of_node, bcmasp_mdio_of_match, NULL, dev); |
| @@ -1307,13 +1307,17 @@ static int bcmasp_probe(struct platform_device *pdev) · 5 unmodified lines | ||
| 1307 | 1307 | |
| 1308 | 1308 | priv->mda_filters = devm_kcalloc(dev, priv->num_mda_filters, |
| 1309 | 1309 | sizeof(*priv->mda_filters), GFP_KERNEL); |
| 1310 | − if (!priv->mda_filters) | |
| 1311 | − return -ENOMEM; | |
| 1310 | + if (!priv->mda_filters) { | |
| 1311 | + ret = -ENOMEM; | |
| 1312 | + goto err_clock_disable; | |
| 1313 | + } | |
| 1312 | 1314 | |
| 1313 | 1315 | priv->net_filters = devm_kcalloc(dev, priv->num_net_filters, |
| 1314 | 1316 | sizeof(*priv->net_filters), GFP_KERNEL); |
| 1315 | − if (!priv->net_filters) | |
| 1316 | − return -ENOMEM; | |
| 1317 | + if (!priv->net_filters) { | |
| 1318 | + ret = -ENOMEM; | |
| 1319 | + goto err_clock_disable; | |
| 1320 | + } | |
| 1317 | 1321 | |
| 1318 | 1322 | bcmasp_core_init_filters(priv); |
| 1319 | 1323 | |
| @@ -1322,7 +1326,8 @@ static int bcmasp_probe(struct platform_device *pdev) · 2 unmodified lines | ||
| 1322 | 1326 | ports_node = of_find_node_by_name(dev->of_node, "ethernet-ports"); |
| 1323 | 1327 | if (!ports_node) { |
| 1324 | 1328 | dev_warn(dev, "No ports found\n"); |
| 1325 | − return -EINVAL; | |
| 1329 | + ret = -EINVAL; | |
| 1330 | + goto err_clock_disable; | |
| 1326 | 1331 | } |
| 1327 | 1332 | |
| 1328 | 1333 | i = 0; |
| @@ -1344,8 +1349,6 @@ static int bcmasp_probe(struct platform_device *pdev) · 15 unmodified lines | ||
| 1344 | 1349 | */ |
| 1345 | 1350 | bcmasp_core_clock_set(priv, 0, ASP_CTRL_CLOCK_CTRL_ASP_ALL_DISABLE); |
| 1346 | 1351 | |
| 1347 | − clk_disable_unprepare(priv->clk); | |
| 1348 | − | |
| 1349 | 1352 | /* Now do the registration of the network ports which will take care |
| 1350 | 1353 | * of managing the clock properly. |
| 1351 | 1354 | */ |
| @@ -1358,13 +1361,16 @@ static int bcmasp_probe(struct platform_device *pdev) · 6 unmodified lines | ||
| 1358 | 1361 | count++; |
| 1359 | 1362 | } |
| 1360 | 1363 | |
| 1364 | + clk_disable_unprepare(priv->clk); | |
| 1365 | + | |
| 1361 | 1366 | dev_info(dev, "Initialized %d port(s)\n", count); |
| 1362 | 1367 | |
| 1363 | 1368 | return ret; |
| 1364 | 1369 | |
| 1365 | 1370 | err_cleanup: |
| 1366 | − bcmasp_wol_irq_destroy(priv); | |
| 1367 | 1371 | bcmasp_remove_intfs(priv); |
| 1372 | +err_clock_disable: | |
| 1373 | + clk_disable_unprepare(priv->clk); | |
| 1368 | 1374 | |
| 1369 | 1375 | return ret; |
| 1370 | 1376 | } |
| @@ -1376,7 +1382,6 @@ static void bcmasp_remove(struct platform_device *pdev) · 5 unmodified lines | ||
| 1376 | 1382 | if (!priv) |
| 1377 | 1383 | return; |
| 1378 | 1384 | |
| 1379 | − bcmasp_wol_irq_destroy(priv); | |
| 1380 | 1385 | bcmasp_remove_intfs(priv); |
| 1381 | 1386 | } |
| 1382 | 1387 | |
| @@ -1071,7 +1071,7 @@ static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budge · 1070 unmodified lines | ||
| 1071 | 1071 | } |
| 1072 | 1072 | |
| 1073 | 1073 | if (tx_skb->skb) { |
| 1074 | − napi_consume_skb(tx_skb->skb, budget); | |
| 1074 | + dev_consume_skb_any(tx_skb->skb); | |
| 1075 | 1075 | tx_skb->skb = NULL; |
| 1076 | 1076 | } |
| 1077 | 1077 | } |
| @@ -3224,7 +3224,7 @@ static void gem_get_ethtool_stats(struct net_device *dev, · 2146 unmodified lines | ||
| 3224 | 3224 | spin_lock_irq(&bp->stats_lock); |
| 3225 | 3225 | gem_update_stats(bp); |
| 3226 | 3226 | memcpy(data, &bp->ethtool_stats, sizeof(u64) |
| 3227 | − * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); | |
| 3227 | + * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); | |
| 3228 | 3228 | spin_unlock_irq(&bp->stats_lock); |
| 3229 | 3229 | } |
| 3230 | 3230 | |
| @@ -5776,9 +5776,9 @@ static int __maybe_unused macb_suspend(struct device *dev) · 2545 unmodified lines | ||
| 5776 | 5776 | struct macb_queue *queue; |
| 5777 | 5777 | struct in_device *idev; |
| 5778 | 5778 | unsigned long flags; |
| 5779 | + u32 tmp, ifa_local; | |
| 5779 | 5780 | unsigned int q; |
| 5780 | 5781 | int err; |
| 5781 | − u32 tmp; | |
| 5782 | 5782 | |
| 5783 | 5783 | if (!device_may_wakeup(&bp->dev->dev)) |
| 5784 | 5784 | phy_exit(bp->phy); |
| @@ -5787,14 +5787,21 @@ static int __maybe_unused macb_suspend(struct device *dev) · 2 unmodified lines | ||
| 5787 | 5787 | return 0; |
| 5788 | 5788 | |
| 5789 | 5789 | if (bp->wol & MACB_WOL_ENABLED) { |
| 5790 | − /* Check for IP address in WOL ARP mode */ | |
| 5791 | − idev = __in_dev_get_rcu(bp->dev); | |
| 5792 | − if (idev) | |
| 5793 | − ifa = rcu_dereference(idev->ifa_list); | |
| 5794 | − if ((bp->wolopts & WAKE_ARP) && !ifa) { | |
| 5795 | − netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); | |
| 5796 | − return -EOPNOTSUPP; | |
| 5790 | + if (bp->wolopts & WAKE_ARP) { | |
| 5791 | + /* Check for IP address in WOL ARP mode */ | |
| 5792 | + rcu_read_lock(); | |
| 5793 | + idev = __in_dev_get_rcu(bp->dev); | |
| 5794 | + if (idev) | |
| 5795 | + ifa = rcu_dereference(idev->ifa_list); | |
| 5796 | + if (!ifa) { | |
| 5797 | + rcu_read_unlock(); | |
| 5798 | + netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); | |
| 5799 | + return -EOPNOTSUPP; | |
| 5800 | + } | |
| 5801 | + ifa_local = be32_to_cpu(ifa->ifa_local); | |
| 5802 | + rcu_read_unlock(); | |
| 5797 | 5803 | } |
| 5804 | + | |
| 5798 | 5805 | spin_lock_irqsave(&bp->lock, flags); |
| 5799 | 5806 | |
| 5800 | 5807 | /* Disable Tx and Rx engines before disabling the queues, |
| @@ -5833,8 +5840,9 @@ static int __maybe_unused macb_suspend(struct device *dev) · 32 unmodified lines | ||
| 5833 | 5840 | if (bp->wolopts & WAKE_ARP) { |
| 5834 | 5841 | tmp |= MACB_BIT(ARP); |
| 5835 | 5842 | /* write IP address into register */ |
| 5836 | − tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local)); | |
| 5843 | + tmp |= MACB_BFEXT(IP, ifa_local); | |
| 5837 | 5844 | } |
| 5845 | + spin_unlock_irqrestore(&bp->lock, flags); | |
| 5838 | 5846 | |
| 5839 | 5847 | /* Change interrupt handler and |
| 5840 | 5848 | * Enable WoL IRQ on queue 0 |
| @@ -5847,11 +5855,12 @@ static int __maybe_unused macb_suspend(struct device *dev) · 6 unmodified lines | ||
| 5847 | 5855 | dev_err(dev, |
| 5848 | 5856 | "Unable to request IRQ %d (error %d)\n", |
| 5849 | 5857 | bp->queues[0].irq, err); |
| 5850 | − spin_unlock_irqrestore(&bp->lock, flags); | |
| 5851 | 5858 | return err; |
| 5852 | 5859 | } |
| 5860 | + spin_lock_irqsave(&bp->lock, flags); | |
| 5853 | 5861 | queue_writel(bp->queues, IER, GEM_BIT(WOL)); |
| 5854 | 5862 | gem_writel(bp, WOL, tmp); |
| 5863 | + spin_unlock_irqrestore(&bp->lock, flags); | |
| 5855 | 5864 | } else { |
| 5856 | 5865 | err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, |
| 5857 | 5866 | IRQF_SHARED, netdev->name, bp->queues); |
| @@ -5859,13 +5868,13 @@ static int __maybe_unused macb_suspend(struct device *dev) · 1 unmodified line | ||
| 5859 | 5868 | dev_err(dev, |
| 5860 | 5869 | "Unable to request IRQ %d (error %d)\n", |
| 5861 | 5870 | bp->queues[0].irq, err); |
| 5862 | − spin_unlock_irqrestore(&bp->lock, flags); | |
| 5863 | 5871 | return err; |
| 5864 | 5872 | } |
| 5873 | + spin_lock_irqsave(&bp->lock, flags); | |
| 5865 | 5874 | queue_writel(bp->queues, IER, MACB_BIT(WOL)); |
| 5866 | 5875 | macb_writel(bp, WOL, tmp); |
| 5876 | + spin_unlock_irqrestore(&bp->lock, flags); | |
| 5867 | 5877 | } |
| 5868 | − spin_unlock_irqrestore(&bp->lock, flags); | |
| 5869 | 5878 | |
| 5870 | 5879 | enable_irq_wake(bp->queues[0].irq); |
| 5871 | 5880 | } |
| @@ -5932,6 +5941,8 @@ static int __maybe_unused macb_resume(struct device *dev) · 60 unmodified lines | ||
| 5932 | 5941 | queue_readl(bp->queues, ISR); |
| 5933 | 5942 | if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) |
| 5934 | 5943 | queue_writel(bp->queues, ISR, -1); |
| 5944 | + spin_unlock_irqrestore(&bp->lock, flags); | |
| 5945 | + | |
| 5935 | 5946 | /* Replace interrupt handler on queue 0 */ |
| 5936 | 5947 | devm_free_irq(dev, bp->queues[0].irq, bp->queues); |
| 5937 | 5948 | err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, |
| @@ -5940,10 +5951,8 @@ static int __maybe_unused macb_resume(struct device *dev) · 2 unmodified lines | ||
| 5940 | 5951 | dev_err(dev, |
| 5941 | 5952 | "Unable to request IRQ %d (error %d)\n", |
| 5942 | 5953 | bp->queues[0].irq, err); |
| 5943 | − spin_unlock_irqrestore(&bp->lock, flags); | |
| 5944 | 5954 | return err; |
| 5945 | 5955 | } |
| 5946 | − spin_unlock_irqrestore(&bp->lock, flags); | |
| 5947 | 5956 | |
| 5948 | 5957 | disable_irq_wake(bp->queues[0].irq); |
| 5949 | 5958 | |
| @@ -813,6 +813,8 @@ static void enetc_get_ringparam(struct net_device *ndev, · 812 unmodified lines | ||
| 813 | 813 | { |
| 814 | 814 | struct enetc_ndev_priv *priv = netdev_priv(ndev); |
| 815 | 815 | |
| 816 | + ring->rx_max_pending = priv->rx_bd_count; | |
| 817 | + ring->tx_max_pending = priv->tx_bd_count; | |
| 816 | 818 | ring->rx_pending = priv->rx_bd_count; |
| 817 | 819 | ring->tx_pending = priv->tx_bd_count; |
| 818 | 820 | |
| @@ -313,14 +313,13 @@ static int iavf_get_sset_count(struct net_device *netdev, int sset) · 312 unmodified lines | ||
| 313 | 313 | { |
| 314 | 314 | /* Report the maximum number queues, even if not every queue is |
| 315 | 315 | * currently configured. Since allocation of queues is in pairs, |
| 316 | − * use netdev->real_num_tx_queues * 2. The real_num_tx_queues is set | |
| 317 | − * at device creation and never changes. | |
| 316 | + * use netdev->num_tx_queues * 2. The num_tx_queues is set at | |
| 317 | + * device creation and never changes. | |
| 318 | 318 | */ |
| 319 | 319 | |
| 320 | 320 | if (sset == ETH_SS_STATS) |
| 321 | 321 | return IAVF_STATS_LEN + |
| 322 | − (IAVF_QUEUE_STATS_LEN * 2 * | |
| 323 | − netdev->real_num_tx_queues); | |
| 322 | + (IAVF_QUEUE_STATS_LEN * 2 * netdev->num_tx_queues); | |
| 324 | 323 | else |
| 325 | 324 | return -EINVAL; |
| 326 | 325 | } |
| @@ -345,19 +344,19 @@ static void iavf_get_ethtool_stats(struct net_device *netdev, · 18 unmodified lines | ||
| 345 | 344 | iavf_add_ethtool_stats(&data, adapter, iavf_gstrings_stats); |
| 346 | 345 | |
| 347 | 346 | rcu_read_lock(); |
| 348 | − /* As num_active_queues describe both tx and rx queues, we can use | |
| 349 | − * it to iterate over rings' stats. | |
| 347 | + /* Use num_tx_queues to report stats for the maximum number of queues. | |
| 348 | + * Queues beyond num_active_queues will report zero. | |
| 350 | 349 | */ |
| 351 | − for (i = 0; i < adapter->num_active_queues; i++) { | |
| 352 | − struct iavf_ring *ring; | |
| 350 | + for (i = 0; i < netdev->num_tx_queues; i++) { | |
| 351 | + struct iavf_ring *tx_ring = NULL, *rx_ring = NULL; | |
| 353 | 352 | |
| 354 | − /* Tx rings stats */ | |
| 355 | − ring = &adapter->tx_rings[i]; | |
| 356 | − iavf_add_queue_stats(&data, ring); | |
| 353 | + if (i < adapter->num_active_queues) { | |
| 354 | + tx_ring = &adapter->tx_rings[i]; | |
| 355 | + rx_ring = &adapter->rx_rings[i]; | |
| 356 | + } | |
| 357 | 357 | |
| 358 | − /* Rx rings stats */ | |
| 359 | − ring = &adapter->rx_rings[i]; | |
| 360 | − iavf_add_queue_stats(&data, ring); | |
| 358 | + iavf_add_queue_stats(&data, tx_ring); | |
| 359 | + iavf_add_queue_stats(&data, rx_ring); | |
| 361 | 360 | } |
| 362 | 361 | rcu_read_unlock(); |
| 363 | 362 | } |
| @@ -376,9 +375,9 @@ static void iavf_get_stat_strings(struct net_device *netdev, u8 *data) · 12 unmodified lines | ||
| 376 | 375 | iavf_add_stat_strings(&data, iavf_gstrings_stats); |
| 377 | 376 | |
| 378 | 377 | /* Queues are always allocated in pairs, so we just use |
| 379 | − * real_num_tx_queues for both Tx and Rx queues. | |
| 378 | + * num_tx_queues for both Tx and Rx queues. | |
| 380 | 379 | */ |
| 381 | − for (i = 0; i < netdev->real_num_tx_queues; i++) { | |
| 380 | + for (i = 0; i < netdev->num_tx_queues; i++) { | |
| 382 | 381 | iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, |
| 383 | 382 | "tx", i); |
| 384 | 383 | iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, |
| @@ -839,6 +839,28 @@ static inline void ice_tx_xsk_pool(struct ice_vsi *vsi, u16 qid) · 838 unmodified lines | ||
| 839 | 839 | WRITE_ONCE(ring->xsk_pool, ice_get_xp_from_qid(vsi, qid)); |
| 840 | 840 | } |
| 841 | 841 | |
| 842 | +/** | |
| 843 | + * ice_get_max_txq - return the maximum number of Tx queues for in a PF | |
| 844 | + * @pf: PF structure | |
| 845 | + * | |
| 846 | + * Return: maximum number of Tx queues | |
| 847 | + */ | |
| 848 | +static inline int ice_get_max_txq(struct ice_pf *pf) | |
| 849 | +{ | |
| 850 | + return min(num_online_cpus(), pf->hw.func_caps.common_cap.num_txq); | |
| 851 | +} | |
| 852 | + | |
| 853 | +/** | |
| 854 | + * ice_get_max_rxq - return the maximum number of Rx queues for in a PF | |
| 855 | + * @pf: PF structure | |
| 856 | + * | |
| 857 | + * Return: maximum number of Rx queues | |
| 858 | + */ | |
| 859 | +static inline int ice_get_max_rxq(struct ice_pf *pf) | |
| 860 | +{ | |
| 861 | + return min(num_online_cpus(), pf->hw.func_caps.common_cap.num_rxq); | |
| 862 | +} | |
| 863 | + | |
| 842 | 864 | /** |
| 843 | 865 | * ice_get_main_vsi - Get the PF VSI |
| 844 | 866 | * @pf: PF instance |
| @@ -1930,6 +1930,17 @@ __ice_get_ethtool_stats(struct net_device *netdev, · 1929 unmodified lines | ||
| 1930 | 1930 | int i = 0; |
| 1931 | 1931 | char *p; |
| 1932 | 1932 | |
| 1933 | + if (ice_is_port_repr_netdev(netdev)) { | |
| 1934 | + ice_update_eth_stats(vsi); | |
| 1935 | + | |
| 1936 | + for (j = 0; j < ICE_VSI_STATS_LEN; j++) { | |
| 1937 | + p = (char *)vsi + ice_gstrings_vsi_stats[j].stat_offset; | |
| 1938 | + data[i++] = (ice_gstrings_vsi_stats[j].sizeof_stat == | |
| 1939 | + sizeof(u64)) ? *(u64 *)p : *(u32 *)p; | |
| 1940 | + } | |
| 1941 | + return; | |
| 1942 | + } | |
| 1943 | + | |
| 1933 | 1944 | ice_update_pf_stats(pf); |
| 1934 | 1945 | ice_update_vsi_stats(vsi); |
| 1935 | 1946 | |
| @@ -1939,9 +1950,6 @@ __ice_get_ethtool_stats(struct net_device *netdev, · 3 unmodified lines | ||
| 1939 | 1950 | sizeof(u64)) ? *(u64 *)p : *(u32 *)p; |
| 1940 | 1951 | } |
| 1941 | 1952 | |
| 1942 | − if (ice_is_port_repr_netdev(netdev)) | |
| 1943 | − return; | |
| 1944 | − | |
| 1945 | 1953 | /* populate per queue stats */ |
| 1946 | 1954 | rcu_read_lock(); |
| 1947 | 1955 | |
| @@ -3773,24 +3781,6 @@ ice_get_ts_info(struct net_device *dev, struct kernel_ethtool_ts_info *info) · 1825 unmodified lines | ||
| 3773 | 3781 | return 0; |
| 3774 | 3782 | } |
| 3775 | 3783 | |
| 3776 | −/** | |
| 3777 | − * ice_get_max_txq - return the maximum number of Tx queues for in a PF | |
| 3778 | − * @pf: PF structure | |
| 3779 | − */ | |
| 3780 | −static int ice_get_max_txq(struct ice_pf *pf) | |
| 3781 | −{ | |
| 3782 | − return min(num_online_cpus(), pf->hw.func_caps.common_cap.num_txq); | |
| 3783 | −} | |
| 3784 | − | |
| 3785 | −/** | |
| 3786 | − * ice_get_max_rxq - return the maximum number of Rx queues for in a PF | |
| 3787 | − * @pf: PF structure | |
| 3788 | − */ | |
| 3789 | −static int ice_get_max_rxq(struct ice_pf *pf) | |
| 3790 | −{ | |
| 3791 | − return min(num_online_cpus(), pf->hw.func_caps.common_cap.num_rxq); | |
| 3792 | −} | |
| 3793 | − | |
| 3794 | 3784 | /** |
| 3795 | 3785 | * ice_get_combined_cnt - return the current number of combined channels |
| 3796 | 3786 | * @vsi: PF VSI pointer |
| @@ -4699,8 +4699,8 @@ static int ice_cfg_netdev(struct ice_vsi *vsi) · 4698 unmodified lines | ||
| 4699 | 4699 | struct net_device *netdev; |
| 4700 | 4700 | u8 mac_addr[ETH_ALEN]; |
| 4701 | 4701 | |
| 4702 | − netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, | |
| 4703 | − vsi->alloc_rxq); | |
| 4702 | + netdev = alloc_etherdev_mqs(sizeof(*np), ice_get_max_txq(vsi->back), | |
| 4703 | + ice_get_max_rxq(vsi->back)); | |
| 4704 | 4704 | if (!netdev) |
| 4705 | 4705 | return -ENOMEM; |
| 4706 | 4706 | |
| @@ -2,6 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | /* Copyright (C) 2019-2021, Intel Corporation. */ |
| 3 | 3 | |
| 4 | 4 | #include "ice.h" |
| 5 | +#include "ice_lib.h" | |
| 5 | 6 | #include "ice_eswitch.h" |
| 6 | 7 | #include "devlink/devlink.h" |
| 7 | 8 | #include "devlink/port.h" |
| @@ -67,7 +68,7 @@ ice_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) · 59 unmodified lines | ||
| 67 | 68 | return; |
| 68 | 69 | vsi = repr->src_vsi; |
| 69 | 70 | |
| 70 | − ice_update_vsi_stats(vsi); | |
| 71 | + ice_update_eth_stats(vsi); | |
| 71 | 72 | eth_stats = &vsi->eth_stats; |
| 72 | 73 | |
| 73 | 74 | stats->tx_packets = eth_stats->tx_unicast + eth_stats->tx_broadcast + |
| @@ -315,7 +316,7 @@ ice_repr_reg_netdev(struct net_device *netdev, const struct net_device_ops *ops) · 241 unmodified lines | ||
| 315 | 316 | |
| 316 | 317 | static int ice_repr_ready_vf(struct ice_repr *repr) |
| 317 | 318 | { |
| 318 | − return !ice_check_vf_ready_for_cfg(repr->vf); | |
| 319 | + return ice_check_vf_ready_for_cfg(repr->vf); | |
| 319 | 320 | } |
| 320 | 321 | |
| 321 | 322 | static int ice_repr_ready_sf(struct ice_repr *repr) |
| @@ -1066,7 +1066,7 @@ bool idpf_vport_set_hsplit(const struct idpf_vport *vport, u8 val); · 1065 unmodified lines | ||
| 1066 | 1066 | int idpf_idc_init(struct idpf_adapter *adapter); |
| 1067 | 1067 | int idpf_idc_init_aux_core_dev(struct idpf_adapter *adapter, |
| 1068 | 1068 | enum iidc_function_type ftype); |
| 1069 | −void idpf_idc_deinit_core_aux_device(struct iidc_rdma_core_dev_info *cdev_info); | |
| 1069 | +void idpf_idc_deinit_core_aux_device(struct idpf_adapter *adapter); | |
| 1070 | 1070 | void idpf_idc_deinit_vport_aux_device(struct iidc_rdma_vport_dev_info *vdev_info); |
| 1071 | 1071 | void idpf_idc_issue_reset_event(struct iidc_rdma_core_dev_info *cdev_info); |
| 1072 | 1072 | void idpf_idc_vdev_mtu_event(struct iidc_rdma_vport_dev_info *vdev_info, |
| @@ -470,10 +470,11 @@ err_privd_alloc: · 469 unmodified lines | ||
| 470 | 470 | |
| 471 | 471 | /** |
| 472 | 472 | * idpf_idc_deinit_core_aux_device - de-initialize Auxiliary Device(s) |
| 473 | − * @cdev_info: IDC core device info pointer | |
| 473 | + * @adapter: driver private data structure | |
| 474 | 474 | */ |
| 475 | −void idpf_idc_deinit_core_aux_device(struct iidc_rdma_core_dev_info *cdev_info) | |
| 475 | +void idpf_idc_deinit_core_aux_device(struct idpf_adapter *adapter) | |
| 476 | 476 | { |
| 477 | + struct iidc_rdma_core_dev_info *cdev_info = adapter->cdev_info; | |
| 477 | 478 | struct iidc_rdma_priv_dev_info *privd; |
| 478 | 479 | |
| 479 | 480 | if (!cdev_info) |
| @@ -485,6 +486,7 @@ void idpf_idc_deinit_core_aux_device(struct iidc_rdma_core_dev_info *cdev_info) · 5 unmodified lines | ||
| 485 | 486 | kfree(privd->mapped_mem_regions); |
| 486 | 487 | kfree(privd); |
| 487 | 488 | kfree(cdev_info); |
| 489 | + adapter->cdev_info = NULL; | |
| 488 | 490 | } |
| 489 | 491 | |
| 490 | 492 | /** |
| @@ -1860,13 +1860,13 @@ static int idpf_rxq_group_alloc(struct idpf_vport *vport, · 1859 unmodified lines | ||
| 1860 | 1860 | idpf_queue_assign(HSPLIT_EN, q, hs); |
| 1861 | 1861 | idpf_queue_assign(RSC_EN, q, rsc); |
| 1862 | 1862 | |
| 1863 | − bufq_set->num_refillqs = num_rxq; | |
| 1864 | 1863 | bufq_set->refillqs = kcalloc(num_rxq, swq_size, |
| 1865 | 1864 | GFP_KERNEL); |
| 1866 | 1865 | if (!bufq_set->refillqs) { |
| 1867 | 1866 | err = -ENOMEM; |
| 1868 | 1867 | goto err_alloc; |
| 1869 | 1868 | } |
| 1869 | + bufq_set->num_refillqs = num_rxq; | |
| 1870 | 1870 | for (unsigned int k = 0; k < bufq_set->num_refillqs; k++) { |
| 1871 | 1871 | struct idpf_sw_queue *refillq = |
| 1872 | 1872 | &bufq_set->refillqs[k]; |
| @@ -3668,7 +3668,7 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter) · 3667 unmodified lines | ||
| 3668 | 3668 | |
| 3669 | 3669 | idpf_ptp_release(adapter); |
| 3670 | 3670 | idpf_deinit_task(adapter); |
| 3671 | − idpf_idc_deinit_core_aux_device(adapter->cdev_info); | |
| 3671 | + idpf_idc_deinit_core_aux_device(adapter); | |
| 3672 | 3672 | idpf_rel_rx_pt_lkup(adapter); |
| 3673 | 3673 | idpf_intr_rel(adapter); |
| 3674 | 3674 | |
| @@ -3053,6 +3053,11 @@ static void lan743x_phylink_mac_link_up(struct phylink_config *config, · 3052 unmodified lines | ||
| 3053 | 3053 | else if (speed == SPEED_100) |
| 3054 | 3054 | mac_cr |= MAC_CR_CFG_L_; |
| 3055 | 3055 | |
| 3056 | + if (duplex == DUPLEX_FULL) | |
| 3057 | + mac_cr |= MAC_CR_DPX_; | |
| 3058 | + else | |
| 3059 | + mac_cr &= ~MAC_CR_DPX_; | |
| 3060 | + | |
| 3056 | 3061 | lan743x_csr_write(adapter, MAC_CR, mac_cr); |
| 3057 | 3062 | |
| 3058 | 3063 | lan743x_ptp_update_latency(adapter, speed); |
| @@ -3425,6 +3425,7 @@ static int add_adev(struct gdma_dev *gd, const char *name) · 3424 unmodified lines | ||
| 3425 | 3425 | struct auxiliary_device *adev; |
| 3426 | 3426 | struct mana_adev *madev; |
| 3427 | 3427 | int ret; |
| 3428 | + int id; | |
| 3428 | 3429 | |
| 3429 | 3430 | madev = kzalloc_obj(*madev); |
| 3430 | 3431 | if (!madev) |
| @@ -3434,7 +3435,8 @@ static int add_adev(struct gdma_dev *gd, const char *name) · 3 unmodified lines | ||
| 3434 | 3435 | ret = mana_adev_idx_alloc(); |
| 3435 | 3436 | if (ret < 0) |
| 3436 | 3437 | goto idx_fail; |
| 3437 | − adev->id = ret; | |
| 3438 | + id = ret; | |
| 3439 | + adev->id = id; | |
| 3438 | 3440 | |
| 3439 | 3441 | adev->name = name; |
| 3440 | 3442 | adev->dev.parent = gd->gdma_context->dev; |
| @@ -3460,7 +3462,7 @@ add_fail: · 19 unmodified lines | ||
| 3460 | 3462 | auxiliary_device_uninit(adev); |
| 3461 | 3463 | |
| 3462 | 3464 | init_fail: |
| 3463 | − mana_adev_idx_free(adev->id); | |
| 3465 | + mana_adev_idx_free(id); | |
| 3464 | 3466 | |
| 3465 | 3467 | idx_fail: |
| 3466 | 3468 | kfree(madev); |
| @@ -1719,13 +1719,18 @@ static int ionic_set_mac_address(struct net_device *netdev, void *sa) · 1718 unmodified lines | ||
| 1719 | 1719 | if (ether_addr_equal(netdev->dev_addr, mac)) |
| 1720 | 1720 | return 0; |
| 1721 | 1721 | |
| 1722 | − err = ionic_program_mac(lif, mac); | |
| 1723 | − if (err < 0) | |
| 1724 | − return err; | |
| 1722 | + /* Only program macs for virtual functions to avoid losing the permanent | |
| 1723 | + * Mac across warm reset/reboot. | |
| 1724 | + */ | |
| 1725 | + if (lif->ionic->pdev->is_virtfn) { | |
| 1726 | + err = ionic_program_mac(lif, mac); | |
| 1727 | + if (err < 0) | |
| 1728 | + return err; | |
| 1725 | 1729 | |
| 1726 | − if (err > 0) | |
| 1727 | − netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n", | |
| 1728 | − __func__); | |
| 1730 | + if (err > 0) | |
| 1731 | + netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n", | |
| 1732 | + __func__); | |
| 1733 | + } | |
| 1729 | 1734 | |
| 1730 | 1735 | err = eth_prepare_mac_addr_change(netdev, addr); |
| 1731 | 1736 | if (err) |
| @@ -962,7 +962,6 @@ static int emac_rx_packet_zc(struct prueth_emac *emac, u32 flow_id, · 961 unmodified lines | ||
| 962 | 962 | pkt_len -= 4; |
| 963 | 963 | cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); |
| 964 | 964 | psdata = cppi5_hdesc_get_psdata(desc_rx); |
| 965 | − k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); | |
| 966 | 965 | count++; |
| 967 | 966 | xsk_buff_set_size(xdp, pkt_len); |
| 968 | 967 | xsk_buff_dma_sync_for_cpu(xdp); |
| @@ -988,6 +987,7 @@ static int emac_rx_packet_zc(struct prueth_emac *emac, u32 flow_id, · 19 unmodified lines | ||
| 988 | 987 | emac_dispatch_skb_zc(emac, xdp, psdata); |
| 989 | 988 | xsk_buff_free(xdp); |
| 990 | 989 | } |
| 990 | + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); | |
| 991 | 991 | } |
| 992 | 992 | |
| 993 | 993 | if (xdp_status & ICSSG_XDP_REDIR) |
| @@ -1057,7 +1057,6 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) · 63 unmodified lines | ||
| 1057 | 1057 | /* firmware adds 4 CRC bytes, strip them */ |
| 1058 | 1058 | pkt_len -= 4; |
| 1059 | 1059 | cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); |
| 1060 | − k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); | |
| 1061 | 1060 | |
| 1062 | 1061 | /* if allocation fails we drop the packet but push the |
| 1063 | 1062 | * descriptor back to the ring with old page to prevent a stall |
| @@ -1115,6 +1114,7 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) · 51 unmodified lines | ||
| 1115 | 1114 | ndev->stats.rx_packets++; |
| 1116 | 1115 | |
| 1117 | 1116 | requeue: |
| 1117 | + k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx); | |
| 1118 | 1118 | /* queue another RX DMA */ |
| 1119 | 1119 | ret = prueth_dma_rx_push_mapped(emac, &emac->rx_chns, new_page, |
| 1120 | 1120 | PRUETH_MAX_PKT_SIZE); |
| @@ -2058,6 +2058,68 @@ static const struct ethtool_ops team_ethtool_ops = { · 2057 unmodified lines | ||
| 2058 | 2058 | * rt netlink interface |
| 2059 | 2059 | ***********************/ |
| 2060 | 2060 | |
| 2061 | +/* For tx path we need a linkup && enabled port and for parse any port | |
| 2062 | + * suffices. | |
| 2063 | + */ | |
| 2064 | +static struct team_port *team_header_port_get_rcu(struct team *team, | |
| 2065 | + bool txable) | |
| 2066 | +{ | |
| 2067 | + struct team_port *port; | |
| 2068 | + | |
| 2069 | + list_for_each_entry_rcu(port, &team->port_list, list) { | |
| 2070 | + if (!txable || team_port_txable(port)) | |
| 2071 | + return port; | |
| 2072 | + } | |
| 2073 | + | |
| 2074 | + return NULL; | |
| 2075 | +} | |
| 2076 | + | |
| 2077 | +static int team_header_create(struct sk_buff *skb, struct net_device *team_dev, | |
| 2078 | + unsigned short type, const void *daddr, | |
| 2079 | + const void *saddr, unsigned int len) | |
| 2080 | +{ | |
| 2081 | + struct team *team = netdev_priv(team_dev); | |
| 2082 | + const struct header_ops *port_ops; | |
| 2083 | + struct team_port *port; | |
| 2084 | + int ret = 0; | |
| 2085 | + | |
| 2086 | + rcu_read_lock(); | |
| 2087 | + port = team_header_port_get_rcu(team, true); | |
| 2088 | + if (port) { | |
| 2089 | + port_ops = READ_ONCE(port->dev->header_ops); | |
| 2090 | + if (port_ops && port_ops->create) | |
| 2091 | + ret = port_ops->create(skb, port->dev, | |
| 2092 | + type, daddr, saddr, len); | |
| 2093 | + } | |
| 2094 | + rcu_read_unlock(); | |
| 2095 | + return ret; | |
| 2096 | +} | |
| 2097 | + | |
| 2098 | +static int team_header_parse(const struct sk_buff *skb, | |
| 2099 | + const struct net_device *team_dev, | |
| 2100 | + unsigned char *haddr) | |
| 2101 | +{ | |
| 2102 | + struct team *team = netdev_priv(team_dev); | |
| 2103 | + const struct header_ops *port_ops; | |
| 2104 | + struct team_port *port; | |
| 2105 | + int ret = 0; | |
| 2106 | + | |
| 2107 | + rcu_read_lock(); | |
| 2108 | + port = team_header_port_get_rcu(team, false); | |
| 2109 | + if (port) { | |
| 2110 | + port_ops = READ_ONCE(port->dev->header_ops); | |
| 2111 | + if (port_ops && port_ops->parse) | |
| 2112 | + ret = port_ops->parse(skb, port->dev, haddr); | |
| 2113 | + } | |
| 2114 | + rcu_read_unlock(); | |
| 2115 | + return ret; | |
| 2116 | +} | |
| 2117 | + | |
| 2118 | +static const struct header_ops team_header_ops = { | |
| 2119 | + .create = team_header_create, | |
| 2120 | + .parse = team_header_parse, | |
| 2121 | +}; | |
| 2122 | + | |
| 2061 | 2123 | static void team_setup_by_port(struct net_device *dev, |
| 2062 | 2124 | struct net_device *port_dev) |
| 2063 | 2125 | { |
| @@ -2066,7 +2128,8 @@ static void team_setup_by_port(struct net_device *dev, · 2 unmodified lines | ||
| 2066 | 2128 | if (port_dev->type == ARPHRD_ETHER) |
| 2067 | 2129 | dev->header_ops = team->header_ops_cache; |
| 2068 | 2130 | else |
| 2069 | − dev->header_ops = port_dev->header_ops; | |
| 2131 | + dev->header_ops = port_dev->header_ops ? | |
| 2132 | + &team_header_ops : NULL; | |
| 2070 | 2133 | dev->type = port_dev->type; |
| 2071 | 2134 | dev->hard_header_len = port_dev->hard_header_len; |
| 2072 | 2135 | dev->needed_headroom = port_dev->needed_headroom; |
| @@ -244,7 +244,7 @@ tun_vnet_hdr_tnl_from_skb(unsigned int flags, · 243 unmodified lines | ||
| 244 | 244 | |
| 245 | 245 | if (virtio_net_hdr_tnl_from_skb(skb, tnl_hdr, has_tnl_offload, |
| 246 | 246 | tun_vnet_is_little_endian(flags), |
| 247 | − vlan_hlen, true)) { | |
| 247 | + vlan_hlen, true, false)) { | |
| 248 | 248 | struct virtio_net_hdr_v1 *hdr = &tnl_hdr->hash_hdr.hdr; |
| 249 | 249 | struct skb_shared_info *sinfo = skb_shinfo(skb); |
| 250 | 250 | |
| @@ -3267,8 +3267,12 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) · 3266 unmodified lines | ||
| 3267 | 3267 | struct virtio_net_hdr_v1_hash_tunnel *hdr; |
| 3268 | 3268 | int num_sg; |
| 3269 | 3269 | unsigned hdr_len = vi->hdr_len; |
| 3270 | + bool feature_hdrlen; | |
| 3270 | 3271 | bool can_push; |
| 3271 | 3272 | |
| 3273 | + feature_hdrlen = virtio_has_feature(vi->vdev, | |
| 3274 | + VIRTIO_NET_F_GUEST_HDRLEN); | |
| 3275 | + | |
| 3272 | 3276 | pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); |
| 3273 | 3277 | |
| 3274 | 3278 | /* Make sure it's safe to cast between formats */ |
| @@ -3288,7 +3292,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) · 13 unmodified lines | ||
| 3288 | 3292 | |
| 3289 | 3293 | if (virtio_net_hdr_tnl_from_skb(skb, hdr, vi->tx_tnl, |
| 3290 | 3294 | virtio_is_little_endian(vi->vdev), 0, |
| 3291 | − false)) | |
| 3295 | + false, feature_hdrlen)) | |
| 3292 | 3296 | return -EPROTO; |
| 3293 | 3297 | |
| 3294 | 3298 | if (vi->mergeable_rx_bufs) |
| @@ -3351,6 +3355,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) · 56 unmodified lines | ||
| 3351 | 3355 | /* Don't wait up for transmitted skbs to be freed. */ |
| 3352 | 3356 | if (!use_napi) { |
| 3353 | 3357 | skb_orphan(skb); |
| 3358 | + skb_dst_drop(skb); | |
| 3354 | 3359 | nf_reset_ct(skb); |
| 3355 | 3360 | } |
| 3356 | 3361 | |
| @@ -299,8 +299,10 @@ static bool pci_pwrctrl_is_required(struct device_node *np) · 298 unmodified lines | ||
| 299 | 299 | struct device_node *remote __free(device_node) = |
| 300 | 300 | of_graph_get_remote_port_parent(endpoint); |
| 301 | 301 | if (remote) { |
| 302 | − if (of_pci_supply_present(remote)) | |
| 302 | + if (of_pci_supply_present(remote)) { | |
| 303 | + of_node_put(endpoint); | |
| 303 | 304 | return true; |
| 305 | + } | |
| 304 | 306 | } |
| 305 | 307 | } |
| 306 | 308 | } |
| @@ -68,13 +68,6 @@ static int pwrseq_pwrctrl_power_off(struct pci_pwrctrl *pwrctrl) · 67 unmodified lines | ||
| 68 | 68 | return pwrseq_power_off(pwrseq->pwrseq); |
| 69 | 69 | } |
| 70 | 70 | |
| 71 | −static void devm_pwrseq_pwrctrl_power_off(void *data) | |
| 72 | −{ | |
| 73 | − struct pwrseq_pwrctrl *pwrseq = data; | |
| 74 | − | |
| 75 | − pwrseq_pwrctrl_power_off(&pwrseq->pwrctrl); | |
| 76 | −} | |
| 77 | − | |
| 78 | 71 | static int pwrseq_pwrctrl_probe(struct platform_device *pdev) |
| 79 | 72 | { |
| 80 | 73 | const struct pwrseq_pwrctrl_pdata *pdata; |
| @@ -101,11 +94,6 @@ static int pwrseq_pwrctrl_probe(struct platform_device *pdev) · 20 unmodified lines | ||
| 101 | 94 | return dev_err_probe(dev, PTR_ERR(pwrseq->pwrseq), |
| 102 | 95 | "Failed to get the power sequencer\n"); |
| 103 | 96 | |
| 104 | − ret = devm_add_action_or_reset(dev, devm_pwrseq_pwrctrl_power_off, | |
| 105 | − pwrseq); | |
| 106 | − if (ret) | |
| 107 | − return ret; | |
| 108 | − | |
| 109 | 97 | pwrseq->pwrctrl.power_on = pwrseq_pwrctrl_power_on; |
| 110 | 98 | pwrseq->pwrctrl.power_off = pwrseq_pwrctrl_power_off; |
| 111 | 99 | |
| @@ -63,7 +63,6 @@ static void devm_slot_pwrctrl_release(void *data) · 62 unmodified lines | ||
| 63 | 63 | { |
| 64 | 64 | struct slot_pwrctrl *slot = data; |
| 65 | 65 | |
| 66 | − slot_pwrctrl_power_off(&slot->pwrctrl); | |
| 67 | 66 | regulator_bulk_free(slot->num_supplies, slot->supplies); |
| 68 | 67 | } |
| 69 | 68 | |
| @@ -6,7 +6,7 @@ · 5 unmodified lines | ||
| 6 | 6 | menu "PHY Subsystem" |
| 7 | 7 | |
| 8 | 8 | config PHY_COMMON_PROPS |
| 9 | − bool | |
| 9 | + bool "PHY common properties" if KUNIT_ALL_TESTS | |
| 10 | 10 | help |
| 11 | 11 | This parses properties common between generic PHYs and Ethernet PHYs. |
| 12 | 12 | |
| @@ -16,8 +16,7 @@ config PHY_COMMON_PROPS · 3 unmodified lines | ||
| 16 | 16 | |
| 17 | 17 | config PHY_COMMON_PROPS_TEST |
| 18 | 18 | tristate "KUnit tests for PHY common props" if !KUNIT_ALL_TESTS |
| 19 | − select PHY_COMMON_PROPS | |
| 20 | − depends on KUNIT | |
| 19 | + depends on KUNIT && PHY_COMMON_PROPS | |
| 21 | 20 | default KUNIT_ALL_TESTS |
| 22 | 21 | help |
| 23 | 22 | This builds KUnit tests for the PHY common property API. |
| @@ -1069,6 +1069,8 @@ static void lynx_28g_cdr_lock_check(struct work_struct *work) · 1068 unmodified lines | ||
| 1069 | 1069 | |
| 1070 | 1070 | for (i = 0; i < LYNX_28G_NUM_LANE; i++) { |
| 1071 | 1071 | lane = &priv->lane[i]; |
| 1072 | + if (!lane->phy) | |
| 1073 | + continue; | |
| 1072 | 1074 | |
| 1073 | 1075 | mutex_lock(&lane->phy->mutex); |
| 1074 | 1076 | |
| @@ -990,6 +990,7 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = { · 989 unmodified lines | ||
| 990 | 990 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02), |
| 991 | 991 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43), |
| 992 | 992 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0xc1), |
| 993 | + QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), | |
| 993 | 994 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f), |
| 994 | 995 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x68), |
| 995 | 996 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4, 0x0e), |
| @@ -999,13 +1000,11 @@ static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = { · 3 unmodified lines | ||
| 999 | 1000 | }; |
| 1000 | 1001 | |
| 1001 | 1002 | static const struct qmp_phy_init_tbl sm8650_ufsphy_g4_pcs[] = { |
| 1002 | − QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x13), | |
| 1003 | 1003 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x04), |
| 1004 | 1004 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x04), |
| 1005 | 1005 | }; |
| 1006 | 1006 | |
| 1007 | 1007 | static const struct qmp_phy_init_tbl sm8650_ufsphy_g5_pcs[] = { |
| 1008 | − QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33), | |
| 1009 | 1008 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x05), |
| 1010 | 1009 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x05), |
| 1011 | 1010 | QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HS_G5_SYNC_LENGTH_CAPABILITY, 0x4d), |
| @@ -48,6 +48,9 @@ · 47 unmodified lines | ||
| 48 | 48 | #define PHY_CLK_HSTXP_EN BIT(3) /* clock hstxp enable */ |
| 49 | 49 | #define PHY_HSTXP_MODE BIT(4) /* 0: force en_txp to be 1; 1: no force */ |
| 50 | 50 | |
| 51 | +#define PHY_K1_HS_HOST_DISC 0x40 | |
| 52 | +#define PHY_K1_HS_HOST_DISC_CLR BIT(0) | |
| 53 | + | |
| 51 | 54 | #define PHY_PLL_DIV_CFG 0x98 |
| 52 | 55 | #define PHY_FDIV_FRACT_8_15 GENMASK(7, 0) |
| 53 | 56 | #define PHY_FDIV_FRACT_16_19 GENMASK(11, 8) |
| @@ -142,9 +145,20 @@ static int spacemit_usb2phy_exit(struct phy *phy) · 88 unmodified lines | ||
| 142 | 145 | return 0; |
| 143 | 146 | } |
| 144 | 147 | |
| 148 | +static int spacemit_usb2phy_disconnect(struct phy *phy, int port) | |
| 149 | +{ | |
| 150 | + struct spacemit_usb2phy *sphy = phy_get_drvdata(phy); | |
| 151 | + | |
| 152 | + regmap_update_bits(sphy->regmap_base, PHY_K1_HS_HOST_DISC, | |
| 153 | + PHY_K1_HS_HOST_DISC_CLR, PHY_K1_HS_HOST_DISC_CLR); | |
| 154 | + | |
| 155 | + return 0; | |
| 156 | +} | |
| 157 | + | |
| 145 | 158 | static const struct phy_ops spacemit_usb2phy_ops = { |
| 146 | 159 | .init = spacemit_usb2phy_init, |
| 147 | 160 | .exit = spacemit_usb2phy_exit, |
| 161 | + .disconnect = spacemit_usb2phy_disconnect, | |
| 148 | 162 | .owner = THIS_MODULE, |
| 149 | 163 | }; |
| 150 | 164 | |
| @@ -1425,6 +1425,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz) · 1424 unmodified lines | ||
| 1425 | 1425 | dev_err(dev, |
| 1426 | 1426 | "%s: Reading \"reg\" from \"%s\" failed: %d\n", |
| 1427 | 1427 | __func__, subnode->name, ret); |
| 1428 | + of_node_put(serdes); | |
| 1428 | 1429 | return ret; |
| 1429 | 1430 | } |
| 1430 | 1431 | of_property_read_u32(subnode, "cdns,num-lanes", &num_lanes); |
| @@ -1439,6 +1440,7 @@ static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz) · 8 unmodified lines | ||
| 1439 | 1440 | } |
| 1440 | 1441 | } |
| 1441 | 1442 | |
| 1443 | + of_node_put(serdes); | |
| 1442 | 1444 | return 0; |
| 1443 | 1445 | } |
| 1444 | 1446 | |
| @@ -1135,9 +1135,12 @@ int mtk_pctrl_init(struct platform_device *pdev, · 1134 unmodified lines | ||
| 1135 | 1135 | goto chip_error; |
| 1136 | 1136 | } |
| 1137 | 1137 | |
| 1138 | − ret = mtk_eint_init(pctl, pdev); | |
| 1139 | − if (ret) | |
| 1140 | − goto chip_error; | |
| 1138 | + /* Only initialize EINT if we have EINT pins */ | |
| 1139 | + if (data->eint_hw.ap_num > 0) { | |
| 1140 | + ret = mtk_eint_init(pctl, pdev); | |
| 1141 | + if (ret) | |
| 1142 | + goto chip_error; | |
| 1143 | + } | |
| 1141 | 1144 | |
| 1142 | 1145 | return 0; |
| 1143 | 1146 | |
| @@ -723,6 +723,21 @@ static const struct pinconf_ops pmic_gpio_pinconf_ops = { · 722 unmodified lines | ||
| 723 | 723 | .pin_config_group_dbg_show = pmic_gpio_config_dbg_show, |
| 724 | 724 | }; |
| 725 | 725 | |
| 726 | +static int pmic_gpio_get_direction(struct gpio_chip *chip, unsigned pin) | |
| 727 | +{ | |
| 728 | + struct pmic_gpio_state *state = gpiochip_get_data(chip); | |
| 729 | + struct pmic_gpio_pad *pad; | |
| 730 | + | |
| 731 | + pad = state->ctrl->desc->pins[pin].drv_data; | |
| 732 | + | |
| 733 | + if (!pad->is_enabled || pad->analog_pass || | |
| 734 | + (!pad->input_enabled && !pad->output_enabled)) | |
| 735 | + return -EINVAL; | |
| 736 | + | |
| 737 | + /* Make sure the state is aligned on what pmic_gpio_get() returns */ | |
| 738 | + return pad->input_enabled ? GPIO_LINE_DIRECTION_IN : GPIO_LINE_DIRECTION_OUT; | |
| 739 | +} | |
| 740 | + | |
| 726 | 741 | static int pmic_gpio_direction_input(struct gpio_chip *chip, unsigned pin) |
| 727 | 742 | { |
| 728 | 743 | struct pmic_gpio_state *state = gpiochip_get_data(chip); |
| @@ -801,6 +816,7 @@ static void pmic_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip) · 72 unmodified lines | ||
| 801 | 816 | } |
| 802 | 817 | |
| 803 | 818 | static const struct gpio_chip pmic_gpio_gpio_template = { |
| 819 | + .get_direction = pmic_gpio_get_direction, | |
| 804 | 820 | .direction_input = pmic_gpio_direction_input, |
| 805 | 821 | .direction_output = pmic_gpio_direction_output, |
| 806 | 822 | .get = pmic_gpio_get, |
| @@ -589,7 +589,7 @@ static inline unsigned int rza1_get_bit(struct rza1_port *port, · 588 unmodified lines | ||
| 589 | 589 | { |
| 590 | 590 | void __iomem *mem = RZA1_ADDR(port->base, reg, port->id); |
| 591 | 591 | |
| 592 | − return ioread16(mem) & BIT(bit); | |
| 592 | + return !!(ioread16(mem) & BIT(bit)); | |
| 593 | 593 | } |
| 594 | 594 | |
| 595 | 595 | /** |
| @@ -85,7 +85,7 @@ struct rzt2h_pinctrl { · 84 unmodified lines | ||
| 85 | 85 | struct gpio_chip gpio_chip; |
| 86 | 86 | struct pinctrl_gpio_range gpio_range; |
| 87 | 87 | DECLARE_BITMAP(used_irqs, RZT2H_INTERRUPTS_NUM); |
| 88 | − spinlock_t lock; /* lock read/write registers */ | |
| 88 | + raw_spinlock_t lock; /* lock read/write registers */ | |
| 89 | 89 | struct mutex mutex; /* serialize adding groups and functions */ |
| 90 | 90 | bool safety_port_enabled; |
| 91 | 91 | atomic_t wakeup_path; |
| @@ -145,7 +145,7 @@ static void rzt2h_pinctrl_set_pfc_mode(struct rzt2h_pinctrl *pctrl, · 53 unmodified lines | ||
| 145 | 145 | u64 reg64; |
| 146 | 146 | u16 reg16; |
| 147 | 147 | |
| 148 | − guard(spinlock_irqsave)(&pctrl->lock); | |
| 148 | + guard(raw_spinlock_irqsave)(&pctrl->lock); | |
| 149 | 149 | |
| 150 | 150 | /* Set pin to 'Non-use (Hi-Z input protection)' */ |
| 151 | 151 | reg16 = rzt2h_pinctrl_readw(pctrl, port, PM(port)); |
| @@ -474,7 +474,7 @@ static int rzt2h_gpio_request(struct gpio_chip *chip, unsigned int offset) · 322 unmodified lines | ||
| 474 | 474 | if (ret) |
| 475 | 475 | return ret; |
| 476 | 476 | |
| 477 | − guard(spinlock_irqsave)(&pctrl->lock); | |
| 477 | + guard(raw_spinlock_irqsave)(&pctrl->lock); | |
| 478 | 478 | |
| 479 | 479 | /* Select GPIO mode in PMC Register */ |
| 480 | 480 | rzt2h_pinctrl_set_gpio_en(pctrl, port, bit, true); |
| @@ -487,7 +487,7 @@ static void rzt2h_gpio_set_direction(struct rzt2h_pinctrl *pctrl, u32 port, · 6 unmodified lines | ||
| 487 | 487 | { |
| 488 | 488 | u16 reg; |
| 489 | 489 | |
| 490 | − guard(spinlock_irqsave)(&pctrl->lock); | |
| 490 | + guard(raw_spinlock_irqsave)(&pctrl->lock); | |
| 491 | 491 | |
| 492 | 492 | reg = rzt2h_pinctrl_readw(pctrl, port, PM(port)); |
| 493 | 493 | reg &= ~PM_PIN_MASK(bit); |
| @@ -509,7 +509,7 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) · 15 unmodified lines | ||
| 509 | 509 | if (ret) |
| 510 | 510 | return ret; |
| 511 | 511 | |
| 512 | − guard(spinlock_irqsave)(&pctrl->lock); | |
| 512 | + guard(raw_spinlock_irqsave)(&pctrl->lock); | |
| 513 | 513 | |
| 514 | 514 | if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) { |
| 515 | 515 | /* |
| @@ -547,7 +547,7 @@ static int rzt2h_gpio_set(struct gpio_chip *chip, unsigned int offset, · 31 unmodified lines | ||
| 547 | 547 | u8 bit = RZT2H_PIN_ID_TO_PIN(offset); |
| 548 | 548 | u8 reg; |
| 549 | 549 | |
| 550 | − guard(spinlock_irqsave)(&pctrl->lock); | |
| 550 | + guard(raw_spinlock_irqsave)(&pctrl->lock); | |
| 551 | 551 | |
| 552 | 552 | reg = rzt2h_pinctrl_readb(pctrl, port, P(port)); |
| 553 | 553 | if (value) |
| @@ -833,6 +833,7 @@ static int rzt2h_gpio_register(struct rzt2h_pinctrl *pctrl) · 279 unmodified lines | ||
| 833 | 833 | if (ret) |
| 834 | 834 | return dev_err_probe(dev, ret, "Unable to parse gpio-ranges\n"); |
| 835 | 835 | |
| 836 | + of_node_put(of_args.np); | |
| 836 | 837 | if (of_args.args[0] != 0 || of_args.args[1] != 0 || |
| 837 | 838 | of_args.args[2] != pctrl->data->n_port_pins) |
| 838 | 839 | return dev_err_probe(dev, -EINVAL, |
| @@ -964,7 +965,7 @@ static int rzt2h_pinctrl_probe(struct platform_device *pdev) · 125 unmodified lines | ||
| 964 | 965 | if (ret) |
| 965 | 966 | return ret; |
| 966 | 967 | |
| 967 | − spin_lock_init(&pctrl->lock); | |
| 968 | + raw_spin_lock_init(&pctrl->lock); | |
| 968 | 969 | mutex_init(&pctrl->mutex); |
| 969 | 970 | platform_set_drvdata(pdev, pctrl); |
| 970 | 971 | |
| @@ -65,6 +65,7 @@ config PINCTRL_STM32_HDP · 64 unmodified lines | ||
| 65 | 65 | select PINMUX |
| 66 | 66 | select GENERIC_PINCONF |
| 67 | 67 | select GPIOLIB |
| 68 | + select GPIO_GENERIC | |
| 68 | 69 | help |
| 69 | 70 | The Hardware Debug Port allows the observation of internal signals. |
| 70 | 71 | It uses configurable multiplexer to route signals in a dedicated observation register. |
| @@ -157,6 +157,7 @@ sunxi_pinctrl_desc_find_function_by_name(struct sunxi_pinctrl *pctl, · 156 unmodified lines | ||
| 157 | 157 | const char *pin_name, |
| 158 | 158 | const char *func_name) |
| 159 | 159 | { |
| 160 | + unsigned long variant = pctl->flags & SUNXI_PINCTRL_VARIANT_MASK; | |
| 160 | 161 | int i; |
| 161 | 162 | |
| 162 | 163 | for (i = 0; i < pctl->desc->npins; i++) { |
| @@ -168,7 +169,7 @@ sunxi_pinctrl_desc_find_function_by_name(struct sunxi_pinctrl *pctl, · 5 unmodified lines | ||
| 168 | 169 | while (func->name) { |
| 169 | 170 | if (!strcmp(func->name, func_name) && |
| 170 | 171 | (!func->variant || |
| 171 | − func->variant & pctl->variant)) | |
| 172 | + func->variant & variant)) | |
| 172 | 173 | return func; |
| 173 | 174 | |
| 174 | 175 | func++; |
| @@ -209,6 +210,8 @@ sunxi_pinctrl_desc_find_function_by_pin_and_mux(struct sunxi_pinctrl *pctl, · 34 unmodified lines | ||
| 209 | 210 | const u16 pin_num, |
| 210 | 211 | const u8 muxval) |
| 211 | 212 | { |
| 213 | + unsigned long variant = pctl->flags & SUNXI_PINCTRL_VARIANT_MASK; | |
| 214 | + | |
| 212 | 215 | for (unsigned int i = 0; i < pctl->desc->npins; i++) { |
| 213 | 216 | const struct sunxi_desc_pin *pin = pctl->desc->pins + i; |
| 214 | 217 | struct sunxi_desc_function *func = pin->functions; |
| @@ -216,7 +219,7 @@ sunxi_pinctrl_desc_find_function_by_pin_and_mux(struct sunxi_pinctrl *pctl, · 1 unmodified line | ||
| 216 | 219 | if (pin->pin.number != pin_num) |
| 217 | 220 | continue; |
| 218 | 221 | |
| 219 | − if (pin->variant && !(pctl->variant & pin->variant)) | |
| 222 | + if (pin->variant && !(variant & pin->variant)) | |
| 220 | 223 | continue; |
| 221 | 224 | |
| 222 | 225 | while (func->name) { |
| @@ -1089,6 +1092,9 @@ static int sunxi_pinctrl_irq_request_resources(struct irq_data *d) · 866 unmodified lines | ||
| 1089 | 1092 | { |
| 1090 | 1093 | struct sunxi_pinctrl *pctl = irq_data_get_irq_chip_data(d); |
| 1091 | 1094 | struct sunxi_desc_function *func; |
| 1095 | + unsigned int offset; | |
| 1096 | + u32 reg, shift, mask; | |
| 1097 | + u8 disabled_mux, muxval; | |
| 1092 | 1098 | int ret; |
| 1093 | 1099 | |
| 1094 | 1100 | func = sunxi_pinctrl_desc_find_function_by_pin(pctl, |
| @@ -1096,8 +1102,21 @@ static int sunxi_pinctrl_irq_request_resources(struct irq_data *d) · 1 unmodified line | ||
| 1096 | 1102 | if (!func) |
| 1097 | 1103 | return -EINVAL; |
| 1098 | 1104 | |
| 1099 | − ret = gpiochip_lock_as_irq(pctl->chip, | |
| 1100 | − pctl->irq_array[d->hwirq] - pctl->desc->pin_base); | |
| 1105 | + offset = pctl->irq_array[d->hwirq] - pctl->desc->pin_base; | |
| 1106 | + sunxi_mux_reg(pctl, offset, ®, &shift, &mask); | |
| 1107 | + muxval = (readl(pctl->membase + reg) & mask) >> shift; | |
| 1108 | + | |
| 1109 | + /* Change muxing to GPIO INPUT mode if at reset value */ | |
| 1110 | + if (pctl->flags & SUNXI_PINCTRL_NEW_REG_LAYOUT) | |
| 1111 | + disabled_mux = SUN4I_FUNC_DISABLED_NEW; | |
| 1112 | + else | |
| 1113 | + disabled_mux = SUN4I_FUNC_DISABLED_OLD; | |
| 1114 | + | |
| 1115 | + if (muxval == disabled_mux) | |
| 1116 | + sunxi_pmx_set(pctl->pctl_dev, pctl->irq_array[d->hwirq], | |
| 1117 | + SUN4I_FUNC_INPUT); | |
| 1118 | + | |
| 1119 | + ret = gpiochip_lock_as_irq(pctl->chip, offset); | |
| 1101 | 1120 | if (ret) { |
| 1102 | 1121 | dev_err(pctl->dev, "unable to lock HW IRQ %lu for IRQ\n", |
| 1103 | 1122 | irqd_to_hwirq(d)); |
| @@ -1338,6 +1357,7 @@ static int sunxi_pinctrl_add_function(struct sunxi_pinctrl *pctl, · 234 unmodified lines | ||
| 1338 | 1357 | static int sunxi_pinctrl_build_state(struct platform_device *pdev) |
| 1339 | 1358 | { |
| 1340 | 1359 | struct sunxi_pinctrl *pctl = platform_get_drvdata(pdev); |
| 1360 | + unsigned long variant = pctl->flags & SUNXI_PINCTRL_VARIANT_MASK; | |
| 1341 | 1361 | void *ptr; |
| 1342 | 1362 | int i; |
| 1343 | 1363 | |
| @@ -1362,7 +1382,7 @@ static int sunxi_pinctrl_build_state(struct platform_device *pdev) · 18 unmodified lines | ||
| 1362 | 1382 | const struct sunxi_desc_pin *pin = pctl->desc->pins + i; |
| 1363 | 1383 | struct sunxi_pinctrl_group *group = pctl->groups + pctl->ngroups; |
| 1364 | 1384 | |
| 1365 | − if (pin->variant && !(pctl->variant & pin->variant)) | |
| 1385 | + if (pin->variant && !(variant & pin->variant)) | |
| 1366 | 1386 | continue; |
| 1367 | 1387 | |
| 1368 | 1388 | group->name = pin->pin.name; |
| @@ -1387,11 +1407,11 @@ static int sunxi_pinctrl_build_state(struct platform_device *pdev) · 18 unmodified lines | ||
| 1387 | 1407 | const struct sunxi_desc_pin *pin = pctl->desc->pins + i; |
| 1388 | 1408 | struct sunxi_desc_function *func; |
| 1389 | 1409 | |
| 1390 | − if (pin->variant && !(pctl->variant & pin->variant)) | |
| 1410 | + if (pin->variant && !(variant & pin->variant)) | |
| 1391 | 1411 | continue; |
| 1392 | 1412 | |
| 1393 | 1413 | for (func = pin->functions; func->name; func++) { |
| 1394 | − if (func->variant && !(pctl->variant & func->variant)) | |
| 1414 | + if (func->variant && !(variant & func->variant)) | |
| 1395 | 1415 | continue; |
| 1396 | 1416 | |
| 1397 | 1417 | /* Create interrupt mapping while we're at it */ |
| @@ -1419,14 +1439,14 @@ static int sunxi_pinctrl_build_state(struct platform_device *pdev) · 21 unmodified lines | ||
| 1419 | 1439 | const struct sunxi_desc_pin *pin = pctl->desc->pins + i; |
| 1420 | 1440 | struct sunxi_desc_function *func; |
| 1421 | 1441 | |
| 1422 | − if (pin->variant && !(pctl->variant & pin->variant)) | |
| 1442 | + if (pin->variant && !(variant & pin->variant)) | |
| 1423 | 1443 | continue; |
| 1424 | 1444 | |
| 1425 | 1445 | for (func = pin->functions; func->name; func++) { |
| 1426 | 1446 | struct sunxi_pinctrl_function *func_item; |
| 1427 | 1447 | const char **func_grp; |
| 1428 | 1448 | |
| 1429 | − if (func->variant && !(pctl->variant & func->variant)) | |
| 1449 | + if (func->variant && !(variant & func->variant)) | |
| 1430 | 1450 | continue; |
| 1431 | 1451 | |
| 1432 | 1452 | func_item = sunxi_pinctrl_find_function_by_name(pctl, |
| @@ -1568,7 +1588,7 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, · 135 unmodified lines | ||
| 1568 | 1588 | |
| 1569 | 1589 | pctl->dev = &pdev->dev; |
| 1570 | 1590 | pctl->desc = desc; |
| 1571 | − pctl->variant = flags & SUNXI_PINCTRL_VARIANT_MASK; | |
| 1591 | + pctl->flags = flags; | |
| 1572 | 1592 | if (flags & SUNXI_PINCTRL_NEW_REG_LAYOUT) { |
| 1573 | 1593 | pctl->bank_mem_size = D1_BANK_MEM_SIZE; |
| 1574 | 1594 | pctl->pull_regs_offset = D1_PULL_REGS_OFFSET; |
| @@ -1604,8 +1624,9 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, · 29 unmodified lines | ||
| 1604 | 1624 | |
| 1605 | 1625 | for (i = 0, pin_idx = 0; i < pctl->desc->npins; i++) { |
| 1606 | 1626 | const struct sunxi_desc_pin *pin = pctl->desc->pins + i; |
| 1627 | + unsigned long variant = pctl->flags & SUNXI_PINCTRL_VARIANT_MASK; | |
| 1607 | 1628 | |
| 1608 | − if (pin->variant && !(pctl->variant & pin->variant)) | |
| 1629 | + if (pin->variant && !(variant & pin->variant)) | |
| 1609 | 1630 | continue; |
| 1610 | 1631 | |
| 1611 | 1632 | pins[pin_idx++] = pin->pin; |
| @@ -86,6 +86,8 @@ · 85 unmodified lines | ||
| 86 | 86 | |
| 87 | 87 | #define SUN4I_FUNC_INPUT 0 |
| 88 | 88 | #define SUN4I_FUNC_IRQ 6 |
| 89 | +#define SUN4I_FUNC_DISABLED_OLD 7 | |
| 90 | +#define SUN4I_FUNC_DISABLED_NEW 15 | |
| 89 | 91 | |
| 90 | 92 | #define SUNXI_PINCTRL_VARIANT_MASK GENMASK(7, 0) |
| 91 | 93 | #define SUNXI_PINCTRL_NEW_REG_LAYOUT BIT(8) |
| @@ -174,7 +176,7 @@ struct sunxi_pinctrl { · 82 unmodified lines | ||
| 174 | 176 | unsigned *irq_array; |
| 175 | 177 | raw_spinlock_t lock; |
| 176 | 178 | struct pinctrl_dev *pctl_dev; |
| 177 | − unsigned long variant; | |
| 179 | + unsigned long flags; | |
| 178 | 180 | u32 bank_mem_size; |
| 179 | 181 | u32 pull_regs_offset; |
| 180 | 182 | u32 dlevel_field_width; |
| @@ -482,7 +482,7 @@ static int olpc_xo175_ec_cmd(u8 cmd, u8 *inbuf, size_t inlen, u8 *resp, · 481 unmodified lines | ||
| 482 | 482 | dev_dbg(dev, "CMD %x, %zd bytes expected\n", cmd, resp_len); |
| 483 | 483 | |
| 484 | 484 | if (inlen > 5) { |
| 485 | − dev_err(dev, "command len %zd too big!\n", resp_len); | |
| 485 | + dev_err(dev, "command len %zd too big!\n", inlen); | |
| 486 | 486 | return -EOVERFLOW; |
| 487 | 487 | } |
| 488 | 488 | |
| @@ -117,7 +117,7 @@ static int __hsmp_send_message(struct hsmp_socket *sock, struct hsmp_message *ms · 116 unmodified lines | ||
| 117 | 117 | } |
| 118 | 118 | |
| 119 | 119 | if (unlikely(mbox_status == HSMP_STATUS_NOT_READY)) { |
| 120 | − dev_err(sock->dev, "Message ID 0x%X failure : SMU tmeout (status = 0x%X)\n", | |
| 120 | + dev_err(sock->dev, "Message ID 0x%X failure : SMU timeout (status = 0x%X)\n", | |
| 121 | 121 | msg->msg_id, mbox_status); |
| 122 | 122 | return -ETIMEDOUT; |
| 123 | 123 | } else if (unlikely(mbox_status == HSMP_ERR_INVALID_MSG)) { |
| @@ -1080,6 +1080,20 @@ static const struct dmi_system_id power_limits[] = { · 1079 unmodified lines | ||
| 1080 | 1080 | .requires_fan_curve = true, |
| 1081 | 1081 | }, |
| 1082 | 1082 | }, |
| 1083 | + { | |
| 1084 | + .matches = { | |
| 1085 | + DMI_MATCH(DMI_BOARD_NAME, "GA503QM"), | |
| 1086 | + }, | |
| 1087 | + .driver_data = &(struct power_data) { | |
| 1088 | + .ac_data = &(struct power_limits) { | |
| 1089 | + .ppt_pl1_spl_min = 15, | |
| 1090 | + .ppt_pl1_spl_def = 35, | |
| 1091 | + .ppt_pl1_spl_max = 80, | |
| 1092 | + .ppt_pl2_sppt_min = 65, | |
| 1093 | + .ppt_pl2_sppt_max = 80, | |
| 1094 | + }, | |
| 1095 | + }, | |
| 1096 | + }, | |
| 1083 | 1097 | { |
| 1084 | 1098 | .matches = { |
| 1085 | 1099 | DMI_MATCH(DMI_BOARD_NAME, "GA503QR"), |
| @@ -1518,6 +1532,35 @@ static const struct dmi_system_id power_limits[] = { · 432 unmodified lines | ||
| 1518 | 1532 | }, |
| 1519 | 1533 | }, |
| 1520 | 1534 | }, |
| 1535 | + { | |
| 1536 | + .matches = { | |
| 1537 | + DMI_MATCH(DMI_BOARD_NAME, "GZ302EA"), | |
| 1538 | + }, | |
| 1539 | + .driver_data = &(struct power_data) { | |
| 1540 | + .ac_data = &(struct power_limits) { | |
| 1541 | + .ppt_pl1_spl_min = 28, | |
| 1542 | + .ppt_pl1_spl_def = 60, | |
| 1543 | + .ppt_pl1_spl_max = 80, | |
| 1544 | + .ppt_pl2_sppt_min = 32, | |
| 1545 | + .ppt_pl2_sppt_def = 75, | |
| 1546 | + .ppt_pl2_sppt_max = 92, | |
| 1547 | + .ppt_pl3_fppt_min = 45, | |
| 1548 | + .ppt_pl3_fppt_def = 86, | |
| 1549 | + .ppt_pl3_fppt_max = 93, | |
| 1550 | + }, | |
| 1551 | + .dc_data = &(struct power_limits) { | |
| 1552 | + .ppt_pl1_spl_min = 28, | |
| 1553 | + .ppt_pl1_spl_def = 45, | |
| 1554 | + .ppt_pl1_spl_max = 80, | |
| 1555 | + .ppt_pl2_sppt_min = 32, | |
| 1556 | + .ppt_pl2_sppt_def = 52, | |
| 1557 | + .ppt_pl2_sppt_max = 92, | |
| 1558 | + .ppt_pl3_fppt_min = 45, | |
| 1559 | + .ppt_pl3_fppt_def = 71, | |
| 1560 | + .ppt_pl3_fppt_max = 93, | |
| 1561 | + }, | |
| 1562 | + }, | |
| 1563 | + }, | |
| 1521 | 1564 | { |
| 1522 | 1565 | .matches = { |
| 1523 | 1566 | DMI_MATCH(DMI_BOARD_NAME, "G513I"), |
| @@ -1596,6 +1639,40 @@ static const struct dmi_system_id power_limits[] = { · 72 unmodified lines | ||
| 1596 | 1639 | .requires_fan_curve = true, |
| 1597 | 1640 | }, |
| 1598 | 1641 | }, |
| 1642 | + { | |
| 1643 | + .matches = { | |
| 1644 | + DMI_MATCH(DMI_BOARD_NAME, "G614FP"), | |
| 1645 | + }, | |
| 1646 | + .driver_data = &(struct power_data) { | |
| 1647 | + .ac_data = &(struct power_limits) { | |
| 1648 | + .ppt_pl1_spl_min = 30, | |
| 1649 | + .ppt_pl1_spl_max = 120, | |
| 1650 | + .ppt_pl2_sppt_min = 65, | |
| 1651 | + .ppt_pl2_sppt_def = 140, | |
| 1652 | + .ppt_pl2_sppt_max = 165, | |
| 1653 | + .ppt_pl3_fppt_min = 65, | |
| 1654 | + .ppt_pl3_fppt_def = 140, | |
| 1655 | + .ppt_pl3_fppt_max = 165, | |
| 1656 | + .nv_temp_target_min = 75, | |
| 1657 | + .nv_temp_target_max = 87, | |
| 1658 | + .nv_dynamic_boost_min = 5, | |
| 1659 | + .nv_dynamic_boost_max = 15, | |
| 1660 | + .nv_tgp_min = 50, | |
| 1661 | + .nv_tgp_max = 100, | |
| 1662 | + }, | |
| 1663 | + .dc_data = &(struct power_limits) { | |
| 1664 | + .ppt_pl1_spl_min = 25, | |
| 1665 | + .ppt_pl1_spl_max = 65, | |
| 1666 | + .ppt_pl2_sppt_min = 25, | |
| 1667 | + .ppt_pl2_sppt_max = 65, | |
| 1668 | + .ppt_pl3_fppt_min = 35, | |
| 1669 | + .ppt_pl3_fppt_max = 75, | |
| 1670 | + .nv_temp_target_min = 75, | |
| 1671 | + .nv_temp_target_max = 87, | |
| 1672 | + }, | |
| 1673 | + .requires_fan_curve = true, | |
| 1674 | + }, | |
| 1675 | + }, | |
| 1599 | 1676 | { |
| 1600 | 1677 | .matches = { |
| 1601 | 1678 | DMI_MATCH(DMI_BOARD_NAME, "G614J"), |
| @@ -548,7 +548,7 @@ static const struct dmi_system_id asus_quirks[] = { · 547 unmodified lines | ||
| 548 | 548 | .callback = dmi_matched, |
| 549 | 549 | .ident = "ASUS ROG Z13", |
| 550 | 550 | .matches = { |
| 551 | − DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), | |
| 551 | + DMI_MATCH(DMI_SYS_VENDOR, "ASUS"), | |
| 552 | 552 | DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow Z13"), |
| 553 | 553 | }, |
| 554 | 554 | .driver_data = &quirk_asus_z13, |
| @@ -120,6 +120,13 @@ static const struct thermal_profile_params omen_v1_thermal_params = { · 119 unmodified lines | ||
| 120 | 120 | .ec_tp_offset = HP_VICTUS_S_EC_THERMAL_PROFILE_OFFSET, |
| 121 | 121 | }; |
| 122 | 122 | |
| 123 | +static const struct thermal_profile_params omen_v1_legacy_thermal_params = { | |
| 124 | + .performance = HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE, | |
| 125 | + .balanced = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, | |
| 126 | + .low_power = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, | |
| 127 | + .ec_tp_offset = HP_OMEN_EC_THERMAL_PROFILE_OFFSET, | |
| 128 | +}; | |
| 129 | + | |
| 123 | 130 | /* |
| 124 | 131 | * A generic pointer for the currently-active board's thermal profile |
| 125 | 132 | * parameters. |
| @@ -175,6 +182,10 @@ static const char * const victus_thermal_profile_boards[] = { · 49 unmodified lines | ||
| 175 | 182 | |
| 176 | 183 | /* DMI Board names of Victus 16-r and Victus 16-s laptops */ |
| 177 | 184 | static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = { |
| 185 | + { | |
| 186 | + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8A4D") }, | |
| 187 | + .driver_data = (void *)&omen_v1_legacy_thermal_params, | |
| 188 | + }, | |
| 178 | 189 | { |
| 179 | 190 | .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BAB") }, |
| 180 | 191 | .driver_data = (void *)&omen_v1_thermal_params, |
| @@ -183,6 +194,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst · 2 unmodified lines | ||
| 183 | 194 | .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") }, |
| 184 | 195 | .driver_data = (void *)&victus_s_thermal_params, |
| 185 | 196 | }, |
| 197 | + { | |
| 198 | + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCA") }, | |
| 199 | + .driver_data = (void *)&omen_v1_thermal_params, | |
| 200 | + }, | |
| 186 | 201 | { |
| 187 | 202 | .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCD") }, |
| 188 | 203 | .driver_data = (void *)&omen_v1_thermal_params, |
| @@ -195,6 +210,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst · 6 unmodified lines | ||
| 195 | 210 | .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") }, |
| 196 | 211 | .driver_data = (void *)&victus_s_thermal_params, |
| 197 | 212 | }, |
| 213 | + { | |
| 214 | + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C76") }, | |
| 215 | + .driver_data = (void *)&omen_v1_thermal_params, | |
| 216 | + }, | |
| 198 | 217 | { |
| 199 | 218 | .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") }, |
| 200 | 219 | .driver_data = (void *)&omen_v1_thermal_params, |
| @@ -438,6 +438,14 @@ static int intel_hid_pl_suspend_handler(struct device *device) · 437 unmodified lines | ||
| 438 | 438 | return 0; |
| 439 | 439 | } |
| 440 | 440 | |
| 441 | +static int intel_hid_pl_freeze_handler(struct device *device) | |
| 442 | +{ | |
| 443 | + struct intel_hid_priv *priv = dev_get_drvdata(device); | |
| 444 | + | |
| 445 | + priv->wakeup_mode = false; | |
| 446 | + return intel_hid_pl_suspend_handler(device); | |
| 447 | +} | |
| 448 | + | |
| 441 | 449 | static int intel_hid_pl_resume_handler(struct device *device) |
| 442 | 450 | { |
| 443 | 451 | intel_hid_pm_complete(device); |
| @@ -452,7 +460,7 @@ static int intel_hid_pl_resume_handler(struct device *device) · 8 unmodified lines | ||
| 452 | 460 | static const struct dev_pm_ops intel_hid_pl_pm_ops = { |
| 453 | 461 | .prepare = intel_hid_pm_prepare, |
| 454 | 462 | .complete = intel_hid_pm_complete, |
| 455 | − .freeze = intel_hid_pl_suspend_handler, | |
| 463 | + .freeze = intel_hid_pl_freeze_handler, | |
| 456 | 464 | .thaw = intel_hid_pl_resume_handler, |
| 457 | 465 | .restore = intel_hid_pl_resume_handler, |
| 458 | 466 | .suspend = intel_hid_pl_suspend_handler, |
| @@ -558,6 +558,9 @@ static bool disable_dynamic_sst_features(void) · 557 unmodified lines | ||
| 558 | 558 | { |
| 559 | 559 | u64 value; |
| 560 | 560 | |
| 561 | + if (!static_cpu_has(X86_FEATURE_HWP)) | |
| 562 | + return true; | |
| 563 | + | |
| 561 | 564 | rdmsrq(MSR_PM_ENABLE, value); |
| 562 | 565 | return !(value & 0x1); |
| 563 | 566 | } |
| @@ -869,7 +872,7 @@ static int isst_if_get_perf_level(void __user *argp) · 305 unmodified lines | ||
| 869 | 872 | _read_pp_info("current_level", perf_level.current_level, SST_PP_STATUS_OFFSET, |
| 870 | 873 | SST_PP_LEVEL_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE) |
| 871 | 874 | _read_pp_info("locked", perf_level.locked, SST_PP_STATUS_OFFSET, |
| 872 | − SST_PP_LOCK_START, SST_PP_LEVEL_WIDTH, SST_MUL_FACTOR_NONE) | |
| 875 | + SST_PP_LOCK_START, SST_PP_LOCK_WIDTH, SST_MUL_FACTOR_NONE) | |
| 873 | 876 | _read_pp_info("feature_state", perf_level.feature_state, SST_PP_STATUS_OFFSET, |
| 874 | 877 | SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE) |
| 875 | 878 | perf_level.enabled = !!(power_domain_info->sst_header.cap_mask & BIT(1)); |
| @@ -31,8 +31,6 @@ · 30 unmodified lines | ||
| 31 | 31 | #define LWMI_GZ_METHOD_ID_SMARTFAN_SET 44 |
| 32 | 32 | #define LWMI_GZ_METHOD_ID_SMARTFAN_GET 45 |
| 33 | 33 | |
| 34 | −static BLOCKING_NOTIFIER_HEAD(gz_chain_head); | |
| 35 | − | |
| 36 | 34 | struct lwmi_gz_priv { |
| 37 | 35 | enum thermal_mode current_mode; |
| 38 | 36 | struct notifier_block event_nb; |
| @@ -4966,7 +4966,8 @@ static void ibmvfc_discover_targets_done(struct ibmvfc_event *evt) · 4965 unmodified lines | ||
| 4966 | 4966 | switch (mad_status) { |
| 4967 | 4967 | case IBMVFC_MAD_SUCCESS: |
| 4968 | 4968 | ibmvfc_dbg(vhost, "Discover Targets succeeded\n"); |
| 4969 | − vhost->num_targets = be32_to_cpu(rsp->num_written); | |
| 4969 | + vhost->num_targets = min_t(u32, be32_to_cpu(rsp->num_written), | |
| 4970 | + max_targets); | |
| 4970 | 4971 | ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_ALLOC_TGTS); |
| 4971 | 4972 | break; |
| 4972 | 4973 | case IBMVFC_MAD_FAILED: |
| @@ -1734,7 +1734,7 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel, · 1733 unmodified lines | ||
| 1734 | 1734 | break; |
| 1735 | 1735 | |
| 1736 | 1736 | default: |
| 1737 | − if (channel < shost->max_channel) { | |
| 1737 | + if (channel <= shost->max_channel) { | |
| 1738 | 1738 | res = scsi_scan_host_selected(shost, channel, id, lun, |
| 1739 | 1739 | SCSI_SCAN_MANUAL); |
| 1740 | 1740 | } else { |
| @@ -215,7 +215,7 @@ static unsigned char *ses_get_page2_descriptor(struct enclosure_device *edev, · 214 unmodified lines | ||
| 215 | 215 | unsigned char *type_ptr = ses_dev->page1_types; |
| 216 | 216 | unsigned char *desc_ptr = ses_dev->page2 + 8; |
| 217 | 217 | |
| 218 | − if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len) < 0) | |
| 218 | + if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len)) | |
| 219 | 219 | return NULL; |
| 220 | 220 | |
| 221 | 221 | for (i = 0; i < ses_dev->page1_num_types; i++, type_ptr += 4) { |
| @@ -1009,7 +1009,7 @@ static int fsl_lpspi_probe(struct platform_device *pdev) · 1008 unmodified lines | ||
| 1009 | 1009 | enable_irq(irq); |
| 1010 | 1010 | } |
| 1011 | 1011 | |
| 1012 | − ret = devm_spi_register_controller(&pdev->dev, controller); | |
| 1012 | + ret = spi_register_controller(controller); | |
| 1013 | 1013 | if (ret < 0) { |
| 1014 | 1014 | dev_err_probe(&pdev->dev, ret, "spi_register_controller error\n"); |
| 1015 | 1015 | goto free_dma; |
| @@ -1035,6 +1035,7 @@ static void fsl_lpspi_remove(struct platform_device *pdev) · 19 unmodified lines | ||
| 1035 | 1035 | struct fsl_lpspi_data *fsl_lpspi = |
| 1036 | 1036 | spi_controller_get_devdata(controller); |
| 1037 | 1037 | |
| 1038 | + spi_unregister_controller(controller); | |
| 1038 | 1039 | fsl_lpspi_dma_exit(controller); |
| 1039 | 1040 | |
| 1040 | 1041 | pm_runtime_dont_use_autosuspend(fsl_lpspi->dev); |
| @@ -1101,8 +1101,6 @@ static void meson_spicc_remove(struct platform_device *pdev) · 1100 unmodified lines | ||
| 1101 | 1101 | |
| 1102 | 1102 | /* Disable SPI */ |
| 1103 | 1103 | writel(0, spicc->base + SPICC_CONREG); |
| 1104 | − | |
| 1105 | − spi_controller_put(spicc->host); | |
| 1106 | 1104 | } |
| 1107 | 1105 | |
| 1108 | 1106 | static const struct meson_spicc_data meson_spicc_gx_data = { |
| @@ -612,7 +612,7 @@ static int f_ospi_probe(struct platform_device *pdev) · 611 unmodified lines | ||
| 612 | 612 | u32 num_cs = OSPI_NUM_CS; |
| 613 | 613 | int ret; |
| 614 | 614 | |
| 615 | − ctlr = spi_alloc_host(dev, sizeof(*ospi)); | |
| 615 | + ctlr = devm_spi_alloc_host(dev, sizeof(*ospi)); | |
| 616 | 616 | if (!ctlr) |
| 617 | 617 | return -ENOMEM; |
| 618 | 618 | |
| @@ -635,43 +635,22 @@ static int f_ospi_probe(struct platform_device *pdev) · 16 unmodified lines | ||
| 635 | 635 | platform_set_drvdata(pdev, ospi); |
| 636 | 636 | |
| 637 | 637 | ospi->base = devm_platform_ioremap_resource(pdev, 0); |
| 638 | − if (IS_ERR(ospi->base)) { | |
| 639 | − ret = PTR_ERR(ospi->base); | |
| 640 | − goto err_put_ctlr; | |
| 641 | − } | |
| 638 | + if (IS_ERR(ospi->base)) | |
| 639 | + return PTR_ERR(ospi->base); | |
| 642 | 640 | |
| 643 | 641 | ospi->clk = devm_clk_get_enabled(dev, NULL); |
| 644 | − if (IS_ERR(ospi->clk)) { | |
| 645 | − ret = PTR_ERR(ospi->clk); | |
| 646 | − goto err_put_ctlr; | |
| 647 | − } | |
| 642 | + if (IS_ERR(ospi->clk)) | |
| 643 | + return PTR_ERR(ospi->clk); | |
| 648 | 644 | |
| 649 | − mutex_init(&ospi->mlock); | |
| 650 | − | |
| 651 | − ret = f_ospi_init(ospi); | |
| 645 | + ret = devm_mutex_init(dev, &ospi->mlock); | |
| 652 | 646 | if (ret) |
| 653 | − goto err_destroy_mutex; | |
| 647 | + return ret; | |
| 654 | 648 | |
| 655 | − ret = devm_spi_register_controller(dev, ctlr); | |
| 649 | + ret = f_ospi_init(ospi); | |
| 656 | 650 | if (ret) |
| 657 | − goto err_destroy_mutex; | |
| 658 | − | |
| 659 | − return 0; | |
| 660 | − | |
| 661 | −err_destroy_mutex: | |
| 662 | − mutex_destroy(&ospi->mlock); | |
| 663 | − | |
| 664 | −err_put_ctlr: | |
| 665 | − spi_controller_put(ctlr); | |
| 666 | − | |
| 667 | − return ret; | |
| 668 | −} | |
| 669 | − | |
| 670 | −static void f_ospi_remove(struct platform_device *pdev) | |
| 671 | −{ | |
| 672 | − struct f_ospi *ospi = platform_get_drvdata(pdev); | |
| 651 | + return ret; | |
| 673 | 652 | |
| 674 | − mutex_destroy(&ospi->mlock); | |
| 653 | + return devm_spi_register_controller(dev, ctlr); | |
| 675 | 654 | } |
| 676 | 655 | |
| 677 | 656 | static const struct of_device_id f_ospi_dt_ids[] = { |
| @@ -686,7 +665,6 @@ static struct platform_driver f_ospi_driver = { · 8 unmodified lines | ||
| 686 | 665 | .of_match_table = f_ospi_dt_ids, |
| 687 | 666 | }, |
| 688 | 667 | .probe = f_ospi_probe, |
| 689 | − .remove = f_ospi_remove, | |
| 690 | 668 | }; |
| 691 | 669 | module_platform_driver(f_ospi_driver); |
| 692 | 670 | |
| @@ -50,7 +50,6 @@ static void spidev_release(struct device *dev) · 49 unmodified lines | ||
| 50 | 50 | struct spi_device *spi = to_spi_device(dev); |
| 51 | 51 | |
| 52 | 52 | spi_controller_put(spi->controller); |
| 53 | − kfree(spi->driver_override); | |
| 54 | 53 | free_percpu(spi->pcpu_statistics); |
| 55 | 54 | kfree(spi); |
| 56 | 55 | } |
| @@ -73,10 +72,9 @@ static ssize_t driver_override_store(struct device *dev, · 16 unmodified lines | ||
| 73 | 72 | struct device_attribute *a, |
| 74 | 73 | const char *buf, size_t count) |
| 75 | 74 | { |
| 76 | − struct spi_device *spi = to_spi_device(dev); | |
| 77 | 75 | int ret; |
| 78 | 76 | |
| 79 | − ret = driver_set_override(dev, &spi->driver_override, buf, count); | |
| 77 | + ret = __device_set_driver_override(dev, buf, count); | |
| 80 | 78 | if (ret) |
| 81 | 79 | return ret; |
| 82 | 80 | |
| @@ -86,13 +84,8 @@ static ssize_t driver_override_store(struct device *dev, · 3 unmodified lines | ||
| 86 | 84 | static ssize_t driver_override_show(struct device *dev, |
| 87 | 85 | struct device_attribute *a, char *buf) |
| 88 | 86 | { |
| 89 | − const struct spi_device *spi = to_spi_device(dev); | |
| 90 | − ssize_t len; | |
| 91 | − | |
| 92 | − device_lock(dev); | |
| 93 | − len = sysfs_emit(buf, "%s\n", spi->driver_override ? : ""); | |
| 94 | − device_unlock(dev); | |
| 95 | − return len; | |
| 87 | + guard(spinlock)(&dev->driver_override.lock); | |
| 88 | + return sysfs_emit(buf, "%s\n", dev->driver_override.name ?: ""); | |
| 96 | 89 | } |
| 97 | 90 | static DEVICE_ATTR_RW(driver_override); |
| 98 | 91 | |
| @@ -376,10 +369,12 @@ static int spi_match_device(struct device *dev, const struct device_driver *drv) · 277 unmodified lines | ||
| 376 | 369 | { |
| 377 | 370 | const struct spi_device *spi = to_spi_device(dev); |
| 378 | 371 | const struct spi_driver *sdrv = to_spi_driver(drv); |
| 372 | + int ret; | |
| 379 | 373 | |
| 380 | 374 | /* Check override first, and if set, only use the named driver */ |
| 381 | − if (spi->driver_override) | |
| 382 | − return strcmp(spi->driver_override, drv->name) == 0; | |
| 375 | + ret = device_match_driver_override(dev, drv); | |
| 376 | + if (ret >= 0) | |
| 377 | + return ret; | |
| 383 | 378 | |
| 384 | 379 | /* Attempt an OF style match */ |
| 385 | 380 | if (of_driver_match_device(dev, drv)) |
| @@ -3539,8 +3534,19 @@ int devm_spi_register_controller(struct device *dev, · 3153 unmodified lines | ||
| 3539 | 3534 | if (ret) |
| 3540 | 3535 | return ret; |
| 3541 | 3536 | |
| 3542 | − return devm_add_action_or_reset(dev, devm_spi_unregister_controller, ctlr); | |
| 3537 | + /* | |
| 3538 | + * Prevent controller from being freed by spi_unregister_controller() | |
| 3539 | + * if devm_add_action_or_reset() fails for a non-devres allocated | |
| 3540 | + * controller. | |
| 3541 | + */ | |
| 3542 | + spi_controller_get(ctlr); | |
| 3543 | + | |
| 3544 | + ret = devm_add_action_or_reset(dev, devm_spi_unregister_controller, ctlr); | |
| 3543 | 3545 | |
| 3546 | + if (ret == 0 || ctlr->devm_allocated) | |
| 3547 | + spi_controller_put(ctlr); | |
| 3548 | + | |
| 3549 | + return ret; | |
| 3544 | 3550 | } |
| 3545 | 3551 | EXPORT_SYMBOL_GPL(devm_spi_register_controller); |
| 3546 | 3552 | |
| @@ -26,6 +26,7 @@ · 25 unmodified lines | ||
| 26 | 26 | #include <linux/slab.h> |
| 27 | 27 | #include <linux/types.h> |
| 28 | 28 | #include <linux/configfs.h> |
| 29 | +#include <linux/blk-mq.h> | |
| 29 | 30 | #include <scsi/scsi.h> |
| 30 | 31 | #include <scsi/scsi_tcq.h> |
| 31 | 32 | #include <scsi/scsi_host.h> |
| @@ -269,15 +270,27 @@ static int tcm_loop_device_reset(struct scsi_cmnd *sc) · 237 unmodified lines | ||
| 269 | 270 | return (ret == TMR_FUNCTION_COMPLETE) ? SUCCESS : FAILED; |
| 270 | 271 | } |
| 271 | 272 | |
| 273 | +static bool tcm_loop_flush_work_iter(struct request *rq, void *data) | |
| 274 | +{ | |
| 275 | + struct scsi_cmnd *sc = blk_mq_rq_to_pdu(rq); | |
| 276 | + struct tcm_loop_cmd *tl_cmd = scsi_cmd_priv(sc); | |
| 277 | + struct se_cmd *se_cmd = &tl_cmd->tl_se_cmd; | |
| 278 | + | |
| 279 | + flush_work(&se_cmd->work); | |
| 280 | + return true; | |
| 281 | +} | |
| 282 | + | |
| 272 | 283 | static int tcm_loop_target_reset(struct scsi_cmnd *sc) |
| 273 | 284 | { |
| 274 | 285 | struct tcm_loop_hba *tl_hba; |
| 275 | 286 | struct tcm_loop_tpg *tl_tpg; |
| 287 | + struct Scsi_Host *sh = sc->device->host; | |
| 288 | + int ret; | |
| 276 | 289 | |
| 277 | 290 | /* |
| 278 | 291 | * Locate the tcm_loop_hba_t pointer |
| 279 | 292 | */ |
| 280 | − tl_hba = *(struct tcm_loop_hba **)shost_priv(sc->device->host); | |
| 293 | + tl_hba = *(struct tcm_loop_hba **)shost_priv(sh); | |
| 281 | 294 | if (!tl_hba) { |
| 282 | 295 | pr_err("Unable to perform device reset without active I_T Nexus\n"); |
| 283 | 296 | return FAILED; |
| @@ -286,11 +299,38 @@ static int tcm_loop_target_reset(struct scsi_cmnd *sc) · 2 unmodified lines | ||
| 286 | 299 | * Locate the tl_tpg pointer from TargetID in sc->device->id |
| 287 | 300 | */ |
| 288 | 301 | tl_tpg = &tl_hba->tl_hba_tpgs[sc->device->id]; |
| 289 | − if (tl_tpg) { | |
| 290 | − tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE; | |
| 291 | − return SUCCESS; | |
| 292 | − } | |
| 293 | − return FAILED; | |
| 302 | + if (!tl_tpg) | |
| 303 | + return FAILED; | |
| 304 | + | |
| 305 | + /* | |
| 306 | + * Issue a LUN_RESET to drain all commands that the target core | |
| 307 | + * knows about. This handles commands not yet marked CMD_T_COMPLETE. | |
| 308 | + */ | |
| 309 | + ret = tcm_loop_issue_tmr(tl_tpg, sc->device->lun, 0, TMR_LUN_RESET); | |
| 310 | + if (ret != TMR_FUNCTION_COMPLETE) | |
| 311 | + return FAILED; | |
| 312 | + | |
| 313 | + /* | |
| 314 | + * Flush any deferred target core completion work that may still be | |
| 315 | + * queued. Commands that already had CMD_T_COMPLETE set before the TMR | |
| 316 | + * are skipped by the TMR drain, but their async completion work | |
| 317 | + * (transport_lun_remove_cmd → percpu_ref_put, release_cmd → scsi_done) | |
| 318 | + * may still be pending in target_completion_wq. | |
| 319 | + * | |
| 320 | + * The SCSI EH will reuse in-flight scsi_cmnd structures for recovery | |
| 321 | + * commands (e.g. TUR) immediately after this handler returns SUCCESS — | |
| 322 | + * if deferred work is still pending, the memset in queuecommand would | |
| 323 | + * zero the se_cmd while the work accesses it, leaking the LUN | |
| 324 | + * percpu_ref and hanging configfs unlink forever. | |
| 325 | + * | |
| 326 | + * Use blk_mq_tagset_busy_iter() to find all started requests and | |
| 327 | + * flush_work() on each — the same pattern used by mpi3mr, scsi_debug, | |
| 328 | + * and other SCSI drivers to drain outstanding commands during reset. | |
| 329 | + */ | |
| 330 | + blk_mq_tagset_busy_iter(&sh->tag_set, tcm_loop_flush_work_iter, NULL); | |
| 331 | + | |
| 332 | + tl_tpg->tl_transport_status = TCM_TRANSPORT_ONLINE; | |
| 333 | + return SUCCESS; | |
| 294 | 334 | } |
| 295 | 335 | |
| 296 | 336 | static const struct scsi_host_template tcm_loop_driver_template = { |
| @@ -276,7 +276,7 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, · 275 unmodified lines | ||
| 276 | 276 | ssize_t len = 0; |
| 277 | 277 | int ret = 0, i; |
| 278 | 278 | |
| 279 | − aio_cmd = kmalloc_flex(*aio_cmd, bvecs, sgl_nents); | |
| 279 | + aio_cmd = kzalloc_flex(*aio_cmd, bvecs, sgl_nents); | |
| 280 | 280 | if (!aio_cmd) |
| 281 | 281 | return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; |
| 282 | 282 | |
| @@ -176,15 +176,21 @@ static inline void write_soc_slider(struct proc_thermal_device *proc_priv, u64 v · 175 unmodified lines | ||
| 176 | 176 | |
| 177 | 177 | static void set_soc_power_profile(struct proc_thermal_device *proc_priv, int slider) |
| 178 | 178 | { |
| 179 | + u8 offset; | |
| 179 | 180 | u64 val; |
| 180 | 181 | |
| 181 | 182 | val = read_soc_slider(proc_priv); |
| 182 | 183 | val &= ~SLIDER_MASK; |
| 183 | 184 | val |= FIELD_PREP(SLIDER_MASK, slider) | BIT(SLIDER_ENABLE_BIT); |
| 184 | 185 | |
| 186 | + if (slider == SOC_SLIDER_VALUE_MINIMUM || slider == SOC_SLIDER_VALUE_MAXIMUM) | |
| 187 | + offset = 0; | |
| 188 | + else | |
| 189 | + offset = slider_offset; | |
| 190 | + | |
| 185 | 191 | /* Set the slider offset from module params */ |
| 186 | 192 | val &= ~SLIDER_OFFSET_MASK; |
| 187 | − val |= FIELD_PREP(SLIDER_OFFSET_MASK, slider_offset); | |
| 193 | + val |= FIELD_PREP(SLIDER_OFFSET_MASK, offset); | |
| 188 | 194 | |
| 189 | 195 | write_soc_slider(proc_priv, val); |
| 190 | 196 | } |
| @@ -301,11 +301,10 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, · 300 unmodified lines | ||
| 301 | 301 | */ |
| 302 | 302 | ret = dma_buf_fd(priv->dmabuf, get_dma_buf.open_flags); |
| 303 | 303 | if (ret < 0) |
| 304 | − goto err_dma_buf; | |
| 304 | + dma_buf_put(priv->dmabuf); | |
| 305 | + | |
| 305 | 306 | return ret; |
| 306 | 307 | |
| 307 | −err_dma_buf: | |
| 308 | − dma_buf_put(priv->dmabuf); | |
| 309 | 308 | err_dev_put: |
| 310 | 309 | vfio_device_put_registration(&vdev->vdev); |
| 311 | 310 | err_free_phys: |
| @@ -171,6 +171,8 @@ static void tdx_mr_deinit(const struct attribute_group *mr_grp) · 170 unmodified lines | ||
| 171 | 171 | #define GET_QUOTE_SUCCESS 0 |
| 172 | 172 | #define GET_QUOTE_IN_FLIGHT 0xffffffffffffffff |
| 173 | 173 | |
| 174 | +#define TDX_QUOTE_MAX_LEN (GET_QUOTE_BUF_SIZE - sizeof(struct tdx_quote_buf)) | |
| 175 | + | |
| 174 | 176 | /* struct tdx_quote_buf: Format of Quote request buffer. |
| 175 | 177 | * @version: Quote format version, filled by TD. |
| 176 | 178 | * @status: Status code of Quote request, filled by VMM. |
| @@ -269,6 +271,7 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data) · 92 unmodified lines | ||
| 269 | 271 | u8 *buf; |
| 270 | 272 | struct tdx_quote_buf *quote_buf = quote_data; |
| 271 | 273 | struct tsm_report_desc *desc = &report->desc; |
| 274 | + u32 out_len; | |
| 272 | 275 | int ret; |
| 273 | 276 | u64 err; |
| 274 | 277 | |
| @@ -306,12 +309,17 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data) · 31 unmodified lines | ||
| 306 | 309 | return ret; |
| 307 | 310 | } |
| 308 | 311 | |
| 309 | − buf = kvmemdup(quote_buf->data, quote_buf->out_len, GFP_KERNEL); | |
| 312 | + out_len = READ_ONCE(quote_buf->out_len); | |
| 313 | + | |
| 314 | + if (out_len > TDX_QUOTE_MAX_LEN) | |
| 315 | + return -EFBIG; | |
| 316 | + | |
| 317 | + buf = kvmemdup(quote_buf->data, out_len, GFP_KERNEL); | |
| 310 | 318 | if (!buf) |
| 311 | 319 | return -ENOMEM; |
| 312 | 320 | |
| 313 | 321 | report->outblob = buf; |
| 314 | − report->outblob_len = quote_buf->out_len; | |
| 322 | + report->outblob_len = out_len; | |
| 315 | 323 | |
| 316 | 324 | /* |
| 317 | 325 | * TODO: parse the PEM-formatted cert chain out of the quote buffer when |
| @@ -2912,10 +2912,10 @@ EXPORT_SYMBOL_GPL(virtqueue_add_inbuf); · 2911 unmodified lines | ||
| 2912 | 2912 | * @data: the token identifying the buffer. |
| 2913 | 2913 | * @gfp: how to do memory allocations (if necessary). |
| 2914 | 2914 | * |
| 2915 | − * Same as virtqueue_add_inbuf but passes DMA_ATTR_CPU_CACHE_CLEAN to indicate | |
| 2916 | − * that the CPU will not dirty any cacheline overlapping this buffer while it | |
| 2917 | − * is available, and to suppress overlapping cacheline warnings in DMA debug | |
| 2918 | − * builds. | |
| 2915 | + * Same as virtqueue_add_inbuf but passes DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | |
| 2916 | + * to indicate that the CPU will not dirty any cacheline overlapping this buffer | |
| 2917 | + * while it is available, and to suppress overlapping cacheline warnings in DMA | |
| 2918 | + * debug builds. | |
| 2919 | 2919 | * |
| 2920 | 2920 | * Caller must ensure we don't call this with other virtqueue operations |
| 2921 | 2921 | * at the same time (except where noted). |
| @@ -2928,7 +2928,7 @@ int virtqueue_add_inbuf_cache_clean(struct virtqueue *vq, · 6 unmodified lines | ||
| 2928 | 2928 | gfp_t gfp) |
| 2929 | 2929 | { |
| 2930 | 2930 | return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, false, gfp, |
| 2931 | − DMA_ATTR_CPU_CACHE_CLEAN); | |
| 2931 | + DMA_ATTR_DEBUGGING_IGNORE_CACHELINES); | |
| 2932 | 2932 | } |
| 2933 | 2933 | EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_cache_clean); |
| 2934 | 2934 | |
| @@ -12,6 +12,7 @@ · 11 unmodified lines | ||
| 12 | 12 | #include <linux/eventfd.h> |
| 13 | 13 | #include <linux/file.h> |
| 14 | 14 | #include <linux/kernel.h> |
| 15 | +#include <linux/kstrtox.h> | |
| 15 | 16 | #include <linux/module.h> |
| 16 | 17 | #include <linux/mutex.h> |
| 17 | 18 | #include <linux/poll.h> |
| @@ -30,7 +31,10 @@ · 12 unmodified lines | ||
| 30 | 31 | #include <linux/seq_file.h> |
| 31 | 32 | #include <linux/miscdevice.h> |
| 32 | 33 | #include <linux/moduleparam.h> |
| 34 | +#include <linux/notifier.h> | |
| 35 | +#include <linux/security.h> | |
| 33 | 36 | #include <linux/virtio_mmio.h> |
| 37 | +#include <linux/wait.h> | |
| 34 | 38 | |
| 35 | 39 | #include <asm/xen/hypervisor.h> |
| 36 | 40 | #include <asm/xen/hypercall.h> |
| @@ -46,6 +50,7 @@ · 9 unmodified lines | ||
| 46 | 50 | #include <xen/page.h> |
| 47 | 51 | #include <xen/xen-ops.h> |
| 48 | 52 | #include <xen/balloon.h> |
| 53 | +#include <xen/xenbus.h> | |
| 49 | 54 | #ifdef CONFIG_XEN_ACPI |
| 50 | 55 | #include <xen/acpi.h> |
| 51 | 56 | #endif |
| @@ -68,10 +73,20 @@ module_param_named(dm_op_buf_max_size, privcmd_dm_op_buf_max_size, uint, · 16 unmodified lines | ||
| 68 | 73 | MODULE_PARM_DESC(dm_op_buf_max_size, |
| 69 | 74 | "Maximum size of a dm_op hypercall buffer"); |
| 70 | 75 | |
| 76 | +static bool unrestricted; | |
| 77 | +module_param(unrestricted, bool, 0); | |
| 78 | +MODULE_PARM_DESC(unrestricted, | |
| 79 | + "Don't restrict hypercalls to target domain if running in a domU"); | |
| 80 | + | |
| 71 | 81 | struct privcmd_data { |
| 72 | 82 | domid_t domid; |
| 73 | 83 | }; |
| 74 | 84 | |
| 85 | +/* DOMID_INVALID implies no restriction */ | |
| 86 | +static domid_t target_domain = DOMID_INVALID; | |
| 87 | +static bool restrict_wait; | |
| 88 | +static DECLARE_WAIT_QUEUE_HEAD(restrict_wait_wq); | |
| 89 | + | |
| 75 | 90 | static int privcmd_vma_range_is_mapped( |
| 76 | 91 | struct vm_area_struct *vma, |
| 77 | 92 | unsigned long addr, |
| @@ -1563,13 +1578,16 @@ static long privcmd_ioctl(struct file *file, · 1485 unmodified lines | ||
| 1563 | 1578 | |
| 1564 | 1579 | static int privcmd_open(struct inode *ino, struct file *file) |
| 1565 | 1580 | { |
| 1566 | − struct privcmd_data *data = kzalloc_obj(*data); | |
| 1581 | + struct privcmd_data *data; | |
| 1567 | 1582 | |
| 1583 | + if (wait_event_interruptible(restrict_wait_wq, !restrict_wait) < 0) | |
| 1584 | + return -EINTR; | |
| 1585 | + | |
| 1586 | + data = kzalloc_obj(*data); | |
| 1568 | 1587 | if (!data) |
| 1569 | 1588 | return -ENOMEM; |
| 1570 | 1589 | |
| 1571 | − /* DOMID_INVALID implies no restriction */ | |
| 1572 | − data->domid = DOMID_INVALID; | |
| 1590 | + data->domid = target_domain; | |
| 1573 | 1591 | |
| 1574 | 1592 | file->private_data = data; |
| 1575 | 1593 | return 0; |
| @@ -1662,6 +1680,52 @@ static struct miscdevice privcmd_dev = { · 86 unmodified lines | ||
| 1662 | 1680 | .fops = &xen_privcmd_fops, |
| 1663 | 1681 | }; |
| 1664 | 1682 | |
| 1683 | +static int init_restrict(struct notifier_block *notifier, | |
| 1684 | + unsigned long event, | |
| 1685 | + void *data) | |
| 1686 | +{ | |
| 1687 | + char *target; | |
| 1688 | + unsigned int domid; | |
| 1689 | + | |
| 1690 | + /* Default to an guaranteed unused domain-id. */ | |
| 1691 | + target_domain = DOMID_IDLE; | |
| 1692 | + | |
| 1693 | + target = xenbus_read(XBT_NIL, "target", "", NULL); | |
| 1694 | + if (IS_ERR(target) || kstrtouint(target, 10, &domid)) { | |
| 1695 | + pr_err("No target domain found, blocking all hypercalls\n"); | |
| 1696 | + goto out; | |
| 1697 | + } | |
| 1698 | + | |
| 1699 | + target_domain = domid; | |
| 1700 | + | |
| 1701 | + out: | |
| 1702 | + if (!IS_ERR(target)) | |
| 1703 | + kfree(target); | |
| 1704 | + | |
| 1705 | + restrict_wait = false; | |
| 1706 | + wake_up_all(&restrict_wait_wq); | |
| 1707 | + | |
| 1708 | + return NOTIFY_DONE; | |
| 1709 | +} | |
| 1710 | + | |
| 1711 | +static struct notifier_block xenstore_notifier = { | |
| 1712 | + .notifier_call = init_restrict, | |
| 1713 | +}; | |
| 1714 | + | |
| 1715 | +static void __init restrict_driver(void) | |
| 1716 | +{ | |
| 1717 | + if (unrestricted) { | |
| 1718 | + if (security_locked_down(LOCKDOWN_XEN_USER_ACTIONS)) | |
| 1719 | + pr_warn("Kernel is locked down, parameter \"unrestricted\" ignored\n"); | |
| 1720 | + else | |
| 1721 | + return; | |
| 1722 | + } | |
| 1723 | + | |
| 1724 | + restrict_wait = true; | |
| 1725 | + | |
| 1726 | + register_xenstore_notifier(&xenstore_notifier); | |
| 1727 | +} | |
| 1728 | + | |
| 1665 | 1729 | static int __init privcmd_init(void) |
| 1666 | 1730 | { |
| 1667 | 1731 | int err; |
| @@ -1669,6 +1733,9 @@ static int __init privcmd_init(void) · 1 unmodified line | ||
| 1669 | 1733 | if (!xen_domain()) |
| 1670 | 1734 | return -ENODEV; |
| 1671 | 1735 | |
| 1736 | + if (!xen_initial_domain()) | |
| 1737 | + restrict_driver(); | |
| 1738 | + | |
| 1672 | 1739 | err = misc_register(&privcmd_dev); |
| 1673 | 1740 | if (err != 0) { |
| 1674 | 1741 | pr_err("Could not register Xen privcmd device\n"); |
| @@ -1698,6 +1765,9 @@ err_privcmdbuf: · 23 unmodified lines | ||
| 1698 | 1765 | |
| 1699 | 1766 | static void __exit privcmd_exit(void) |
| 1700 | 1767 | { |
| 1768 | + if (!xen_initial_domain()) | |
| 1769 | + unregister_xenstore_notifier(&xenstore_notifier); | |
| 1770 | + | |
| 1701 | 1771 | privcmd_ioeventfd_exit(); |
| 1702 | 1772 | privcmd_irqfd_exit(); |
| 1703 | 1773 | misc_deregister(&privcmd_dev); |
| @@ -4583,7 +4583,7 @@ static void check_removing_space_info(struct btrfs_space_info *space_info) · 4582 unmodified lines | ||
| 4583 | 4583 | for (int i = 0; i < BTRFS_SPACE_INFO_SUB_GROUP_MAX; i++) { |
| 4584 | 4584 | if (space_info->sub_group[i]) { |
| 4585 | 4585 | check_removing_space_info(space_info->sub_group[i]); |
| 4586 | − kfree(space_info->sub_group[i]); | |
| 4586 | + btrfs_sysfs_remove_space_info(space_info->sub_group[i]); | |
| 4587 | 4587 | space_info->sub_group[i] = NULL; |
| 4588 | 4588 | } |
| 4589 | 4589 | } |
| @@ -2531,8 +2531,8 @@ int btrfs_validate_super(const struct btrfs_fs_info *fs_info, · 2530 unmodified lines | ||
| 2531 | 2531 | |
| 2532 | 2532 | if (mirror_num >= 0 && |
| 2533 | 2533 | btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) { |
| 2534 | − btrfs_err(fs_info, "super offset mismatch %llu != %u", | |
| 2535 | − btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET); | |
| 2534 | + btrfs_err(fs_info, "super offset mismatch %llu != %llu", | |
| 2535 | + btrfs_super_bytenr(sb), btrfs_sb_offset(mirror_num)); | |
| 2536 | 2536 | ret = -EINVAL; |
| 2537 | 2537 | } |
| 2538 | 2538 | |
| @@ -4616,21 +4616,32 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, · 4615 unmodified lines | ||
| 4616 | 4616 | struct inode *inode, bool log_inode_only, |
| 4617 | 4617 | u64 logged_isize) |
| 4618 | 4618 | { |
| 4619 | + u64 gen = BTRFS_I(inode)->generation; | |
| 4619 | 4620 | u64 flags; |
| 4620 | 4621 | |
| 4621 | 4622 | if (log_inode_only) { |
| 4622 | − /* set the generation to zero so the recover code | |
| 4623 | − * can tell the difference between an logging | |
| 4624 | − * just to say 'this inode exists' and a logging | |
| 4625 | − * to say 'update this inode with these values' | |
| 4623 | + /* | |
| 4624 | + * Set the generation to zero so the recover code can tell the | |
| 4625 | + * difference between a logging just to say 'this inode exists' | |
| 4626 | + * and a logging to say 'update this inode with these values'. | |
| 4627 | + * But only if the inode was not already logged before. | |
| 4628 | + * We access ->logged_trans directly since it was already set | |
| 4629 | + * up in the call chain by btrfs_log_inode(), and data_race() | |
| 4630 | + * to avoid false alerts from KCSAN and since it was set already | |
| 4631 | + * and one can set it to 0 since that only happens on eviction | |
| 4632 | + * and we are holding a ref on the inode. | |
| 4626 | 4633 | */ |
| 4627 | − btrfs_set_inode_generation(leaf, item, 0); | |
| 4634 | + ASSERT(data_race(BTRFS_I(inode)->logged_trans) > 0); | |
| 4635 | + if (data_race(BTRFS_I(inode)->logged_trans) < trans->transid) | |
| 4636 | + gen = 0; | |
| 4637 | + | |
| 4628 | 4638 | btrfs_set_inode_size(leaf, item, logged_isize); |
| 4629 | 4639 | } else { |
| 4630 | − btrfs_set_inode_generation(leaf, item, BTRFS_I(inode)->generation); | |
| 4631 | 4640 | btrfs_set_inode_size(leaf, item, inode->i_size); |
| 4632 | 4641 | } |
| 4633 | 4642 | |
| 4643 | + btrfs_set_inode_generation(leaf, item, gen); | |
| 4644 | + | |
| 4634 | 4645 | btrfs_set_inode_uid(leaf, item, i_uid_read(inode)); |
| 4635 | 4646 | btrfs_set_inode_gid(leaf, item, i_gid_read(inode)); |
| 4636 | 4647 | btrfs_set_inode_mode(leaf, item, inode->i_mode); |
| @@ -5448,42 +5459,63 @@ process: · 811 unmodified lines | ||
| 5448 | 5459 | return 0; |
| 5449 | 5460 | } |
| 5450 | 5461 | |
| 5451 | −static int logged_inode_size(struct btrfs_root *log, struct btrfs_inode *inode, | |
| 5452 | − struct btrfs_path *path, u64 *size_ret) | |
| 5462 | +static int get_inode_size_to_log(struct btrfs_trans_handle *trans, | |
| 5463 | + struct btrfs_inode *inode, | |
| 5464 | + struct btrfs_path *path, u64 *size_ret) | |
| 5453 | 5465 | { |
| 5454 | 5466 | struct btrfs_key key; |
| 5467 | + struct btrfs_inode_item *item; | |
| 5455 | 5468 | int ret; |
| 5456 | 5469 | |
| 5457 | 5470 | key.objectid = btrfs_ino(inode); |
| 5458 | 5471 | key.type = BTRFS_INODE_ITEM_KEY; |
| 5459 | 5472 | key.offset = 0; |
| 5460 | 5473 | |
| 5461 | − ret = btrfs_search_slot(NULL, log, &key, path, 0, 0); | |
| 5462 | − if (ret < 0) { | |
| 5463 | − return ret; | |
| 5464 | − } else if (ret > 0) { | |
| 5465 | − *size_ret = 0; | |
| 5466 | − } else { | |
| 5467 | − struct btrfs_inode_item *item; | |
| 5474 | + /* | |
| 5475 | + * Our caller called inode_logged(), so logged_trans is up to date. | |
| 5476 | + * Use data_race() to silence any warning from KCSAN. Once logged_trans | |
| 5477 | + * is set, it can only be reset to 0 after inode eviction. | |
| 5478 | + */ | |
| 5479 | + if (data_race(inode->logged_trans) == trans->transid) { | |
| 5480 | + ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); | |
| 5481 | + } else if (inode->generation < trans->transid) { | |
| 5482 | + path->search_commit_root = true; | |
| 5483 | + path->skip_locking = true; | |
| 5484 | + ret = btrfs_search_slot(NULL, inode->root, &key, path, 0, 0); | |
| 5485 | + path->search_commit_root = false; | |
| 5486 | + path->skip_locking = false; | |
| 5468 | 5487 | |
| 5469 | − item = btrfs_item_ptr(path->nodes[0], path->slots[0], | |
| 5470 | − struct btrfs_inode_item); | |
| 5471 | − *size_ret = btrfs_inode_size(path->nodes[0], item); | |
| 5472 | − /* | |
| 5473 | − * If the in-memory inode's i_size is smaller then the inode | |
| 5474 | − * size stored in the btree, return the inode's i_size, so | |
| 5475 | − * that we get a correct inode size after replaying the log | |
| 5476 | − * when before a power failure we had a shrinking truncate | |
| 5477 | − * followed by addition of a new name (rename / new hard link). | |
| 5478 | − * Otherwise return the inode size from the btree, to avoid | |
| 5479 | − * data loss when replaying a log due to previously doing a | |
| 5480 | − * write that expands the inode's size and logging a new name | |
| 5481 | − * immediately after. | |
| 5482 | − */ | |
| 5483 | − if (*size_ret > inode->vfs_inode.i_size) | |
| 5484 | − *size_ret = inode->vfs_inode.i_size; | |
| 5488 | + } else { | |
| 5489 | + *size_ret = 0; | |
| 5490 | + return 0; | |
| 5485 | 5491 | } |
| 5486 | 5492 | |
| 5493 | + /* | |
| 5494 | + * If the inode was logged before or is from a past transaction, then | |
| 5495 | + * its inode item must exist in the log root or in the commit root. | |
| 5496 | + */ | |
| 5497 | + ASSERT(ret <= 0); | |
| 5498 | + if (WARN_ON_ONCE(ret > 0)) | |
| 5499 | + ret = -ENOENT; | |
| 5500 | + | |
| 5501 | + if (ret < 0) | |
| 5502 | + return ret; | |
| 5503 | + | |
| 5504 | + item = btrfs_item_ptr(path->nodes[0], path->slots[0], | |
| 5505 | + struct btrfs_inode_item); | |
| 5506 | + *size_ret = btrfs_inode_size(path->nodes[0], item); | |
| 5507 | + /* | |
| 5508 | + * If the in-memory inode's i_size is smaller then the inode size stored | |
| 5509 | + * in the btree, return the inode's i_size, so that we get a correct | |
| 5510 | + * inode size after replaying the log when before a power failure we had | |
| 5511 | + * a shrinking truncate followed by addition of a new name (rename / new | |
| 5512 | + * hard link). Otherwise return the inode size from the btree, to avoid | |
| 5513 | + * data loss when replaying a log due to previously doing a write that | |
| 5514 | + * expands the inode's size and logging a new name immediately after. | |
| 5515 | + */ | |
| 5516 | + if (*size_ret > inode->vfs_inode.i_size) | |
| 5517 | + *size_ret = inode->vfs_inode.i_size; | |
| 5518 | + | |
| 5487 | 5519 | btrfs_release_path(path); |
| 5488 | 5520 | return 0; |
| 5489 | 5521 | } |
| @@ -6996,7 +7028,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, · 1506 unmodified lines | ||
| 6996 | 7028 | ret = drop_inode_items(trans, log, path, inode, |
| 6997 | 7029 | BTRFS_XATTR_ITEM_KEY); |
| 6998 | 7030 | } else { |
| 6999 | − if (inode_only == LOG_INODE_EXISTS && ctx->logged_before) { | |
| 7031 | + if (inode_only == LOG_INODE_EXISTS) { | |
| 7000 | 7032 | /* |
| 7001 | 7033 | * Make sure the new inode item we write to the log has |
| 7002 | 7034 | * the same isize as the current one (if it exists). |
| @@ -7010,7 +7042,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, · 7 unmodified lines | ||
| 7010 | 7042 | * (zeroes), as if an expanding truncate happened, |
| 7011 | 7043 | * instead of getting a file of 4Kb only. |
| 7012 | 7044 | */ |
| 7013 | − ret = logged_inode_size(log, inode, path, &logged_isize); | |
| 7045 | + ret = get_inode_size_to_log(trans, inode, path, &logged_isize); | |
| 7014 | 7046 | if (ret) |
| 7015 | 7047 | goto out_unlock; |
| 7016 | 7048 | } |
| @@ -8099,8 +8099,9 @@ int btrfs_run_dev_stats(struct btrfs_trans_handle *trans) · 8098 unmodified lines | ||
| 8099 | 8099 | smp_rmb(); |
| 8100 | 8100 | |
| 8101 | 8101 | ret = update_dev_stat_item(trans, device); |
| 8102 | − if (!ret) | |
| 8103 | − atomic_sub(stats_cnt, &device->dev_stats_ccnt); | |
| 8102 | + if (ret) | |
| 8103 | + break; | |
| 8104 | + atomic_sub(stats_cnt, &device->dev_stats_ccnt); | |
| 8104 | 8105 | } |
| 8105 | 8106 | mutex_unlock(&fs_devices->device_list_mutex); |
| 8106 | 8107 | |
| @@ -308,7 +308,9 @@ int zlib_compress_bio(struct list_head *ws, struct compressed_bio *cb) · 307 unmodified lines | ||
| 308 | 308 | } |
| 309 | 309 | /* Queue the remaining part of the folio. */ |
| 310 | 310 | if (workspace->strm.total_out > bio->bi_iter.bi_size) { |
| 311 | − u32 cur_len = offset_in_folio(out_folio, workspace->strm.total_out); | |
| 311 | + const u32 cur_len = workspace->strm.total_out - bio->bi_iter.bi_size; | |
| 312 | + | |
| 313 | + ASSERT(cur_len <= folio_size(out_folio)); | |
| 312 | 314 | |
| 313 | 315 | if (!bio_add_folio(bio, out_folio, cur_len, 0)) { |
| 314 | 316 | ret = -E2BIG; |
| @@ -16,22 +16,36 @@ config EROFS_FS · 15 unmodified lines | ||
| 16 | 16 | select ZLIB_INFLATE if EROFS_FS_ZIP_DEFLATE |
| 17 | 17 | select ZSTD_DECOMPRESS if EROFS_FS_ZIP_ZSTD |
| 18 | 18 | help |
| 19 | − EROFS (Enhanced Read-Only File System) is a lightweight read-only | |
| 20 | − file system with modern designs (e.g. no buffer heads, inline | |
| 21 | − xattrs/data, chunk-based deduplication, multiple devices, etc.) for | |
| 22 | − scenarios which need high-performance read-only solutions, e.g. | |
| 23 | − smartphones with Android OS, LiveCDs and high-density hosts with | |
| 24 | − numerous containers; | |
| 25 | − | |
| 26 | − It also provides transparent compression and deduplication support to | |
| 27 | − improve storage density and maintain relatively high compression | |
| 28 | − ratios, and it implements in-place decompression to temporarily reuse | |
| 29 | − page cache for compressed data using proper strategies, which is | |
| 30 | − quite useful for ensuring guaranteed end-to-end runtime decompression | |
| 19 | + EROFS (Enhanced Read-Only File System) is a modern, lightweight, | |
| 20 | + secure read-only filesystem for various use cases, such as immutable | |
| 21 | + system images, container images, application sandboxes, and datasets. | |
| 22 | + | |
| 23 | + EROFS uses a flexible, hierarchical on-disk design so that features | |
| 24 | + can be enabled on demand: the core on-disk format is block-aligned in | |
| 25 | + order to perform optimally on all kinds of devices, including block | |
| 26 | + and memory-backed devices; the format is easy to parse and has zero | |
| 27 | + metadata redundancy, unlike generic filesystems, making it ideal for | |
| 28 | + filesystem auditing and remote access; inline data, random-access | |
| 29 | + friendly directory data, inline/shared extended attributes and | |
| 30 | + chunk-based deduplication ensure space efficiency while maintaining | |
| 31 | + high performance. | |
| 32 | + | |
| 33 | + Optionally, it supports multiple devices to reference external data, | |
| 34 | + enabling data sharing for container images. | |
| 35 | + | |
| 36 | + It also has advanced encoded on-disk layouts, particularly for data | |
| 37 | + compression and fine-grained deduplication. It utilizes fixed-size | |
| 38 | + output compression to improve storage density while keeping relatively | |
| 39 | + high compression ratios. Furthermore, it implements in-place | |
| 40 | + decompression to reuse file pages to keep compressed data temporarily | |
| 41 | + with proper strategies, which ensures guaranteed end-to-end runtime | |
| 31 | 42 | performance under extreme memory pressure without extra cost. |
| 32 | 43 | |
| 33 | − See the documentation at <file:Documentation/filesystems/erofs.rst> | |
| 34 | − and the web pages at <https://erofs.docs.kernel.org> for more details. | |
| 44 | + For more details, see the web pages at <https://erofs.docs.kernel.org> | |
| 45 | + and the documentation at <file:Documentation/filesystems/erofs.rst>. | |
| 46 | + | |
| 47 | + To compile EROFS filesystem support as a module, choose M here. The | |
| 48 | + module will be called erofs. | |
| 35 | 49 | |
| 36 | 50 | If unsure, say N. |
| 37 | 51 | |
| @@ -105,7 +119,8 @@ config EROFS_FS_ZIP · 67 unmodified lines | ||
| 105 | 119 | depends on EROFS_FS |
| 106 | 120 | default y |
| 107 | 121 | help |
| 108 | − Enable transparent compression support for EROFS file systems. | |
| 122 | + Enable EROFS compression layouts so that filesystems containing | |
| 123 | + compressed files can be parsed by the kernel. | |
| 109 | 124 | |
| 110 | 125 | If you don't want to enable compression feature, say N. |
| 111 | 126 | |
| @@ -25,10 +25,8 @@ static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret) · 24 unmodified lines | ||
| 25 | 25 | container_of(iocb, struct erofs_fileio_rq, iocb); |
| 26 | 26 | struct folio_iter fi; |
| 27 | 27 | |
| 28 | − if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) { | |
| 29 | − bio_advance(&rq->bio, ret); | |
| 30 | − zero_fill_bio(&rq->bio); | |
| 31 | − } | |
| 28 | + if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) | |
| 29 | + ret = -EIO; | |
| 32 | 30 | if (!rq->bio.bi_end_io) { |
| 33 | 31 | bio_for_each_folio_all(fi, &rq->bio) { |
| 34 | 32 | DBG_BUGON(folio_test_uptodate(fi.folio)); |
| @@ -200,8 +200,19 @@ struct inode *erofs_real_inode(struct inode *inode, bool *need_iput) · 199 unmodified lines | ||
| 200 | 200 | |
| 201 | 201 | int __init erofs_init_ishare(void) |
| 202 | 202 | { |
| 203 | − erofs_ishare_mnt = kern_mount(&erofs_anon_fs_type); | |
| 204 | − return PTR_ERR_OR_ZERO(erofs_ishare_mnt); | |
| 203 | + struct vfsmount *mnt; | |
| 204 | + int ret; | |
| 205 | + | |
| 206 | + mnt = kern_mount(&erofs_anon_fs_type); | |
| 207 | + if (IS_ERR(mnt)) | |
| 208 | + return PTR_ERR(mnt); | |
| 209 | + /* generic_fadvise() doesn't work if s_bdi == &noop_backing_dev_info */ | |
| 210 | + ret = super_setup_bdi(mnt->mnt_sb); | |
| 211 | + if (ret) | |
| 212 | + kern_unmount(mnt); | |
| 213 | + else | |
| 214 | + erofs_ishare_mnt = mnt; | |
| 215 | + return ret; | |
| 205 | 216 | } |
| 206 | 217 | |
| 207 | 218 | void erofs_exit_ishare(void) |
| @@ -1445,6 +1445,7 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, · 1444 unmodified lines | ||
| 1445 | 1445 | int bios) |
| 1446 | 1446 | { |
| 1447 | 1447 | struct erofs_sb_info *const sbi = EROFS_SB(io->sb); |
| 1448 | + int gfp_flag; | |
| 1448 | 1449 | |
| 1449 | 1450 | /* wake up the caller thread for sync decompression */ |
| 1450 | 1451 | if (io->sync) { |
| @@ -1477,7 +1478,9 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, · 26 unmodified lines | ||
| 1477 | 1478 | sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; |
| 1478 | 1479 | return; |
| 1479 | 1480 | } |
| 1481 | + gfp_flag = memalloc_noio_save(); | |
| 1480 | 1482 | z_erofs_decompressqueue_work(&io->u.work); |
| 1483 | + memalloc_noio_restore(gfp_flag); | |
| 1481 | 1484 | } |
| 1482 | 1485 | |
| 1483 | 1486 | static void z_erofs_fill_bio_vec(struct bio_vec *bvec, |
| @@ -14,7 +14,8 @@ ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \ · 13 unmodified lines | ||
| 14 | 14 | |
| 15 | 15 | ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o |
| 16 | 16 | ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o |
| 17 | −ext4-inode-test-objs += inode-test.o | |
| 18 | −obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-inode-test.o | |
| 17 | +ext4-test-objs += inode-test.o mballoc-test.o \ | |
| 18 | + extents-test.o | |
| 19 | +obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-test.o | |
| 19 | 20 | ext4-$(CONFIG_FS_VERITY) += verity.o |
| 20 | 21 | ext4-$(CONFIG_FS_ENCRYPTION) += crypto.o |
| @@ -163,10 +163,17 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, · 162 unmodified lines | ||
| 163 | 163 | */ |
| 164 | 164 | |
| 165 | 165 | if (handle) { |
| 166 | + /* | |
| 167 | + * Since the inode is new it is ok to pass the | |
| 168 | + * XATTR_CREATE flag. This is necessary to match the | |
| 169 | + * remaining journal credits check in the set_handle | |
| 170 | + * function with the credits allocated for the new | |
| 171 | + * inode. | |
| 172 | + */ | |
| 166 | 173 | res = ext4_xattr_set_handle(handle, inode, |
| 167 | 174 | EXT4_XATTR_INDEX_ENCRYPTION, |
| 168 | 175 | EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, |
| 169 | − ctx, len, 0); | |
| 176 | + ctx, len, XATTR_CREATE); | |
| 170 | 177 | if (!res) { |
| 171 | 178 | ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); |
| 172 | 179 | ext4_clear_inode_state(inode, |
| @@ -1570,6 +1570,7 @@ struct ext4_sb_info { · 1569 unmodified lines | ||
| 1570 | 1570 | struct proc_dir_entry *s_proc; |
| 1571 | 1571 | struct kobject s_kobj; |
| 1572 | 1572 | struct completion s_kobj_unregister; |
| 1573 | + struct mutex s_error_notify_mutex; /* protects sysfs_notify vs kobject_del */ | |
| 1573 | 1574 | struct super_block *s_sb; |
| 1574 | 1575 | struct buffer_head *s_mmp_bh; |
| 1575 | 1576 | |
| @@ -3944,6 +3945,11 @@ static inline bool ext4_inode_can_atomic_write(struct inode *inode) · 2368 unmodified lines | ||
| 3944 | 3945 | extern int ext4_block_write_begin(handle_t *handle, struct folio *folio, |
| 3945 | 3946 | loff_t pos, unsigned len, |
| 3946 | 3947 | get_block_t *get_block); |
| 3948 | + | |
| 3949 | +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) | |
| 3950 | +#define EXPORT_SYMBOL_FOR_EXT4_TEST(sym) \ | |
| 3951 | + EXPORT_SYMBOL_FOR_MODULES(sym, "ext4-test") | |
| 3952 | +#endif | |
| 3947 | 3953 | #endif /* __KERNEL__ */ |
| 3948 | 3954 | |
| 3949 | 3955 | #endif /* _EXT4_H */ |
| @@ -264,5 +264,17 @@ static inline void ext4_idx_store_pblock(struct ext4_extent_idx *ix, · 263 unmodified lines | ||
| 264 | 264 | 0xffff); |
| 265 | 265 | } |
| 266 | 266 | |
| 267 | +extern int __ext4_ext_dirty(const char *where, unsigned int line, | |
| 268 | + handle_t *handle, struct inode *inode, | |
| 269 | + struct ext4_ext_path *path); | |
| 270 | +extern int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex); | |
| 271 | +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) | |
| 272 | +extern int ext4_ext_space_root_idx_test(struct inode *inode, int check); | |
| 273 | +extern struct ext4_ext_path *ext4_split_convert_extents_test( | |
| 274 | + handle_t *handle, struct inode *inode, | |
| 275 | + struct ext4_map_blocks *map, | |
| 276 | + struct ext4_ext_path *path, | |
| 277 | + int flags, unsigned int *allocated); | |
| 278 | +#endif | |
| 267 | 279 | #endif /* _EXT4_EXTENTS */ |
| 268 | 280 | |
| @@ -142,8 +142,10 @@ static struct file_system_type ext_fs_type = { · 141 unmodified lines | ||
| 142 | 142 | |
| 143 | 143 | static void extents_kunit_exit(struct kunit *test) |
| 144 | 144 | { |
| 145 | − struct ext4_sb_info *sbi = k_ctx.k_ei->vfs_inode.i_sb->s_fs_info; | |
| 145 | + struct super_block *sb = k_ctx.k_ei->vfs_inode.i_sb; | |
| 146 | + struct ext4_sb_info *sbi = sb->s_fs_info; | |
| 146 | 147 | |
| 148 | + ext4_es_unregister_shrinker(sbi); | |
| 147 | 149 | kfree(sbi); |
| 148 | 150 | kfree(k_ctx.k_ei); |
| 149 | 151 | kfree(k_ctx.k_data); |
| @@ -280,8 +282,8 @@ static int extents_kunit_init(struct kunit *test) · 130 unmodified lines | ||
| 280 | 282 | eh->eh_depth = 0; |
| 281 | 283 | eh->eh_entries = cpu_to_le16(1); |
| 282 | 284 | eh->eh_magic = EXT4_EXT_MAGIC; |
| 283 | − eh->eh_max = | |
| 284 | − cpu_to_le16(ext4_ext_space_root_idx(&k_ctx.k_ei->vfs_inode, 0)); | |
| 285 | + eh->eh_max = cpu_to_le16(ext4_ext_space_root_idx_test( | |
| 286 | + &k_ctx.k_ei->vfs_inode, 0)); | |
| 285 | 287 | eh->eh_generation = 0; |
| 286 | 288 | |
| 287 | 289 | /* |
| @@ -384,8 +386,8 @@ static void test_split_convert(struct kunit *test) · 96 unmodified lines | ||
| 384 | 386 | |
| 385 | 387 | switch (param->type) { |
| 386 | 388 | case TEST_SPLIT_CONVERT: |
| 387 | − path = ext4_split_convert_extents(NULL, inode, &map, path, | |
| 388 | − param->split_flags, NULL); | |
| 389 | + path = ext4_split_convert_extents_test(NULL, inode, &map, | |
| 390 | + path, param->split_flags, NULL); | |
| 389 | 391 | break; |
| 390 | 392 | case TEST_CREATE_BLOCKS: |
| 391 | 393 | ext4_map_create_blocks_helper(test, inode, &map, param->split_flags); |
| @@ -184,9 +184,9 @@ static int ext4_ext_get_access(handle_t *handle, struct inode *inode, · 183 unmodified lines | ||
| 184 | 184 | * - ENOMEM |
| 185 | 185 | * - EIO |
| 186 | 186 | */ |
| 187 | −static int __ext4_ext_dirty(const char *where, unsigned int line, | |
| 188 | − handle_t *handle, struct inode *inode, | |
| 189 | − struct ext4_ext_path *path) | |
| 187 | +int __ext4_ext_dirty(const char *where, unsigned int line, | |
| 188 | + handle_t *handle, struct inode *inode, | |
| 189 | + struct ext4_ext_path *path) | |
| 190 | 190 | { |
| 191 | 191 | int err; |
| 192 | 192 | |
| @@ -1736,6 +1736,13 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, · 1543 unmodified lines | ||
| 1736 | 1736 | err = ext4_ext_get_access(handle, inode, path + k); |
| 1737 | 1737 | if (err) |
| 1738 | 1738 | return err; |
| 1739 | + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { | |
| 1740 | + EXT4_ERROR_INODE(inode, | |
| 1741 | + "path[%d].p_idx %p > EXT_LAST_INDEX %p", | |
| 1742 | + k, path[k].p_idx, | |
| 1743 | + EXT_LAST_INDEX(path[k].p_hdr)); | |
| 1744 | + return -EFSCORRUPTED; | |
| 1745 | + } | |
| 1739 | 1746 | path[k].p_idx->ei_block = border; |
| 1740 | 1747 | err = ext4_ext_dirty(handle, inode, path + k); |
| 1741 | 1748 | if (err) |
| @@ -1748,6 +1755,14 @@ static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, · 6 unmodified lines | ||
| 1748 | 1755 | err = ext4_ext_get_access(handle, inode, path + k); |
| 1749 | 1756 | if (err) |
| 1750 | 1757 | goto clean; |
| 1758 | + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { | |
| 1759 | + EXT4_ERROR_INODE(inode, | |
| 1760 | + "path[%d].p_idx %p > EXT_LAST_INDEX %p", | |
| 1761 | + k, path[k].p_idx, | |
| 1762 | + EXT_LAST_INDEX(path[k].p_hdr)); | |
| 1763 | + err = -EFSCORRUPTED; | |
| 1764 | + goto clean; | |
| 1765 | + } | |
| 1751 | 1766 | path[k].p_idx->ei_block = border; |
| 1752 | 1767 | err = ext4_ext_dirty(handle, inode, path + k); |
| 1753 | 1768 | if (err) |
| @@ -3144,7 +3159,7 @@ static void ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex) · 1390 unmodified lines | ||
| 3144 | 3159 | } |
| 3145 | 3160 | |
| 3146 | 3161 | /* FIXME!! we need to try to merge to left or right after zero-out */ |
| 3147 | −static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex) | |
| 3162 | +int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex) | |
| 3148 | 3163 | { |
| 3149 | 3164 | ext4_fsblk_t ee_pblock; |
| 3150 | 3165 | unsigned int ee_len; |
| @@ -3239,6 +3254,9 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, · 88 unmodified lines | ||
| 3239 | 3254 | |
| 3240 | 3255 | insert_err = PTR_ERR(path); |
| 3241 | 3256 | err = 0; |
| 3257 | + if (insert_err != -ENOSPC && insert_err != -EDQUOT && | |
| 3258 | + insert_err != -ENOMEM) | |
| 3259 | + goto out_path; | |
| 3242 | 3260 | |
| 3243 | 3261 | /* |
| 3244 | 3262 | * Get a new path to try to zeroout or fix the extent length. |
| @@ -3255,13 +3273,20 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, · 10 unmodified lines | ||
| 3255 | 3273 | goto out_path; |
| 3256 | 3274 | } |
| 3257 | 3275 | |
| 3276 | + depth = ext_depth(inode); | |
| 3277 | + ex = path[depth].p_ext; | |
| 3278 | + if (!ex) { | |
| 3279 | + EXT4_ERROR_INODE(inode, | |
| 3280 | + "bad extent address lblock: %lu, depth: %d pblock %llu", | |
| 3281 | + (unsigned long)ee_block, depth, path[depth].p_block); | |
| 3282 | + err = -EFSCORRUPTED; | |
| 3283 | + goto out; | |
| 3284 | + } | |
| 3285 | + | |
| 3258 | 3286 | err = ext4_ext_get_access(handle, inode, path + depth); |
| 3259 | 3287 | if (err) |
| 3260 | 3288 | goto out; |
| 3261 | 3289 | |
| 3262 | − depth = ext_depth(inode); | |
| 3263 | − ex = path[depth].p_ext; | |
| 3264 | − | |
| 3265 | 3290 | fix_extent_len: |
| 3266 | 3291 | ex->ee_len = orig_ex.ee_len; |
| 3267 | 3292 | err = ext4_ext_dirty(handle, inode, path + path->p_depth); |
| @@ -3363,7 +3388,7 @@ static int ext4_split_extent_zeroout(handle_t *handle, struct inode *inode, · 95 unmodified lines | ||
| 3363 | 3388 | |
| 3364 | 3389 | ext4_ext_mark_initialized(ex); |
| 3365 | 3390 | |
| 3366 | − ext4_ext_dirty(handle, inode, path + depth); | |
| 3391 | + err = ext4_ext_dirty(handle, inode, path + depth); | |
| 3367 | 3392 | if (err) |
| 3368 | 3393 | return err; |
| 3369 | 3394 | |
| @@ -4457,9 +4482,13 @@ got_allocated_blocks: · 1087 unmodified lines | ||
| 4457 | 4482 | path = ext4_ext_insert_extent(handle, inode, path, &newex, flags); |
| 4458 | 4483 | if (IS_ERR(path)) { |
| 4459 | 4484 | err = PTR_ERR(path); |
| 4460 | − if (allocated_clusters) { | |
| 4485 | + /* | |
| 4486 | + * Gracefully handle out of space conditions. If the filesystem | |
| 4487 | + * is inconsistent, we'll just leak allocated blocks to avoid | |
| 4488 | + * causing even more damage. | |
| 4489 | + */ | |
| 4490 | + if (allocated_clusters && (err == -EDQUOT || err == -ENOSPC)) { | |
| 4461 | 4491 | int fb_flags = 0; |
| 4462 | − | |
| 4463 | 4492 | /* |
| 4464 | 4493 | * free data blocks we just allocated. |
| 4465 | 4494 | * not a good idea to call discard here directly, |
| @@ -6238,6 +6267,33 @@ out: · 1772 unmodified lines | ||
| 6238 | 6267 | return 0; |
| 6239 | 6268 | } |
| 6240 | 6269 | |
| 6241 | −#ifdef CONFIG_EXT4_KUNIT_TESTS | |
| 6242 | −#include "extents-test.c" | |
| 6270 | +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) | |
| 6271 | +int ext4_ext_space_root_idx_test(struct inode *inode, int check) | |
| 6272 | +{ | |
| 6273 | + return ext4_ext_space_root_idx(inode, check); | |
| 6274 | +} | |
| 6275 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_space_root_idx_test); | |
| 6276 | + | |
| 6277 | +struct ext4_ext_path *ext4_split_convert_extents_test(handle_t *handle, | |
| 6278 | + struct inode *inode, struct ext4_map_blocks *map, | |
| 6279 | + struct ext4_ext_path *path, int flags, | |
| 6280 | + unsigned int *allocated) | |
| 6281 | +{ | |
| 6282 | + return ext4_split_convert_extents(handle, inode, map, path, | |
| 6283 | + flags, allocated); | |
| 6284 | +} | |
| 6285 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_split_convert_extents_test); | |
| 6286 | + | |
| 6287 | +EXPORT_SYMBOL_FOR_EXT4_TEST(__ext4_ext_dirty); | |
| 6288 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_zeroout); | |
| 6289 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_register_shrinker); | |
| 6290 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_unregister_shrinker); | |
| 6291 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_create_blocks); | |
| 6292 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_init_tree); | |
| 6293 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_lookup_extent); | |
| 6294 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_insert_extent); | |
| 6295 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_insert_extent); | |
| 6296 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_find_extent); | |
| 6297 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_issue_zeroout); | |
| 6298 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_query_blocks); | |
| 6243 | 6299 | #endif |
| @@ -975,13 +975,13 @@ static int ext4_fc_flush_data(journal_t *journal) · 974 unmodified lines | ||
| 975 | 975 | int ret = 0; |
| 976 | 976 | |
| 977 | 977 | list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { |
| 978 | − ret = jbd2_submit_inode_data(journal, ei->jinode); | |
| 978 | + ret = jbd2_submit_inode_data(journal, READ_ONCE(ei->jinode)); | |
| 979 | 979 | if (ret) |
| 980 | 980 | return ret; |
| 981 | 981 | } |
| 982 | 982 | |
| 983 | 983 | list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { |
| 984 | − ret = jbd2_wait_inode_data(journal, ei->jinode); | |
| 984 | + ret = jbd2_wait_inode_data(journal, READ_ONCE(ei->jinode)); | |
| 985 | 985 | if (ret) |
| 986 | 986 | return ret; |
| 987 | 987 | } |
| @@ -1613,19 +1613,21 @@ static int ext4_fc_replay_inode(struct super_block *sb, · 625 unmodified lines | ||
| 1613 | 1613 | /* Immediately update the inode on disk. */ |
| 1614 | 1614 | ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); |
| 1615 | 1615 | if (ret) |
| 1616 | − goto out; | |
| 1616 | + goto out_brelse; | |
| 1617 | 1617 | ret = sync_dirty_buffer(iloc.bh); |
| 1618 | 1618 | if (ret) |
| 1619 | − goto out; | |
| 1619 | + goto out_brelse; | |
| 1620 | 1620 | ret = ext4_mark_inode_used(sb, ino); |
| 1621 | 1621 | if (ret) |
| 1622 | − goto out; | |
| 1622 | + goto out_brelse; | |
| 1623 | 1623 | |
| 1624 | 1624 | /* Given that we just wrote the inode on disk, this SHOULD succeed. */ |
| 1625 | 1625 | inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); |
| 1626 | 1626 | if (IS_ERR(inode)) { |
| 1627 | 1627 | ext4_debug("Inode not found."); |
| 1628 | − return -EFSCORRUPTED; | |
| 1628 | + inode = NULL; | |
| 1629 | + ret = -EFSCORRUPTED; | |
| 1630 | + goto out_brelse; | |
| 1629 | 1631 | } |
| 1630 | 1632 | |
| 1631 | 1633 | /* |
| @@ -1642,13 +1644,14 @@ static int ext4_fc_replay_inode(struct super_block *sb, · 10 unmodified lines | ||
| 1642 | 1644 | ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode)); |
| 1643 | 1645 | ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); |
| 1644 | 1646 | sync_dirty_buffer(iloc.bh); |
| 1647 | +out_brelse: | |
| 1645 | 1648 | brelse(iloc.bh); |
| 1646 | 1649 | out: |
| 1647 | 1650 | iput(inode); |
| 1648 | 1651 | if (!ret) |
| 1649 | 1652 | blkdev_issue_flush(sb->s_bdev); |
| 1650 | 1653 | |
| 1651 | − return 0; | |
| 1654 | + return ret; | |
| 1652 | 1655 | } |
| 1653 | 1656 | |
| 1654 | 1657 | /* |
| @@ -83,11 +83,23 @@ static int ext4_fsync_nojournal(struct file *file, loff_t start, loff_t end, · 82 unmodified lines | ||
| 83 | 83 | int datasync, bool *needs_barrier) |
| 84 | 84 | { |
| 85 | 85 | struct inode *inode = file->f_inode; |
| 86 | + struct writeback_control wbc = { | |
| 87 | + .sync_mode = WB_SYNC_ALL, | |
| 88 | + .nr_to_write = 0, | |
| 89 | + }; | |
| 86 | 90 | int ret; |
| 87 | 91 | |
| 88 | 92 | ret = generic_buffers_fsync_noflush(file, start, end, datasync); |
| 89 | − if (!ret) | |
| 90 | − ret = ext4_sync_parent(inode); | |
| 93 | + if (ret) | |
| 94 | + return ret; | |
| 95 | + | |
| 96 | + /* Force writeout of inode table buffer to disk */ | |
| 97 | + ret = ext4_write_inode(inode, &wbc); | |
| 98 | + if (ret) | |
| 99 | + return ret; | |
| 100 | + | |
| 101 | + ret = ext4_sync_parent(inode); | |
| 102 | + | |
| 91 | 103 | if (test_opt(inode->i_sb, BARRIER)) |
| 92 | 104 | *needs_barrier = true; |
| 93 | 105 | |
| @@ -686,6 +686,12 @@ static int recently_deleted(struct super_block *sb, ext4_group_t group, int ino) · 685 unmodified lines | ||
| 686 | 686 | if (unlikely(!gdp)) |
| 687 | 687 | return 0; |
| 688 | 688 | |
| 689 | + /* Inode was never used in this filesystem? */ | |
| 690 | + if (ext4_has_group_desc_csum(sb) && | |
| 691 | + (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT) || | |
| 692 | + ino >= EXT4_INODES_PER_GROUP(sb) - ext4_itable_unused_count(sb, gdp))) | |
| 693 | + return 0; | |
| 694 | + | |
| 689 | 695 | bh = sb_find_get_block(sb, ext4_inode_table(sb, gdp) + |
| 690 | 696 | (ino / inodes_per_block)); |
| 691 | 697 | if (!bh || !buffer_uptodate(bh)) |
| @@ -522,7 +522,15 @@ static int ext4_read_inline_folio(struct inode *inode, struct folio *folio) · 521 unmodified lines | ||
| 522 | 522 | goto out; |
| 523 | 523 | |
| 524 | 524 | len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode)); |
| 525 | − BUG_ON(len > PAGE_SIZE); | |
| 525 | + | |
| 526 | + if (len > PAGE_SIZE) { | |
| 527 | + ext4_error_inode(inode, __func__, __LINE__, 0, | |
| 528 | + "inline size %zu exceeds PAGE_SIZE", len); | |
| 529 | + ret = -EFSCORRUPTED; | |
| 530 | + brelse(iloc.bh); | |
| 531 | + goto out; | |
| 532 | + } | |
| 533 | + | |
| 526 | 534 | kaddr = kmap_local_folio(folio, 0); |
| 527 | 535 | ret = ext4_read_inline_data(inode, kaddr, len, &iloc); |
| 528 | 536 | kaddr = folio_zero_tail(folio, len, kaddr + len); |
| @@ -128,6 +128,8 @@ void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, · 127 unmodified lines | ||
| 128 | 128 | static inline int ext4_begin_ordered_truncate(struct inode *inode, |
| 129 | 129 | loff_t new_size) |
| 130 | 130 | { |
| 131 | + struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode); | |
| 132 | + | |
| 131 | 133 | trace_ext4_begin_ordered_truncate(inode, new_size); |
| 132 | 134 | /* |
| 133 | 135 | * If jinode is zero, then we never opened the file for |
| @@ -135,10 +137,10 @@ static inline int ext4_begin_ordered_truncate(struct inode *inode, · 1 unmodified line | ||
| 135 | 137 | * jbd2_journal_begin_ordered_truncate() since there's no |
| 136 | 138 | * outstanding writes we need to flush. |
| 137 | 139 | */ |
| 138 | − if (!EXT4_I(inode)->jinode) | |
| 140 | + if (!jinode) | |
| 139 | 141 | return 0; |
| 140 | 142 | return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), |
| 141 | − EXT4_I(inode)->jinode, | |
| 143 | + jinode, | |
| 142 | 144 | new_size); |
| 143 | 145 | } |
| 144 | 146 | |
| @@ -184,6 +186,14 @@ void ext4_evict_inode(struct inode *inode) · 39 unmodified lines | ||
| 184 | 186 | if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL) |
| 185 | 187 | ext4_evict_ea_inode(inode); |
| 186 | 188 | if (inode->i_nlink) { |
| 189 | + /* | |
| 190 | + * If there's dirty page will lead to data loss, user | |
| 191 | + * could see stale data. | |
| 192 | + */ | |
| 193 | + if (unlikely(!ext4_emergency_state(inode->i_sb) && | |
| 194 | + mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY))) | |
| 195 | + ext4_warning_inode(inode, "data will be lost"); | |
| 196 | + | |
| 187 | 197 | truncate_inode_pages_final(&inode->i_data); |
| 188 | 198 | |
| 189 | 199 | goto no_delete; |
| @@ -4451,8 +4461,13 @@ int ext4_inode_attach_jinode(struct inode *inode) · 4261 unmodified lines | ||
| 4451 | 4461 | spin_unlock(&inode->i_lock); |
| 4452 | 4462 | return -ENOMEM; |
| 4453 | 4463 | } |
| 4454 | − ei->jinode = jinode; | |
| 4455 | − jbd2_journal_init_jbd_inode(ei->jinode, inode); | |
| 4464 | + jbd2_journal_init_jbd_inode(jinode, inode); | |
| 4465 | + /* | |
| 4466 | + * Publish ->jinode only after it is fully initialized so that | |
| 4467 | + * readers never observe a partially initialized jbd2_inode. | |
| 4468 | + */ | |
| 4469 | + smp_wmb(); | |
| 4470 | + WRITE_ONCE(ei->jinode, jinode); | |
| 4456 | 4471 | jinode = NULL; |
| 4457 | 4472 | } |
| 4458 | 4473 | spin_unlock(&inode->i_lock); |
| @@ -5401,18 +5416,36 @@ struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, · 942 unmodified lines | ||
| 5401 | 5416 | inode->i_op = &ext4_encrypted_symlink_inode_operations; |
| 5402 | 5417 | } else if (ext4_inode_is_fast_symlink(inode)) { |
| 5403 | 5418 | inode->i_op = &ext4_fast_symlink_inode_operations; |
| 5404 | − if (inode->i_size == 0 || | |
| 5405 | − inode->i_size >= sizeof(ei->i_data) || | |
| 5406 | − strnlen((char *)ei->i_data, inode->i_size + 1) != | |
| 5407 | − inode->i_size) { | |
| 5408 | − ext4_error_inode(inode, function, line, 0, | |
| 5409 | − "invalid fast symlink length %llu", | |
| 5410 | − (unsigned long long)inode->i_size); | |
| 5411 | − ret = -EFSCORRUPTED; | |
| 5412 | − goto bad_inode; | |
| 5419 | + | |
| 5420 | + /* | |
| 5421 | + * Orphan cleanup can see inodes with i_size == 0 | |
| 5422 | + * and i_data uninitialized. Skip size checks in | |
| 5423 | + * that case. This is safe because the first thing | |
| 5424 | + * ext4_evict_inode() does for fast symlinks is | |
| 5425 | + * clearing of i_data and i_size. | |
| 5426 | + */ | |
| 5427 | + if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) { | |
| 5428 | + if (inode->i_nlink != 0) { | |
| 5429 | + ext4_error_inode(inode, function, line, 0, | |
| 5430 | + "invalid orphan symlink nlink %d", | |
| 5431 | + inode->i_nlink); | |
| 5432 | + ret = -EFSCORRUPTED; | |
| 5433 | + goto bad_inode; | |
| 5434 | + } | |
| 5435 | + } else { | |
| 5436 | + if (inode->i_size == 0 || | |
| 5437 | + inode->i_size >= sizeof(ei->i_data) || | |
| 5438 | + strnlen((char *)ei->i_data, inode->i_size + 1) != | |
| 5439 | + inode->i_size) { | |
| 5440 | + ext4_error_inode(inode, function, line, 0, | |
| 5441 | + "invalid fast symlink length %llu", | |
| 5442 | + (unsigned long long)inode->i_size); | |
| 5443 | + ret = -EFSCORRUPTED; | |
| 5444 | + goto bad_inode; | |
| 5445 | + } | |
| 5446 | + inode_set_cached_link(inode, (char *)ei->i_data, | |
| 5447 | + inode->i_size); | |
| 5413 | 5448 | } |
| 5414 | − inode_set_cached_link(inode, (char *)ei->i_data, | |
| 5415 | − inode->i_size); | |
| 5416 | 5449 | } else { |
| 5417 | 5450 | inode->i_op = &ext4_symlink_inode_operations; |
| 5418 | 5451 | } |
| @@ -5849,6 +5882,18 @@ int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, · 430 unmodified lines | ||
| 5849 | 5882 | if (attr->ia_size == inode->i_size) |
| 5850 | 5883 | inc_ivers = false; |
| 5851 | 5884 | |
| 5885 | + /* | |
| 5886 | + * If file has inline data but new size exceeds inline capacity, | |
| 5887 | + * convert to extent-based storage first to prevent inconsistent | |
| 5888 | + * state (inline flag set but size exceeds inline capacity). | |
| 5889 | + */ | |
| 5890 | + if (ext4_has_inline_data(inode) && | |
| 5891 | + attr->ia_size > EXT4_I(inode)->i_inline_size) { | |
| 5892 | + error = ext4_convert_inline_data(inode); | |
| 5893 | + if (error) | |
| 5894 | + goto err_out; | |
| 5895 | + } | |
| 5896 | + | |
| 5852 | 5897 | if (shrink) { |
| 5853 | 5898 | if (ext4_should_order_data(inode)) { |
| 5854 | 5899 | error = ext4_begin_ordered_truncate(inode, |
| @@ -8,6 +8,7 @@ · 7 unmodified lines | ||
| 8 | 8 | #include <linux/random.h> |
| 9 | 9 | |
| 10 | 10 | #include "ext4.h" |
| 11 | +#include "mballoc.h" | |
| 11 | 12 | |
| 12 | 13 | struct mbt_grp_ctx { |
| 13 | 14 | struct buffer_head bitmap_bh; |
| @@ -336,7 +337,7 @@ ext4_mb_mark_context_stub(handle_t *handle, struct super_block *sb, bool state, · 322 unmodified lines | ||
| 336 | 337 | if (state) |
| 337 | 338 | mb_set_bits(bitmap_bh->b_data, blkoff, len); |
| 338 | 339 | else |
| 339 | − mb_clear_bits(bitmap_bh->b_data, blkoff, len); | |
| 340 | + mb_clear_bits_test(bitmap_bh->b_data, blkoff, len); | |
| 340 | 341 | |
| 341 | 342 | return 0; |
| 342 | 343 | } |
| @@ -413,14 +414,14 @@ static void test_new_blocks_simple(struct kunit *test) · 70 unmodified lines | ||
| 413 | 414 | |
| 414 | 415 | /* get block at goal */ |
| 415 | 416 | ar.goal = ext4_group_first_block_no(sb, goal_group); |
| 416 | − found = ext4_mb_new_blocks_simple(&ar, &err); | |
| 417 | + found = ext4_mb_new_blocks_simple_test(&ar, &err); | |
| 417 | 418 | KUNIT_ASSERT_EQ_MSG(test, ar.goal, found, |
| 418 | 419 | "failed to alloc block at goal, expected %llu found %llu", |
| 419 | 420 | ar.goal, found); |
| 420 | 421 | |
| 421 | 422 | /* get block after goal in goal group */ |
| 422 | 423 | ar.goal = ext4_group_first_block_no(sb, goal_group); |
| 423 | − found = ext4_mb_new_blocks_simple(&ar, &err); | |
| 424 | + found = ext4_mb_new_blocks_simple_test(&ar, &err); | |
| 424 | 425 | KUNIT_ASSERT_EQ_MSG(test, ar.goal + EXT4_C2B(sbi, 1), found, |
| 425 | 426 | "failed to alloc block after goal in goal group, expected %llu found %llu", |
| 426 | 427 | ar.goal + 1, found); |
| @@ -428,7 +429,7 @@ static void test_new_blocks_simple(struct kunit *test) · 1 unmodified line | ||
| 428 | 429 | /* get block after goal group */ |
| 429 | 430 | mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); |
| 430 | 431 | ar.goal = ext4_group_first_block_no(sb, goal_group); |
| 431 | − found = ext4_mb_new_blocks_simple(&ar, &err); | |
| 432 | + found = ext4_mb_new_blocks_simple_test(&ar, &err); | |
| 432 | 433 | KUNIT_ASSERT_EQ_MSG(test, |
| 433 | 434 | ext4_group_first_block_no(sb, goal_group + 1), found, |
| 434 | 435 | "failed to alloc block after goal group, expected %llu found %llu", |
| @@ -438,7 +439,7 @@ static void test_new_blocks_simple(struct kunit *test) · 3 unmodified lines | ||
| 438 | 439 | for (i = goal_group; i < ext4_get_groups_count(sb); i++) |
| 439 | 440 | mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); |
| 440 | 441 | ar.goal = ext4_group_first_block_no(sb, goal_group); |
| 441 | − found = ext4_mb_new_blocks_simple(&ar, &err); | |
| 442 | + found = ext4_mb_new_blocks_simple_test(&ar, &err); | |
| 442 | 443 | KUNIT_ASSERT_EQ_MSG(test, |
| 443 | 444 | ext4_group_first_block_no(sb, 0) + EXT4_C2B(sbi, 1), found, |
| 444 | 445 | "failed to alloc block before goal group, expected %llu found %llu", |
| @@ -448,7 +449,7 @@ static void test_new_blocks_simple(struct kunit *test) · 3 unmodified lines | ||
| 448 | 449 | for (i = 0; i < ext4_get_groups_count(sb); i++) |
| 449 | 450 | mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); |
| 450 | 451 | ar.goal = ext4_group_first_block_no(sb, goal_group); |
| 451 | − found = ext4_mb_new_blocks_simple(&ar, &err); | |
| 452 | + found = ext4_mb_new_blocks_simple_test(&ar, &err); | |
| 452 | 453 | KUNIT_ASSERT_NE_MSG(test, err, 0, |
| 453 | 454 | "unexpectedly get block when no block is available"); |
| 454 | 455 | } |
| @@ -492,16 +493,16 @@ validate_free_blocks_simple(struct kunit *test, struct super_block *sb, · 37 unmodified lines | ||
| 492 | 493 | continue; |
| 493 | 494 | |
| 494 | 495 | bitmap = mbt_ctx_bitmap(sb, i); |
| 495 | − bit = mb_find_next_zero_bit(bitmap, max, 0); | |
| 496 | + bit = mb_find_next_zero_bit_test(bitmap, max, 0); | |
| 496 | 497 | KUNIT_ASSERT_EQ_MSG(test, bit, max, |
| 497 | 498 | "free block on unexpected group %d", i); |
| 498 | 499 | } |
| 499 | 500 | |
| 500 | 501 | bitmap = mbt_ctx_bitmap(sb, goal_group); |
| 501 | − bit = mb_find_next_zero_bit(bitmap, max, 0); | |
| 502 | + bit = mb_find_next_zero_bit_test(bitmap, max, 0); | |
| 502 | 503 | KUNIT_ASSERT_EQ(test, bit, start); |
| 503 | 504 | |
| 504 | − bit = mb_find_next_bit(bitmap, max, bit + 1); | |
| 505 | + bit = mb_find_next_bit_test(bitmap, max, bit + 1); | |
| 505 | 506 | KUNIT_ASSERT_EQ(test, bit, start + len); |
| 506 | 507 | } |
| 507 | 508 | |
| @@ -524,7 +525,7 @@ test_free_blocks_simple_range(struct kunit *test, ext4_group_t goal_group, · 16 unmodified lines | ||
| 524 | 525 | |
| 525 | 526 | block = ext4_group_first_block_no(sb, goal_group) + |
| 526 | 527 | EXT4_C2B(sbi, start); |
| 527 | − ext4_free_blocks_simple(inode, block, len); | |
| 528 | + ext4_free_blocks_simple_test(inode, block, len); | |
| 528 | 529 | validate_free_blocks_simple(test, sb, goal_group, start, len); |
| 529 | 530 | mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); |
| 530 | 531 | } |
| @@ -566,15 +567,15 @@ test_mark_diskspace_used_range(struct kunit *test, · 35 unmodified lines | ||
| 566 | 567 | |
| 567 | 568 | bitmap = mbt_ctx_bitmap(sb, TEST_GOAL_GROUP); |
| 568 | 569 | memset(bitmap, 0, sb->s_blocksize); |
| 569 | − ret = ext4_mb_mark_diskspace_used(ac, NULL); | |
| 570 | + ret = ext4_mb_mark_diskspace_used_test(ac, NULL); | |
| 570 | 571 | KUNIT_ASSERT_EQ(test, ret, 0); |
| 571 | 572 | |
| 572 | 573 | max = EXT4_CLUSTERS_PER_GROUP(sb); |
| 573 | − i = mb_find_next_bit(bitmap, max, 0); | |
| 574 | + i = mb_find_next_bit_test(bitmap, max, 0); | |
| 574 | 575 | KUNIT_ASSERT_EQ(test, i, start); |
| 575 | − i = mb_find_next_zero_bit(bitmap, max, i + 1); | |
| 576 | + i = mb_find_next_zero_bit_test(bitmap, max, i + 1); | |
| 576 | 577 | KUNIT_ASSERT_EQ(test, i, start + len); |
| 577 | − i = mb_find_next_bit(bitmap, max, i + 1); | |
| 578 | + i = mb_find_next_bit_test(bitmap, max, i + 1); | |
| 578 | 579 | KUNIT_ASSERT_EQ(test, max, i); |
| 579 | 580 | } |
| 580 | 581 | |
| @@ -617,54 +618,54 @@ static void mbt_generate_buddy(struct super_block *sb, void *buddy, · 36 unmodified lines | ||
| 617 | 618 | max = EXT4_CLUSTERS_PER_GROUP(sb); |
| 618 | 619 | bb_h = buddy + sbi->s_mb_offsets[1]; |
| 619 | 620 | |
| 620 | − off = mb_find_next_zero_bit(bb, max, 0); | |
| 621 | + off = mb_find_next_zero_bit_test(bb, max, 0); | |
| 621 | 622 | grp->bb_first_free = off; |
| 622 | 623 | while (off < max) { |
| 623 | 624 | grp->bb_counters[0]++; |
| 624 | 625 | grp->bb_free++; |
| 625 | 626 | |
| 626 | − if (!(off & 1) && !mb_test_bit(off + 1, bb)) { | |
| 627 | + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { | |
| 627 | 628 | grp->bb_free++; |
| 628 | 629 | grp->bb_counters[0]--; |
| 629 | − mb_clear_bit(off >> 1, bb_h); | |
| 630 | + mb_clear_bit_test(off >> 1, bb_h); | |
| 630 | 631 | grp->bb_counters[1]++; |
| 631 | 632 | grp->bb_largest_free_order = 1; |
| 632 | 633 | off++; |
| 633 | 634 | } |
| 634 | 635 | |
| 635 | − off = mb_find_next_zero_bit(bb, max, off + 1); | |
| 636 | + off = mb_find_next_zero_bit_test(bb, max, off + 1); | |
| 636 | 637 | } |
| 637 | 638 | |
| 638 | 639 | for (order = 1; order < MB_NUM_ORDERS(sb) - 1; order++) { |
| 639 | 640 | bb = buddy + sbi->s_mb_offsets[order]; |
| 640 | 641 | bb_h = buddy + sbi->s_mb_offsets[order + 1]; |
| 641 | 642 | max = max >> 1; |
| 642 | − off = mb_find_next_zero_bit(bb, max, 0); | |
| 643 | + off = mb_find_next_zero_bit_test(bb, max, 0); | |
| 643 | 644 | |
| 644 | 645 | while (off < max) { |
| 645 | − if (!(off & 1) && !mb_test_bit(off + 1, bb)) { | |
| 646 | + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { | |
| 646 | 647 | mb_set_bits(bb, off, 2); |
| 647 | 648 | grp->bb_counters[order] -= 2; |
| 648 | − mb_clear_bit(off >> 1, bb_h); | |
| 649 | + mb_clear_bit_test(off >> 1, bb_h); | |
| 649 | 650 | grp->bb_counters[order + 1]++; |
| 650 | 651 | grp->bb_largest_free_order = order + 1; |
| 651 | 652 | off++; |
| 652 | 653 | } |
| 653 | 654 | |
| 654 | − off = mb_find_next_zero_bit(bb, max, off + 1); | |
| 655 | + off = mb_find_next_zero_bit_test(bb, max, off + 1); | |
| 655 | 656 | } |
| 656 | 657 | } |
| 657 | 658 | |
| 658 | 659 | max = EXT4_CLUSTERS_PER_GROUP(sb); |
| 659 | − off = mb_find_next_zero_bit(bitmap, max, 0); | |
| 660 | + off = mb_find_next_zero_bit_test(bitmap, max, 0); | |
| 660 | 661 | while (off < max) { |
| 661 | 662 | grp->bb_fragments++; |
| 662 | 663 | |
| 663 | − off = mb_find_next_bit(bitmap, max, off + 1); | |
| 664 | + off = mb_find_next_bit_test(bitmap, max, off + 1); | |
| 664 | 665 | if (off + 1 >= max) |
| 665 | 666 | break; |
| 666 | 667 | |
| 667 | − off = mb_find_next_zero_bit(bitmap, max, off + 1); | |
| 668 | + off = mb_find_next_zero_bit_test(bitmap, max, off + 1); | |
| 668 | 669 | } |
| 669 | 670 | } |
| 670 | 671 | |
| @@ -706,7 +707,7 @@ do_test_generate_buddy(struct kunit *test, struct super_block *sb, void *bitmap, · 35 unmodified lines | ||
| 706 | 707 | /* needed by validation in ext4_mb_generate_buddy */ |
| 707 | 708 | ext4_grp->bb_free = mbt_grp->bb_free; |
| 708 | 709 | memset(ext4_buddy, 0xff, sb->s_blocksize); |
| 709 | − ext4_mb_generate_buddy(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, | |
| 710 | + ext4_mb_generate_buddy_test(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, | |
| 710 | 711 | ext4_grp); |
| 711 | 712 | |
| 712 | 713 | KUNIT_ASSERT_EQ(test, memcmp(mbt_buddy, ext4_buddy, sb->s_blocksize), |
| @@ -760,7 +761,7 @@ test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, · 47 unmodified lines | ||
| 760 | 761 | ex.fe_group = TEST_GOAL_GROUP; |
| 761 | 762 | |
| 762 | 763 | ext4_lock_group(sb, TEST_GOAL_GROUP); |
| 763 | − mb_mark_used(e4b, &ex); | |
| 764 | + mb_mark_used_test(e4b, &ex); | |
| 764 | 765 | ext4_unlock_group(sb, TEST_GOAL_GROUP); |
| 765 | 766 | |
| 766 | 767 | mb_set_bits(bitmap, start, len); |
| @@ -769,7 +770,7 @@ test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, · 2 unmodified lines | ||
| 769 | 770 | memset(buddy, 0xff, sb->s_blocksize); |
| 770 | 771 | for (i = 0; i < MB_NUM_ORDERS(sb); i++) |
| 771 | 772 | grp->bb_counters[i] = 0; |
| 772 | − ext4_mb_generate_buddy(sb, buddy, bitmap, 0, grp); | |
| 773 | + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); | |
| 773 | 774 | |
| 774 | 775 | KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), |
| 775 | 776 | 0); |
| @@ -798,7 +799,7 @@ static void test_mb_mark_used(struct kunit *test) · 22 unmodified lines | ||
| 798 | 799 | bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); |
| 799 | 800 | KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); |
| 800 | 801 | |
| 801 | − ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); | |
| 802 | + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); | |
| 802 | 803 | KUNIT_ASSERT_EQ(test, ret, 0); |
| 803 | 804 | |
| 804 | 805 | grp->bb_free = EXT4_CLUSTERS_PER_GROUP(sb); |
| @@ -809,7 +810,7 @@ static void test_mb_mark_used(struct kunit *test) · 4 unmodified lines | ||
| 809 | 810 | test_mb_mark_used_range(test, &e4b, ranges[i].start, |
| 810 | 811 | ranges[i].len, bitmap, buddy, grp); |
| 811 | 812 | |
| 812 | − ext4_mb_unload_buddy(&e4b); | |
| 813 | + ext4_mb_unload_buddy_test(&e4b); | |
| 813 | 814 | } |
| 814 | 815 | |
| 815 | 816 | static void |
| @@ -825,16 +826,16 @@ test_mb_free_blocks_range(struct kunit *test, struct ext4_buddy *e4b, · 9 unmodified lines | ||
| 825 | 826 | return; |
| 826 | 827 | |
| 827 | 828 | ext4_lock_group(sb, e4b->bd_group); |
| 828 | − mb_free_blocks(NULL, e4b, start, len); | |
| 829 | + mb_free_blocks_test(NULL, e4b, start, len); | |
| 829 | 830 | ext4_unlock_group(sb, e4b->bd_group); |
| 830 | 831 | |
| 831 | − mb_clear_bits(bitmap, start, len); | |
| 832 | + mb_clear_bits_test(bitmap, start, len); | |
| 832 | 833 | /* bypass bb_free validatoin in ext4_mb_generate_buddy */ |
| 833 | 834 | grp->bb_free += len; |
| 834 | 835 | memset(buddy, 0xff, sb->s_blocksize); |
| 835 | 836 | for (i = 0; i < MB_NUM_ORDERS(sb); i++) |
| 836 | 837 | grp->bb_counters[i] = 0; |
| 837 | − ext4_mb_generate_buddy(sb, buddy, bitmap, 0, grp); | |
| 838 | + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); | |
| 838 | 839 | |
| 839 | 840 | KUNIT_ASSERT_EQ(test, memcmp(buddy, e4b->bd_buddy, sb->s_blocksize), |
| 840 | 841 | 0); |
| @@ -865,7 +866,7 @@ static void test_mb_free_blocks(struct kunit *test) · 24 unmodified lines | ||
| 865 | 866 | bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); |
| 866 | 867 | KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); |
| 867 | 868 | |
| 868 | − ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); | |
| 869 | + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); | |
| 869 | 870 | KUNIT_ASSERT_EQ(test, ret, 0); |
| 870 | 871 | |
| 871 | 872 | ex.fe_start = 0; |
| @@ -873,7 +874,7 @@ static void test_mb_free_blocks(struct kunit *test) · 1 unmodified line | ||
| 873 | 874 | ex.fe_group = TEST_GOAL_GROUP; |
| 874 | 875 | |
| 875 | 876 | ext4_lock_group(sb, TEST_GOAL_GROUP); |
| 876 | − mb_mark_used(&e4b, &ex); | |
| 877 | + mb_mark_used_test(&e4b, &ex); | |
| 877 | 878 | ext4_unlock_group(sb, TEST_GOAL_GROUP); |
| 878 | 879 | |
| 879 | 880 | grp->bb_free = 0; |
| @@ -886,7 +887,7 @@ static void test_mb_free_blocks(struct kunit *test) · 6 unmodified lines | ||
| 886 | 887 | test_mb_free_blocks_range(test, &e4b, ranges[i].start, |
| 887 | 888 | ranges[i].len, bitmap, buddy, grp); |
| 888 | 889 | |
| 889 | − ext4_mb_unload_buddy(&e4b); | |
| 890 | + ext4_mb_unload_buddy_test(&e4b); | |
| 890 | 891 | } |
| 891 | 892 | |
| 892 | 893 | #define COUNT_FOR_ESTIMATE 100000 |
| @@ -904,7 +905,7 @@ static void test_mb_mark_used_cost(struct kunit *test) · 11 unmodified lines | ||
| 904 | 905 | if (sb->s_blocksize > PAGE_SIZE) |
| 905 | 906 | kunit_skip(test, "blocksize exceeds pagesize"); |
| 906 | 907 | |
| 907 | − ret = ext4_mb_load_buddy(sb, TEST_GOAL_GROUP, &e4b); | |
| 908 | + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); | |
| 908 | 909 | KUNIT_ASSERT_EQ(test, ret, 0); |
| 909 | 910 | |
| 910 | 911 | ex.fe_group = TEST_GOAL_GROUP; |
| @@ -918,7 +919,7 @@ static void test_mb_mark_used_cost(struct kunit *test) · 7 unmodified lines | ||
| 918 | 919 | ex.fe_start = ranges[i].start; |
| 919 | 920 | ex.fe_len = ranges[i].len; |
| 920 | 921 | ext4_lock_group(sb, TEST_GOAL_GROUP); |
| 921 | − mb_mark_used(&e4b, &ex); | |
| 922 | + mb_mark_used_test(&e4b, &ex); | |
| 922 | 923 | ext4_unlock_group(sb, TEST_GOAL_GROUP); |
| 923 | 924 | } |
| 924 | 925 | end = jiffies; |
| @@ -929,14 +930,14 @@ static void test_mb_mark_used_cost(struct kunit *test) · 4 unmodified lines | ||
| 929 | 930 | continue; |
| 930 | 931 | |
| 931 | 932 | ext4_lock_group(sb, TEST_GOAL_GROUP); |
| 932 | − mb_free_blocks(NULL, &e4b, ranges[i].start, | |
| 933 | + mb_free_blocks_test(NULL, &e4b, ranges[i].start, | |
| 933 | 934 | ranges[i].len); |
| 934 | 935 | ext4_unlock_group(sb, TEST_GOAL_GROUP); |
| 935 | 936 | } |
| 936 | 937 | } |
| 937 | 938 | |
| 938 | 939 | kunit_info(test, "costed jiffies %lu\n", all); |
| 939 | − ext4_mb_unload_buddy(&e4b); | |
| 940 | + ext4_mb_unload_buddy_test(&e4b); | |
| 940 | 941 | } |
| 941 | 942 | |
| 942 | 943 | static const struct mbt_ext4_block_layout mbt_test_layouts[] = { |
| @@ -1199,6 +1199,8 @@ static int ext4_mb_scan_groups(struct ext4_allocation_context *ac) · 1198 unmodified lines | ||
| 1199 | 1199 | |
| 1200 | 1200 | /* searching for the right group start from the goal value specified */ |
| 1201 | 1201 | start = ac->ac_g_ex.fe_group; |
| 1202 | + if (start >= ngroups) | |
| 1203 | + start = 0; | |
| 1202 | 1204 | ac->ac_prefetch_grp = start; |
| 1203 | 1205 | ac->ac_prefetch_nr = 0; |
| 1204 | 1206 | |
| @@ -2443,8 +2445,12 @@ int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, · 1238 unmodified lines | ||
| 2443 | 2445 | return 0; |
| 2444 | 2446 | |
| 2445 | 2447 | err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); |
| 2446 | − if (err) | |
| 2448 | + if (err) { | |
| 2449 | + if (EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info) && | |
| 2450 | + !(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) | |
| 2451 | + return 0; | |
| 2447 | 2452 | return err; |
| 2453 | + } | |
| 2448 | 2454 | |
| 2449 | 2455 | ext4_lock_group(ac->ac_sb, group); |
| 2450 | 2456 | if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) |
| @@ -3580,9 +3586,7 @@ err_freebuddy: · 1129 unmodified lines | ||
| 3580 | 3586 | rcu_read_unlock(); |
| 3581 | 3587 | iput(sbi->s_buddy_cache); |
| 3582 | 3588 | err_freesgi: |
| 3583 | − rcu_read_lock(); | |
| 3584 | − kvfree(rcu_dereference(sbi->s_group_info)); | |
| 3585 | − rcu_read_unlock(); | |
| 3589 | + kvfree(rcu_access_pointer(sbi->s_group_info)); | |
| 3586 | 3590 | return -ENOMEM; |
| 3587 | 3591 | } |
| 3588 | 3592 | |
| @@ -3889,15 +3893,14 @@ void ext4_mb_release(struct super_block *sb) · 300 unmodified lines | ||
| 3889 | 3893 | struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); |
| 3890 | 3894 | int count; |
| 3891 | 3895 | |
| 3892 | − if (test_opt(sb, DISCARD)) { | |
| 3893 | − /* | |
| 3894 | − * wait the discard work to drain all of ext4_free_data | |
| 3895 | − */ | |
| 3896 | − flush_work(&sbi->s_discard_work); | |
| 3897 | − WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); | |
| 3898 | − } | |
| 3896 | + /* | |
| 3897 | + * wait the discard work to drain all of ext4_free_data | |
| 3898 | + */ | |
| 3899 | + flush_work(&sbi->s_discard_work); | |
| 3900 | + WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); | |
| 3899 | 3901 | |
| 3900 | − if (sbi->s_group_info) { | |
| 3902 | + group_info = rcu_access_pointer(sbi->s_group_info); | |
| 3903 | + if (group_info) { | |
| 3901 | 3904 | for (i = 0; i < ngroups; i++) { |
| 3902 | 3905 | cond_resched(); |
| 3903 | 3906 | grinfo = ext4_get_group_info(sb, i); |
| @@ -3915,12 +3918,9 @@ void ext4_mb_release(struct super_block *sb) · 11 unmodified lines | ||
| 3915 | 3918 | num_meta_group_infos = (ngroups + |
| 3916 | 3919 | EXT4_DESC_PER_BLOCK(sb) - 1) >> |
| 3917 | 3920 | EXT4_DESC_PER_BLOCK_BITS(sb); |
| 3918 | − rcu_read_lock(); | |
| 3919 | − group_info = rcu_dereference(sbi->s_group_info); | |
| 3920 | 3921 | for (i = 0; i < num_meta_group_infos; i++) |
| 3921 | 3922 | kfree(group_info[i]); |
| 3922 | 3923 | kvfree(group_info); |
| 3923 | − rcu_read_unlock(); | |
| 3924 | 3924 | } |
| 3925 | 3925 | ext4_mb_avg_fragment_size_destroy(sbi); |
| 3926 | 3926 | ext4_mb_largest_free_orders_destroy(sbi); |
| @@ -4084,7 +4084,7 @@ void ext4_exit_mballoc(void) · 157 unmodified lines | ||
| 4084 | 4084 | |
| 4085 | 4085 | #define EXT4_MB_BITMAP_MARKED_CHECK 0x0001 |
| 4086 | 4086 | #define EXT4_MB_SYNC_UPDATE 0x0002 |
| 4087 | −static int | |
| 4087 | +int | |
| 4088 | 4088 | ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, |
| 4089 | 4089 | ext4_group_t group, ext4_grpblk_t blkoff, |
| 4090 | 4090 | ext4_grpblk_t len, int flags, ext4_grpblk_t *ret_changed) |
| @@ -7188,6 +7188,102 @@ out_unload: · 3097 unmodified lines | ||
| 7188 | 7188 | return error; |
| 7189 | 7189 | } |
| 7190 | 7190 | |
| 7191 | −#ifdef CONFIG_EXT4_KUNIT_TESTS | |
| 7192 | −#include "mballoc-test.c" | |
| 7191 | +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) | |
| 7192 | +void mb_clear_bits_test(void *bm, int cur, int len) | |
| 7193 | +{ | |
| 7194 | + mb_clear_bits(bm, cur, len); | |
| 7195 | +} | |
| 7196 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bits_test); | |
| 7197 | + | |
| 7198 | +ext4_fsblk_t | |
| 7199 | +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, | |
| 7200 | + int *errp) | |
| 7201 | +{ | |
| 7202 | + return ext4_mb_new_blocks_simple(ar, errp); | |
| 7203 | +} | |
| 7204 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_new_blocks_simple_test); | |
| 7205 | + | |
| 7206 | +int mb_find_next_zero_bit_test(void *addr, int max, int start) | |
| 7207 | +{ | |
| 7208 | + return mb_find_next_zero_bit(addr, max, start); | |
| 7209 | +} | |
| 7210 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_zero_bit_test); | |
| 7211 | + | |
| 7212 | +int mb_find_next_bit_test(void *addr, int max, int start) | |
| 7213 | +{ | |
| 7214 | + return mb_find_next_bit(addr, max, start); | |
| 7215 | +} | |
| 7216 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_bit_test); | |
| 7217 | + | |
| 7218 | +void mb_clear_bit_test(int bit, void *addr) | |
| 7219 | +{ | |
| 7220 | + mb_clear_bit(bit, addr); | |
| 7221 | +} | |
| 7222 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bit_test); | |
| 7223 | + | |
| 7224 | +int mb_test_bit_test(int bit, void *addr) | |
| 7225 | +{ | |
| 7226 | + return mb_test_bit(bit, addr); | |
| 7227 | +} | |
| 7228 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_test_bit_test); | |
| 7229 | + | |
| 7230 | +int ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, | |
| 7231 | + handle_t *handle) | |
| 7232 | +{ | |
| 7233 | + return ext4_mb_mark_diskspace_used(ac, handle); | |
| 7234 | +} | |
| 7235 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_diskspace_used_test); | |
| 7236 | + | |
| 7237 | +int mb_mark_used_test(struct ext4_buddy *e4b, struct ext4_free_extent *ex) | |
| 7238 | +{ | |
| 7239 | + return mb_mark_used(e4b, ex); | |
| 7240 | +} | |
| 7241 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_mark_used_test); | |
| 7242 | + | |
| 7243 | +void ext4_mb_generate_buddy_test(struct super_block *sb, void *buddy, | |
| 7244 | + void *bitmap, ext4_group_t group, | |
| 7245 | + struct ext4_group_info *grp) | |
| 7246 | +{ | |
| 7247 | + ext4_mb_generate_buddy(sb, buddy, bitmap, group, grp); | |
| 7248 | +} | |
| 7249 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_generate_buddy_test); | |
| 7250 | + | |
| 7251 | +int ext4_mb_load_buddy_test(struct super_block *sb, ext4_group_t group, | |
| 7252 | + struct ext4_buddy *e4b) | |
| 7253 | +{ | |
| 7254 | + return ext4_mb_load_buddy(sb, group, e4b); | |
| 7255 | +} | |
| 7256 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_load_buddy_test); | |
| 7257 | + | |
| 7258 | +void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b) | |
| 7259 | +{ | |
| 7260 | + ext4_mb_unload_buddy(e4b); | |
| 7261 | +} | |
| 7262 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_unload_buddy_test); | |
| 7263 | + | |
| 7264 | +void mb_free_blocks_test(struct inode *inode, struct ext4_buddy *e4b, | |
| 7265 | + int first, int count) | |
| 7266 | +{ | |
| 7267 | + mb_free_blocks(inode, e4b, first, count); | |
| 7268 | +} | |
| 7269 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_free_blocks_test); | |
| 7270 | + | |
| 7271 | +void ext4_free_blocks_simple_test(struct inode *inode, ext4_fsblk_t block, | |
| 7272 | + unsigned long count) | |
| 7273 | +{ | |
| 7274 | + return ext4_free_blocks_simple(inode, block, count); | |
| 7275 | +} | |
| 7276 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_blocks_simple_test); | |
| 7277 | + | |
| 7278 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_wait_block_bitmap); | |
| 7279 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_init); | |
| 7280 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_desc); | |
| 7281 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_count_free_clusters); | |
| 7282 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_info); | |
| 7283 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_group_clusters_set); | |
| 7284 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_release); | |
| 7285 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_read_block_bitmap_nowait); | |
| 7286 | +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_set_bits); | |
| 7287 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_fc_init_inode); | |
| 7288 | +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_context); | |
| 7193 | 7289 | #endif |
| @@ -270,4 +270,34 @@ ext4_mballoc_query_range( · 269 unmodified lines | ||
| 270 | 270 | ext4_mballoc_query_range_fn formatter, |
| 271 | 271 | void *priv); |
| 272 | 272 | |
| 273 | +extern int ext4_mb_mark_context(handle_t *handle, | |
| 274 | + struct super_block *sb, bool state, | |
| 275 | + ext4_group_t group, ext4_grpblk_t blkoff, | |
| 276 | + ext4_grpblk_t len, int flags, | |
| 277 | + ext4_grpblk_t *ret_changed); | |
| 278 | +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) | |
| 279 | +extern void mb_clear_bits_test(void *bm, int cur, int len); | |
| 280 | +extern ext4_fsblk_t | |
| 281 | +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, | |
| 282 | + int *errp); | |
| 283 | +extern int mb_find_next_zero_bit_test(void *addr, int max, int start); | |
| 284 | +extern int mb_find_next_bit_test(void *addr, int max, int start); | |
| 285 | +extern void mb_clear_bit_test(int bit, void *addr); | |
| 286 | +extern int mb_test_bit_test(int bit, void *addr); | |
| 287 | +extern int | |
| 288 | +ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, | |
| 289 | + handle_t *handle); | |
| 290 | +extern int mb_mark_used_test(struct ext4_buddy *e4b, | |
| 291 | + struct ext4_free_extent *ex); | |
| 292 | +extern void ext4_mb_generate_buddy_test(struct super_block *sb, | |
| 293 | + void *buddy, void *bitmap, ext4_group_t group, | |
| 294 | + struct ext4_group_info *grp); | |
| 295 | +extern int ext4_mb_load_buddy_test(struct super_block *sb, | |
| 296 | + ext4_group_t group, struct ext4_buddy *e4b); | |
| 297 | +extern void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b); | |
| 298 | +extern void mb_free_blocks_test(struct inode *inode, | |
| 299 | + struct ext4_buddy *e4b, int first, int count); | |
| 300 | +extern void ext4_free_blocks_simple_test(struct inode *inode, | |
| 301 | + ext4_fsblk_t block, unsigned long count); | |
| 302 | +#endif | |
| 273 | 303 | #endif |
| @@ -524,9 +524,15 @@ int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, · 523 unmodified lines | ||
| 524 | 524 | nr_to_submit++; |
| 525 | 525 | } while ((bh = bh->b_this_page) != head); |
| 526 | 526 | |
| 527 | − /* Nothing to submit? Just unlock the folio... */ | |
| 528 | − if (!nr_to_submit) | |
| 527 | + if (!nr_to_submit) { | |
| 528 | + /* | |
| 529 | + * We have nothing to submit. Just cycle the folio through | |
| 530 | + * writeback state to properly update xarray tags. | |
| 531 | + */ | |
| 532 | + __folio_start_writeback(folio, keep_towrite); | |
| 533 | + folio_end_writeback(folio); | |
| 529 | 534 | return 0; |
| 535 | + } | |
| 530 | 536 | |
| 531 | 537 | bh = head = folio_buffers(folio); |
| 532 | 538 | |
| @@ -1254,12 +1254,10 @@ static void ext4_group_desc_free(struct ext4_sb_info *sbi) · 1253 unmodified lines | ||
| 1254 | 1254 | struct buffer_head **group_desc; |
| 1255 | 1255 | int i; |
| 1256 | 1256 | |
| 1257 | − rcu_read_lock(); | |
| 1258 | − group_desc = rcu_dereference(sbi->s_group_desc); | |
| 1257 | + group_desc = rcu_access_pointer(sbi->s_group_desc); | |
| 1259 | 1258 | for (i = 0; i < sbi->s_gdb_count; i++) |
| 1260 | 1259 | brelse(group_desc[i]); |
| 1261 | 1260 | kvfree(group_desc); |
| 1262 | − rcu_read_unlock(); | |
| 1263 | 1261 | } |
| 1264 | 1262 | |
| 1265 | 1263 | static void ext4_flex_groups_free(struct ext4_sb_info *sbi) |
| @@ -1267,14 +1265,12 @@ static void ext4_flex_groups_free(struct ext4_sb_info *sbi) · 1 unmodified line | ||
| 1267 | 1265 | struct flex_groups **flex_groups; |
| 1268 | 1266 | int i; |
| 1269 | 1267 | |
| 1270 | − rcu_read_lock(); | |
| 1271 | − flex_groups = rcu_dereference(sbi->s_flex_groups); | |
| 1268 | + flex_groups = rcu_access_pointer(sbi->s_flex_groups); | |
| 1272 | 1269 | if (flex_groups) { |
| 1273 | 1270 | for (i = 0; i < sbi->s_flex_groups_allocated; i++) |
| 1274 | 1271 | kvfree(flex_groups[i]); |
| 1275 | 1272 | kvfree(flex_groups); |
| 1276 | 1273 | } |
| 1277 | − rcu_read_unlock(); | |
| 1278 | 1274 | } |
| 1279 | 1275 | |
| 1280 | 1276 | static void ext4_put_super(struct super_block *sb) |
| @@ -1527,6 +1523,27 @@ void ext4_clear_inode(struct inode *inode) · 246 unmodified lines | ||
| 1527 | 1523 | invalidate_inode_buffers(inode); |
| 1528 | 1524 | clear_inode(inode); |
| 1529 | 1525 | ext4_discard_preallocations(inode); |
| 1526 | + /* | |
| 1527 | + * We must remove the inode from the hash before ext4_free_inode() | |
| 1528 | + * clears the bit in inode bitmap as otherwise another process reusing | |
| 1529 | + * the inode will block in insert_inode_hash() waiting for inode | |
| 1530 | + * eviction to complete while holding transaction handle open, but | |
| 1531 | + * ext4_evict_inode() still running for that inode could block waiting | |
| 1532 | + * for transaction commit if the inode is marked as IS_SYNC => deadlock. | |
| 1533 | + * | |
| 1534 | + * Removing the inode from the hash here is safe. There are two cases | |
| 1535 | + * to consider: | |
| 1536 | + * 1) The inode still has references to it (i_nlink > 0). In that case | |
| 1537 | + * we are keeping the inode and once we remove the inode from the hash, | |
| 1538 | + * iget() can create the new inode structure for the same inode number | |
| 1539 | + * and we are fine with that as all IO on behalf of the inode is | |
| 1540 | + * finished. | |
| 1541 | + * 2) We are deleting the inode (i_nlink == 0). In that case inode | |
| 1542 | + * number cannot be reused until ext4_free_inode() clears the bit in | |
| 1543 | + * the inode bitmap, at which point all IO is done and reuse is fine | |
| 1544 | + * again. | |
| 1545 | + */ | |
| 1546 | + remove_inode_hash(inode); | |
| 1530 | 1547 | ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS); |
| 1531 | 1548 | dquot_drop(inode); |
| 1532 | 1549 | if (EXT4_I(inode)->jinode) { |
| @@ -3633,6 +3650,13 @@ int ext4_feature_set_ok(struct super_block *sb, int readonly) · 2100 unmodified lines | ||
| 3633 | 3650 | "extents feature\n"); |
| 3634 | 3651 | return 0; |
| 3635 | 3652 | } |
| 3653 | + if (ext4_has_feature_bigalloc(sb) && | |
| 3654 | + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) { | |
| 3655 | + ext4_msg(sb, KERN_WARNING, | |
| 3656 | + "bad geometry: bigalloc file system with non-zero " | |
| 3657 | + "first_data_block\n"); | |
| 3658 | + return 0; | |
| 3659 | + } | |
| 3636 | 3660 | |
| 3637 | 3661 | #if !IS_ENABLED(CONFIG_QUOTA) || !IS_ENABLED(CONFIG_QFMT_V2) |
| 3638 | 3662 | if (!readonly && (ext4_has_feature_quota(sb) || |
| @@ -5403,6 +5427,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) · 1764 unmodified lines | ||
| 5403 | 5427 | |
| 5404 | 5428 | timer_setup(&sbi->s_err_report, print_daily_error_info, 0); |
| 5405 | 5429 | spin_lock_init(&sbi->s_error_lock); |
| 5430 | + mutex_init(&sbi->s_error_notify_mutex); | |
| 5406 | 5431 | INIT_WORK(&sbi->s_sb_upd_work, update_super_work); |
| 5407 | 5432 | |
| 5408 | 5433 | err = ext4_group_desc_init(sb, es, logical_sb_block, &first_not_zeroed); |
| @@ -597,7 +597,10 @@ static const struct kobj_type ext4_feat_ktype = { · 596 unmodified lines | ||
| 597 | 597 | |
| 598 | 598 | void ext4_notify_error_sysfs(struct ext4_sb_info *sbi) |
| 599 | 599 | { |
| 600 | − sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); | |
| 600 | + mutex_lock(&sbi->s_error_notify_mutex); | |
| 601 | + if (sbi->s_kobj.state_in_sysfs) | |
| 602 | + sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); | |
| 603 | + mutex_unlock(&sbi->s_error_notify_mutex); | |
| 601 | 604 | } |
| 602 | 605 | |
| 603 | 606 | static struct kobject *ext4_root; |
| @@ -610,8 +613,10 @@ int ext4_register_sysfs(struct super_block *sb) · 6 unmodified lines | ||
| 610 | 613 | int err; |
| 611 | 614 | |
| 612 | 615 | init_completion(&sbi->s_kobj_unregister); |
| 616 | + mutex_lock(&sbi->s_error_notify_mutex); | |
| 613 | 617 | err = kobject_init_and_add(&sbi->s_kobj, &ext4_sb_ktype, ext4_root, |
| 614 | 618 | "%s", sb->s_id); |
| 619 | + mutex_unlock(&sbi->s_error_notify_mutex); | |
| 615 | 620 | if (err) { |
| 616 | 621 | kobject_put(&sbi->s_kobj); |
| 617 | 622 | wait_for_completion(&sbi->s_kobj_unregister); |
| @@ -644,7 +649,10 @@ void ext4_unregister_sysfs(struct super_block *sb) · 26 unmodified lines | ||
| 644 | 649 | |
| 645 | 650 | if (sbi->s_proc) |
| 646 | 651 | remove_proc_subtree(sb->s_id, ext4_proc_root); |
| 652 | + | |
| 653 | + mutex_lock(&sbi->s_error_notify_mutex); | |
| 647 | 654 | kobject_del(&sbi->s_kobj); |
| 655 | + mutex_unlock(&sbi->s_error_notify_mutex); | |
| 648 | 656 | } |
| 649 | 657 | |
| 650 | 658 | int __init ext4_init_sysfs(void) |
| @@ -1711,6 +1711,19 @@ static void requeue_inode(struct inode *inode, struct bdi_writeback *wb, · 1710 unmodified lines | ||
| 1711 | 1711 | } |
| 1712 | 1712 | } |
| 1713 | 1713 | |
| 1714 | +static bool __sync_lazytime(struct inode *inode) | |
| 1715 | +{ | |
| 1716 | + spin_lock(&inode->i_lock); | |
| 1717 | + if (!(inode_state_read(inode) & I_DIRTY_TIME)) { | |
| 1718 | + spin_unlock(&inode->i_lock); | |
| 1719 | + return false; | |
| 1720 | + } | |
| 1721 | + inode_state_clear(inode, I_DIRTY_TIME); | |
| 1722 | + spin_unlock(&inode->i_lock); | |
| 1723 | + inode->i_op->sync_lazytime(inode); | |
| 1724 | + return true; | |
| 1725 | +} | |
| 1726 | + | |
| 1714 | 1727 | bool sync_lazytime(struct inode *inode) |
| 1715 | 1728 | { |
| 1716 | 1729 | if (!(inode_state_read_once(inode) & I_DIRTY_TIME)) |
| @@ -1718,9 +1731,8 @@ bool sync_lazytime(struct inode *inode) · 1 unmodified line | ||
| 1718 | 1731 | |
| 1719 | 1732 | trace_writeback_lazytime(inode); |
| 1720 | 1733 | if (inode->i_op->sync_lazytime) |
| 1721 | − inode->i_op->sync_lazytime(inode); | |
| 1722 | − else | |
| 1723 | − mark_inode_dirty_sync(inode); | |
| 1734 | + return __sync_lazytime(inode); | |
| 1735 | + mark_inode_dirty_sync(inode); | |
| 1724 | 1736 | return true; |
| 1725 | 1737 | } |
| 1726 | 1738 | |
| @@ -2775,13 +2787,8 @@ static void wait_sb_inodes(struct super_block *sb) · 1048 unmodified lines | ||
| 2775 | 2787 | * The mapping can appear untagged while still on-list since we |
| 2776 | 2788 | * do not have the mapping lock. Skip it here, wb completion |
| 2777 | 2789 | * will remove it. |
| 2778 | − * | |
| 2779 | − * If the mapping does not have data integrity semantics, | |
| 2780 | − * there's no need to wait for the writeout to complete, as the | |
| 2781 | − * mapping cannot guarantee that data is persistently stored. | |
| 2782 | 2790 | */ |
| 2783 | − if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK) || | |
| 2784 | − mapping_no_data_integrity(mapping)) | |
| 2791 | + if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) | |
| 2785 | 2792 | continue; |
| 2786 | 2793 | |
| 2787 | 2794 | spin_unlock_irq(&sb->s_inode_wblist_lock); |
| @@ -2916,6 +2923,17 @@ void sync_inodes_sb(struct super_block *sb) · 128 unmodified lines | ||
| 2916 | 2923 | */ |
| 2917 | 2924 | if (bdi == &noop_backing_dev_info) |
| 2918 | 2925 | return; |
| 2926 | + | |
| 2927 | + /* | |
| 2928 | + * If the superblock has SB_I_NO_DATA_INTEGRITY set, there's no need to | |
| 2929 | + * wait for the writeout to complete, as the filesystem cannot guarantee | |
| 2930 | + * data persistence on sync. Just kick off writeback and return. | |
| 2931 | + */ | |
| 2932 | + if (sb->s_iflags & SB_I_NO_DATA_INTEGRITY) { | |
| 2933 | + wakeup_flusher_threads_bdi(bdi, WB_REASON_SYNC); | |
| 2934 | + return; | |
| 2935 | + } | |
| 2936 | + | |
| 2919 | 2937 | WARN_ON(!rwsem_is_locked(&sb->s_umount)); |
| 2920 | 2938 | |
| 2921 | 2939 | /* protect against inode wb switch, see inode_switch_wbs_work_fn() */ |
| @@ -3201,10 +3201,8 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) · 3200 unmodified lines | ||
| 3201 | 3201 | |
| 3202 | 3202 | inode->i_fop = &fuse_file_operations; |
| 3203 | 3203 | inode->i_data.a_ops = &fuse_file_aops; |
| 3204 | − if (fc->writeback_cache) { | |
| 3204 | + if (fc->writeback_cache) | |
| 3205 | 3205 | mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data); |
| 3206 | − mapping_set_no_data_integrity(&inode->i_data); | |
| 3207 | − } | |
| 3208 | 3206 | |
| 3209 | 3207 | INIT_LIST_HEAD(&fi->write_files); |
| 3210 | 3208 | INIT_LIST_HEAD(&fi->queued_writes); |
| @@ -1709,6 +1709,7 @@ static void fuse_sb_defaults(struct super_block *sb) · 1708 unmodified lines | ||
| 1709 | 1709 | sb->s_export_op = &fuse_export_operations; |
| 1710 | 1710 | sb->s_iflags |= SB_I_IMA_UNVERIFIABLE_SIGNATURE; |
| 1711 | 1711 | sb->s_iflags |= SB_I_NOIDMAP; |
| 1712 | + sb->s_iflags |= SB_I_NO_DATA_INTEGRITY; | |
| 1712 | 1713 | if (sb->s_user_ns != &init_user_ns) |
| 1713 | 1714 | sb->s_iflags |= SB_I_UNTRUSTED_MOUNTER; |
| 1714 | 1715 | sb->s_flags &= ~(SB_NOSEC | SB_I_VERSION); |
| @@ -8,7 +8,10 @@ · 7 unmodified lines | ||
| 8 | 8 | #include "internal.h" |
| 9 | 9 | #include "trace.h" |
| 10 | 10 | |
| 11 | −static void iomap_read_end_io(struct bio *bio) | |
| 11 | +static DEFINE_SPINLOCK(failed_read_lock); | |
| 12 | +static struct bio_list failed_read_list = BIO_EMPTY_LIST; | |
| 13 | + | |
| 14 | +static void __iomap_read_end_io(struct bio *bio) | |
| 12 | 15 | { |
| 13 | 16 | int error = blk_status_to_errno(bio->bi_status); |
| 14 | 17 | struct folio_iter fi; |
| @@ -18,6 +21,52 @@ static void iomap_read_end_io(struct bio *bio) · 3 unmodified lines | ||
| 18 | 21 | bio_put(bio); |
| 19 | 22 | } |
| 20 | 23 | |
| 24 | +static void | |
| 25 | +iomap_fail_reads( | |
| 26 | + struct work_struct *work) | |
| 27 | +{ | |
| 28 | + struct bio *bio; | |
| 29 | + struct bio_list tmp = BIO_EMPTY_LIST; | |
| 30 | + unsigned long flags; | |
| 31 | + | |
| 32 | + spin_lock_irqsave(&failed_read_lock, flags); | |
| 33 | + bio_list_merge_init(&tmp, &failed_read_list); | |
| 34 | + spin_unlock_irqrestore(&failed_read_lock, flags); | |
| 35 | + | |
| 36 | + while ((bio = bio_list_pop(&tmp)) != NULL) { | |
| 37 | + __iomap_read_end_io(bio); | |
| 38 | + cond_resched(); | |
| 39 | + } | |
| 40 | +} | |
| 41 | + | |
| 42 | +static DECLARE_WORK(failed_read_work, iomap_fail_reads); | |
| 43 | + | |
| 44 | +static void iomap_fail_buffered_read(struct bio *bio) | |
| 45 | +{ | |
| 46 | + unsigned long flags; | |
| 47 | + | |
| 48 | + /* | |
| 49 | + * Bounce I/O errors to a workqueue to avoid nested i_lock acquisitions | |
| 50 | + * in the fserror code. The caller no longer owns the bio reference | |
| 51 | + * after the spinlock drops. | |
| 52 | + */ | |
| 53 | + spin_lock_irqsave(&failed_read_lock, flags); | |
| 54 | + if (bio_list_empty(&failed_read_list)) | |
| 55 | + WARN_ON_ONCE(!schedule_work(&failed_read_work)); | |
| 56 | + bio_list_add(&failed_read_list, bio); | |
| 57 | + spin_unlock_irqrestore(&failed_read_lock, flags); | |
| 58 | +} | |
| 59 | + | |
| 60 | +static void iomap_read_end_io(struct bio *bio) | |
| 61 | +{ | |
| 62 | + if (bio->bi_status) { | |
| 63 | + iomap_fail_buffered_read(bio); | |
| 64 | + return; | |
| 65 | + } | |
| 66 | + | |
| 67 | + __iomap_read_end_io(bio); | |
| 68 | +} | |
| 69 | + | |
| 21 | 70 | static void iomap_bio_submit_read(struct iomap_read_folio_ctx *ctx) |
| 22 | 71 | { |
| 23 | 72 | struct bio *bio = ctx->read_ctx; |
| @@ -514,6 +514,7 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, · 513 unmodified lines | ||
| 514 | 514 | loff_t length = iomap_length(iter); |
| 515 | 515 | struct folio *folio = ctx->cur_folio; |
| 516 | 516 | size_t folio_len = folio_size(folio); |
| 517 | + struct iomap_folio_state *ifs; | |
| 517 | 518 | size_t poff, plen; |
| 518 | 519 | loff_t pos_diff; |
| 519 | 520 | int ret; |
| @@ -525,7 +526,7 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, · 5 unmodified lines | ||
| 525 | 526 | return iomap_iter_advance(iter, length); |
| 526 | 527 | } |
| 527 | 528 | |
| 528 | − ifs_alloc(iter->inode, folio, iter->flags); | |
| 529 | + ifs = ifs_alloc(iter->inode, folio, iter->flags); | |
| 529 | 530 | |
| 530 | 531 | length = min_t(loff_t, length, folio_len - offset_in_folio(folio, pos)); |
| 531 | 532 | while (length) { |
| @@ -560,11 +561,15 @@ static int iomap_read_folio_iter(struct iomap_iter *iter, · 28 unmodified lines | ||
| 560 | 561 | |
| 561 | 562 | *bytes_submitted += plen; |
| 562 | 563 | /* |
| 563 | − * If the entire folio has been read in by the IO | |
| 564 | − * helper, then the helper owns the folio and will end | |
| 565 | − * the read on it. | |
| 564 | + * Hand off folio ownership to the IO helper when: | |
| 565 | + * 1) The entire folio has been submitted for IO, or | |
| 566 | + * 2) There is no ifs attached to the folio | |
| 567 | + * | |
| 568 | + * Case (2) occurs when 1 << i_blkbits matches the folio | |
| 569 | + * size but the underlying filesystem or block device | |
| 570 | + * uses a smaller granularity for IO. | |
| 566 | 571 | */ |
| 567 | − if (*bytes_submitted == folio_len) | |
| 572 | + if (*bytes_submitted == folio_len || !ifs) | |
| 568 | 573 | ctx->cur_folio = NULL; |
| 569 | 574 | } |
| 570 | 575 | |
| @@ -267,7 +267,15 @@ restart: · 266 unmodified lines | ||
| 267 | 267 | */ |
| 268 | 268 | BUFFER_TRACE(bh, "queue"); |
| 269 | 269 | get_bh(bh); |
| 270 | − J_ASSERT_BH(bh, !buffer_jwrite(bh)); | |
| 270 | + if (WARN_ON_ONCE(buffer_jwrite(bh))) { | |
| 271 | + put_bh(bh); /* drop the ref we just took */ | |
| 272 | + spin_unlock(&journal->j_list_lock); | |
| 273 | + /* Clean up any previously batched buffers */ | |
| 274 | + if (batch_count) | |
| 275 | + __flush_batch(journal, &batch_count); | |
| 276 | + jbd2_journal_abort(journal, -EFSCORRUPTED); | |
| 277 | + return -EFSCORRUPTED; | |
| 278 | + } | |
| 271 | 279 | journal->j_chkpt_bhs[batch_count++] = bh; |
| 272 | 280 | transaction->t_chp_stats.cs_written++; |
| 273 | 281 | transaction->t_checkpoint_list = jh->b_cpnext; |
| @@ -325,7 +333,10 @@ int jbd2_cleanup_journal_tail(journal_t *journal) · 51 unmodified lines | ||
| 325 | 333 | |
| 326 | 334 | if (!jbd2_journal_get_log_tail(journal, &first_tid, &blocknr)) |
| 327 | 335 | return 1; |
| 328 | − J_ASSERT(blocknr != 0); | |
| 336 | + if (WARN_ON_ONCE(blocknr == 0)) { | |
| 337 | + jbd2_journal_abort(journal, -EFSCORRUPTED); | |
| 338 | + return -EFSCORRUPTED; | |
| 339 | + } | |
| 329 | 340 | |
| 330 | 341 | /* |
| 331 | 342 | * We need to make sure that any blocks that were recently written out |
| @@ -2437,8 +2437,14 @@ inside: · 2436 unmodified lines | ||
| 2437 | 2437 | EXPORT_SYMBOL(hashlen_string); |
| 2438 | 2438 | |
| 2439 | 2439 | /* |
| 2440 | − * Calculate the length and hash of the path component, and | |
| 2441 | − * return the length as the result. | |
| 2440 | + * hash_name - Calculate the length and hash of the path component | |
| 2441 | + * @nd: the path resolution state | |
| 2442 | + * @name: the pathname to read the component from | |
| 2443 | + * @lastword: if the component fits in a single word, LAST_WORD_IS_DOT, | |
| 2444 | + * LAST_WORD_IS_DOTDOT, or some other value depending on whether the | |
| 2445 | + * component is '.', '..', or something else. Otherwise, @lastword is 0. | |
| 2446 | + * | |
| 2447 | + * Returns: a pointer to the terminating '/' or NUL character in @name. | |
| 2442 | 2448 | */ |
| 2443 | 2449 | static inline const char *hash_name(struct nameidata *nd, |
| 2444 | 2450 | const char *name, |
| @@ -171,9 +171,8 @@ static void netfs_queue_read(struct netfs_io_request *rreq, · 170 unmodified lines | ||
| 171 | 171 | spin_lock(&rreq->lock); |
| 172 | 172 | list_add_tail(&subreq->rreq_link, &stream->subrequests); |
| 173 | 173 | if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { |
| 174 | − stream->front = subreq; | |
| 175 | 174 | if (!stream->active) { |
| 176 | − stream->collected_to = stream->front->start; | |
| 175 | + stream->collected_to = subreq->start; | |
| 177 | 176 | /* Store list pointers before active flag */ |
| 178 | 177 | smp_store_release(&stream->active, true); |
| 179 | 178 | } |
| @@ -71,9 +71,8 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) · 70 unmodified lines | ||
| 71 | 71 | spin_lock(&rreq->lock); |
| 72 | 72 | list_add_tail(&subreq->rreq_link, &stream->subrequests); |
| 73 | 73 | if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { |
| 74 | − stream->front = subreq; | |
| 75 | 74 | if (!stream->active) { |
| 76 | − stream->collected_to = stream->front->start; | |
| 75 | + stream->collected_to = subreq->start; | |
| 77 | 76 | /* Store list pointers before active flag */ |
| 78 | 77 | smp_store_release(&stream->active, true); |
| 79 | 78 | } |
| @@ -111,7 +111,6 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) · 110 unmodified lines | ||
| 111 | 111 | netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred); |
| 112 | 112 | subreq = stream->construct; |
| 113 | 113 | stream->construct = NULL; |
| 114 | − stream->front = NULL; | |
| 115 | 114 | } |
| 116 | 115 | |
| 117 | 116 | /* Check if (re-)preparation failed. */ |
| @@ -186,10 +185,18 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) · 68 unmodified lines | ||
| 186 | 185 | stream->sreq_max_segs = INT_MAX; |
| 187 | 186 | |
| 188 | 187 | netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); |
| 189 | − stream->prepare_write(subreq); | |
| 190 | 188 | |
| 191 | − __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); | |
| 192 | − netfs_stat(&netfs_n_wh_retry_write_subreq); | |
| 189 | + if (stream->prepare_write) { | |
| 190 | + stream->prepare_write(subreq); | |
| 191 | + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); | |
| 192 | + netfs_stat(&netfs_n_wh_retry_write_subreq); | |
| 193 | + } else { | |
| 194 | + struct iov_iter source; | |
| 195 | + | |
| 196 | + netfs_reset_iter(subreq); | |
| 197 | + source = subreq->io_iter; | |
| 198 | + netfs_reissue_write(stream, subreq, &source); | |
| 199 | + } | |
| 193 | 200 | } |
| 194 | 201 | |
| 195 | 202 | netfs_unbuffered_write_done(wreq); |
| @@ -142,6 +142,47 @@ static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset, · 141 unmodified lines | ||
| 142 | 142 | return min(span, max_size); |
| 143 | 143 | } |
| 144 | 144 | |
| 145 | +/* | |
| 146 | + * Select the span of a kvec iterator we're going to use. Limit it by both | |
| 147 | + * maximum size and maximum number of segments. Returns the size of the span | |
| 148 | + * in bytes. | |
| 149 | + */ | |
| 150 | +static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset, | |
| 151 | + size_t max_size, size_t max_segs) | |
| 152 | +{ | |
| 153 | + const struct kvec *kvecs = iter->kvec; | |
| 154 | + unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0; | |
| 155 | + size_t len, span = 0, n = iter->count; | |
| 156 | + size_t skip = iter->iov_offset + start_offset; | |
| 157 | + | |
| 158 | + if (WARN_ON(!iov_iter_is_kvec(iter)) || | |
| 159 | + WARN_ON(start_offset > n) || | |
| 160 | + n == 0) | |
| 161 | + return 0; | |
| 162 | + | |
| 163 | + while (n && ix < nkv && skip) { | |
| 164 | + len = kvecs[ix].iov_len; | |
| 165 | + if (skip < len) | |
| 166 | + break; | |
| 167 | + skip -= len; | |
| 168 | + n -= len; | |
| 169 | + ix++; | |
| 170 | + } | |
| 171 | + | |
| 172 | + while (n && ix < nkv) { | |
| 173 | + len = min3(n, kvecs[ix].iov_len - skip, max_size); | |
| 174 | + span += len; | |
| 175 | + nsegs++; | |
| 176 | + ix++; | |
| 177 | + if (span >= max_size || nsegs >= max_segs) | |
| 178 | + break; | |
| 179 | + skip = 0; | |
| 180 | + n -= len; | |
| 181 | + } | |
| 182 | + | |
| 183 | + return min(span, max_size); | |
| 184 | +} | |
| 185 | + | |
| 145 | 186 | /* |
| 146 | 187 | * Select the span of an xarray iterator we're going to use. Limit it by both |
| 147 | 188 | * maximum size and maximum number of segments. It is assumed that segments |
| @@ -245,6 +286,8 @@ size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, · 97 unmodified lines | ||
| 245 | 286 | return netfs_limit_bvec(iter, start_offset, max_size, max_segs); |
| 246 | 287 | if (iov_iter_is_xarray(iter)) |
| 247 | 288 | return netfs_limit_xarray(iter, start_offset, max_size, max_segs); |
| 289 | + if (iov_iter_is_kvec(iter)) | |
| 290 | + return netfs_limit_kvec(iter, start_offset, max_size, max_segs); | |
| 248 | 291 | BUG(); |
| 249 | 292 | } |
| 250 | 293 | EXPORT_SYMBOL(netfs_limit_iter); |
| @@ -205,7 +205,8 @@ reassess: · 204 unmodified lines | ||
| 205 | 205 | * in progress. The issuer thread may be adding stuff to the tail |
| 206 | 206 | * whilst we're doing this. |
| 207 | 207 | */ |
| 208 | − front = READ_ONCE(stream->front); | |
| 208 | + front = list_first_entry_or_null(&stream->subrequests, | |
| 209 | + struct netfs_io_subrequest, rreq_link); | |
| 209 | 210 | while (front) { |
| 210 | 211 | size_t transferred; |
| 211 | 212 | |
| @@ -301,7 +302,6 @@ reassess: · 89 unmodified lines | ||
| 301 | 302 | list_del_init(&front->rreq_link); |
| 302 | 303 | front = list_first_entry_or_null(&stream->subrequests, |
| 303 | 304 | struct netfs_io_subrequest, rreq_link); |
| 304 | − stream->front = front; | |
| 305 | 305 | spin_unlock(&rreq->lock); |
| 306 | 306 | netfs_put_subrequest(remove, |
| 307 | 307 | notes & ABANDON_SREQ ? |
| @@ -93,8 +93,10 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) · 92 unmodified lines | ||
| 93 | 93 | from->start, from->transferred, from->len); |
| 94 | 94 | |
| 95 | 95 | if (test_bit(NETFS_SREQ_FAILED, &from->flags) || |
| 96 | − !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) | |
| 96 | + !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) { | |
| 97 | + subreq = from; | |
| 97 | 98 | goto abandon; |
| 99 | + } | |
| 98 | 100 | |
| 99 | 101 | list_for_each_continue(next, &stream->subrequests) { |
| 100 | 102 | subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); |
| @@ -178,6 +180,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) · 77 unmodified lines | ||
| 178 | 180 | if (subreq == to) |
| 179 | 181 | break; |
| 180 | 182 | } |
| 183 | + subreq = NULL; | |
| 181 | 184 | continue; |
| 182 | 185 | } |
| 183 | 186 | |
| @@ -107,7 +107,6 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) · 106 unmodified lines | ||
| 107 | 107 | spin_lock(&rreq->lock); |
| 108 | 108 | list_add_tail(&subreq->rreq_link, &stream->subrequests); |
| 109 | 109 | trace_netfs_sreq(subreq, netfs_sreq_trace_added); |
| 110 | − stream->front = subreq; | |
| 111 | 110 | /* Store list pointers before active flag */ |
| 112 | 111 | smp_store_release(&stream->active, true); |
| 113 | 112 | spin_unlock(&rreq->lock); |
| @@ -228,7 +228,8 @@ reassess_streams: · 227 unmodified lines | ||
| 228 | 228 | if (!smp_load_acquire(&stream->active)) |
| 229 | 229 | continue; |
| 230 | 230 | |
| 231 | − front = stream->front; | |
| 231 | + front = list_first_entry_or_null(&stream->subrequests, | |
| 232 | + struct netfs_io_subrequest, rreq_link); | |
| 232 | 233 | while (front) { |
| 233 | 234 | trace_netfs_collect_sreq(wreq, front); |
| 234 | 235 | //_debug("sreq [%x] %llx %zx/%zx", |
| @@ -279,7 +280,6 @@ reassess_streams: · 44 unmodified lines | ||
| 279 | 280 | list_del_init(&front->rreq_link); |
| 280 | 281 | front = list_first_entry_or_null(&stream->subrequests, |
| 281 | 282 | struct netfs_io_subrequest, rreq_link); |
| 282 | − stream->front = front; | |
| 283 | 283 | spin_unlock(&wreq->lock); |
| 284 | 284 | netfs_put_subrequest(remove, |
| 285 | 285 | notes & SAW_FAILURE ? |
| @@ -206,9 +206,8 @@ void netfs_prepare_write(struct netfs_io_request *wreq, · 205 unmodified lines | ||
| 206 | 206 | spin_lock(&wreq->lock); |
| 207 | 207 | list_add_tail(&subreq->rreq_link, &stream->subrequests); |
| 208 | 208 | if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { |
| 209 | − stream->front = subreq; | |
| 210 | 209 | if (!stream->active) { |
| 211 | − stream->collected_to = stream->front->start; | |
| 210 | + stream->collected_to = subreq->start; | |
| 212 | 211 | /* Write list pointers before active flag */ |
| 213 | 212 | smp_store_release(&stream->active, true); |
| 214 | 213 | } |
| @@ -1146,15 +1146,15 @@ static int ovl_copy_up_one(struct dentry *parent, struct dentry *dentry, · 1145 unmodified lines | ||
| 1146 | 1146 | return -EOVERFLOW; |
| 1147 | 1147 | |
| 1148 | 1148 | /* |
| 1149 | − * With metacopy disabled, we fsync after final metadata copyup, for | |
| 1149 | + * With "fsync=strict", we fsync after final metadata copyup, for | |
| 1150 | 1150 | * both regular files and directories to get atomic copyup semantics |
| 1151 | 1151 | * on filesystems that do not use strict metadata ordering (e.g. ubifs). |
| 1152 | 1152 | * |
| 1153 | − * With metacopy enabled we want to avoid fsync on all meta copyup | |
| 1153 | + * By default, we want to avoid fsync on all meta copyup, because | |
| 1154 | 1154 | * that will hurt performance of workloads such as chown -R, so we |
| 1155 | 1155 | * only fsync on data copyup as legacy behavior. |
| 1156 | 1156 | */ |
| 1157 | − ctx.metadata_fsync = !OVL_FS(dentry->d_sb)->config.metacopy && | |
| 1157 | + ctx.metadata_fsync = ovl_should_sync_metadata(OVL_FS(dentry->d_sb)) && | |
| 1158 | 1158 | (S_ISREG(ctx.stat.mode) || S_ISDIR(ctx.stat.mode)); |
| 1159 | 1159 | ctx.metacopy = ovl_need_meta_copy_up(dentry, ctx.stat.mode, flags); |
| 1160 | 1160 | |
| @@ -99,6 +99,12 @@ enum { · 98 unmodified lines | ||
| 99 | 99 | OVL_VERITY_REQUIRE, |
| 100 | 100 | }; |
| 101 | 101 | |
| 102 | +enum { | |
| 103 | + OVL_FSYNC_VOLATILE, | |
| 104 | + OVL_FSYNC_AUTO, | |
| 105 | + OVL_FSYNC_STRICT, | |
| 106 | +}; | |
| 107 | + | |
| 102 | 108 | /* |
| 103 | 109 | * The tuple (fh,uuid) is a universal unique identifier for a copy up origin, |
| 104 | 110 | * where: |
| @@ -656,6 +662,21 @@ static inline bool ovl_xino_warn(struct ovl_fs *ofs) · 551 unmodified lines | ||
| 656 | 662 | return ofs->config.xino == OVL_XINO_ON; |
| 657 | 663 | } |
| 658 | 664 | |
| 665 | +static inline bool ovl_should_sync(struct ovl_fs *ofs) | |
| 666 | +{ | |
| 667 | + return ofs->config.fsync_mode != OVL_FSYNC_VOLATILE; | |
| 668 | +} | |
| 669 | + | |
| 670 | +static inline bool ovl_should_sync_metadata(struct ovl_fs *ofs) | |
| 671 | +{ | |
| 672 | + return ofs->config.fsync_mode == OVL_FSYNC_STRICT; | |
| 673 | +} | |
| 674 | + | |
| 675 | +static inline bool ovl_is_volatile(struct ovl_config *config) | |
| 676 | +{ | |
| 677 | + return config->fsync_mode == OVL_FSYNC_VOLATILE; | |
| 678 | +} | |
| 679 | + | |
| 659 | 680 | /* |
| 660 | 681 | * To avoid regressions in existing setups with overlay lower offline changes, |
| 661 | 682 | * we allow lower changes only if none of the new features are used. |
| @@ -18,7 +18,7 @@ struct ovl_config { · 17 unmodified lines | ||
| 18 | 18 | int xino; |
| 19 | 19 | bool metacopy; |
| 20 | 20 | bool userxattr; |
| 21 | − bool ovl_volatile; | |
| 21 | + int fsync_mode; | |
| 22 | 22 | }; |
| 23 | 23 | |
| 24 | 24 | struct ovl_sb { |
| @@ -120,11 +120,6 @@ static inline struct ovl_fs *OVL_FS(struct super_block *sb) · 95 unmodified lines | ||
| 120 | 120 | return (struct ovl_fs *)sb->s_fs_info; |
| 121 | 121 | } |
| 122 | 122 | |
| 123 | −static inline bool ovl_should_sync(struct ovl_fs *ofs) | |
| 124 | −{ | |
| 125 | − return !ofs->config.ovl_volatile; | |
| 126 | −} | |
| 127 | − | |
| 128 | 123 | static inline unsigned int ovl_numlower(struct ovl_entry *oe) |
| 129 | 124 | { |
| 130 | 125 | return oe ? oe->__numlower : 0; |
| @@ -58,6 +58,7 @@ enum ovl_opt { · 57 unmodified lines | ||
| 58 | 58 | Opt_xino, |
| 59 | 59 | Opt_metacopy, |
| 60 | 60 | Opt_verity, |
| 61 | + Opt_fsync, | |
| 61 | 62 | Opt_volatile, |
| 62 | 63 | Opt_override_creds, |
| 63 | 64 | }; |
| @@ -140,6 +141,23 @@ static int ovl_verity_mode_def(void) · 76 unmodified lines | ||
| 140 | 141 | return OVL_VERITY_OFF; |
| 141 | 142 | } |
| 142 | 143 | |
| 144 | +static const struct constant_table ovl_parameter_fsync[] = { | |
| 145 | + { "volatile", OVL_FSYNC_VOLATILE }, | |
| 146 | + { "auto", OVL_FSYNC_AUTO }, | |
| 147 | + { "strict", OVL_FSYNC_STRICT }, | |
| 148 | + {} | |
| 149 | +}; | |
| 150 | + | |
| 151 | +static const char *ovl_fsync_mode(struct ovl_config *config) | |
| 152 | +{ | |
| 153 | + return ovl_parameter_fsync[config->fsync_mode].name; | |
| 154 | +} | |
| 155 | + | |
| 156 | +static int ovl_fsync_mode_def(void) | |
| 157 | +{ | |
| 158 | + return OVL_FSYNC_AUTO; | |
| 159 | +} | |
| 160 | + | |
| 143 | 161 | const struct fs_parameter_spec ovl_parameter_spec[] = { |
| 144 | 162 | fsparam_string_empty("lowerdir", Opt_lowerdir), |
| 145 | 163 | fsparam_file_or_string("lowerdir+", Opt_lowerdir_add), |
| @@ -155,6 +173,7 @@ const struct fs_parameter_spec ovl_parameter_spec[] = { · 9 unmodified lines | ||
| 155 | 173 | fsparam_enum("xino", Opt_xino, ovl_parameter_xino), |
| 156 | 174 | fsparam_enum("metacopy", Opt_metacopy, ovl_parameter_bool), |
| 157 | 175 | fsparam_enum("verity", Opt_verity, ovl_parameter_verity), |
| 176 | + fsparam_enum("fsync", Opt_fsync, ovl_parameter_fsync), | |
| 158 | 177 | fsparam_flag("volatile", Opt_volatile), |
| 159 | 178 | fsparam_flag_no("override_creds", Opt_override_creds), |
| 160 | 179 | {} |
| @@ -665,8 +684,11 @@ static int ovl_parse_param(struct fs_context *fc, struct fs_parameter *param) · 504 unmodified lines | ||
| 665 | 684 | case Opt_verity: |
| 666 | 685 | config->verity_mode = result.uint_32; |
| 667 | 686 | break; |
| 687 | + case Opt_fsync: | |
| 688 | + config->fsync_mode = result.uint_32; | |
| 689 | + break; | |
| 668 | 690 | case Opt_volatile: |
| 669 | − config->ovl_volatile = true; | |
| 691 | + config->fsync_mode = OVL_FSYNC_VOLATILE; | |
| 670 | 692 | break; |
| 671 | 693 | case Opt_userxattr: |
| 672 | 694 | config->userxattr = true; |
| @@ -800,6 +822,7 @@ int ovl_init_fs_context(struct fs_context *fc) · 127 unmodified lines | ||
| 800 | 822 | ofs->config.nfs_export = ovl_nfs_export_def; |
| 801 | 823 | ofs->config.xino = ovl_xino_def(); |
| 802 | 824 | ofs->config.metacopy = ovl_metacopy_def; |
| 825 | + ofs->config.fsync_mode = ovl_fsync_mode_def(); | |
| 803 | 826 | |
| 804 | 827 | fc->s_fs_info = ofs; |
| 805 | 828 | fc->fs_private = ctx; |
| @@ -870,9 +893,9 @@ int ovl_fs_params_verify(const struct ovl_fs_context *ctx, · 64 unmodified lines | ||
| 870 | 893 | config->index = false; |
| 871 | 894 | } |
| 872 | 895 | |
| 873 | − if (!config->upperdir && config->ovl_volatile) { | |
| 896 | + if (!config->upperdir && ovl_is_volatile(config)) { | |
| 874 | 897 | pr_info("option \"volatile\" is meaningless in a non-upper mount, ignoring it.\n"); |
| 875 | − config->ovl_volatile = false; | |
| 898 | + config->fsync_mode = ovl_fsync_mode_def(); | |
| 876 | 899 | } |
| 877 | 900 | |
| 878 | 901 | if (!config->upperdir && config->uuid == OVL_UUID_ON) { |
| @@ -1070,8 +1093,8 @@ int ovl_show_options(struct seq_file *m, struct dentry *dentry) · 191 unmodified lines | ||
| 1070 | 1093 | seq_printf(m, ",xino=%s", ovl_xino_mode(&ofs->config)); |
| 1071 | 1094 | if (ofs->config.metacopy != ovl_metacopy_def) |
| 1072 | 1095 | seq_printf(m, ",metacopy=%s", str_on_off(ofs->config.metacopy)); |
| 1073 | − if (ofs->config.ovl_volatile) | |
| 1074 | − seq_puts(m, ",volatile"); | |
| 1096 | + if (ofs->config.fsync_mode != ovl_fsync_mode_def()) | |
| 1097 | + seq_printf(m, ",fsync=%s", ovl_fsync_mode(&ofs->config)); | |
| 1075 | 1098 | if (ofs->config.userxattr) |
| 1076 | 1099 | seq_puts(m, ",userxattr"); |
| 1077 | 1100 | if (ofs->config.verity_mode != ovl_verity_mode_def()) |
| @@ -776,7 +776,7 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, · 775 unmodified lines | ||
| 776 | 776 | * For volatile mount, create a incompat/volatile/dirty file to keep |
| 777 | 777 | * track of it. |
| 778 | 778 | */ |
| 779 | − if (ofs->config.ovl_volatile) { | |
| 779 | + if (ovl_is_volatile(&ofs->config)) { | |
| 780 | 780 | err = ovl_create_volatile_dirty(ofs); |
| 781 | 781 | if (err < 0) { |
| 782 | 782 | pr_err("Failed to create volatile/dirty file.\n"); |
| @@ -85,7 +85,10 @@ int ovl_can_decode_fh(struct super_block *sb) · 84 unmodified lines | ||
| 85 | 85 | if (!exportfs_can_decode_fh(sb->s_export_op)) |
| 86 | 86 | return 0; |
| 87 | 87 | |
| 88 | − return sb->s_export_op->encode_fh ? -1 : FILEID_INO32_GEN; | |
| 88 | + if (sb->s_export_op->encode_fh == generic_encode_ino32_fh) | |
| 89 | + return FILEID_INO32_GEN; | |
| 90 | + | |
| 91 | + return -1; | |
| 89 | 92 | } |
| 90 | 93 | |
| 91 | 94 | struct dentry *ovl_indexdir(struct super_block *sb) |
| @@ -48,8 +48,8 @@ cifs-$(CONFIG_CIFS_COMPRESSION) += compress.o compress/lz77.o · 47 unmodified lines | ||
| 48 | 48 | # Build the SMB2 error mapping table from smb2status.h |
| 49 | 49 | # |
| 50 | 50 | $(obj)/smb2_mapping_table.c: $(src)/../common/smb2status.h \ |
| 51 | − $(src)/gen_smb2_mapping | |
| 52 | − $(call cmd,gen_smb2_mapping) | |
| 51 | + $(src)/gen_smb2_mapping FORCE | |
| 52 | + $(call if_changed,gen_smb2_mapping) | |
| 53 | 53 | |
| 54 | 54 | $(obj)/smb2maperror.o: $(obj)/smb2_mapping_table.c |
| 55 | 55 | |
| @@ -58,4 +58,5 @@ quiet_cmd_gen_smb2_mapping = GEN $@ · 2 unmodified lines | ||
| 58 | 58 | |
| 59 | 59 | obj-$(CONFIG_SMB_KUNIT_TESTS) += smb2maperror_test.o |
| 60 | 60 | |
| 61 | −clean-files += smb2_mapping_table.c | |
| 61 | +# Let Kbuild handle tracking and cleaning | |
| 62 | +targets += smb2_mapping_table.c | |
| @@ -82,11 +82,19 @@ static void lease_del_list(struct oplock_info *opinfo) · 81 unmodified lines | ||
| 82 | 82 | spin_unlock(&lb->lb_lock); |
| 83 | 83 | } |
| 84 | 84 | |
| 85 | −static void lb_add(struct lease_table *lb) | |
| 85 | +static struct lease_table *alloc_lease_table(struct oplock_info *opinfo) | |
| 86 | 86 | { |
| 87 | − write_lock(&lease_list_lock); | |
| 88 | − list_add(&lb->l_entry, &lease_table_list); | |
| 89 | − write_unlock(&lease_list_lock); | |
| 87 | + struct lease_table *lb; | |
| 88 | + | |
| 89 | + lb = kmalloc_obj(struct lease_table, KSMBD_DEFAULT_GFP); | |
| 90 | + if (!lb) | |
| 91 | + return NULL; | |
| 92 | + | |
| 93 | + memcpy(lb->client_guid, opinfo->conn->ClientGUID, | |
| 94 | + SMB2_CLIENT_GUID_SIZE); | |
| 95 | + INIT_LIST_HEAD(&lb->lease_list); | |
| 96 | + spin_lock_init(&lb->lb_lock); | |
| 97 | + return lb; | |
| 90 | 98 | } |
| 91 | 99 | |
| 92 | 100 | static int alloc_lease(struct oplock_info *opinfo, struct lease_ctx_info *lctx) |
| @@ -1042,34 +1050,27 @@ static void copy_lease(struct oplock_info *op1, struct oplock_info *op2) · 949 unmodified lines | ||
| 1042 | 1050 | lease2->version = lease1->version; |
| 1043 | 1051 | } |
| 1044 | 1052 | |
| 1045 | −static int add_lease_global_list(struct oplock_info *opinfo) | |
| 1053 | +static void add_lease_global_list(struct oplock_info *opinfo, | |
| 1054 | + struct lease_table *new_lb) | |
| 1046 | 1055 | { |
| 1047 | 1056 | struct lease_table *lb; |
| 1048 | 1057 | |
| 1049 | − read_lock(&lease_list_lock); | |
| 1058 | + write_lock(&lease_list_lock); | |
| 1050 | 1059 | list_for_each_entry(lb, &lease_table_list, l_entry) { |
| 1051 | 1060 | if (!memcmp(lb->client_guid, opinfo->conn->ClientGUID, |
| 1052 | 1061 | SMB2_CLIENT_GUID_SIZE)) { |
| 1053 | 1062 | opinfo->o_lease->l_lb = lb; |
| 1054 | 1063 | lease_add_list(opinfo); |
| 1055 | − read_unlock(&lease_list_lock); | |
| 1056 | − return 0; | |
| 1064 | + write_unlock(&lease_list_lock); | |
| 1065 | + kfree(new_lb); | |
| 1066 | + return; | |
| 1057 | 1067 | } |
| 1058 | 1068 | } |
| 1059 | − read_unlock(&lease_list_lock); | |
| 1060 | 1069 | |
| 1061 | − lb = kmalloc_obj(struct lease_table, KSMBD_DEFAULT_GFP); | |
| 1062 | − if (!lb) | |
| 1063 | − return -ENOMEM; | |
| 1064 | − | |
| 1065 | − memcpy(lb->client_guid, opinfo->conn->ClientGUID, | |
| 1066 | − SMB2_CLIENT_GUID_SIZE); | |
| 1067 | − INIT_LIST_HEAD(&lb->lease_list); | |
| 1068 | − spin_lock_init(&lb->lb_lock); | |
| 1069 | − opinfo->o_lease->l_lb = lb; | |
| 1070 | + opinfo->o_lease->l_lb = new_lb; | |
| 1070 | 1071 | lease_add_list(opinfo); |
| 1071 | − lb_add(lb); | |
| 1072 | − return 0; | |
| 1072 | + list_add(&new_lb->l_entry, &lease_table_list); | |
| 1073 | + write_unlock(&lease_list_lock); | |
| 1073 | 1074 | } |
| 1074 | 1075 | |
| 1075 | 1076 | static void set_oplock_level(struct oplock_info *opinfo, int level, |
| @@ -1189,6 +1190,7 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, · 113 unmodified lines | ||
| 1189 | 1190 | int err = 0; |
| 1190 | 1191 | struct oplock_info *opinfo = NULL, *prev_opinfo = NULL; |
| 1191 | 1192 | struct ksmbd_inode *ci = fp->f_ci; |
| 1193 | + struct lease_table *new_lb = NULL; | |
| 1192 | 1194 | bool prev_op_has_lease; |
| 1193 | 1195 | __le32 prev_op_state = 0; |
| 1194 | 1196 | |
| @@ -1291,21 +1293,37 @@ set_lev: · 96 unmodified lines | ||
| 1291 | 1293 | set_oplock_level(opinfo, req_op_level, lctx); |
| 1292 | 1294 | |
| 1293 | 1295 | out: |
| 1294 | − opinfo_count_inc(fp); | |
| 1295 | − opinfo_add(opinfo, fp); | |
| 1296 | − | |
| 1296 | + /* | |
| 1297 | + * Set o_fp before any publication so that concurrent readers | |
| 1298 | + * (e.g. find_same_lease_key() on the lease list) that | |
| 1299 | + * dereference opinfo->o_fp don't hit a NULL pointer. | |
| 1300 | + * | |
| 1301 | + * Keep the original publication order so concurrent opens can | |
| 1302 | + * still observe the in-flight grant via ci->m_op_list, but make | |
| 1303 | + * everything after opinfo_add() no-fail by preallocating any new | |
| 1304 | + * lease_table first. | |
| 1305 | + */ | |
| 1306 | + opinfo->o_fp = fp; | |
| 1297 | 1307 | if (opinfo->is_lease) { |
| 1298 | − err = add_lease_global_list(opinfo); | |
| 1299 | − if (err) | |
| 1308 | + new_lb = alloc_lease_table(opinfo); | |
| 1309 | + if (!new_lb) { | |
| 1310 | + err = -ENOMEM; | |
| 1300 | 1311 | goto err_out; |
| 1312 | + } | |
| 1301 | 1313 | } |
| 1302 | 1314 | |
| 1315 | + opinfo_count_inc(fp); | |
| 1316 | + opinfo_add(opinfo, fp); | |
| 1317 | + | |
| 1318 | + if (opinfo->is_lease) | |
| 1319 | + add_lease_global_list(opinfo, new_lb); | |
| 1320 | + | |
| 1303 | 1321 | rcu_assign_pointer(fp->f_opinfo, opinfo); |
| 1304 | − opinfo->o_fp = fp; | |
| 1305 | 1322 | |
| 1306 | 1323 | return 0; |
| 1307 | 1324 | err_out: |
| 1308 | − __free_opinfo(opinfo); | |
| 1325 | + kfree(new_lb); | |
| 1326 | + opinfo_put(opinfo); | |
| 1309 | 1327 | return err; |
| 1310 | 1328 | } |
| 1311 | 1329 | |
| @@ -1939,8 +1939,14 @@ out_err: · 1938 unmodified lines | ||
| 1939 | 1939 | if (sess->user && sess->user->flags & KSMBD_USER_FLAG_DELAY_SESSION) |
| 1940 | 1940 | try_delay = true; |
| 1941 | 1941 | |
| 1942 | − sess->last_active = jiffies; | |
| 1943 | − sess->state = SMB2_SESSION_EXPIRED; | |
| 1942 | + /* | |
| 1943 | + * For binding requests, session belongs to another | |
| 1944 | + * connection. Do not expire it. | |
| 1945 | + */ | |
| 1946 | + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { | |
| 1947 | + sess->last_active = jiffies; | |
| 1948 | + sess->state = SMB2_SESSION_EXPIRED; | |
| 1949 | + } | |
| 1944 | 1950 | ksmbd_user_session_put(sess); |
| 1945 | 1951 | work->sess = NULL; |
| 1946 | 1952 | if (try_delay) { |
| @@ -4446,8 +4452,9 @@ int smb2_query_dir(struct ksmbd_work *work) · 2499 unmodified lines | ||
| 4446 | 4452 | d_info.wptr = (char *)rsp->Buffer; |
| 4447 | 4453 | d_info.rptr = (char *)rsp->Buffer; |
| 4448 | 4454 | d_info.out_buf_len = |
| 4449 | − smb2_calc_max_out_buf_len(work, 8, | |
| 4450 | − le32_to_cpu(req->OutputBufferLength)); | |
| 4455 | + smb2_calc_max_out_buf_len(work, | |
| 4456 | + offsetof(struct smb2_query_directory_rsp, Buffer), | |
| 4457 | + le32_to_cpu(req->OutputBufferLength)); | |
| 4451 | 4458 | if (d_info.out_buf_len < 0) { |
| 4452 | 4459 | rc = -EINVAL; |
| 4453 | 4460 | goto err_out; |
| @@ -4714,8 +4721,9 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, · 260 unmodified lines | ||
| 4714 | 4721 | } |
| 4715 | 4722 | |
| 4716 | 4723 | buf_free_len = |
| 4717 | − smb2_calc_max_out_buf_len(work, 8, | |
| 4718 | − le32_to_cpu(req->OutputBufferLength)); | |
| 4724 | + smb2_calc_max_out_buf_len(work, | |
| 4725 | + offsetof(struct smb2_query_info_rsp, Buffer), | |
| 4726 | + le32_to_cpu(req->OutputBufferLength)); | |
| 4719 | 4727 | if (buf_free_len < 0) |
| 4720 | 4728 | return -EINVAL; |
| 4721 | 4729 | |
| @@ -4932,7 +4940,8 @@ static int get_file_all_info(struct ksmbd_work *work, · 210 unmodified lines | ||
| 4932 | 4940 | int conv_len; |
| 4933 | 4941 | char *filename; |
| 4934 | 4942 | u64 time; |
| 4935 | − int ret; | |
| 4943 | + int ret, buf_free_len, filename_len; | |
| 4944 | + struct smb2_query_info_req *req = ksmbd_req_buf_next(work); | |
| 4936 | 4945 | |
| 4937 | 4946 | if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { |
| 4938 | 4947 | ksmbd_debug(SMB, "no right to read the attributes : 0x%x\n", |
| @@ -4944,6 +4953,16 @@ static int get_file_all_info(struct ksmbd_work *work, · 5 unmodified lines | ||
| 4944 | 4953 | if (IS_ERR(filename)) |
| 4945 | 4954 | return PTR_ERR(filename); |
| 4946 | 4955 | |
| 4956 | + filename_len = strlen(filename); | |
| 4957 | + buf_free_len = smb2_calc_max_out_buf_len(work, | |
| 4958 | + offsetof(struct smb2_query_info_rsp, Buffer) + | |
| 4959 | + offsetof(struct smb2_file_all_info, FileName), | |
| 4960 | + le32_to_cpu(req->OutputBufferLength)); | |
| 4961 | + if (buf_free_len < (filename_len + 1) * 2) { | |
| 4962 | + kfree(filename); | |
| 4963 | + return -EINVAL; | |
| 4964 | + } | |
| 4965 | + | |
| 4947 | 4966 | ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, |
| 4948 | 4967 | AT_STATX_SYNC_AS_STAT); |
| 4949 | 4968 | if (ret) { |
| @@ -4987,7 +5006,8 @@ static int get_file_all_info(struct ksmbd_work *work, · 37 unmodified lines | ||
| 4987 | 5006 | file_info->Mode = fp->coption; |
| 4988 | 5007 | file_info->AlignmentRequirement = 0; |
| 4989 | 5008 | conv_len = smbConvertToUTF16((__le16 *)file_info->FileName, filename, |
| 4990 | − PATH_MAX, conn->local_nls, 0); | |
| 5009 | + min(filename_len, PATH_MAX), | |
| 5010 | + conn->local_nls, 0); | |
| 4991 | 5011 | conv_len *= 2; |
| 4992 | 5012 | file_info->FileNameLength = cpu_to_le32(conv_len); |
| 4993 | 5013 | rsp->OutputBufferLength = |
| @@ -5041,8 +5061,9 @@ static int get_file_stream_info(struct ksmbd_work *work, · 47 unmodified lines | ||
| 5041 | 5061 | file_info = (struct smb2_file_stream_info *)rsp->Buffer; |
| 5042 | 5062 | |
| 5043 | 5063 | buf_free_len = |
| 5044 | − smb2_calc_max_out_buf_len(work, 8, | |
| 5045 | − le32_to_cpu(req->OutputBufferLength)); | |
| 5064 | + smb2_calc_max_out_buf_len(work, | |
| 5065 | + offsetof(struct smb2_query_info_rsp, Buffer), | |
| 5066 | + le32_to_cpu(req->OutputBufferLength)); | |
| 5046 | 5067 | if (buf_free_len < 0) |
| 5047 | 5068 | goto out; |
| 5048 | 5069 | |
| @@ -7586,14 +7607,15 @@ retry: · 2537 unmodified lines | ||
| 7586 | 7607 | rc = vfs_lock_file(filp, smb_lock->cmd, flock, NULL); |
| 7587 | 7608 | skip: |
| 7588 | 7609 | if (smb_lock->flags & SMB2_LOCKFLAG_UNLOCK) { |
| 7610 | + locks_free_lock(flock); | |
| 7611 | + kfree(smb_lock); | |
| 7589 | 7612 | if (!rc) { |
| 7590 | 7613 | ksmbd_debug(SMB, "File unlocked\n"); |
| 7591 | 7614 | } else if (rc == -ENOENT) { |
| 7592 | 7615 | rsp->hdr.Status = STATUS_NOT_LOCKED; |
| 7616 | + err = rc; | |
| 7593 | 7617 | goto out; |
| 7594 | 7618 | } |
| 7595 | − locks_free_lock(flock); | |
| 7596 | − kfree(smb_lock); | |
| 7597 | 7619 | } else { |
| 7598 | 7620 | if (rc == FILE_LOCK_DEFERRED) { |
| 7599 | 7621 | void **argv; |
| @@ -7662,6 +7684,9 @@ skip: · 62 unmodified lines | ||
| 7662 | 7684 | spin_unlock(&work->conn->llist_lock); |
| 7663 | 7685 | ksmbd_debug(SMB, "successful in taking lock\n"); |
| 7664 | 7686 | } else { |
| 7687 | + locks_free_lock(flock); | |
| 7688 | + kfree(smb_lock); | |
| 7689 | + err = rc; | |
| 7665 | 7690 | goto out; |
| 7666 | 7691 | } |
| 7667 | 7692 | } |
| @@ -7692,13 +7717,17 @@ out: · 24 unmodified lines | ||
| 7692 | 7717 | struct file_lock *rlock = NULL; |
| 7693 | 7718 | |
| 7694 | 7719 | rlock = smb_flock_init(filp); |
| 7695 | − rlock->c.flc_type = F_UNLCK; | |
| 7696 | − rlock->fl_start = smb_lock->start; | |
| 7697 | − rlock->fl_end = smb_lock->end; | |
| 7720 | + if (rlock) { | |
| 7721 | + rlock->c.flc_type = F_UNLCK; | |
| 7722 | + rlock->fl_start = smb_lock->start; | |
| 7723 | + rlock->fl_end = smb_lock->end; | |
| 7698 | 7724 | |
| 7699 | − rc = vfs_lock_file(filp, F_SETLK, rlock, NULL); | |
| 7700 | − if (rc) | |
| 7701 | − pr_err("rollback unlock fail : %d\n", rc); | |
| 7725 | + rc = vfs_lock_file(filp, F_SETLK, rlock, NULL); | |
| 7726 | + if (rc) | |
| 7727 | + pr_err("rollback unlock fail : %d\n", rc); | |
| 7728 | + } else { | |
| 7729 | + pr_err("rollback unlock alloc failed\n"); | |
| 7730 | + } | |
| 7702 | 7731 | |
| 7703 | 7732 | list_del(&smb_lock->llist); |
| 7704 | 7733 | spin_lock(&work->conn->llist_lock); |
| @@ -7708,7 +7737,8 @@ out: · 3 unmodified lines | ||
| 7708 | 7737 | spin_unlock(&work->conn->llist_lock); |
| 7709 | 7738 | |
| 7710 | 7739 | locks_free_lock(smb_lock->fl); |
| 7711 | − locks_free_lock(rlock); | |
| 7740 | + if (rlock) | |
| 7741 | + locks_free_lock(rlock); | |
| 7712 | 7742 | kfree(smb_lock); |
| 7713 | 7743 | } |
| 7714 | 7744 | out2: |
| @@ -8191,8 +8221,9 @@ int smb2_ioctl(struct ksmbd_work *work) · 476 unmodified lines | ||
| 8191 | 8221 | buffer = (char *)req + le32_to_cpu(req->InputOffset); |
| 8192 | 8222 | |
| 8193 | 8223 | cnt_code = le32_to_cpu(req->CtlCode); |
| 8194 | − ret = smb2_calc_max_out_buf_len(work, 48, | |
| 8195 | − le32_to_cpu(req->MaxOutputResponse)); | |
| 8224 | + ret = smb2_calc_max_out_buf_len(work, | |
| 8225 | + offsetof(struct smb2_ioctl_rsp, Buffer), | |
| 8226 | + le32_to_cpu(req->MaxOutputResponse)); | |
| 8196 | 8227 | if (ret < 0) { |
| 8197 | 8228 | rsp->hdr.Status = STATUS_INVALID_PARAMETER; |
| 8198 | 8229 | goto out; |
| @@ -55,7 +55,8 @@ struct xfs_attr_list_context { · 54 unmodified lines | ||
| 55 | 55 | struct xfs_trans *tp; |
| 56 | 56 | struct xfs_inode *dp; /* inode */ |
| 57 | 57 | struct xfs_attrlist_cursor_kern cursor; /* position in list */ |
| 58 | − void *buffer; /* output buffer */ | |
| 58 | + /* output buffer */ | |
| 59 | + void *buffer __counted_by_ptr(bufsize); | |
| 59 | 60 | |
| 60 | 61 | /* |
| 61 | 62 | * Abort attribute list iteration if non-zero. Can be used to pass |
| @@ -1415,6 +1415,28 @@ xfs_attr3_leaf_create( · 1414 unmodified lines | ||
| 1415 | 1415 | return 0; |
| 1416 | 1416 | } |
| 1417 | 1417 | |
| 1418 | +/* | |
| 1419 | + * Reinitialize an existing attr fork block as an empty leaf, and attach | |
| 1420 | + * the buffer to tp. | |
| 1421 | + */ | |
| 1422 | +int | |
| 1423 | +xfs_attr3_leaf_init( | |
| 1424 | + struct xfs_trans *tp, | |
| 1425 | + struct xfs_inode *dp, | |
| 1426 | + xfs_dablk_t blkno) | |
| 1427 | +{ | |
| 1428 | + struct xfs_buf *bp = NULL; | |
| 1429 | + struct xfs_da_args args = { | |
| 1430 | + .trans = tp, | |
| 1431 | + .dp = dp, | |
| 1432 | + .owner = dp->i_ino, | |
| 1433 | + .geo = dp->i_mount->m_attr_geo, | |
| 1434 | + }; | |
| 1435 | + | |
| 1436 | + ASSERT(tp != NULL); | |
| 1437 | + | |
| 1438 | + return xfs_attr3_leaf_create(&args, blkno, &bp); | |
| 1439 | +} | |
| 1418 | 1440 | /* |
| 1419 | 1441 | * Split the leaf node, rebalance, then add the new entry. |
| 1420 | 1442 | * |
| @@ -87,6 +87,9 @@ int xfs_attr3_leaf_list_int(struct xfs_buf *bp, · 86 unmodified lines | ||
| 87 | 87 | /* |
| 88 | 88 | * Routines used for shrinking the Btree. |
| 89 | 89 | */ |
| 90 | + | |
| 91 | +int xfs_attr3_leaf_init(struct xfs_trans *tp, struct xfs_inode *dp, | |
| 92 | + xfs_dablk_t blkno); | |
| 90 | 93 | int xfs_attr3_leaf_toosmall(struct xfs_da_state *state, int *retval); |
| 91 | 94 | void xfs_attr3_leaf_unbalance(struct xfs_da_state *state, |
| 92 | 95 | struct xfs_da_state_blk *drop_blk, |
| @@ -1506,21 +1506,20 @@ xfs_da3_fixhashpath( · 1505 unmodified lines | ||
| 1506 | 1506 | } |
| 1507 | 1507 | |
| 1508 | 1508 | /* |
| 1509 | − * Remove an entry from an intermediate node. | |
| 1509 | + * Internal implementation to remove an entry from an intermediate node. | |
| 1510 | 1510 | */ |
| 1511 | 1511 | STATIC void |
| 1512 | −xfs_da3_node_remove( | |
| 1513 | − struct xfs_da_state *state, | |
| 1514 | − struct xfs_da_state_blk *drop_blk) | |
| 1512 | +__xfs_da3_node_remove( | |
| 1513 | + struct xfs_trans *tp, | |
| 1514 | + struct xfs_inode *dp, | |
| 1515 | + struct xfs_da_geometry *geo, | |
| 1516 | + struct xfs_da_state_blk *drop_blk) | |
| 1515 | 1517 | { |
| 1516 | 1518 | struct xfs_da_intnode *node; |
| 1517 | 1519 | struct xfs_da3_icnode_hdr nodehdr; |
| 1518 | 1520 | struct xfs_da_node_entry *btree; |
| 1519 | 1521 | int index; |
| 1520 | 1522 | int tmp; |
| 1521 | − struct xfs_inode *dp = state->args->dp; | |
| 1522 | − | |
| 1523 | − trace_xfs_da_node_remove(state->args); | |
| 1524 | 1523 | |
| 1525 | 1524 | node = drop_blk->bp->b_addr; |
| 1526 | 1525 | xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); |
| @@ -1536,17 +1535,17 @@ xfs_da3_node_remove( · 9 unmodified lines | ||
| 1536 | 1535 | tmp = nodehdr.count - index - 1; |
| 1537 | 1536 | tmp *= (uint)sizeof(xfs_da_node_entry_t); |
| 1538 | 1537 | memmove(&btree[index], &btree[index + 1], tmp); |
| 1539 | − xfs_trans_log_buf(state->args->trans, drop_blk->bp, | |
| 1538 | + xfs_trans_log_buf(tp, drop_blk->bp, | |
| 1540 | 1539 | XFS_DA_LOGRANGE(node, &btree[index], tmp)); |
| 1541 | 1540 | index = nodehdr.count - 1; |
| 1542 | 1541 | } |
| 1543 | 1542 | memset(&btree[index], 0, sizeof(xfs_da_node_entry_t)); |
| 1544 | − xfs_trans_log_buf(state->args->trans, drop_blk->bp, | |
| 1543 | + xfs_trans_log_buf(tp, drop_blk->bp, | |
| 1545 | 1544 | XFS_DA_LOGRANGE(node, &btree[index], sizeof(btree[index]))); |
| 1546 | 1545 | nodehdr.count -= 1; |
| 1547 | 1546 | xfs_da3_node_hdr_to_disk(dp->i_mount, node, &nodehdr); |
| 1548 | − xfs_trans_log_buf(state->args->trans, drop_blk->bp, | |
| 1549 | − XFS_DA_LOGRANGE(node, &node->hdr, state->args->geo->node_hdr_size)); | |
| 1547 | + xfs_trans_log_buf(tp, drop_blk->bp, | |
| 1548 | + XFS_DA_LOGRANGE(node, &node->hdr, geo->node_hdr_size)); | |
| 1550 | 1549 | |
| 1551 | 1550 | /* |
| 1552 | 1551 | * Copy the last hash value from the block to propagate upwards. |
| @@ -1554,6 +1553,38 @@ xfs_da3_node_remove( · 1 unmodified line | ||
| 1554 | 1553 | drop_blk->hashval = be32_to_cpu(btree[index - 1].hashval); |
| 1555 | 1554 | } |
| 1556 | 1555 | |
| 1556 | +/* | |
| 1557 | + * Remove an entry from an intermediate node. | |
| 1558 | + */ | |
| 1559 | +STATIC void | |
| 1560 | +xfs_da3_node_remove( | |
| 1561 | + struct xfs_da_state *state, | |
| 1562 | + struct xfs_da_state_blk *drop_blk) | |
| 1563 | +{ | |
| 1564 | + trace_xfs_da_node_remove(state->args); | |
| 1565 | + | |
| 1566 | + __xfs_da3_node_remove(state->args->trans, state->args->dp, | |
| 1567 | + state->args->geo, drop_blk); | |
| 1568 | +} | |
| 1569 | + | |
| 1570 | +/* | |
| 1571 | + * Remove an entry from an intermediate attr node at the specified index. | |
| 1572 | + */ | |
| 1573 | +void | |
| 1574 | +xfs_attr3_node_entry_remove( | |
| 1575 | + struct xfs_trans *tp, | |
| 1576 | + struct xfs_inode *dp, | |
| 1577 | + struct xfs_buf *bp, | |
| 1578 | + int index) | |
| 1579 | +{ | |
| 1580 | + struct xfs_da_state_blk blk = { | |
| 1581 | + .index = index, | |
| 1582 | + .bp = bp, | |
| 1583 | + }; | |
| 1584 | + | |
| 1585 | + __xfs_da3_node_remove(tp, dp, dp->i_mount->m_attr_geo, &blk); | |
| 1586 | +} | |
| 1587 | + | |
| 1557 | 1588 | /* |
| 1558 | 1589 | * Unbalance the elements between two intermediate nodes, |
| 1559 | 1590 | * move all Btree elements from one node into another. |
| @@ -184,6 +184,8 @@ int xfs_da3_split(xfs_da_state_t *state); · 183 unmodified lines | ||
| 184 | 184 | int xfs_da3_join(xfs_da_state_t *state); |
| 185 | 185 | void xfs_da3_fixhashpath(struct xfs_da_state *state, |
| 186 | 186 | struct xfs_da_state_path *path_to_to_fix); |
| 187 | +void xfs_attr3_node_entry_remove(struct xfs_trans *tp, struct xfs_inode *dp, | |
| 188 | + struct xfs_buf *bp, int index); | |
| 187 | 189 | |
| 188 | 190 | /* |
| 189 | 191 | * Routines used for finding things in the Btree. |
| @@ -171,8 +171,10 @@ xchk_quota_item( · 170 unmodified lines | ||
| 171 | 171 | |
| 172 | 172 | error = xchk_quota_item_bmap(sc, dq, offset); |
| 173 | 173 | xchk_iunlock(sc, XFS_ILOCK_SHARED); |
| 174 | − if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) | |
| 174 | + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) { | |
| 175 | + mutex_unlock(&dq->q_qlock); | |
| 175 | 176 | return error; |
| 177 | + } | |
| 176 | 178 | |
| 177 | 179 | /* |
| 178 | 180 | * Warn if the hard limits are larger than the fs. |
| @@ -972,20 +972,12 @@ TRACE_EVENT(xfile_create, · 971 unmodified lines | ||
| 972 | 972 | TP_STRUCT__entry( |
| 973 | 973 | __field(dev_t, dev) |
| 974 | 974 | __field(unsigned long, ino) |
| 975 | − __array(char, pathname, MAXNAMELEN) | |
| 976 | 975 | ), |
| 977 | 976 | TP_fast_assign( |
| 978 | − char *path; | |
| 979 | − | |
| 980 | 977 | __entry->ino = file_inode(xf->file)->i_ino; |
| 981 | − path = file_path(xf->file, __entry->pathname, MAXNAMELEN); | |
| 982 | − if (IS_ERR(path)) | |
| 983 | − strncpy(__entry->pathname, "(unknown)", | |
| 984 | − sizeof(__entry->pathname)); | |
| 985 | 978 | ), |
| 986 | − TP_printk("xfino 0x%lx path '%s'", | |
| 987 | − __entry->ino, | |
| 988 | − __entry->pathname) | |
| 979 | + TP_printk("xfino 0x%lx", | |
| 980 | + __entry->ino) | |
| 989 | 981 | ); |
| 990 | 982 | |
| 991 | 983 | TRACE_EVENT(xfile_destroy, |
| @@ -140,7 +140,7 @@ xfs_attr3_node_inactive( · 139 unmodified lines | ||
| 140 | 140 | xfs_daddr_t parent_blkno, child_blkno; |
| 141 | 141 | struct xfs_buf *child_bp; |
| 142 | 142 | struct xfs_da3_icnode_hdr ichdr; |
| 143 | − int error, i; | |
| 143 | + int error; | |
| 144 | 144 | |
| 145 | 145 | /* |
| 146 | 146 | * Since this code is recursive (gasp!) we must protect ourselves. |
| @@ -152,7 +152,7 @@ xfs_attr3_node_inactive( · 5 unmodified lines | ||
| 152 | 152 | return -EFSCORRUPTED; |
| 153 | 153 | } |
| 154 | 154 | |
| 155 | − xfs_da3_node_hdr_from_disk(dp->i_mount, &ichdr, bp->b_addr); | |
| 155 | + xfs_da3_node_hdr_from_disk(mp, &ichdr, bp->b_addr); | |
| 156 | 156 | parent_blkno = xfs_buf_daddr(bp); |
| 157 | 157 | if (!ichdr.count) { |
| 158 | 158 | xfs_trans_brelse(*trans, bp); |
| @@ -167,7 +167,7 @@ xfs_attr3_node_inactive( · 8 unmodified lines | ||
| 167 | 167 | * over the leaves removing all of them. If this is higher up |
| 168 | 168 | * in the tree, recurse downward. |
| 169 | 169 | */ |
| 170 | − for (i = 0; i < ichdr.count; i++) { | |
| 170 | + while (ichdr.count > 0) { | |
| 171 | 171 | /* |
| 172 | 172 | * Read the subsidiary block to see what we have to work with. |
| 173 | 173 | * Don't do this in a transaction. This is a depth-first |
| @@ -218,29 +218,32 @@ xfs_attr3_node_inactive( · 44 unmodified lines | ||
| 218 | 218 | xfs_trans_binval(*trans, child_bp); |
| 219 | 219 | child_bp = NULL; |
| 220 | 220 | |
| 221 | + error = xfs_da3_node_read_mapped(*trans, dp, | |
| 222 | + parent_blkno, &bp, XFS_ATTR_FORK); | |
| 223 | + if (error) | |
| 224 | + return error; | |
| 225 | + | |
| 221 | 226 | /* |
| 222 | − * If we're not done, re-read the parent to get the next | |
| 223 | − * child block number. | |
| 227 | + * Remove entry from parent node, prevents being indexed to. | |
| 224 | 228 | */ |
| 225 | − if (i + 1 < ichdr.count) { | |
| 226 | − struct xfs_da3_icnode_hdr phdr; | |
| 229 | + xfs_attr3_node_entry_remove(*trans, dp, bp, 0); | |
| 230 | + | |
| 231 | + xfs_da3_node_hdr_from_disk(mp, &ichdr, bp->b_addr); | |
| 232 | + bp = NULL; | |
| 227 | 233 | |
| 228 | − error = xfs_da3_node_read_mapped(*trans, dp, | |
| 229 | − parent_blkno, &bp, XFS_ATTR_FORK); | |
| 234 | + if (ichdr.count > 0) { | |
| 235 | + /* | |
| 236 | + * If we're not done, get the next child block number. | |
| 237 | + */ | |
| 238 | + child_fsb = be32_to_cpu(ichdr.btree[0].before); | |
| 239 | + | |
| 240 | + /* | |
| 241 | + * Atomically commit the whole invalidate stuff. | |
| 242 | + */ | |
| 243 | + error = xfs_trans_roll_inode(trans, dp); | |
| 230 | 244 | if (error) |
| 231 | 245 | return error; |
| 232 | − xfs_da3_node_hdr_from_disk(dp->i_mount, &phdr, | |
| 233 | − bp->b_addr); | |
| 234 | − child_fsb = be32_to_cpu(phdr.btree[i + 1].before); | |
| 235 | − xfs_trans_brelse(*trans, bp); | |
| 236 | − bp = NULL; | |
| 237 | 246 | } |
| 238 | − /* | |
| 239 | − * Atomically commit the whole invalidate stuff. | |
| 240 | − */ | |
| 241 | − error = xfs_trans_roll_inode(trans, dp); | |
| 242 | − if (error) | |
| 243 | − return error; | |
| 244 | 247 | } |
| 245 | 248 | |
| 246 | 249 | return 0; |
| @@ -257,10 +260,8 @@ xfs_attr3_root_inactive( · 10 unmodified lines | ||
| 257 | 260 | struct xfs_trans **trans, |
| 258 | 261 | struct xfs_inode *dp) |
| 259 | 262 | { |
| 260 | − struct xfs_mount *mp = dp->i_mount; | |
| 261 | 263 | struct xfs_da_blkinfo *info; |
| 262 | 264 | struct xfs_buf *bp; |
| 263 | − xfs_daddr_t blkno; | |
| 264 | 265 | int error; |
| 265 | 266 | |
| 266 | 267 | /* |
| @@ -272,7 +273,6 @@ xfs_attr3_root_inactive( · 5 unmodified lines | ||
| 272 | 273 | error = xfs_da3_node_read(*trans, dp, 0, &bp, XFS_ATTR_FORK); |
| 273 | 274 | if (error) |
| 274 | 275 | return error; |
| 275 | − blkno = xfs_buf_daddr(bp); | |
| 276 | 276 | |
| 277 | 277 | /* |
| 278 | 278 | * Invalidate the tree, even if the "tree" is only a single leaf block. |
| @@ -283,10 +283,26 @@ xfs_attr3_root_inactive( · 4 unmodified lines | ||
| 283 | 283 | case cpu_to_be16(XFS_DA_NODE_MAGIC): |
| 284 | 284 | case cpu_to_be16(XFS_DA3_NODE_MAGIC): |
| 285 | 285 | error = xfs_attr3_node_inactive(trans, dp, bp, 1); |
| 286 | + /* | |
| 287 | + * Empty root node block are not allowed, convert it to leaf. | |
| 288 | + */ | |
| 289 | + if (!error) | |
| 290 | + error = xfs_attr3_leaf_init(*trans, dp, 0); | |
| 291 | + if (!error) | |
| 292 | + error = xfs_trans_roll_inode(trans, dp); | |
| 286 | 293 | break; |
| 287 | 294 | case cpu_to_be16(XFS_ATTR_LEAF_MAGIC): |
| 288 | 295 | case cpu_to_be16(XFS_ATTR3_LEAF_MAGIC): |
| 289 | 296 | error = xfs_attr3_leaf_inactive(trans, dp, bp); |
| 297 | + /* | |
| 298 | + * Reinit the leaf before truncating extents so that a crash | |
| 299 | + * mid-truncation leaves an empty leaf rather than one with | |
| 300 | + * entries that may reference freed remote value blocks. | |
| 301 | + */ | |
| 302 | + if (!error) | |
| 303 | + error = xfs_attr3_leaf_init(*trans, dp, 0); | |
| 304 | + if (!error) | |
| 305 | + error = xfs_trans_roll_inode(trans, dp); | |
| 290 | 306 | break; |
| 291 | 307 | default: |
| 292 | 308 | xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); |
| @@ -295,21 +311,6 @@ xfs_attr3_root_inactive( · 2 unmodified lines | ||
| 295 | 311 | xfs_trans_brelse(*trans, bp); |
| 296 | 312 | break; |
| 297 | 313 | } |
| 298 | − if (error) | |
| 299 | − return error; | |
| 300 | − | |
| 301 | − /* | |
| 302 | − * Invalidate the incore copy of the root block. | |
| 303 | − */ | |
| 304 | − error = xfs_trans_get_buf(*trans, mp->m_ddev_targp, blkno, | |
| 305 | − XFS_FSB_TO_BB(mp, mp->m_attr_geo->fsbcount), 0, &bp); | |
| 306 | − if (error) | |
| 307 | − return error; | |
| 308 | − xfs_trans_binval(*trans, bp); /* remove from cache */ | |
| 309 | − /* | |
| 310 | − * Commit the invalidate and start the next transaction. | |
| 311 | − */ | |
| 312 | − error = xfs_trans_roll_inode(trans, dp); | |
| 313 | 314 | |
| 314 | 315 | return error; |
| 315 | 316 | } |
| @@ -328,6 +329,7 @@ xfs_attr_inactive( · 12 unmodified lines | ||
| 328 | 329 | { |
| 329 | 330 | struct xfs_trans *trans; |
| 330 | 331 | struct xfs_mount *mp; |
| 332 | + struct xfs_buf *bp; | |
| 331 | 333 | int lock_mode = XFS_ILOCK_SHARED; |
| 332 | 334 | int error = 0; |
| 333 | 335 | |
| @@ -363,10 +365,27 @@ xfs_attr_inactive( · 29 unmodified lines | ||
| 363 | 365 | * removal below. |
| 364 | 366 | */ |
| 365 | 367 | if (dp->i_af.if_nextents > 0) { |
| 368 | + /* | |
| 369 | + * Invalidate and truncate all blocks but leave the root block. | |
| 370 | + */ | |
| 366 | 371 | error = xfs_attr3_root_inactive(&trans, dp); |
| 367 | 372 | if (error) |
| 368 | 373 | goto out_cancel; |
| 369 | 374 | |
| 375 | + error = xfs_itruncate_extents(&trans, dp, XFS_ATTR_FORK, | |
| 376 | + XFS_FSB_TO_B(mp, mp->m_attr_geo->fsbcount)); | |
| 377 | + if (error) | |
| 378 | + goto out_cancel; | |
| 379 | + | |
| 380 | + /* | |
| 381 | + * Invalidate and truncate the root block and ensure that the | |
| 382 | + * operation is completed within a single transaction. | |
| 383 | + */ | |
| 384 | + error = xfs_da_get_buf(trans, dp, 0, &bp, XFS_ATTR_FORK); | |
| 385 | + if (error) | |
| 386 | + goto out_cancel; | |
| 387 | + | |
| 388 | + xfs_trans_binval(trans, bp); | |
| 370 | 389 | error = xfs_itruncate_extents(&trans, dp, XFS_ATTR_FORK, 0); |
| 371 | 390 | if (error) |
| 372 | 391 | goto out_cancel; |
| @@ -653,7 +653,6 @@ xfs_attri_recover_work( · 652 unmodified lines | ||
| 653 | 653 | break; |
| 654 | 654 | } |
| 655 | 655 | if (error) { |
| 656 | − xfs_irele(ip); | |
| 657 | 656 | XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attrp, |
| 658 | 657 | sizeof(*attrp)); |
| 659 | 658 | return ERR_PTR(-EFSCORRUPTED); |
| @@ -1047,8 +1046,8 @@ xlog_recover_attri_commit_pass2( · 387 unmodified lines | ||
| 1047 | 1046 | break; |
| 1048 | 1047 | case XFS_ATTRI_OP_FLAGS_SET: |
| 1049 | 1048 | case XFS_ATTRI_OP_FLAGS_REPLACE: |
| 1050 | − /* Log item, attr name, attr value */ | |
| 1051 | − if (item->ri_total != 3) { | |
| 1049 | + /* Log item, attr name, optional attr value */ | |
| 1050 | + if (item->ri_total != 2 + !!attri_formatp->alfi_value_len) { | |
| 1052 | 1051 | XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, |
| 1053 | 1052 | attri_formatp, len); |
| 1054 | 1053 | return -EFSCORRUPTED; |
| @@ -1132,52 +1131,6 @@ xlog_recover_attri_commit_pass2( · 77 unmodified lines | ||
| 1132 | 1131 | return -EFSCORRUPTED; |
| 1133 | 1132 | } |
| 1134 | 1133 | |
| 1135 | − switch (op) { | |
| 1136 | − case XFS_ATTRI_OP_FLAGS_REMOVE: | |
| 1137 | − /* Regular remove operations operate only on names. */ | |
| 1138 | − if (attr_value != NULL || value_len != 0) { | |
| 1139 | − XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, | |
| 1140 | − attri_formatp, len); | |
| 1141 | − return -EFSCORRUPTED; | |
| 1142 | − } | |
| 1143 | − fallthrough; | |
| 1144 | − case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: | |
| 1145 | − case XFS_ATTRI_OP_FLAGS_PPTR_SET: | |
| 1146 | − case XFS_ATTRI_OP_FLAGS_SET: | |
| 1147 | − case XFS_ATTRI_OP_FLAGS_REPLACE: | |
| 1148 | − /* | |
| 1149 | − * Regular xattr set/remove/replace operations require a name | |
| 1150 | − * and do not take a newname. Values are optional for set and | |
| 1151 | − * replace. | |
| 1152 | − * | |
| 1153 | − * Name-value set/remove operations must have a name, do not | |
| 1154 | − * take a newname, and can take a value. | |
| 1155 | − */ | |
| 1156 | − if (attr_name == NULL || name_len == 0) { | |
| 1157 | − XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, | |
| 1158 | − attri_formatp, len); | |
| 1159 | − return -EFSCORRUPTED; | |
| 1160 | − } | |
| 1161 | − break; | |
| 1162 | − case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: | |
| 1163 | − /* | |
| 1164 | − * Name-value replace operations require the caller to | |
| 1165 | − * specify the old and new names and values explicitly. | |
| 1166 | − * Values are optional. | |
| 1167 | − */ | |
| 1168 | − if (attr_name == NULL || name_len == 0) { | |
| 1169 | − XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, | |
| 1170 | − attri_formatp, len); | |
| 1171 | − return -EFSCORRUPTED; | |
| 1172 | − } | |
| 1173 | − if (attr_new_name == NULL || new_name_len == 0) { | |
| 1174 | − XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, | |
| 1175 | − attri_formatp, len); | |
| 1176 | − return -EFSCORRUPTED; | |
| 1177 | − } | |
| 1178 | − break; | |
| 1179 | − } | |
| 1180 | − | |
| 1181 | 1134 | /* |
| 1182 | 1135 | * Memory alloc failure will cause replay to abort. We attach the |
| 1183 | 1136 | * name/value buffer to the recovered incore log item and drop our |
| @@ -125,6 +125,7 @@ xfs_qm_dquot_logitem_push( · 124 unmodified lines | ||
| 125 | 125 | struct xfs_dq_logitem *qlip = DQUOT_ITEM(lip); |
| 126 | 126 | struct xfs_dquot *dqp = qlip->qli_dquot; |
| 127 | 127 | struct xfs_buf *bp; |
| 128 | + struct xfs_ail *ailp = lip->li_ailp; | |
| 128 | 129 | uint rval = XFS_ITEM_SUCCESS; |
| 129 | 130 | int error; |
| 130 | 131 | |
| @@ -153,7 +154,7 @@ xfs_qm_dquot_logitem_push( · 22 unmodified lines | ||
| 153 | 154 | goto out_unlock; |
| 154 | 155 | } |
| 155 | 156 | |
| 156 | − spin_unlock(&lip->li_ailp->ail_lock); | |
| 157 | + spin_unlock(&ailp->ail_lock); | |
| 157 | 158 | |
| 158 | 159 | error = xfs_dquot_use_attached_buf(dqp, &bp); |
| 159 | 160 | if (error == -EAGAIN) { |
| @@ -172,9 +173,13 @@ xfs_qm_dquot_logitem_push( · 12 unmodified lines | ||
| 172 | 173 | rval = XFS_ITEM_FLUSHING; |
| 173 | 174 | } |
| 174 | 175 | xfs_buf_relse(bp); |
| 176 | + /* | |
| 177 | + * The buffer no longer protects the log item from reclaim, so | |
| 178 | + * do not reference lip after this point. | |
| 179 | + */ | |
| 175 | 180 | |
| 176 | 181 | out_relock_ail: |
| 177 | − spin_lock(&lip->li_ailp->ail_lock); | |
| 182 | + spin_lock(&ailp->ail_lock); | |
| 178 | 183 | out_unlock: |
| 179 | 184 | mutex_unlock(&dqp->q_qlock); |
| 180 | 185 | return rval; |
| @@ -443,8 +443,8 @@ xfs_ioc_attr_list( · 442 unmodified lines | ||
| 443 | 443 | context.dp = dp; |
| 444 | 444 | context.resynch = 1; |
| 445 | 445 | context.attr_filter = xfs_attr_filter(flags); |
| 446 | − context.buffer = buffer; | |
| 447 | 446 | context.bufsize = round_down(bufsize, sizeof(uint32_t)); |
| 447 | + context.buffer = buffer; | |
| 448 | 448 | context.firstu = context.bufsize; |
| 449 | 449 | context.put_listent = xfs_ioc_attr_put_listent; |
| 450 | 450 | |
| @@ -1048,7 +1048,8 @@ xfs_itruncate_extents_flags( · 1047 unmodified lines | ||
| 1048 | 1048 | xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); |
| 1049 | 1049 | if (icount_read(VFS_I(ip))) |
| 1050 | 1050 | xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL); |
| 1051 | − ASSERT(new_size <= XFS_ISIZE(ip)); | |
| 1051 | + if (whichfork == XFS_DATA_FORK) | |
| 1052 | + ASSERT(new_size <= XFS_ISIZE(ip)); | |
| 1052 | 1053 | ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); |
| 1053 | 1054 | ASSERT(ip->i_itemp != NULL); |
| 1054 | 1055 | ASSERT(ip->i_itemp->ili_lock_flags == 0); |
| @@ -746,6 +746,7 @@ xfs_inode_item_push( · 745 unmodified lines | ||
| 746 | 746 | struct xfs_inode_log_item *iip = INODE_ITEM(lip); |
| 747 | 747 | struct xfs_inode *ip = iip->ili_inode; |
| 748 | 748 | struct xfs_buf *bp = lip->li_buf; |
| 749 | + struct xfs_ail *ailp = lip->li_ailp; | |
| 749 | 750 | uint rval = XFS_ITEM_SUCCESS; |
| 750 | 751 | int error; |
| 751 | 752 | |
| @@ -771,7 +772,7 @@ xfs_inode_item_push( · 19 unmodified lines | ||
| 771 | 772 | if (!xfs_buf_trylock(bp)) |
| 772 | 773 | return XFS_ITEM_LOCKED; |
| 773 | 774 | |
| 774 | − spin_unlock(&lip->li_ailp->ail_lock); | |
| 775 | + spin_unlock(&ailp->ail_lock); | |
| 775 | 776 | |
| 776 | 777 | /* |
| 777 | 778 | * We need to hold a reference for flushing the cluster buffer as it may |
| @@ -795,7 +796,11 @@ xfs_inode_item_push( · 17 unmodified lines | ||
| 795 | 796 | rval = XFS_ITEM_LOCKED; |
| 796 | 797 | } |
| 797 | 798 | |
| 798 | − spin_lock(&lip->li_ailp->ail_lock); | |
| 799 | + /* | |
| 800 | + * The buffer no longer protects the log item from reclaim, so | |
| 801 | + * do not reference lip after this point. | |
| 802 | + */ | |
| 803 | + spin_lock(&ailp->ail_lock); | |
| 799 | 804 | return rval; |
| 800 | 805 | } |
| 801 | 806 | |
| @@ -608,8 +608,9 @@ xfs_unmount_check( · 607 unmodified lines | ||
| 608 | 608 | * have been retrying in the background. This will prevent never-ending |
| 609 | 609 | * retries in AIL pushing from hanging the unmount. |
| 610 | 610 | * |
| 611 | − * Finally, we can push the AIL to clean all the remaining dirty objects, then | |
| 612 | − * reclaim the remaining inodes that are still in memory at this point in time. | |
| 611 | + * Stop inodegc and background reclaim before pushing the AIL so that they | |
| 612 | + * are not running while the AIL is being flushed. Then push the AIL to | |
| 613 | + * clean all the remaining dirty objects and reclaim the remaining inodes. | |
| 613 | 614 | */ |
| 614 | 615 | static void |
| 615 | 616 | xfs_unmount_flush_inodes( |
| @@ -621,9 +622,9 @@ xfs_unmount_flush_inodes( · 5 unmodified lines | ||
| 621 | 622 | |
| 622 | 623 | xfs_set_unmounting(mp); |
| 623 | 624 | |
| 624 | − xfs_ail_push_all_sync(mp->m_ail); | |
| 625 | 625 | xfs_inodegc_stop(mp); |
| 626 | 626 | cancel_delayed_work_sync(&mp->m_reclaim_work); |
| 627 | + xfs_ail_push_all_sync(mp->m_ail); | |
| 627 | 628 | xfs_reclaim_inodes(mp); |
| 628 | 629 | xfs_health_unmount(mp); |
| 629 | 630 | xfs_healthmon_unmount(mp); |
| @@ -56,6 +56,7 @@ · 55 unmodified lines | ||
| 56 | 56 | #include <linux/tracepoint.h> |
| 57 | 57 | |
| 58 | 58 | struct xfs_agf; |
| 59 | +struct xfs_ail; | |
| 59 | 60 | struct xfs_alloc_arg; |
| 60 | 61 | struct xfs_attr_list_context; |
| 61 | 62 | struct xfs_buf_log_item; |
| @@ -1650,16 +1651,43 @@ TRACE_EVENT(xfs_log_force, · 1588 unmodified lines | ||
| 1650 | 1651 | DEFINE_EVENT(xfs_log_item_class, name, \ |
| 1651 | 1652 | TP_PROTO(struct xfs_log_item *lip), \ |
| 1652 | 1653 | TP_ARGS(lip)) |
| 1653 | −DEFINE_LOG_ITEM_EVENT(xfs_ail_push); | |
| 1654 | −DEFINE_LOG_ITEM_EVENT(xfs_ail_pinned); | |
| 1655 | −DEFINE_LOG_ITEM_EVENT(xfs_ail_locked); | |
| 1656 | −DEFINE_LOG_ITEM_EVENT(xfs_ail_flushing); | |
| 1657 | 1654 | DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_mark); |
| 1658 | 1655 | DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_skip); |
| 1659 | 1656 | DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_unpin); |
| 1660 | 1657 | DEFINE_LOG_ITEM_EVENT(xlog_ail_insert_abort); |
| 1661 | 1658 | DEFINE_LOG_ITEM_EVENT(xfs_trans_free_abort); |
| 1662 | 1659 | |
| 1660 | +DECLARE_EVENT_CLASS(xfs_ail_push_class, | |
| 1661 | + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), | |
| 1662 | + TP_ARGS(ailp, type, flags, lsn), | |
| 1663 | + TP_STRUCT__entry( | |
| 1664 | + __field(dev_t, dev) | |
| 1665 | + __field(uint, type) | |
| 1666 | + __field(unsigned long, flags) | |
| 1667 | + __field(xfs_lsn_t, lsn) | |
| 1668 | + ), | |
| 1669 | + TP_fast_assign( | |
| 1670 | + __entry->dev = ailp->ail_log->l_mp->m_super->s_dev; | |
| 1671 | + __entry->type = type; | |
| 1672 | + __entry->flags = flags; | |
| 1673 | + __entry->lsn = lsn; | |
| 1674 | + ), | |
| 1675 | + TP_printk("dev %d:%d lsn %d/%d type %s flags %s", | |
| 1676 | + MAJOR(__entry->dev), MINOR(__entry->dev), | |
| 1677 | + CYCLE_LSN(__entry->lsn), BLOCK_LSN(__entry->lsn), | |
| 1678 | + __print_symbolic(__entry->type, XFS_LI_TYPE_DESC), | |
| 1679 | + __print_flags(__entry->flags, "|", XFS_LI_FLAGS)) | |
| 1680 | +) | |
| 1681 | + | |
| 1682 | +#define DEFINE_AIL_PUSH_EVENT(name) \ | |
| 1683 | +DEFINE_EVENT(xfs_ail_push_class, name, \ | |
| 1684 | + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), \ | |
| 1685 | + TP_ARGS(ailp, type, flags, lsn)) | |
| 1686 | +DEFINE_AIL_PUSH_EVENT(xfs_ail_push); | |
| 1687 | +DEFINE_AIL_PUSH_EVENT(xfs_ail_pinned); | |
| 1688 | +DEFINE_AIL_PUSH_EVENT(xfs_ail_locked); | |
| 1689 | +DEFINE_AIL_PUSH_EVENT(xfs_ail_flushing); | |
| 1690 | + | |
| 1663 | 1691 | DECLARE_EVENT_CLASS(xfs_ail_class, |
| 1664 | 1692 | TP_PROTO(struct xfs_log_item *lip, xfs_lsn_t old_lsn, xfs_lsn_t new_lsn), |
| 1665 | 1693 | TP_ARGS(lip, old_lsn, new_lsn), |
| @@ -5091,23 +5119,16 @@ TRACE_EVENT(xmbuf_create, · 3425 unmodified lines | ||
| 5091 | 5119 | TP_STRUCT__entry( |
| 5092 | 5120 | __field(dev_t, dev) |
| 5093 | 5121 | __field(unsigned long, ino) |
| 5094 | − __array(char, pathname, MAXNAMELEN) | |
| 5095 | 5122 | ), |
| 5096 | 5123 | TP_fast_assign( |
| 5097 | − char *path; | |
| 5098 | 5124 | struct file *file = btp->bt_file; |
| 5099 | 5125 | |
| 5100 | 5126 | __entry->dev = btp->bt_mount->m_super->s_dev; |
| 5101 | 5127 | __entry->ino = file_inode(file)->i_ino; |
| 5102 | − path = file_path(file, __entry->pathname, MAXNAMELEN); | |
| 5103 | − if (IS_ERR(path)) | |
| 5104 | − strncpy(__entry->pathname, "(unknown)", | |
| 5105 | − sizeof(__entry->pathname)); | |
| 5106 | 5128 | ), |
| 5107 | − TP_printk("dev %d:%d xmino 0x%lx path '%s'", | |
| 5129 | + TP_printk("dev %d:%d xmino 0x%lx", | |
| 5108 | 5130 | MAJOR(__entry->dev), MINOR(__entry->dev), |
| 5109 | − __entry->ino, | |
| 5110 | − __entry->pathname) | |
| 5131 | + __entry->ino) | |
| 5111 | 5132 | ); |
| 5112 | 5133 | |
| 5113 | 5134 | TRACE_EVENT(xmbuf_free, |
| @@ -365,6 +365,12 @@ xfsaild_resubmit_item( · 364 unmodified lines | ||
| 365 | 365 | return XFS_ITEM_SUCCESS; |
| 366 | 366 | } |
| 367 | 367 | |
| 368 | +/* | |
| 369 | + * Push a single log item from the AIL. | |
| 370 | + * | |
| 371 | + * @lip may have been released and freed by the time this function returns, | |
| 372 | + * so callers must not dereference the log item afterwards. | |
| 373 | + */ | |
| 368 | 374 | static inline uint |
| 369 | 375 | xfsaild_push_item( |
| 370 | 376 | struct xfs_ail *ailp, |
| @@ -458,6 +464,74 @@ xfs_ail_calc_push_target( · 87 unmodified lines | ||
| 458 | 464 | return target_lsn; |
| 459 | 465 | } |
| 460 | 466 | |
| 467 | +static void | |
| 468 | +xfsaild_process_logitem( | |
| 469 | + struct xfs_ail *ailp, | |
| 470 | + struct xfs_log_item *lip, | |
| 471 | + int *stuck, | |
| 472 | + int *flushing) | |
| 473 | +{ | |
| 474 | + struct xfs_mount *mp = ailp->ail_log->l_mp; | |
| 475 | + uint type = lip->li_type; | |
| 476 | + unsigned long flags = lip->li_flags; | |
| 477 | + xfs_lsn_t item_lsn = lip->li_lsn; | |
| 478 | + int lock_result; | |
| 479 | + | |
| 480 | + /* | |
| 481 | + * Note that iop_push may unlock and reacquire the AIL lock. We | |
| 482 | + * rely on the AIL cursor implementation to be able to deal with | |
| 483 | + * the dropped lock. | |
| 484 | + * | |
| 485 | + * The log item may have been freed by the push, so it must not | |
| 486 | + * be accessed or dereferenced below this line. | |
| 487 | + */ | |
| 488 | + lock_result = xfsaild_push_item(ailp, lip); | |
| 489 | + switch (lock_result) { | |
| 490 | + case XFS_ITEM_SUCCESS: | |
| 491 | + XFS_STATS_INC(mp, xs_push_ail_success); | |
| 492 | + trace_xfs_ail_push(ailp, type, flags, item_lsn); | |
| 493 | + | |
| 494 | + ailp->ail_last_pushed_lsn = item_lsn; | |
| 495 | + break; | |
| 496 | + | |
| 497 | + case XFS_ITEM_FLUSHING: | |
| 498 | + /* | |
| 499 | + * The item or its backing buffer is already being | |
| 500 | + * flushed. The typical reason for that is that an | |
| 501 | + * inode buffer is locked because we already pushed the | |
| 502 | + * updates to it as part of inode clustering. | |
| 503 | + * | |
| 504 | + * We do not want to stop flushing just because lots | |
| 505 | + * of items are already being flushed, but we need to | |
| 506 | + * re-try the flushing relatively soon if most of the | |
| 507 | + * AIL is being flushed. | |
| 508 | + */ | |
| 509 | + XFS_STATS_INC(mp, xs_push_ail_flushing); | |
| 510 | + trace_xfs_ail_flushing(ailp, type, flags, item_lsn); | |
| 511 | + | |
| 512 | + (*flushing)++; | |
| 513 | + ailp->ail_last_pushed_lsn = item_lsn; | |
| 514 | + break; | |
| 515 | + | |
| 516 | + case XFS_ITEM_PINNED: | |
| 517 | + XFS_STATS_INC(mp, xs_push_ail_pinned); | |
| 518 | + trace_xfs_ail_pinned(ailp, type, flags, item_lsn); | |
| 519 | + | |
| 520 | + (*stuck)++; | |
| 521 | + ailp->ail_log_flush++; | |
| 522 | + break; | |
| 523 | + case XFS_ITEM_LOCKED: | |
| 524 | + XFS_STATS_INC(mp, xs_push_ail_locked); | |
| 525 | + trace_xfs_ail_locked(ailp, type, flags, item_lsn); | |
| 526 | + | |
| 527 | + (*stuck)++; | |
| 528 | + break; | |
| 529 | + default: | |
| 530 | + ASSERT(0); | |
| 531 | + break; | |
| 532 | + } | |
| 533 | +} | |
| 534 | + | |
| 461 | 535 | static long |
| 462 | 536 | xfsaild_push( |
| 463 | 537 | struct xfs_ail *ailp) |
| @@ -505,62 +579,11 @@ xfsaild_push( · 41 unmodified lines | ||
| 505 | 579 | |
| 506 | 580 | lsn = lip->li_lsn; |
| 507 | 581 | while ((XFS_LSN_CMP(lip->li_lsn, ailp->ail_target) <= 0)) { |
| 508 | − int lock_result; | |
| 509 | 582 | |
| 510 | 583 | if (test_bit(XFS_LI_FLUSHING, &lip->li_flags)) |
| 511 | 584 | goto next_item; |
| 512 | 585 | |
| 513 | − /* | |
| 514 | − * Note that iop_push may unlock and reacquire the AIL lock. We | |
| 515 | − * rely on the AIL cursor implementation to be able to deal with | |
| 516 | − * the dropped lock. | |
| 517 | − */ | |
| 518 | − lock_result = xfsaild_push_item(ailp, lip); | |
| 519 | − switch (lock_result) { | |
| 520 | − case XFS_ITEM_SUCCESS: | |
| 521 | − XFS_STATS_INC(mp, xs_push_ail_success); | |
| 522 | − trace_xfs_ail_push(lip); | |
| 523 | − | |
| 524 | − ailp->ail_last_pushed_lsn = lsn; | |
| 525 | − break; | |
| 526 | − | |
| 527 | − case XFS_ITEM_FLUSHING: | |
| 528 | − /* | |
| 529 | − * The item or its backing buffer is already being | |
| 530 | − * flushed. The typical reason for that is that an | |
| 531 | − * inode buffer is locked because we already pushed the | |
| 532 | − * updates to it as part of inode clustering. | |
| 533 | − * | |
| 534 | − * We do not want to stop flushing just because lots | |
| 535 | − * of items are already being flushed, but we need to | |
| 536 | − * re-try the flushing relatively soon if most of the | |
| 537 | − * AIL is being flushed. | |
| 538 | − */ | |
| 539 | − XFS_STATS_INC(mp, xs_push_ail_flushing); | |
| 540 | − trace_xfs_ail_flushing(lip); | |
| 541 | − | |
| 542 | − flushing++; | |
| 543 | − ailp->ail_last_pushed_lsn = lsn; | |
| 544 | − break; | |
| 545 | − | |
| 546 | − case XFS_ITEM_PINNED: | |
| 547 | − XFS_STATS_INC(mp, xs_push_ail_pinned); | |
| 548 | − trace_xfs_ail_pinned(lip); | |
| 549 | − | |
| 550 | − stuck++; | |
| 551 | − ailp->ail_log_flush++; | |
| 552 | − break; | |
| 553 | − case XFS_ITEM_LOCKED: | |
| 554 | − XFS_STATS_INC(mp, xs_push_ail_locked); | |
| 555 | − trace_xfs_ail_locked(lip); | |
| 556 | − | |
| 557 | − stuck++; | |
| 558 | − break; | |
| 559 | − default: | |
| 560 | − ASSERT(0); | |
| 561 | − break; | |
| 562 | − } | |
| 563 | − | |
| 586 | + xfsaild_process_logitem(ailp, lip, &stuck, &flushing); | |
| 564 | 587 | count++; |
| 565 | 588 | |
| 566 | 589 | /* |
| @@ -183,10 +183,9 @@ xfs_verify_iosize( · 182 unmodified lines | ||
| 183 | 183 | min_not_zero(SZ_1M, me->me_max_io_size); |
| 184 | 184 | |
| 185 | 185 | BUILD_BUG_ON(BBSHIFT != SECTOR_SHIFT); |
| 186 | − ASSERT(BBTOB(bbcount) >= bdev_logical_block_size(btp->bt_bdev)); | |
| 186 | + ASSERT(BBTOB(bbcount) >= btp->bt_logical_sectorsize); | |
| 187 | 187 | |
| 188 | − return clamp(iosize, bdev_logical_block_size(btp->bt_bdev), | |
| 189 | − BBTOB(bbcount)); | |
| 188 | + return clamp(iosize, btp->bt_logical_sectorsize, BBTOB(bbcount)); | |
| 190 | 189 | } |
| 191 | 190 | |
| 192 | 191 | /* Allocate as much memory as we can get for verification buffer. */ |
| @@ -218,8 +217,8 @@ xfs_verify_media_error( · 25 unmodified lines | ||
| 218 | 217 | unsigned int bio_bbcount, |
| 219 | 218 | blk_status_t bio_status) |
| 220 | 219 | { |
| 221 | − trace_xfs_verify_media_error(mp, me, btp->bt_bdev->bd_dev, daddr, | |
| 222 | − bio_bbcount, bio_status); | |
| 220 | + trace_xfs_verify_media_error(mp, me, btp->bt_dev, daddr, bio_bbcount, | |
| 221 | + bio_status); | |
| 223 | 222 | |
| 224 | 223 | /* |
| 225 | 224 | * Pass any error, I/O or otherwise, up to the caller if we didn't |
| @@ -280,7 +279,7 @@ xfs_verify_media( · 54 unmodified lines | ||
| 280 | 279 | btp = mp->m_ddev_targp; |
| 281 | 280 | break; |
| 282 | 281 | case XFS_DEV_LOG: |
| 283 | − if (mp->m_logdev_targp->bt_bdev != mp->m_ddev_targp->bt_bdev) | |
| 282 | + if (mp->m_logdev_targp != mp->m_ddev_targp) | |
| 284 | 283 | btp = mp->m_logdev_targp; |
| 285 | 284 | break; |
| 286 | 285 | case XFS_DEV_RT: |
| @@ -299,7 +298,7 @@ xfs_verify_media( · 12 unmodified lines | ||
| 299 | 298 | |
| 300 | 299 | /* start and end have to be aligned to the lba size */ |
| 301 | 300 | if (!IS_ALIGNED(BBTOB(me->me_start_daddr | me->me_end_daddr), |
| 302 | − bdev_logical_block_size(btp->bt_bdev))) | |
| 301 | + btp->bt_logical_sectorsize)) | |
| 303 | 302 | return -EINVAL; |
| 304 | 303 | |
| 305 | 304 | /* |
| @@ -331,8 +330,7 @@ xfs_verify_media( · 25 unmodified lines | ||
| 331 | 330 | if (!folio) |
| 332 | 331 | return -ENOMEM; |
| 333 | 332 | |
| 334 | − trace_xfs_verify_media(mp, me, btp->bt_bdev->bd_dev, daddr, bbcount, | |
| 335 | − folio); | |
| 333 | + trace_xfs_verify_media(mp, me, btp->bt_dev, daddr, bbcount, folio); | |
| 336 | 334 | |
| 337 | 335 | bio = bio_alloc(btp->bt_bdev, 1, REQ_OP_READ, GFP_KERNEL); |
| 338 | 336 | if (!bio) { |
| @@ -400,7 +398,7 @@ out_folio: · 61 unmodified lines | ||
| 400 | 398 | * an operational error. |
| 401 | 399 | */ |
| 402 | 400 | me->me_start_daddr = daddr; |
| 403 | − trace_xfs_verify_media_end(mp, me, btp->bt_bdev->bd_dev); | |
| 401 | + trace_xfs_verify_media_end(mp, me, btp->bt_dev); | |
| 404 | 402 | return 0; |
| 405 | 403 | } |
| 406 | 404 | |
| @@ -332,8 +332,8 @@ xfs_vn_listxattr( · 331 unmodified lines | ||
| 332 | 332 | memset(&context, 0, sizeof(context)); |
| 333 | 333 | context.dp = XFS_I(inode); |
| 334 | 334 | context.resynch = 1; |
| 335 | − context.buffer = size ? data : NULL; | |
| 336 | 335 | context.bufsize = size; |
| 336 | + context.buffer = size ? data : NULL; | |
| 337 | 337 | context.firstu = context.bufsize; |
| 338 | 338 | context.put_listent = xfs_xattr_put_listent; |
| 339 | 339 | |
| @@ -810,6 +810,12 @@ struct damon_ctx { · 809 unmodified lines | ||
| 810 | 810 | struct damos_walk_control *walk_control; |
| 811 | 811 | struct mutex walk_control_lock; |
| 812 | 812 | |
| 813 | + /* | |
| 814 | + * indicate if this may be corrupted. Currentonly this is set only for | |
| 815 | + * damon_commit_ctx() failure. | |
| 816 | + */ | |
| 817 | + bool maybe_corrupted; | |
| 818 | + | |
| 813 | 819 | /* Working thread of the given DAMON context */ |
| 814 | 820 | struct task_struct *kdamond; |
| 815 | 821 | /* Protects @kdamond field access */ |
| @@ -80,11 +80,18 @@ · 79 unmodified lines | ||
| 80 | 80 | #define DMA_ATTR_MMIO (1UL << 10) |
| 81 | 81 | |
| 82 | 82 | /* |
| 83 | − * DMA_ATTR_CPU_CACHE_CLEAN: Indicates the CPU will not dirty any cacheline | |
| 84 | − * overlapping this buffer while it is mapped for DMA. All mappings sharing | |
| 85 | − * a cacheline must have this attribute for this to be considered safe. | |
| 83 | + * DMA_ATTR_DEBUGGING_IGNORE_CACHELINES: Indicates the CPU cache line can be | |
| 84 | + * overlapped. All mappings sharing a cacheline must have this attribute for | |
| 85 | + * this to be considered safe. | |
| 86 | 86 | */ |
| 87 | −#define DMA_ATTR_CPU_CACHE_CLEAN (1UL << 11) | |
| 87 | +#define DMA_ATTR_DEBUGGING_IGNORE_CACHELINES (1UL << 11) | |
| 88 | + | |
| 89 | +/* | |
| 90 | + * DMA_ATTR_REQUIRE_COHERENT: Indicates that DMA coherency is required. | |
| 91 | + * All mappings that carry this attribute can't work with SWIOTLB and cache | |
| 92 | + * flushing. | |
| 93 | + */ | |
| 94 | +#define DMA_ATTR_REQUIRE_COHERENT (1UL << 12) | |
| 88 | 95 | |
| 89 | 96 | /* |
| 90 | 97 | * A dma_addr_t can hold any valid DMA or bus address for the platform. It can |
| @@ -248,8 +255,8 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, · 157 unmodified lines | ||
| 248 | 255 | { |
| 249 | 256 | return NULL; |
| 250 | 257 | } |
| 251 | −static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, | |
| 252 | − dma_addr_t dma_handle, unsigned long attrs) | |
| 258 | +static inline void dma_free_attrs(struct device *dev, size_t size, | |
| 259 | + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) | |
| 253 | 260 | { |
| 254 | 261 | } |
| 255 | 262 | static inline void *dmam_alloc_attrs(struct device *dev, size_t size, |
| @@ -338,5 +338,6 @@ struct super_block { · 337 unmodified lines | ||
| 338 | 338 | #define SB_I_NOUMASK 0x00001000 /* VFS does not apply umask */ |
| 339 | 339 | #define SB_I_NOIDMAP 0x00002000 /* No idmapped mounts on this superblock */ |
| 340 | 340 | #define SB_I_ALLOW_HSM 0x00004000 /* Allow HSM events on this superblock */ |
| 341 | +#define SB_I_NO_DATA_INTEGRITY 0x00008000 /* fs cannot guarantee data persistence on sync */ | |
| 341 | 342 | |
| 342 | 343 | #endif /* _LINUX_FS_SUPER_TYPES_H */ |
| @@ -363,6 +363,23 @@ static inline unsigned long softleaf_to_pfn(softleaf_t entry) · 362 unmodified lines | ||
| 363 | 363 | return swp_offset(entry) & SWP_PFN_MASK; |
| 364 | 364 | } |
| 365 | 365 | |
| 366 | +static inline void softleaf_migration_sync(softleaf_t entry, | |
| 367 | + struct folio *folio) | |
| 368 | +{ | |
| 369 | + /* | |
| 370 | + * Ensure we do not race with split, which might alter tail pages into new | |
| 371 | + * folios and thus result in observing an unlocked folio. | |
| 372 | + * This matches the write barrier in __split_folio_to_order(). | |
| 373 | + */ | |
| 374 | + smp_rmb(); | |
| 375 | + | |
| 376 | + /* | |
| 377 | + * Any use of migration entries may only occur while the | |
| 378 | + * corresponding page is locked | |
| 379 | + */ | |
| 380 | + VM_WARN_ON_ONCE(!folio_test_locked(folio)); | |
| 381 | +} | |
| 382 | + | |
| 366 | 383 | /** |
| 367 | 384 | * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry. |
| 368 | 385 | * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true. |
| @@ -374,11 +391,8 @@ static inline struct page *softleaf_to_page(softleaf_t entry) · 5 unmodified lines | ||
| 374 | 391 | struct page *page = pfn_to_page(softleaf_to_pfn(entry)); |
| 375 | 392 | |
| 376 | 393 | VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); |
| 377 | − /* | |
| 378 | − * Any use of migration entries may only occur while the | |
| 379 | − * corresponding page is locked | |
| 380 | − */ | |
| 381 | − VM_WARN_ON_ONCE(softleaf_is_migration(entry) && !PageLocked(page)); | |
| 394 | + if (softleaf_is_migration(entry)) | |
| 395 | + softleaf_migration_sync(entry, page_folio(page)); | |
| 382 | 396 | |
| 383 | 397 | return page; |
| 384 | 398 | } |
| @@ -394,12 +408,8 @@ static inline struct folio *softleaf_to_folio(softleaf_t entry) · 9 unmodified lines | ||
| 394 | 408 | struct folio *folio = pfn_folio(softleaf_to_pfn(entry)); |
| 395 | 409 | |
| 396 | 410 | VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); |
| 397 | − /* | |
| 398 | − * Any use of migration entries may only occur while the | |
| 399 | − * corresponding folio is locked. | |
| 400 | − */ | |
| 401 | − VM_WARN_ON_ONCE(softleaf_is_migration(entry) && | |
| 402 | − !folio_test_locked(folio)); | |
| 411 | + if (softleaf_is_migration(entry)) | |
| 412 | + softleaf_migration_sync(entry, folio); | |
| 403 | 413 | |
| 404 | 414 | return folio; |
| 405 | 415 | } |
| @@ -55,6 +55,7 @@ struct mempolicy { · 54 unmodified lines | ||
| 55 | 55 | nodemask_t cpuset_mems_allowed; /* relative to these nodes */ |
| 56 | 56 | nodemask_t user_nodemask; /* nodemask passed by user */ |
| 57 | 57 | } w; |
| 58 | + struct rcu_head rcu; | |
| 58 | 59 | }; |
| 59 | 60 | |
| 60 | 61 | /* |
| @@ -140,7 +140,6 @@ struct netfs_io_stream { · 139 unmodified lines | ||
| 140 | 140 | void (*issue_write)(struct netfs_io_subrequest *subreq); |
| 141 | 141 | /* Collection tracking */ |
| 142 | 142 | struct list_head subrequests; /* Contributory I/O operations */ |
| 143 | − struct netfs_io_subrequest *front; /* Op being collected */ | |
| 144 | 143 | unsigned long long collected_to; /* Position we've collected results to */ |
| 145 | 144 | size_t transferred; /* The amount transferred from this stream */ |
| 146 | 145 | unsigned short error; /* Aggregate error for the stream */ |
| @@ -210,7 +210,6 @@ enum mapping_flags { · 209 unmodified lines | ||
| 210 | 210 | AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM = 9, |
| 211 | 211 | AS_KERNEL_FILE = 10, /* mapping for a fake kernel file that shouldn't |
| 212 | 212 | account usage to user cgroups */ |
| 213 | − AS_NO_DATA_INTEGRITY = 11, /* no data integrity guarantees */ | |
| 214 | 213 | /* Bits 16-25 are used for FOLIO_ORDER */ |
| 215 | 214 | AS_FOLIO_ORDER_BITS = 5, |
| 216 | 215 | AS_FOLIO_ORDER_MIN = 16, |
| @@ -346,16 +345,6 @@ static inline bool mapping_writeback_may_deadlock_on_reclaim(const struct addres · 129 unmodified lines | ||
| 346 | 345 | return test_bit(AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM, &mapping->flags); |
| 347 | 346 | } |
| 348 | 347 | |
| 349 | −static inline void mapping_set_no_data_integrity(struct address_space *mapping) | |
| 350 | −{ | |
| 351 | − set_bit(AS_NO_DATA_INTEGRITY, &mapping->flags); | |
| 352 | −} | |
| 353 | − | |
| 354 | −static inline bool mapping_no_data_integrity(const struct address_space *mapping) | |
| 355 | −{ | |
| 356 | − return test_bit(AS_NO_DATA_INTEGRITY, &mapping->flags); | |
| 357 | −} | |
| 358 | − | |
| 359 | 348 | static inline gfp_t mapping_gfp_mask(const struct address_space *mapping) |
| 360 | 349 | { |
| 361 | 350 | return mapping->gfp_mask; |
| @@ -145,6 +145,7 @@ enum lockdown_reason { · 144 unmodified lines | ||
| 145 | 145 | LOCKDOWN_BPF_WRITE_USER, |
| 146 | 146 | LOCKDOWN_DBG_WRITE_KERNEL, |
| 147 | 147 | LOCKDOWN_RTAS_ERROR_INJECTION, |
| 148 | + LOCKDOWN_XEN_USER_ACTIONS, | |
| 148 | 149 | LOCKDOWN_INTEGRITY_MAX, |
| 149 | 150 | LOCKDOWN_KCORE, |
| 150 | 151 | LOCKDOWN_KPROBES, |
| @@ -159,10 +159,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, · 158 unmodified lines | ||
| 159 | 159 | * @modalias: Name of the driver to use with this device, or an alias |
| 160 | 160 | * for that name. This appears in the sysfs "modalias" attribute |
| 161 | 161 | * for driver coldplugging, and in uevents used for hotplugging |
| 162 | − * @driver_override: If the name of a driver is written to this attribute, then | |
| 163 | − * the device will bind to the named driver and only the named driver. | |
| 164 | − * Do not set directly, because core frees it; use driver_set_override() to | |
| 165 | − * set or clear it. | |
| 166 | 162 | * @pcpu_statistics: statistics for the spi_device |
| 167 | 163 | * @word_delay: delay to be inserted between consecutive |
| 168 | 164 | * words of a transfer |
| @@ -224,7 +220,6 @@ struct spi_device { · 55 unmodified lines | ||
| 224 | 220 | void *controller_state; |
| 225 | 221 | void *controller_data; |
| 226 | 222 | char modalias[SPI_NAME_SIZE]; |
| 227 | − const char *driver_override; | |
| 228 | 223 | |
| 229 | 224 | /* The statistics */ |
| 230 | 225 | struct spi_statistics __percpu *pcpu_statistics; |
| @@ -11,6 +11,7 @@ · 10 unmodified lines | ||
| 11 | 11 | #ifndef _LINUX_SRCU_TINY_H |
| 12 | 12 | #define _LINUX_SRCU_TINY_H |
| 13 | 13 | |
| 14 | +#include <linux/irq_work_types.h> | |
| 14 | 15 | #include <linux/swait.h> |
| 15 | 16 | |
| 16 | 17 | struct srcu_struct { |
| @@ -24,18 +25,21 @@ struct srcu_struct { · 7 unmodified lines | ||
| 24 | 25 | struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ |
| 25 | 26 | struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ |
| 26 | 27 | struct work_struct srcu_work; /* For driving grace periods. */ |
| 28 | + struct irq_work srcu_irq_work; /* Defer schedule_work() to irq work. */ | |
| 27 | 29 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 28 | 30 | struct lockdep_map dep_map; |
| 29 | 31 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
| 30 | 32 | }; |
| 31 | 33 | |
| 32 | 34 | void srcu_drive_gp(struct work_struct *wp); |
| 35 | +void srcu_tiny_irq_work(struct irq_work *irq_work); | |
| 33 | 36 | |
| 34 | 37 | #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored) \ |
| 35 | 38 | { \ |
| 36 | 39 | .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ |
| 37 | 40 | .srcu_cb_tail = &name.srcu_cb_head, \ |
| 38 | 41 | .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ |
| 42 | + .srcu_irq_work = { .func = srcu_tiny_irq_work }, \ | |
| 39 | 43 | __SRCU_DEP_MAP_INIT(name) \ |
| 40 | 44 | } |
| 41 | 45 | |
| @@ -34,7 +34,7 @@ struct srcu_data { · 33 unmodified lines | ||
| 34 | 34 | /* Values: SRCU_READ_FLAVOR_.* */ |
| 35 | 35 | |
| 36 | 36 | /* Update-side state. */ |
| 37 | − spinlock_t __private lock ____cacheline_internodealigned_in_smp; | |
| 37 | + raw_spinlock_t __private lock ____cacheline_internodealigned_in_smp; | |
| 38 | 38 | struct rcu_segcblist srcu_cblist; /* List of callbacks.*/ |
| 39 | 39 | unsigned long srcu_gp_seq_needed; /* Furthest future GP needed. */ |
| 40 | 40 | unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ |
| @@ -55,7 +55,7 @@ struct srcu_data { · 14 unmodified lines | ||
| 55 | 55 | * Node in SRCU combining tree, similar in function to rcu_data. |
| 56 | 56 | */ |
| 57 | 57 | struct srcu_node { |
| 58 | − spinlock_t __private lock; | |
| 58 | + raw_spinlock_t __private lock; | |
| 59 | 59 | unsigned long srcu_have_cbs[4]; /* GP seq for children having CBs, but only */ |
| 60 | 60 | /* if greater than ->srcu_gp_seq. */ |
| 61 | 61 | unsigned long srcu_data_have_cbs[4]; /* Which srcu_data structs have CBs for given GP? */ |
| @@ -74,7 +74,7 @@ struct srcu_usage { · 12 unmodified lines | ||
| 74 | 74 | /* First node at each level. */ |
| 75 | 75 | int srcu_size_state; /* Small-to-big transition state. */ |
| 76 | 76 | struct mutex srcu_cb_mutex; /* Serialize CB preparation. */ |
| 77 | − spinlock_t __private lock; /* Protect counters and size state. */ | |
| 77 | + raw_spinlock_t __private lock; /* Protect counters and size state. */ | |
| 78 | 78 | struct mutex srcu_gp_mutex; /* Serialize GP work. */ |
| 79 | 79 | unsigned long srcu_gp_seq; /* Grace-period seq #. */ |
| 80 | 80 | unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */ |
| @@ -95,6 +95,7 @@ struct srcu_usage { · 14 unmodified lines | ||
| 95 | 95 | unsigned long reschedule_jiffies; |
| 96 | 96 | unsigned long reschedule_count; |
| 97 | 97 | struct delayed_work work; |
| 98 | + struct irq_work irq_work; | |
| 98 | 99 | struct srcu_struct *srcu_ssp; |
| 99 | 100 | }; |
| 100 | 101 | |
| @@ -156,7 +157,7 @@ struct srcu_struct { · 55 unmodified lines | ||
| 156 | 157 | |
| 157 | 158 | #define __SRCU_USAGE_INIT(name) \ |
| 158 | 159 | { \ |
| 159 | − .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ | |
| 160 | + .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ | |
| 160 | 161 | .srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL, \ |
| 161 | 162 | .srcu_gp_seq_needed = SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE, \ |
| 162 | 163 | .srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL, \ |
| @@ -207,6 +207,39 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, · 206 unmodified lines | ||
| 207 | 207 | return __virtio_net_hdr_to_skb(skb, hdr, little_endian, hdr->gso_type); |
| 208 | 208 | } |
| 209 | 209 | |
| 210 | +/* This function must be called after virtio_net_hdr_from_skb(). */ | |
| 211 | +static inline void __virtio_net_set_hdrlen(const struct sk_buff *skb, | |
| 212 | + struct virtio_net_hdr *hdr, | |
| 213 | + bool little_endian) | |
| 214 | +{ | |
| 215 | + u16 hdr_len; | |
| 216 | + | |
| 217 | + hdr_len = skb_transport_offset(skb); | |
| 218 | + | |
| 219 | + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) | |
| 220 | + hdr_len += sizeof(struct udphdr); | |
| 221 | + else | |
| 222 | + hdr_len += tcp_hdrlen(skb); | |
| 223 | + | |
| 224 | + hdr->hdr_len = __cpu_to_virtio16(little_endian, hdr_len); | |
| 225 | +} | |
| 226 | + | |
| 227 | +/* This function must be called after virtio_net_hdr_from_skb(). */ | |
| 228 | +static inline void __virtio_net_set_tnl_hdrlen(const struct sk_buff *skb, | |
| 229 | + struct virtio_net_hdr *hdr) | |
| 230 | +{ | |
| 231 | + u16 hdr_len; | |
| 232 | + | |
| 233 | + hdr_len = skb_inner_transport_offset(skb); | |
| 234 | + | |
| 235 | + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) | |
| 236 | + hdr_len += sizeof(struct udphdr); | |
| 237 | + else | |
| 238 | + hdr_len += inner_tcp_hdrlen(skb); | |
| 239 | + | |
| 240 | + hdr->hdr_len = __cpu_to_virtio16(true, hdr_len); | |
| 241 | +} | |
| 242 | + | |
| 210 | 243 | static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, |
| 211 | 244 | struct virtio_net_hdr *hdr, |
| 212 | 245 | bool little_endian, |
| @@ -385,7 +418,8 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, · 172 unmodified lines | ||
| 385 | 418 | bool tnl_hdr_negotiated, |
| 386 | 419 | bool little_endian, |
| 387 | 420 | int vlan_hlen, |
| 388 | − bool has_data_valid) | |
| 421 | + bool has_data_valid, | |
| 422 | + bool feature_hdrlen) | |
| 389 | 423 | { |
| 390 | 424 | struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)vhdr; |
| 391 | 425 | unsigned int inner_nh, outer_th; |
| @@ -394,9 +428,17 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, · 2 unmodified lines | ||
| 394 | 428 | |
| 395 | 429 | tnl_gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_UDP_TUNNEL | |
| 396 | 430 | SKB_GSO_UDP_TUNNEL_CSUM); |
| 397 | − if (!tnl_gso_type) | |
| 398 | − return virtio_net_hdr_from_skb(skb, hdr, little_endian, | |
| 399 | − has_data_valid, vlan_hlen); | |
| 431 | + if (!tnl_gso_type) { | |
| 432 | + ret = virtio_net_hdr_from_skb(skb, hdr, little_endian, | |
| 433 | + has_data_valid, vlan_hlen); | |
| 434 | + if (ret) | |
| 435 | + return ret; | |
| 436 | + | |
| 437 | + if (feature_hdrlen && hdr->hdr_len) | |
| 438 | + __virtio_net_set_hdrlen(skb, hdr, little_endian); | |
| 439 | + | |
| 440 | + return ret; | |
| 441 | + } | |
| 400 | 442 | |
| 401 | 443 | /* Tunnel support not negotiated but skb ask for it. */ |
| 402 | 444 | if (!tnl_hdr_negotiated) |
| @@ -414,6 +456,9 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, · 11 unmodified lines | ||
| 414 | 456 | if (ret) |
| 415 | 457 | return ret; |
| 416 | 458 | |
| 459 | + if (feature_hdrlen && hdr->hdr_len) | |
| 460 | + __virtio_net_set_tnl_hdrlen(skb, hdr); | |
| 461 | + | |
| 417 | 462 | if (skb->protocol == htons(ETH_P_IPV6)) |
| 418 | 463 | hdr->gso_type |= VIRTIO_NET_HDR_GSO_UDP_TUNNEL_IPV6; |
| 419 | 464 | else |
| @@ -658,6 +658,7 @@ struct l2cap_conn { · 657 unmodified lines | ||
| 658 | 658 | struct sk_buff *rx_skb; |
| 659 | 659 | __u32 rx_len; |
| 660 | 660 | struct ida tx_ida; |
| 661 | + __u8 tx_ident; | |
| 661 | 662 | |
| 662 | 663 | struct sk_buff_head pending_rx; |
| 663 | 664 | struct work_struct pending_rx_work; |
| @@ -158,6 +158,7 @@ static struct sk_buff *codel_dequeue(void *ctx, · 157 unmodified lines | ||
| 158 | 158 | bool drop; |
| 159 | 159 | |
| 160 | 160 | if (!skb) { |
| 161 | + vars->first_above_time = 0; | |
| 161 | 162 | vars->dropping = false; |
| 162 | 163 | return skb; |
| 163 | 164 | } |
| @@ -264,6 +264,20 @@ inet_bhashfn_portaddr(const struct inet_hashinfo *hinfo, const struct sock *sk, · 263 unmodified lines | ||
| 264 | 264 | return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)]; |
| 265 | 265 | } |
| 266 | 266 | |
| 267 | +static inline bool inet_use_hash2_on_bind(const struct sock *sk) | |
| 268 | +{ | |
| 269 | +#if IS_ENABLED(CONFIG_IPV6) | |
| 270 | + if (sk->sk_family == AF_INET6) { | |
| 271 | + if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) | |
| 272 | + return false; | |
| 273 | + | |
| 274 | + if (!ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) | |
| 275 | + return true; | |
| 276 | + } | |
| 277 | +#endif | |
| 278 | + return sk->sk_rcv_saddr != htonl(INADDR_ANY); | |
| 279 | +} | |
| 280 | + | |
| 267 | 281 | struct inet_bind_hashbucket * |
| 268 | 282 | inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port); |
| 269 | 283 | |
| @@ -507,12 +507,14 @@ void fib6_rt_update(struct net *net, struct fib6_info *rt, · 506 unmodified lines | ||
| 507 | 507 | void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, |
| 508 | 508 | unsigned int flags); |
| 509 | 509 | |
| 510 | +void fib6_age_exceptions(struct fib6_info *rt, struct fib6_gc_args *gc_args, | |
| 511 | + unsigned long now); | |
| 510 | 512 | void fib6_run_gc(unsigned long expires, struct net *net, bool force); |
| 511 | − | |
| 512 | 513 | void fib6_gc_cleanup(void); |
| 513 | 514 | |
| 514 | 515 | int fib6_init(void); |
| 515 | 516 | |
| 517 | +#if IS_ENABLED(CONFIG_IPV6) | |
| 516 | 518 | /* Add the route to the gc list if it is not already there |
| 517 | 519 | * |
| 518 | 520 | * The callers should hold f6i->fib6_table->tb6_lock. |
| @@ -545,6 +547,23 @@ static inline void fib6_remove_gc_list(struct fib6_info *f6i) · 26 unmodified lines | ||
| 545 | 547 | hlist_del_init(&f6i->gc_link); |
| 546 | 548 | } |
| 547 | 549 | |
| 550 | +static inline void fib6_may_remove_gc_list(struct net *net, | |
| 551 | + struct fib6_info *f6i) | |
| 552 | +{ | |
| 553 | + struct fib6_gc_args gc_args; | |
| 554 | + | |
| 555 | + if (hlist_unhashed(&f6i->gc_link)) | |
| 556 | + return; | |
| 557 | + | |
| 558 | + gc_args.timeout = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_interval); | |
| 559 | + gc_args.more = 0; | |
| 560 | + | |
| 561 | + rcu_read_lock(); | |
| 562 | + fib6_age_exceptions(f6i, &gc_args, jiffies); | |
| 563 | + rcu_read_unlock(); | |
| 564 | +} | |
| 565 | +#endif | |
| 566 | + | |
| 548 | 567 | struct ipv6_route_iter { |
| 549 | 568 | struct seq_net_private p; |
| 550 | 569 | struct fib6_walker w; |
| @@ -83,6 +83,11 @@ void nf_conntrack_lock(spinlock_t *lock); · 82 unmodified lines | ||
| 83 | 83 | |
| 84 | 84 | extern spinlock_t nf_conntrack_expect_lock; |
| 85 | 85 | |
| 86 | +static inline void lockdep_nfct_expect_lock_held(void) | |
| 87 | +{ | |
| 88 | + lockdep_assert_held(&nf_conntrack_expect_lock); | |
| 89 | +} | |
| 90 | + | |
| 86 | 91 | /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ |
| 87 | 92 | |
| 88 | 93 | static inline void __nf_ct_set_timeout(struct nf_conn *ct, u64 timeout) |
| @@ -22,10 +22,16 @@ struct nf_conntrack_expect { · 21 unmodified lines | ||
| 22 | 22 | /* Hash member */ |
| 23 | 23 | struct hlist_node hnode; |
| 24 | 24 | |
| 25 | + /* Network namespace */ | |
| 26 | + possible_net_t net; | |
| 27 | + | |
| 25 | 28 | /* We expect this tuple, with the following mask */ |
| 26 | 29 | struct nf_conntrack_tuple tuple; |
| 27 | 30 | struct nf_conntrack_tuple_mask mask; |
| 28 | 31 | |
| 32 | +#ifdef CONFIG_NF_CONNTRACK_ZONES | |
| 33 | + struct nf_conntrack_zone zone; | |
| 34 | +#endif | |
| 29 | 35 | /* Usage count. */ |
| 30 | 36 | refcount_t use; |
| 31 | 37 | |
| @@ -40,7 +46,7 @@ struct nf_conntrack_expect { · 8 unmodified lines | ||
| 40 | 46 | struct nf_conntrack_expect *this); |
| 41 | 47 | |
| 42 | 48 | /* Helper to assign to new connection */ |
| 43 | − struct nf_conntrack_helper *helper; | |
| 49 | + struct nf_conntrack_helper __rcu *helper; | |
| 44 | 50 | |
| 45 | 51 | /* The conntrack of the master connection */ |
| 46 | 52 | struct nf_conn *master; |
| @@ -62,7 +68,17 @@ struct nf_conntrack_expect { · 15 unmodified lines | ||
| 62 | 68 | |
| 63 | 69 | static inline struct net *nf_ct_exp_net(struct nf_conntrack_expect *exp) |
| 64 | 70 | { |
| 65 | − return nf_ct_net(exp->master); | |
| 71 | + return read_pnet(&exp->net); | |
| 72 | +} | |
| 73 | + | |
| 74 | +static inline bool nf_ct_exp_zone_equal_any(const struct nf_conntrack_expect *a, | |
| 75 | + const struct nf_conntrack_zone *b) | |
| 76 | +{ | |
| 77 | +#ifdef CONFIG_NF_CONNTRACK_ZONES | |
| 78 | + return a->zone.id == b->id; | |
| 79 | +#else | |
| 80 | + return true; | |
| 81 | +#endif | |
| 66 | 82 | } |
| 67 | 83 | |
| 68 | 84 | #define NF_CT_EXP_POLICY_NAME_LEN 16 |
| @@ -59,7 +59,7 @@ struct netns_xfrm { · 58 unmodified lines | ||
| 59 | 59 | struct list_head inexact_bins; |
| 60 | 60 | |
| 61 | 61 | |
| 62 | − struct sock *nlsk; | |
| 62 | + struct sock __rcu *nlsk; | |
| 63 | 63 | struct sock *nlsk_stash; |
| 64 | 64 | |
| 65 | 65 | u32 sysctl_aevent_etime; |
| @@ -26,11 +26,6 @@ struct sdca_function_desc; · 25 unmodified lines | ||
| 26 | 26 | */ |
| 27 | 27 | #define SDCA_MAX_ENTITY_COUNT 128 |
| 28 | 28 | |
| 29 | −/* | |
| 30 | − * Sanity check on number of initialization writes, can be expanded if needed. | |
| 31 | − */ | |
| 32 | −#define SDCA_MAX_INIT_COUNT 2048 | |
| 33 | − | |
| 34 | 29 | /* |
| 35 | 30 | * The Cluster IDs are 16-bit, so a maximum of 65535 Clusters per |
| 36 | 31 | * function can be represented, however limit this to a slightly |
| @@ -769,12 +769,15 @@ TRACE_EVENT(btrfs_sync_file, · 768 unmodified lines | ||
| 769 | 769 | ), |
| 770 | 770 | |
| 771 | 771 | TP_fast_assign( |
| 772 | − const struct dentry *dentry = file->f_path.dentry; | |
| 773 | − const struct inode *inode = d_inode(dentry); | |
| 772 | + struct dentry *dentry = file_dentry(file); | |
| 773 | + struct inode *inode = file_inode(file); | |
| 774 | + struct dentry *parent = dget_parent(dentry); | |
| 775 | + struct inode *parent_inode = d_inode(parent); | |
| 774 | 776 | |
| 775 | − TP_fast_assign_fsid(btrfs_sb(file->f_path.dentry->d_sb)); | |
| 777 | + dput(parent); | |
| 778 | + TP_fast_assign_fsid(btrfs_sb(inode->i_sb)); | |
| 776 | 779 | __entry->ino = btrfs_ino(BTRFS_I(inode)); |
| 777 | − __entry->parent = btrfs_ino(BTRFS_I(d_inode(dentry->d_parent))); | |
| 780 | + __entry->parent = btrfs_ino(BTRFS_I(parent_inode)); | |
| 778 | 781 | __entry->datasync = datasync; |
| 779 | 782 | __entry->root_objectid = btrfs_root_id(BTRFS_I(inode)->root); |
| 780 | 783 | ), |
| @@ -32,7 +32,9 @@ TRACE_DEFINE_ENUM(DMA_NONE); · 31 unmodified lines | ||
| 32 | 32 | { DMA_ATTR_ALLOC_SINGLE_PAGES, "ALLOC_SINGLE_PAGES" }, \ |
| 33 | 33 | { DMA_ATTR_NO_WARN, "NO_WARN" }, \ |
| 34 | 34 | { DMA_ATTR_PRIVILEGED, "PRIVILEGED" }, \ |
| 35 | − { DMA_ATTR_MMIO, "MMIO" }) | |
| 35 | + { DMA_ATTR_MMIO, "MMIO" }, \ | |
| 36 | + { DMA_ATTR_DEBUGGING_IGNORE_CACHELINES, "CACHELINES_OVERLAP" }, \ | |
| 37 | + { DMA_ATTR_REQUIRE_COHERENT, "REQUIRE_COHERENT" }) | |
| 36 | 38 | |
| 37 | 39 | DECLARE_EVENT_CLASS(dma_map, |
| 38 | 40 | TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, |
| @@ -740,19 +740,19 @@ TRACE_EVENT(netfs_collect_stream, · 739 unmodified lines | ||
| 740 | 740 | __field(unsigned int, wreq) |
| 741 | 741 | __field(unsigned char, stream) |
| 742 | 742 | __field(unsigned long long, collected_to) |
| 743 | − __field(unsigned long long, front) | |
| 743 | + __field(unsigned long long, issued_to) | |
| 744 | 744 | ), |
| 745 | 745 | |
| 746 | 746 | TP_fast_assign( |
| 747 | 747 | __entry->wreq = wreq->debug_id; |
| 748 | 748 | __entry->stream = stream->stream_nr; |
| 749 | 749 | __entry->collected_to = stream->collected_to; |
| 750 | − __entry->front = stream->front ? stream->front->start : UINT_MAX; | |
| 750 | + __entry->issued_to = atomic64_read(&wreq->issued_to); | |
| 751 | 751 | ), |
| 752 | 752 | |
| 753 | − TP_printk("R=%08x[%x:] cto=%llx frn=%llx", | |
| 753 | + TP_printk("R=%08x[%x:] cto=%llx ito=%llx", | |
| 754 | 754 | __entry->wreq, __entry->stream, |
| 755 | − __entry->collected_to, __entry->front) | |
| 755 | + __entry->collected_to, __entry->issued_to) | |
| 756 | 756 | ); |
| 757 | 757 | |
| 758 | 758 | TRACE_EVENT(netfs_folioq, |
| @@ -159,5 +159,9 @@ enum ip_conntrack_expect_events { · 158 unmodified lines | ||
| 159 | 159 | #define NF_CT_EXPECT_INACTIVE 0x2 |
| 160 | 160 | #define NF_CT_EXPECT_USERSPACE 0x4 |
| 161 | 161 | |
| 162 | +#ifdef __KERNEL__ | |
| 163 | +#define NF_CT_EXPECT_MASK (NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE | \ | |
| 164 | + NF_CT_EXPECT_USERSPACE) | |
| 165 | +#endif | |
| 162 | 166 | |
| 163 | 167 | #endif /* _UAPI_NF_CONNTRACK_COMMON_H */ |
| @@ -146,7 +146,7 @@ config CC_HAS_COUNTED_BY · 145 unmodified lines | ||
| 146 | 146 | config CC_HAS_COUNTED_BY_PTR |
| 147 | 147 | bool |
| 148 | 148 | # supported since clang 22 |
| 149 | − default y if CC_IS_CLANG && CLANG_VERSION >= 220000 | |
| 149 | + default y if CC_IS_CLANG && CLANG_VERSION >= 220100 | |
| 150 | 150 | # supported since gcc 16.0.0 |
| 151 | 151 | default y if CC_IS_GCC && GCC_VERSION >= 160000 |
| 152 | 152 | |
| @@ -119,12 +119,14 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) · 118 unmodified lines | ||
| 119 | 119 | sq_idx); |
| 120 | 120 | break; |
| 121 | 121 | } |
| 122 | − if ((++sq_head & sq_mask) == 0) { | |
| 122 | + if (sq_idx == sq_mask) { | |
| 123 | 123 | seq_printf(m, |
| 124 | 124 | "%5u: corrupted sqe, wrapping 128B entry\n", |
| 125 | 125 | sq_idx); |
| 126 | 126 | break; |
| 127 | 127 | } |
| 128 | + sq_head++; | |
| 129 | + i++; | |
| 128 | 130 | sqe128 = true; |
| 129 | 131 | } |
| 130 | 132 | seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, " |
| @@ -453,7 +453,7 @@ static int active_cacheline_set_overlap(phys_addr_t cln, int overlap) · 452 unmodified lines | ||
| 453 | 453 | return overlap; |
| 454 | 454 | } |
| 455 | 455 | |
| 456 | −static void active_cacheline_inc_overlap(phys_addr_t cln) | |
| 456 | +static void active_cacheline_inc_overlap(phys_addr_t cln, bool is_cache_clean) | |
| 457 | 457 | { |
| 458 | 458 | int overlap = active_cacheline_read_overlap(cln); |
| 459 | 459 | |
| @@ -462,7 +462,7 @@ static void active_cacheline_inc_overlap(phys_addr_t cln) · 2 unmodified lines | ||
| 462 | 462 | /* If we overflowed the overlap counter then we're potentially |
| 463 | 463 | * leaking dma-mappings. |
| 464 | 464 | */ |
| 465 | − WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP, | |
| 465 | + WARN_ONCE(!is_cache_clean && overlap > ACTIVE_CACHELINE_MAX_OVERLAP, | |
| 466 | 466 | pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"), |
| 467 | 467 | ACTIVE_CACHELINE_MAX_OVERLAP, &cln); |
| 468 | 468 | } |
| @@ -495,7 +495,7 @@ static int active_cacheline_insert(struct dma_debug_entry *entry, · 26 unmodified lines | ||
| 495 | 495 | if (rc == -EEXIST) { |
| 496 | 496 | struct dma_debug_entry *existing; |
| 497 | 497 | |
| 498 | − active_cacheline_inc_overlap(cln); | |
| 498 | + active_cacheline_inc_overlap(cln, entry->is_cache_clean); | |
| 499 | 499 | existing = radix_tree_lookup(&dma_active_cacheline, cln); |
| 500 | 500 | /* A lookup failure here after we got -EEXIST is unexpected. */ |
| 501 | 501 | WARN_ON(!existing); |
| @@ -601,7 +601,8 @@ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) · 99 unmodified lines | ||
| 601 | 601 | unsigned long flags; |
| 602 | 602 | int rc; |
| 603 | 603 | |
| 604 | − entry->is_cache_clean = !!(attrs & DMA_ATTR_CPU_CACHE_CLEAN); | |
| 604 | + entry->is_cache_clean = attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | | |
| 605 | + DMA_ATTR_REQUIRE_COHERENT); | |
| 605 | 606 | |
| 606 | 607 | bucket = get_hash_bucket(entry, &flags); |
| 607 | 608 | hash_bucket_add(bucket, entry); |
| @@ -84,7 +84,7 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev, · 83 unmodified lines | ||
| 84 | 84 | dma_addr_t dma_addr; |
| 85 | 85 | |
| 86 | 86 | if (is_swiotlb_force_bounce(dev)) { |
| 87 | − if (attrs & DMA_ATTR_MMIO) | |
| 87 | + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) | |
| 88 | 88 | return DMA_MAPPING_ERROR; |
| 89 | 89 | |
| 90 | 90 | return swiotlb_map(dev, phys, size, dir, attrs); |
| @@ -98,7 +98,8 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev, · 7 unmodified lines | ||
| 98 | 98 | dma_addr = phys_to_dma(dev, phys); |
| 99 | 99 | if (unlikely(!dma_capable(dev, dma_addr, size, true)) || |
| 100 | 100 | dma_kmalloc_needs_bounce(dev, size, dir)) { |
| 101 | − if (is_swiotlb_active(dev)) | |
| 101 | + if (is_swiotlb_active(dev) && | |
| 102 | + !(attrs & DMA_ATTR_REQUIRE_COHERENT)) | |
| 102 | 103 | return swiotlb_map(dev, phys, size, dir, attrs); |
| 103 | 104 | |
| 104 | 105 | goto err_overflow; |
| @@ -123,7 +124,7 @@ static inline void dma_direct_unmap_phys(struct device *dev, dma_addr_t addr, · 18 unmodified lines | ||
| 123 | 124 | { |
| 124 | 125 | phys_addr_t phys; |
| 125 | 126 | |
| 126 | − if (attrs & DMA_ATTR_MMIO) | |
| 127 | + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) | |
| 127 | 128 | /* nothing to do: uncached and no swiotlb */ |
| 128 | 129 | return; |
| 129 | 130 | |
| @@ -164,6 +164,9 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, · 163 unmodified lines | ||
| 164 | 164 | if (WARN_ON_ONCE(!dev->dma_mask)) |
| 165 | 165 | return DMA_MAPPING_ERROR; |
| 166 | 166 | |
| 167 | + if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT)) | |
| 168 | + return DMA_MAPPING_ERROR; | |
| 169 | + | |
| 167 | 170 | if (dma_map_direct(dev, ops) || |
| 168 | 171 | (!is_mmio && arch_dma_map_phys_direct(dev, phys + size))) |
| 169 | 172 | addr = dma_direct_map_phys(dev, phys, size, dir, attrs); |
| @@ -235,6 +238,9 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, · 65 unmodified lines | ||
| 235 | 238 | |
| 236 | 239 | BUG_ON(!valid_dma_direction(dir)); |
| 237 | 240 | |
| 241 | + if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT)) | |
| 242 | + return -EOPNOTSUPP; | |
| 243 | + | |
| 238 | 244 | if (WARN_ON_ONCE(!dev->dma_mask)) |
| 239 | 245 | return 0; |
| 240 | 246 | |
| @@ -30,6 +30,7 @@ · 29 unmodified lines | ||
| 30 | 30 | #include <linux/gfp.h> |
| 31 | 31 | #include <linux/highmem.h> |
| 32 | 32 | #include <linux/io.h> |
| 33 | +#include <linux/kmsan-checks.h> | |
| 33 | 34 | #include <linux/iommu-helper.h> |
| 34 | 35 | #include <linux/init.h> |
| 35 | 36 | #include <linux/memblock.h> |
| @@ -901,10 +902,19 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size · 865 unmodified lines | ||
| 901 | 902 | |
| 902 | 903 | local_irq_save(flags); |
| 903 | 904 | page = pfn_to_page(pfn); |
| 904 | − if (dir == DMA_TO_DEVICE) | |
| 905 | + if (dir == DMA_TO_DEVICE) { | |
| 906 | + /* | |
| 907 | + * Ideally, kmsan_check_highmem_page() | |
| 908 | + * could be used here to detect infoleaks, | |
| 909 | + * but callers may map uninitialized buffers | |
| 910 | + * that will be written by the device, | |
| 911 | + * causing false positives. | |
| 912 | + */ | |
| 905 | 913 | memcpy_from_page(vaddr, page, offset, sz); |
| 906 | − else | |
| 914 | + } else { | |
| 915 | + kmsan_unpoison_memory(vaddr, sz); | |
| 907 | 916 | memcpy_to_page(page, offset, vaddr, sz); |
| 917 | + } | |
| 908 | 918 | local_irq_restore(flags); |
| 909 | 919 | |
| 910 | 920 | size -= sz; |
| @@ -913,8 +923,15 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size · 2 unmodified lines | ||
| 913 | 923 | offset = 0; |
| 914 | 924 | } |
| 915 | 925 | } else if (dir == DMA_TO_DEVICE) { |
| 926 | + /* | |
| 927 | + * Ideally, kmsan_check_memory() could be used here to detect | |
| 928 | + * infoleaks (uninitialized data being sent to device), but | |
| 929 | + * callers may map uninitialized buffers that will be written | |
| 930 | + * by the device, causing false positives. | |
| 931 | + */ | |
| 916 | 932 | memcpy(vaddr, phys_to_virt(orig_addr), size); |
| 917 | 933 | } else { |
| 934 | + kmsan_unpoison_memory(vaddr, size); | |
| 918 | 935 | memcpy(phys_to_virt(orig_addr), vaddr, size); |
| 919 | 936 | } |
| 920 | 937 | } |
| @@ -342,7 +342,7 @@ static int __futex_key_to_node(struct mm_struct *mm, unsigned long addr) · 341 unmodified lines | ||
| 342 | 342 | if (!vma) |
| 343 | 343 | return FUTEX_NO_NODE; |
| 344 | 344 | |
| 345 | − mpol = vma_policy(vma); | |
| 345 | + mpol = READ_ONCE(vma->vm_policy); | |
| 346 | 346 | if (!mpol) |
| 347 | 347 | return FUTEX_NO_NODE; |
| 348 | 348 | |
| @@ -918,7 +918,7 @@ int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked) · 917 unmodified lines | ||
| 918 | 918 | int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock) |
| 919 | 919 | { |
| 920 | 920 | struct hrtimer_sleeper timeout, *to; |
| 921 | − struct task_struct *exiting = NULL; | |
| 921 | + struct task_struct *exiting; | |
| 922 | 922 | struct rt_mutex_waiter rt_waiter; |
| 923 | 923 | struct futex_q q = futex_q_init; |
| 924 | 924 | DEFINE_WAKE_Q(wake_q); |
| @@ -933,6 +933,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl · 8 unmodified lines | ||
| 933 | 933 | to = futex_setup_timer(time, &timeout, flags, 0); |
| 934 | 934 | |
| 935 | 935 | retry: |
| 936 | + exiting = NULL; | |
| 936 | 937 | ret = get_futex_key(uaddr, flags, &q.key, FUTEX_WRITE); |
| 937 | 938 | if (unlikely(ret != 0)) |
| 938 | 939 | goto out; |
| @@ -459,6 +459,14 @@ SYSCALL_DEFINE4(futex_requeue, · 458 unmodified lines | ||
| 459 | 459 | if (ret) |
| 460 | 460 | return ret; |
| 461 | 461 | |
| 462 | + /* | |
| 463 | + * For now mandate both flags are identical, like the sys_futex() | |
| 464 | + * interface has. If/when we merge the variable sized futex support, | |
| 465 | + * that patch can modify this test to allow a difference in size. | |
| 466 | + */ | |
| 467 | + if (futexes[0].w.flags != futexes[1].w.flags) | |
| 468 | + return -EINVAL; | |
| 469 | + | |
| 462 | 470 | cmpval = futexes[0].w.val; |
| 463 | 471 | |
| 464 | 472 | return futex_requeue(u64_to_user_ptr(futexes[0].w.uaddr), futexes[0].w.flags, |
| @@ -40,7 +40,7 @@ void pm_restore_gfp_mask(void) · 39 unmodified lines | ||
| 40 | 40 | { |
| 41 | 41 | WARN_ON(!mutex_is_locked(&system_transition_mutex)); |
| 42 | 42 | |
| 43 | − if (WARN_ON(!saved_gfp_count) || --saved_gfp_count) | |
| 43 | + if (!saved_gfp_count || --saved_gfp_count) | |
| 44 | 44 | return; |
| 45 | 45 | |
| 46 | 46 | gfp_allowed_mask = saved_gfp_mask; |
| @@ -2855,6 +2855,17 @@ int snapshot_write_finalize(struct snapshot_handle *handle) · 2854 unmodified lines | ||
| 2855 | 2855 | { |
| 2856 | 2856 | int error; |
| 2857 | 2857 | |
| 2858 | + /* | |
| 2859 | + * Call snapshot_write_next() to drain any trailing zero pages, | |
| 2860 | + * but make sure we're in the data page region first. | |
| 2861 | + * This function can return PAGE_SIZE if the kernel was expecting | |
| 2862 | + * another copy page. Return -ENODATA in that situation. | |
| 2863 | + */ | |
| 2864 | + if (handle->cur > nr_meta_pages + 1) { | |
| 2865 | + error = snapshot_write_next(handle); | |
| 2866 | + if (error) | |
| 2867 | + return error > 0 ? -ENODATA : error; | |
| 2868 | + } | |
| 2858 | 2869 | copy_last_highmem_page(); |
| 2859 | 2870 | error = hibernate_restore_protect_page(handle->buffer); |
| 2860 | 2871 | /* Do that only if we have loaded the image entirely */ |
| @@ -502,6 +502,15 @@ do { \ · 501 unmodified lines | ||
| 502 | 502 | ___locked; \ |
| 503 | 503 | }) |
| 504 | 504 | |
| 505 | +#define raw_spin_trylock_irqsave_rcu_node(p, flags) \ | |
| 506 | +({ \ | |
| 507 | + bool ___locked = raw_spin_trylock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \ | |
| 508 | + \ | |
| 509 | + if (___locked) \ | |
| 510 | + smp_mb__after_unlock_lock(); \ | |
| 511 | + ___locked; \ | |
| 512 | +}) | |
| 513 | + | |
| 505 | 514 | #define raw_lockdep_assert_held_rcu_node(p) \ |
| 506 | 515 | lockdep_assert_held(&ACCESS_PRIVATE(p, lock)) |
| 507 | 516 | |
| @@ -9,6 +9,7 @@ · 8 unmodified lines | ||
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | 11 | #include <linux/export.h> |
| 12 | +#include <linux/irq_work.h> | |
| 12 | 13 | #include <linux/mutex.h> |
| 13 | 14 | #include <linux/preempt.h> |
| 14 | 15 | #include <linux/rcupdate_wait.h> |
| @@ -41,6 +42,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) · 26 unmodified lines | ||
| 41 | 42 | ssp->srcu_idx_max = 0; |
| 42 | 43 | INIT_WORK(&ssp->srcu_work, srcu_drive_gp); |
| 43 | 44 | INIT_LIST_HEAD(&ssp->srcu_work.entry); |
| 45 | + init_irq_work(&ssp->srcu_irq_work, srcu_tiny_irq_work); | |
| 44 | 46 | return 0; |
| 45 | 47 | } |
| 46 | 48 | |
| @@ -84,6 +86,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); · 37 unmodified lines | ||
| 84 | 86 | void cleanup_srcu_struct(struct srcu_struct *ssp) |
| 85 | 87 | { |
| 86 | 88 | WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); |
| 89 | + irq_work_sync(&ssp->srcu_irq_work); | |
| 87 | 90 | flush_work(&ssp->srcu_work); |
| 88 | 91 | WARN_ON(ssp->srcu_gp_running); |
| 89 | 92 | WARN_ON(ssp->srcu_gp_waiting); |
| @@ -177,6 +180,20 @@ void srcu_drive_gp(struct work_struct *wp) · 87 unmodified lines | ||
| 177 | 180 | } |
| 178 | 181 | EXPORT_SYMBOL_GPL(srcu_drive_gp); |
| 179 | 182 | |
| 183 | +/* | |
| 184 | + * Use an irq_work to defer schedule_work() to avoid acquiring the workqueue | |
| 185 | + * pool->lock while the caller might hold scheduler locks, causing lockdep | |
| 186 | + * splats due to workqueue_init() doing a wakeup. | |
| 187 | + */ | |
| 188 | +void srcu_tiny_irq_work(struct irq_work *irq_work) | |
| 189 | +{ | |
| 190 | + struct srcu_struct *ssp; | |
| 191 | + | |
| 192 | + ssp = container_of(irq_work, struct srcu_struct, srcu_irq_work); | |
| 193 | + schedule_work(&ssp->srcu_work); | |
| 194 | +} | |
| 195 | +EXPORT_SYMBOL_GPL(srcu_tiny_irq_work); | |
| 196 | + | |
| 180 | 197 | static void srcu_gp_start_if_needed(struct srcu_struct *ssp) |
| 181 | 198 | { |
| 182 | 199 | unsigned long cookie; |
| @@ -189,7 +206,7 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) · 6 unmodified lines | ||
| 189 | 206 | WRITE_ONCE(ssp->srcu_idx_max, cookie); |
| 190 | 207 | if (!READ_ONCE(ssp->srcu_gp_running)) { |
| 191 | 208 | if (likely(srcu_init_done)) |
| 192 | − schedule_work(&ssp->srcu_work); | |
| 209 | + irq_work_queue(&ssp->srcu_irq_work); | |
| 193 | 210 | else if (list_empty(&ssp->srcu_work.entry)) |
| 194 | 211 | list_add(&ssp->srcu_work.entry, &srcu_boot_list); |
| 195 | 212 | } |
| @@ -19,6 +19,7 @@ · 18 unmodified lines | ||
| 19 | 19 | #include <linux/mutex.h> |
| 20 | 20 | #include <linux/percpu.h> |
| 21 | 21 | #include <linux/preempt.h> |
| 22 | +#include <linux/irq_work.h> | |
| 22 | 23 | #include <linux/rcupdate_wait.h> |
| 23 | 24 | #include <linux/sched.h> |
| 24 | 25 | #include <linux/smp.h> |
| @@ -75,44 +76,9 @@ static bool __read_mostly srcu_init_done; · 50 unmodified lines | ||
| 75 | 76 | static void srcu_invoke_callbacks(struct work_struct *work); |
| 76 | 77 | static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay); |
| 77 | 78 | static void process_srcu(struct work_struct *work); |
| 79 | +static void srcu_irq_work(struct irq_work *work); | |
| 78 | 80 | static void srcu_delay_timer(struct timer_list *t); |
| 79 | 81 | |
| 80 | −/* Wrappers for lock acquisition and release, see raw_spin_lock_rcu_node(). */ | |
| 81 | −#define spin_lock_rcu_node(p) \ | |
| 82 | −do { \ | |
| 83 | − spin_lock(&ACCESS_PRIVATE(p, lock)); \ | |
| 84 | − smp_mb__after_unlock_lock(); \ | |
| 85 | −} while (0) | |
| 86 | − | |
| 87 | −#define spin_unlock_rcu_node(p) spin_unlock(&ACCESS_PRIVATE(p, lock)) | |
| 88 | − | |
| 89 | −#define spin_lock_irq_rcu_node(p) \ | |
| 90 | −do { \ | |
| 91 | − spin_lock_irq(&ACCESS_PRIVATE(p, lock)); \ | |
| 92 | − smp_mb__after_unlock_lock(); \ | |
| 93 | −} while (0) | |
| 94 | − | |
| 95 | −#define spin_unlock_irq_rcu_node(p) \ | |
| 96 | − spin_unlock_irq(&ACCESS_PRIVATE(p, lock)) | |
| 97 | − | |
| 98 | −#define spin_lock_irqsave_rcu_node(p, flags) \ | |
| 99 | −do { \ | |
| 100 | − spin_lock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \ | |
| 101 | − smp_mb__after_unlock_lock(); \ | |
| 102 | −} while (0) | |
| 103 | − | |
| 104 | −#define spin_trylock_irqsave_rcu_node(p, flags) \ | |
| 105 | −({ \ | |
| 106 | − bool ___locked = spin_trylock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \ | |
| 107 | − \ | |
| 108 | − if (___locked) \ | |
| 109 | − smp_mb__after_unlock_lock(); \ | |
| 110 | − ___locked; \ | |
| 111 | −}) | |
| 112 | − | |
| 113 | −#define spin_unlock_irqrestore_rcu_node(p, flags) \ | |
| 114 | − spin_unlock_irqrestore(&ACCESS_PRIVATE(p, lock), flags) \ | |
| 115 | − | |
| 116 | 82 | /* |
| 117 | 83 | * Initialize SRCU per-CPU data. Note that statically allocated |
| 118 | 84 | * srcu_struct structures might already have srcu_read_lock() and |
| @@ -131,7 +97,7 @@ static void init_srcu_struct_data(struct srcu_struct *ssp) · 12 unmodified lines | ||
| 131 | 97 | */ |
| 132 | 98 | for_each_possible_cpu(cpu) { |
| 133 | 99 | sdp = per_cpu_ptr(ssp->sda, cpu); |
| 134 | − spin_lock_init(&ACCESS_PRIVATE(sdp, lock)); | |
| 100 | + raw_spin_lock_init(&ACCESS_PRIVATE(sdp, lock)); | |
| 135 | 101 | rcu_segcblist_init(&sdp->srcu_cblist); |
| 136 | 102 | sdp->srcu_cblist_invoking = false; |
| 137 | 103 | sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq; |
| @@ -186,7 +152,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) · 48 unmodified lines | ||
| 186 | 152 | |
| 187 | 153 | /* Each pass through this loop initializes one srcu_node structure. */ |
| 188 | 154 | srcu_for_each_node_breadth_first(ssp, snp) { |
| 189 | − spin_lock_init(&ACCESS_PRIVATE(snp, lock)); | |
| 155 | + raw_spin_lock_init(&ACCESS_PRIVATE(snp, lock)); | |
| 190 | 156 | BUILD_BUG_ON(ARRAY_SIZE(snp->srcu_have_cbs) != |
| 191 | 157 | ARRAY_SIZE(snp->srcu_data_have_cbs)); |
| 192 | 158 | for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) { |
| @@ -242,7 +208,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) · 49 unmodified lines | ||
| 242 | 208 | if (!ssp->srcu_sup) |
| 243 | 209 | return -ENOMEM; |
| 244 | 210 | if (!is_static) |
| 245 | − spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); | |
| 211 | + raw_spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); | |
| 246 | 212 | ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL; |
| 247 | 213 | ssp->srcu_sup->node = NULL; |
| 248 | 214 | mutex_init(&ssp->srcu_sup->srcu_cb_mutex); |
| @@ -252,6 +218,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) · 3 unmodified lines | ||
| 252 | 218 | mutex_init(&ssp->srcu_sup->srcu_barrier_mutex); |
| 253 | 219 | atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0); |
| 254 | 220 | INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu); |
| 221 | + init_irq_work(&ssp->srcu_sup->irq_work, srcu_irq_work); | |
| 255 | 222 | ssp->srcu_sup->sda_is_static = is_static; |
| 256 | 223 | if (!is_static) { |
| 257 | 224 | ssp->sda = alloc_percpu(struct srcu_data); |
| @@ -263,9 +230,12 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) · 5 unmodified lines | ||
| 263 | 230 | ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL; |
| 264 | 231 | ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); |
| 265 | 232 | if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { |
| 266 | − if (!init_srcu_struct_nodes(ssp, is_static ? GFP_ATOMIC : GFP_KERNEL)) | |
| 233 | + if (!preemptible()) | |
| 234 | + WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC); | |
| 235 | + else if (init_srcu_struct_nodes(ssp, GFP_KERNEL)) | |
| 236 | + WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); | |
| 237 | + else | |
| 267 | 238 | goto err_free_sda; |
| 268 | − WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); | |
| 269 | 239 | } |
| 270 | 240 | ssp->srcu_sup->srcu_ssp = ssp; |
| 271 | 241 | smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, |
| @@ -394,20 +364,20 @@ static void srcu_transition_to_big(struct srcu_struct *ssp) · 122 unmodified lines | ||
| 394 | 364 | /* Double-checked locking on ->srcu_size-state. */ |
| 395 | 365 | if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) |
| 396 | 366 | return; |
| 397 | − spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); | |
| 367 | + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); | |
| 398 | 368 | if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) { |
| 399 | − spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 369 | + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 400 | 370 | return; |
| 401 | 371 | } |
| 402 | 372 | __srcu_transition_to_big(ssp); |
| 403 | − spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 373 | + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 404 | 374 | } |
| 405 | 375 | |
| 406 | 376 | /* |
| 407 | 377 | * Check to see if the just-encountered contention event justifies |
| 408 | 378 | * a transition to SRCU_SIZE_BIG. |
| 409 | 379 | */ |
| 410 | −static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp) | |
| 380 | +static void raw_spin_lock_irqsave_check_contention(struct srcu_struct *ssp) | |
| 411 | 381 | { |
| 412 | 382 | unsigned long j; |
| 413 | 383 | |
| @@ -429,16 +399,16 @@ static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp) · 15 unmodified lines | ||
| 429 | 399 | * to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module |
| 430 | 400 | * parameter permits this. |
| 431 | 401 | */ |
| 432 | −static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags) | |
| 402 | +static void raw_spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags) | |
| 433 | 403 | { |
| 434 | 404 | struct srcu_struct *ssp = sdp->ssp; |
| 435 | 405 | |
| 436 | − if (spin_trylock_irqsave_rcu_node(sdp, *flags)) | |
| 406 | + if (raw_spin_trylock_irqsave_rcu_node(sdp, *flags)) | |
| 437 | 407 | return; |
| 438 | − spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); | |
| 439 | − spin_lock_irqsave_check_contention(ssp); | |
| 440 | − spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags); | |
| 441 | − spin_lock_irqsave_rcu_node(sdp, *flags); | |
| 408 | + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); | |
| 409 | + raw_spin_lock_irqsave_check_contention(ssp); | |
| 410 | + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags); | |
| 411 | + raw_spin_lock_irqsave_rcu_node(sdp, *flags); | |
| 442 | 412 | } |
| 443 | 413 | |
| 444 | 414 | /* |
| @@ -447,12 +417,12 @@ static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned lon · 2 unmodified lines | ||
| 447 | 417 | * to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module |
| 448 | 418 | * parameter permits this. |
| 449 | 419 | */ |
| 450 | −static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags) | |
| 420 | +static void raw_spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags) | |
| 451 | 421 | { |
| 452 | − if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags)) | |
| 422 | + if (raw_spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags)) | |
| 453 | 423 | return; |
| 454 | − spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); | |
| 455 | − spin_lock_irqsave_check_contention(ssp); | |
| 424 | + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); | |
| 425 | + raw_spin_lock_irqsave_check_contention(ssp); | |
| 456 | 426 | } |
| 457 | 427 | |
| 458 | 428 | /* |
| @@ -470,13 +440,13 @@ static void check_init_srcu_struct(struct srcu_struct *ssp) · 11 unmodified lines | ||
| 470 | 440 | /* The smp_load_acquire() pairs with the smp_store_release(). */ |
| 471 | 441 | if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/ |
| 472 | 442 | return; /* Already initialized. */ |
| 473 | − spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); | |
| 443 | + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); | |
| 474 | 444 | if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) { |
| 475 | − spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 445 | + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 476 | 446 | return; |
| 477 | 447 | } |
| 478 | 448 | init_srcu_struct_fields(ssp, true); |
| 479 | − spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 449 | + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 480 | 450 | } |
| 481 | 451 | |
| 482 | 452 | /* |
| @@ -742,13 +712,15 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) · 259 unmodified lines | ||
| 742 | 712 | unsigned long delay; |
| 743 | 713 | struct srcu_usage *sup = ssp->srcu_sup; |
| 744 | 714 | |
| 745 | − spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 715 | + raw_spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 746 | 716 | delay = srcu_get_delay(ssp); |
| 747 | − spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 717 | + raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 748 | 718 | if (WARN_ON(!delay)) |
| 749 | 719 | return; /* Just leak it! */ |
| 750 | 720 | if (WARN_ON(srcu_readers_active(ssp))) |
| 751 | 721 | return; /* Just leak it! */ |
| 722 | + /* Wait for irq_work to finish first as it may queue a new work. */ | |
| 723 | + irq_work_sync(&sup->irq_work); | |
| 752 | 724 | flush_delayed_work(&sup->work); |
| 753 | 725 | for_each_possible_cpu(cpu) { |
| 754 | 726 | struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); |
| @@ -960,7 +932,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 205 unmodified lines | ||
| 960 | 932 | mutex_lock(&sup->srcu_cb_mutex); |
| 961 | 933 | |
| 962 | 934 | /* End the current grace period. */ |
| 963 | − spin_lock_irq_rcu_node(sup); | |
| 935 | + raw_spin_lock_irq_rcu_node(sup); | |
| 964 | 936 | idx = rcu_seq_state(sup->srcu_gp_seq); |
| 965 | 937 | WARN_ON_ONCE(idx != SRCU_STATE_SCAN2); |
| 966 | 938 | if (srcu_gp_is_expedited(ssp)) |
| @@ -971,7 +943,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 4 unmodified lines | ||
| 971 | 943 | gpseq = rcu_seq_current(&sup->srcu_gp_seq); |
| 972 | 944 | if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq)) |
| 973 | 945 | WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq); |
| 974 | − spin_unlock_irq_rcu_node(sup); | |
| 946 | + raw_spin_unlock_irq_rcu_node(sup); | |
| 975 | 947 | mutex_unlock(&sup->srcu_gp_mutex); |
| 976 | 948 | /* A new grace period can start at this point. But only one. */ |
| 977 | 949 | |
| @@ -983,7 +955,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 5 unmodified lines | ||
| 983 | 955 | } else { |
| 984 | 956 | idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs); |
| 985 | 957 | srcu_for_each_node_breadth_first(ssp, snp) { |
| 986 | − spin_lock_irq_rcu_node(snp); | |
| 958 | + raw_spin_lock_irq_rcu_node(snp); | |
| 987 | 959 | cbs = false; |
| 988 | 960 | last_lvl = snp >= sup->level[rcu_num_lvls - 1]; |
| 989 | 961 | if (last_lvl) |
| @@ -998,7 +970,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 8 unmodified lines | ||
| 998 | 970 | else |
| 999 | 971 | mask = snp->srcu_data_have_cbs[idx]; |
| 1000 | 972 | snp->srcu_data_have_cbs[idx] = 0; |
| 1001 | − spin_unlock_irq_rcu_node(snp); | |
| 973 | + raw_spin_unlock_irq_rcu_node(snp); | |
| 1002 | 974 | if (cbs) |
| 1003 | 975 | srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay); |
| 1004 | 976 | } |
| @@ -1008,27 +980,27 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 3 unmodified lines | ||
| 1008 | 980 | if (!(gpseq & counter_wrap_check)) |
| 1009 | 981 | for_each_possible_cpu(cpu) { |
| 1010 | 982 | sdp = per_cpu_ptr(ssp->sda, cpu); |
| 1011 | − spin_lock_irq_rcu_node(sdp); | |
| 983 | + raw_spin_lock_irq_rcu_node(sdp); | |
| 1012 | 984 | if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100)) |
| 1013 | 985 | sdp->srcu_gp_seq_needed = gpseq; |
| 1014 | 986 | if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100)) |
| 1015 | 987 | sdp->srcu_gp_seq_needed_exp = gpseq; |
| 1016 | − spin_unlock_irq_rcu_node(sdp); | |
| 988 | + raw_spin_unlock_irq_rcu_node(sdp); | |
| 1017 | 989 | } |
| 1018 | 990 | |
| 1019 | 991 | /* Callback initiation done, allow grace periods after next. */ |
| 1020 | 992 | mutex_unlock(&sup->srcu_cb_mutex); |
| 1021 | 993 | |
| 1022 | 994 | /* Start a new grace period if needed. */ |
| 1023 | − spin_lock_irq_rcu_node(sup); | |
| 995 | + raw_spin_lock_irq_rcu_node(sup); | |
| 1024 | 996 | gpseq = rcu_seq_current(&sup->srcu_gp_seq); |
| 1025 | 997 | if (!rcu_seq_state(gpseq) && |
| 1026 | 998 | ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) { |
| 1027 | 999 | srcu_gp_start(ssp); |
| 1028 | − spin_unlock_irq_rcu_node(sup); | |
| 1000 | + raw_spin_unlock_irq_rcu_node(sup); | |
| 1029 | 1001 | srcu_reschedule(ssp, 0); |
| 1030 | 1002 | } else { |
| 1031 | − spin_unlock_irq_rcu_node(sup); | |
| 1003 | + raw_spin_unlock_irq_rcu_node(sup); | |
| 1032 | 1004 | } |
| 1033 | 1005 | |
| 1034 | 1006 | /* Transition to big if needed. */ |
| @@ -1059,19 +1031,19 @@ static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp · 24 unmodified lines | ||
| 1059 | 1031 | if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) || |
| 1060 | 1032 | (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s))) |
| 1061 | 1033 | return; |
| 1062 | − spin_lock_irqsave_rcu_node(snp, flags); | |
| 1034 | + raw_spin_lock_irqsave_rcu_node(snp, flags); | |
| 1063 | 1035 | sgsne = snp->srcu_gp_seq_needed_exp; |
| 1064 | 1036 | if (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)) { |
| 1065 | − spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1037 | + raw_spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1066 | 1038 | return; |
| 1067 | 1039 | } |
| 1068 | 1040 | WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s); |
| 1069 | − spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1041 | + raw_spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1070 | 1042 | } |
| 1071 | − spin_lock_irqsave_ssp_contention(ssp, &flags); | |
| 1043 | + raw_spin_lock_irqsave_ssp_contention(ssp, &flags); | |
| 1072 | 1044 | if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s)) |
| 1073 | 1045 | WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s); |
| 1074 | − spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 1046 | + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 1075 | 1047 | } |
| 1076 | 1048 | |
| 1077 | 1049 | /* |
| @@ -1109,12 +1081,12 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 31 unmodified lines | ||
| 1109 | 1081 | for (snp = snp_leaf; snp != NULL; snp = snp->srcu_parent) { |
| 1110 | 1082 | if (WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && snp != snp_leaf) |
| 1111 | 1083 | return; /* GP already done and CBs recorded. */ |
| 1112 | − spin_lock_irqsave_rcu_node(snp, flags); | |
| 1084 | + raw_spin_lock_irqsave_rcu_node(snp, flags); | |
| 1113 | 1085 | snp_seq = snp->srcu_have_cbs[idx]; |
| 1114 | 1086 | if (!srcu_invl_snp_seq(snp_seq) && ULONG_CMP_GE(snp_seq, s)) { |
| 1115 | 1087 | if (snp == snp_leaf && snp_seq == s) |
| 1116 | 1088 | snp->srcu_data_have_cbs[idx] |= sdp->grpmask; |
| 1117 | − spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1089 | + raw_spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1118 | 1090 | if (snp == snp_leaf && snp_seq != s) { |
| 1119 | 1091 | srcu_schedule_cbs_sdp(sdp, do_norm ? SRCU_INTERVAL : 0); |
| 1120 | 1092 | return; |
| @@ -1129,11 +1101,11 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 8 unmodified lines | ||
| 1129 | 1101 | sgsne = snp->srcu_gp_seq_needed_exp; |
| 1130 | 1102 | if (!do_norm && (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, s))) |
| 1131 | 1103 | WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s); |
| 1132 | − spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1104 | + raw_spin_unlock_irqrestore_rcu_node(snp, flags); | |
| 1133 | 1105 | } |
| 1134 | 1106 | |
| 1135 | 1107 | /* Top of tree, must ensure the grace period will be started. */ |
| 1136 | − spin_lock_irqsave_ssp_contention(ssp, &flags); | |
| 1108 | + raw_spin_lock_irqsave_ssp_contention(ssp, &flags); | |
| 1137 | 1109 | if (ULONG_CMP_LT(sup->srcu_gp_seq_needed, s)) { |
| 1138 | 1110 | /* |
| 1139 | 1111 | * Record need for grace period s. Pair with load |
| @@ -1154,13 +1126,17 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, · 14 unmodified lines | ||
| 1154 | 1126 | // it isn't. And it does not have to be. After all, it |
| 1155 | 1127 | // can only be executed during early boot when there is only |
| 1156 | 1128 | // the one boot CPU running with interrupts still disabled. |
| 1129 | + // | |
| 1130 | + // Use an irq_work here to avoid acquiring runqueue lock with | |
| 1131 | + // srcu rcu_node::lock held. BPF instrument could introduce the | |
| 1132 | + // opposite dependency, hence we need to break the possible | |
| 1133 | + // locking dependency here. | |
| 1157 | 1134 | if (likely(srcu_init_done)) |
| 1158 | − queue_delayed_work(rcu_gp_wq, &sup->work, | |
| 1159 | − !!srcu_get_delay(ssp)); | |
| 1135 | + irq_work_queue(&sup->irq_work); | |
| 1160 | 1136 | else if (list_empty(&sup->work.work.entry)) |
| 1161 | 1137 | list_add(&sup->work.work.entry, &srcu_boot_list); |
| 1162 | 1138 | } |
| 1163 | − spin_unlock_irqrestore_rcu_node(sup, flags); | |
| 1139 | + raw_spin_unlock_irqrestore_rcu_node(sup, flags); | |
| 1164 | 1140 | } |
| 1165 | 1141 | |
| 1166 | 1142 | /* |
| @@ -1172,9 +1148,9 @@ static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) · 5 unmodified lines | ||
| 1172 | 1148 | { |
| 1173 | 1149 | unsigned long curdelay; |
| 1174 | 1150 | |
| 1175 | − spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1151 | + raw_spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1176 | 1152 | curdelay = !srcu_get_delay(ssp); |
| 1177 | − spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1153 | + raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1178 | 1154 | |
| 1179 | 1155 | for (;;) { |
| 1180 | 1156 | if (srcu_readers_active_idx_check(ssp, idx)) |
| @@ -1285,12 +1261,12 @@ static bool srcu_should_expedite(struct srcu_struct *ssp) · 104 unmodified lines | ||
| 1285 | 1261 | return false; |
| 1286 | 1262 | /* If the local srcu_data structure has callbacks, not idle. */ |
| 1287 | 1263 | sdp = raw_cpu_ptr(ssp->sda); |
| 1288 | − spin_lock_irqsave_rcu_node(sdp, flags); | |
| 1264 | + raw_spin_lock_irqsave_rcu_node(sdp, flags); | |
| 1289 | 1265 | if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) { |
| 1290 | − spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1266 | + raw_spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1291 | 1267 | return false; /* Callbacks already present, so not idle. */ |
| 1292 | 1268 | } |
| 1293 | − spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1269 | + raw_spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1294 | 1270 | |
| 1295 | 1271 | /* |
| 1296 | 1272 | * No local callbacks, so probabilistically probe global state. |
| @@ -1350,7 +1326,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, · 53 unmodified lines | ||
| 1350 | 1326 | sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id()); |
| 1351 | 1327 | else |
| 1352 | 1328 | sdp = raw_cpu_ptr(ssp->sda); |
| 1353 | − spin_lock_irqsave_sdp_contention(sdp, &flags); | |
| 1329 | + raw_spin_lock_irqsave_sdp_contention(sdp, &flags); | |
| 1354 | 1330 | if (rhp) |
| 1355 | 1331 | rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); |
| 1356 | 1332 | /* |
| @@ -1410,7 +1386,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, · 53 unmodified lines | ||
| 1410 | 1386 | sdp->srcu_gp_seq_needed_exp = s; |
| 1411 | 1387 | needexp = true; |
| 1412 | 1388 | } |
| 1413 | − spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1389 | + raw_spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1414 | 1390 | |
| 1415 | 1391 | /* Ensure that snp node tree is fully initialized before traversing it */ |
| 1416 | 1392 | if (ss_state < SRCU_SIZE_WAIT_BARRIER) |
| @@ -1522,7 +1498,7 @@ static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) · 105 unmodified lines | ||
| 1522 | 1498 | |
| 1523 | 1499 | /* |
| 1524 | 1500 | * Make sure that later code is ordered after the SRCU grace |
| 1525 | − * period. This pairs with the spin_lock_irq_rcu_node() | |
| 1501 | + * period. This pairs with the raw_spin_lock_irq_rcu_node() | |
| 1526 | 1502 | * in srcu_invoke_callbacks(). Unlike Tree RCU, this is needed |
| 1527 | 1503 | * because the current CPU might have been totally uninvolved with |
| 1528 | 1504 | * (and thus unordered against) that grace period. |
| @@ -1701,7 +1677,7 @@ static void srcu_barrier_cb(struct rcu_head *rhp) · 172 unmodified lines | ||
| 1701 | 1677 | */ |
| 1702 | 1678 | static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp) |
| 1703 | 1679 | { |
| 1704 | − spin_lock_irq_rcu_node(sdp); | |
| 1680 | + raw_spin_lock_irq_rcu_node(sdp); | |
| 1705 | 1681 | atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt); |
| 1706 | 1682 | sdp->srcu_barrier_head.func = srcu_barrier_cb; |
| 1707 | 1683 | debug_rcu_head_queue(&sdp->srcu_barrier_head); |
| @@ -1710,7 +1686,7 @@ static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp) · 2 unmodified lines | ||
| 1710 | 1686 | debug_rcu_head_unqueue(&sdp->srcu_barrier_head); |
| 1711 | 1687 | atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt); |
| 1712 | 1688 | } |
| 1713 | − spin_unlock_irq_rcu_node(sdp); | |
| 1689 | + raw_spin_unlock_irq_rcu_node(sdp); | |
| 1714 | 1690 | } |
| 1715 | 1691 | |
| 1716 | 1692 | /** |
| @@ -1761,7 +1737,7 @@ static void srcu_expedite_current_cb(struct rcu_head *rhp) · 44 unmodified lines | ||
| 1761 | 1737 | bool needcb = false; |
| 1762 | 1738 | struct srcu_data *sdp = container_of(rhp, struct srcu_data, srcu_ec_head); |
| 1763 | 1739 | |
| 1764 | − spin_lock_irqsave_sdp_contention(sdp, &flags); | |
| 1740 | + raw_spin_lock_irqsave_sdp_contention(sdp, &flags); | |
| 1765 | 1741 | if (sdp->srcu_ec_state == SRCU_EC_IDLE) { |
| 1766 | 1742 | WARN_ON_ONCE(1); |
| 1767 | 1743 | } else if (sdp->srcu_ec_state == SRCU_EC_PENDING) { |
| @@ -1771,7 +1747,7 @@ static void srcu_expedite_current_cb(struct rcu_head *rhp) · 3 unmodified lines | ||
| 1771 | 1747 | sdp->srcu_ec_state = SRCU_EC_PENDING; |
| 1772 | 1748 | needcb = true; |
| 1773 | 1749 | } |
| 1774 | − spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1750 | + raw_spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1775 | 1751 | // If needed, requeue ourselves as an expedited SRCU callback. |
| 1776 | 1752 | if (needcb) |
| 1777 | 1753 | __call_srcu(sdp->ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false); |
| @@ -1795,7 +1771,7 @@ void srcu_expedite_current(struct srcu_struct *ssp) · 17 unmodified lines | ||
| 1795 | 1771 | |
| 1796 | 1772 | migrate_disable(); |
| 1797 | 1773 | sdp = this_cpu_ptr(ssp->sda); |
| 1798 | − spin_lock_irqsave_sdp_contention(sdp, &flags); | |
| 1774 | + raw_spin_lock_irqsave_sdp_contention(sdp, &flags); | |
| 1799 | 1775 | if (sdp->srcu_ec_state == SRCU_EC_IDLE) { |
| 1800 | 1776 | sdp->srcu_ec_state = SRCU_EC_PENDING; |
| 1801 | 1777 | needcb = true; |
| @@ -1804,7 +1780,7 @@ void srcu_expedite_current(struct srcu_struct *ssp) · 2 unmodified lines | ||
| 1804 | 1780 | } else { |
| 1805 | 1781 | WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST); |
| 1806 | 1782 | } |
| 1807 | − spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1783 | + raw_spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 1808 | 1784 | // If needed, queue an expedited SRCU callback. |
| 1809 | 1785 | if (needcb) |
| 1810 | 1786 | __call_srcu(ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false); |
| @@ -1848,17 +1824,17 @@ static void srcu_advance_state(struct srcu_struct *ssp) · 37 unmodified lines | ||
| 1848 | 1824 | */ |
| 1849 | 1825 | idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */ |
| 1850 | 1826 | if (idx == SRCU_STATE_IDLE) { |
| 1851 | − spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1827 | + raw_spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1852 | 1828 | if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { |
| 1853 | 1829 | WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)); |
| 1854 | − spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1830 | + raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1855 | 1831 | mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); |
| 1856 | 1832 | return; |
| 1857 | 1833 | } |
| 1858 | 1834 | idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)); |
| 1859 | 1835 | if (idx == SRCU_STATE_IDLE) |
| 1860 | 1836 | srcu_gp_start(ssp); |
| 1861 | − spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1837 | + raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1862 | 1838 | if (idx != SRCU_STATE_IDLE) { |
| 1863 | 1839 | mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); |
| 1864 | 1840 | return; /* Someone else started the grace period. */ |
| @@ -1872,10 +1848,10 @@ static void srcu_advance_state(struct srcu_struct *ssp) · 7 unmodified lines | ||
| 1872 | 1848 | return; /* readers present, retry later. */ |
| 1873 | 1849 | } |
| 1874 | 1850 | srcu_flip(ssp); |
| 1875 | − spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1851 | + raw_spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1876 | 1852 | rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2); |
| 1877 | 1853 | ssp->srcu_sup->srcu_n_exp_nodelay = 0; |
| 1878 | − spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1854 | + raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1879 | 1855 | } |
| 1880 | 1856 | |
| 1881 | 1857 | if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) { |
| @@ -1913,7 +1889,7 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 31 unmodified lines | ||
| 1913 | 1889 | |
| 1914 | 1890 | ssp = sdp->ssp; |
| 1915 | 1891 | rcu_cblist_init(&ready_cbs); |
| 1916 | − spin_lock_irq_rcu_node(sdp); | |
| 1892 | + raw_spin_lock_irq_rcu_node(sdp); | |
| 1917 | 1893 | WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL)); |
| 1918 | 1894 | rcu_segcblist_advance(&sdp->srcu_cblist, |
| 1919 | 1895 | rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); |
| @@ -1924,7 +1900,7 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 4 unmodified lines | ||
| 1924 | 1900 | */ |
| 1925 | 1901 | if (sdp->srcu_cblist_invoking || |
| 1926 | 1902 | !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) { |
| 1927 | − spin_unlock_irq_rcu_node(sdp); | |
| 1903 | + raw_spin_unlock_irq_rcu_node(sdp); | |
| 1928 | 1904 | return; /* Someone else on the job or nothing to do. */ |
| 1929 | 1905 | } |
| 1930 | 1906 | |
| @@ -1932,7 +1908,7 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 1 unmodified line | ||
| 1932 | 1908 | sdp->srcu_cblist_invoking = true; |
| 1933 | 1909 | rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs); |
| 1934 | 1910 | len = ready_cbs.len; |
| 1935 | − spin_unlock_irq_rcu_node(sdp); | |
| 1911 | + raw_spin_unlock_irq_rcu_node(sdp); | |
| 1936 | 1912 | rhp = rcu_cblist_dequeue(&ready_cbs); |
| 1937 | 1913 | for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) { |
| 1938 | 1914 | debug_rcu_head_unqueue(rhp); |
| @@ -1947,11 +1923,11 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 8 unmodified lines | ||
| 1947 | 1923 | * Update counts, accelerate new callbacks, and if needed, |
| 1948 | 1924 | * schedule another round of callback invocation. |
| 1949 | 1925 | */ |
| 1950 | − spin_lock_irq_rcu_node(sdp); | |
| 1926 | + raw_spin_lock_irq_rcu_node(sdp); | |
| 1951 | 1927 | rcu_segcblist_add_len(&sdp->srcu_cblist, -len); |
| 1952 | 1928 | sdp->srcu_cblist_invoking = false; |
| 1953 | 1929 | more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist); |
| 1954 | − spin_unlock_irq_rcu_node(sdp); | |
| 1930 | + raw_spin_unlock_irq_rcu_node(sdp); | |
| 1955 | 1931 | /* An SRCU barrier or callbacks from previous nesting work pending */ |
| 1956 | 1932 | if (more) |
| 1957 | 1933 | srcu_schedule_cbs_sdp(sdp, 0); |
| @@ -1965,7 +1941,7 @@ static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) · 7 unmodified lines | ||
| 1965 | 1941 | { |
| 1966 | 1942 | bool pushgp = true; |
| 1967 | 1943 | |
| 1968 | − spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1944 | + raw_spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1969 | 1945 | if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { |
| 1970 | 1946 | if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) { |
| 1971 | 1947 | /* All requests fulfilled, time to go idle. */ |
| @@ -1975,7 +1951,7 @@ static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) · 3 unmodified lines | ||
| 1975 | 1951 | /* Outstanding request and no GP. Start one. */ |
| 1976 | 1952 | srcu_gp_start(ssp); |
| 1977 | 1953 | } |
| 1978 | − spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1954 | + raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1979 | 1955 | |
| 1980 | 1956 | if (pushgp) |
| 1981 | 1957 | queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay); |
| @@ -1995,9 +1971,9 @@ static void process_srcu(struct work_struct *work) · 13 unmodified lines | ||
| 1995 | 1971 | ssp = sup->srcu_ssp; |
| 1996 | 1972 | |
| 1997 | 1973 | srcu_advance_state(ssp); |
| 1998 | − spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1974 | + raw_spin_lock_irq_rcu_node(ssp->srcu_sup); | |
| 1999 | 1975 | curdelay = srcu_get_delay(ssp); |
| 2000 | − spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 1976 | + raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); | |
| 2001 | 1977 | if (curdelay) { |
| 2002 | 1978 | WRITE_ONCE(sup->reschedule_count, 0); |
| 2003 | 1979 | } else { |
| @@ -2015,6 +1991,23 @@ static void process_srcu(struct work_struct *work) · 11 unmodified lines | ||
| 2015 | 1991 | srcu_reschedule(ssp, curdelay); |
| 2016 | 1992 | } |
| 2017 | 1993 | |
| 1994 | +static void srcu_irq_work(struct irq_work *work) | |
| 1995 | +{ | |
| 1996 | + struct srcu_struct *ssp; | |
| 1997 | + struct srcu_usage *sup; | |
| 1998 | + unsigned long delay; | |
| 1999 | + unsigned long flags; | |
| 2000 | + | |
| 2001 | + sup = container_of(work, struct srcu_usage, irq_work); | |
| 2002 | + ssp = sup->srcu_ssp; | |
| 2003 | + | |
| 2004 | + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); | |
| 2005 | + delay = srcu_get_delay(ssp); | |
| 2006 | + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); | |
| 2007 | + | |
| 2008 | + queue_delayed_work(rcu_gp_wq, &sup->work, !!delay); | |
| 2009 | +} | |
| 2010 | + | |
| 2018 | 2011 | void srcutorture_get_gp_data(struct srcu_struct *ssp, int *flags, |
| 2019 | 2012 | unsigned long *gp_seq) |
| 2020 | 2013 | { |
| @@ -1118,7 +1118,7 @@ int proc_do_large_bitmap(const struct ctl_table *table, int dir, · 1117 unmodified lines | ||
| 1118 | 1118 | unsigned long bitmap_len = table->maxlen; |
| 1119 | 1119 | unsigned long *bitmap = *(unsigned long **) table->data; |
| 1120 | 1120 | unsigned long *tmp_bitmap = NULL; |
| 1121 | − char tr_a[] = { '-', ',', '\n' }, tr_b[] = { ',', '\n', 0 }, c; | |
| 1121 | + char tr_a[] = { '-', ',', '\n' }, tr_b[] = { ',', '\n', 0 }, c = 0; | |
| 1122 | 1122 | |
| 1123 | 1123 | if (!bitmap || !bitmap_len || !left || (*ppos && SYSCTL_KERN_TO_USER(dir))) { |
| 1124 | 1124 | *lenp = 0; |
| @@ -540,7 +540,7 @@ static s64 alarm_timer_forward(struct k_itimer *timr, ktime_t now) · 539 unmodified lines | ||
| 540 | 540 | { |
| 541 | 541 | struct alarm *alarm = &timr->it.alarm.alarmtimer; |
| 542 | 542 | |
| 543 | − return alarm_forward(alarm, timr->it_interval, now); | |
| 543 | + return alarm_forward(alarm, now, timr->it_interval); | |
| 544 | 544 | } |
| 545 | 545 | |
| 546 | 546 | /** |
| @@ -22,6 +22,39 @@ static struct task_struct *trigger_kthread; · 21 unmodified lines | ||
| 22 | 22 | static struct llist_head trigger_data_free_list; |
| 23 | 23 | static DEFINE_MUTEX(trigger_data_kthread_mutex); |
| 24 | 24 | |
| 25 | +static int trigger_kthread_fn(void *ignore); | |
| 26 | + | |
| 27 | +static void trigger_create_kthread_locked(void) | |
| 28 | +{ | |
| 29 | + lockdep_assert_held(&trigger_data_kthread_mutex); | |
| 30 | + | |
| 31 | + if (!trigger_kthread) { | |
| 32 | + struct task_struct *kthread; | |
| 33 | + | |
| 34 | + kthread = kthread_create(trigger_kthread_fn, NULL, | |
| 35 | + "trigger_data_free"); | |
| 36 | + if (!IS_ERR(kthread)) | |
| 37 | + WRITE_ONCE(trigger_kthread, kthread); | |
| 38 | + } | |
| 39 | +} | |
| 40 | + | |
| 41 | +static void trigger_data_free_queued_locked(void) | |
| 42 | +{ | |
| 43 | + struct event_trigger_data *data, *tmp; | |
| 44 | + struct llist_node *llnodes; | |
| 45 | + | |
| 46 | + lockdep_assert_held(&trigger_data_kthread_mutex); | |
| 47 | + | |
| 48 | + llnodes = llist_del_all(&trigger_data_free_list); | |
| 49 | + if (!llnodes) | |
| 50 | + return; | |
| 51 | + | |
| 52 | + tracepoint_synchronize_unregister(); | |
| 53 | + | |
| 54 | + llist_for_each_entry_safe(data, tmp, llnodes, llist) | |
| 55 | + kfree(data); | |
| 56 | +} | |
| 57 | + | |
| 25 | 58 | /* Bulk garbage collection of event_trigger_data elements */ |
| 26 | 59 | static int trigger_kthread_fn(void *ignore) |
| 27 | 60 | { |
| @@ -56,30 +89,50 @@ void trigger_data_free(struct event_trigger_data *data) · 28 unmodified lines | ||
| 56 | 89 | if (data->cmd_ops->set_filter) |
| 57 | 90 | data->cmd_ops->set_filter(NULL, data, NULL); |
| 58 | 91 | |
| 92 | + /* | |
| 93 | + * Boot-time trigger registration can fail before kthread creation | |
| 94 | + * works. Keep the deferred-free semantics during boot and let late | |
| 95 | + * init start the kthread to drain the list. | |
| 96 | + */ | |
| 97 | + if (system_state == SYSTEM_BOOTING && !trigger_kthread) { | |
| 98 | + llist_add(&data->llist, &trigger_data_free_list); | |
| 99 | + return; | |
| 100 | + } | |
| 101 | + | |
| 59 | 102 | if (unlikely(!trigger_kthread)) { |
| 60 | 103 | guard(mutex)(&trigger_data_kthread_mutex); |
| 104 | + | |
| 105 | + trigger_create_kthread_locked(); | |
| 61 | 106 | /* Check again after taking mutex */ |
| 62 | 107 | if (!trigger_kthread) { |
| 63 | − struct task_struct *kthread; | |
| 64 | − | |
| 65 | − kthread = kthread_create(trigger_kthread_fn, NULL, | |
| 66 | − "trigger_data_free"); | |
| 67 | − if (!IS_ERR(kthread)) | |
| 68 | − WRITE_ONCE(trigger_kthread, kthread); | |
| 108 | + llist_add(&data->llist, &trigger_data_free_list); | |
| 109 | + /* Drain the queued frees synchronously if creation failed. */ | |
| 110 | + trigger_data_free_queued_locked(); | |
| 111 | + return; | |
| 69 | 112 | } |
| 70 | 113 | } |
| 71 | 114 | |
| 72 | − if (!trigger_kthread) { | |
| 73 | − /* Do it the slow way */ | |
| 74 | − tracepoint_synchronize_unregister(); | |
| 75 | − kfree(data); | |
| 76 | − return; | |
| 77 | − } | |
| 78 | − | |
| 79 | 115 | llist_add(&data->llist, &trigger_data_free_list); |
| 80 | 116 | wake_up_process(trigger_kthread); |
| 81 | 117 | } |
| 82 | 118 | |
| 119 | +static int __init trigger_data_free_init(void) | |
| 120 | +{ | |
| 121 | + guard(mutex)(&trigger_data_kthread_mutex); | |
| 122 | + | |
| 123 | + if (llist_empty(&trigger_data_free_list)) | |
| 124 | + return 0; | |
| 125 | + | |
| 126 | + trigger_create_kthread_locked(); | |
| 127 | + if (trigger_kthread) | |
| 128 | + wake_up_process(trigger_kthread); | |
| 129 | + else | |
| 130 | + trigger_data_free_queued_locked(); | |
| 131 | + | |
| 132 | + return 0; | |
| 133 | +} | |
| 134 | +late_initcall(trigger_data_free_init); | |
| 135 | + | |
| 83 | 136 | static inline void data_ops_trigger(struct event_trigger_data *data, |
| 84 | 137 | struct trace_buffer *buffer, void *rec, |
| 85 | 138 | struct ring_buffer_event *event) |
| @@ -2073,8 +2073,8 @@ static void osnoise_hotplug_workfn(struct work_struct *dummy) · 2072 unmodified lines | ||
| 2073 | 2073 | if (!osnoise_has_registered_instances()) |
| 2074 | 2074 | return; |
| 2075 | 2075 | |
| 2076 | − guard(mutex)(&interface_lock); | |
| 2077 | 2076 | guard(cpus_read_lock)(); |
| 2077 | + guard(mutex)(&interface_lock); | |
| 2078 | 2078 | |
| 2079 | 2079 | if (!cpu_online(cpu)) |
| 2080 | 2080 | return; |
| @@ -2237,11 +2237,11 @@ static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf, · 156 unmodified lines | ||
| 2237 | 2237 | if (running) |
| 2238 | 2238 | stop_per_cpu_kthreads(); |
| 2239 | 2239 | |
| 2240 | − mutex_lock(&interface_lock); | |
| 2241 | 2240 | /* |
| 2242 | 2241 | * avoid CPU hotplug operations that might read options. |
| 2243 | 2242 | */ |
| 2244 | 2243 | cpus_read_lock(); |
| 2244 | + mutex_lock(&interface_lock); | |
| 2245 | 2245 | |
| 2246 | 2246 | retval = cnt; |
| 2247 | 2247 | |
| @@ -2257,8 +2257,8 @@ static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf, · 9 unmodified lines | ||
| 2257 | 2257 | clear_bit(option, &osnoise_options); |
| 2258 | 2258 | } |
| 2259 | 2259 | |
| 2260 | − cpus_read_unlock(); | |
| 2261 | 2260 | mutex_unlock(&interface_lock); |
| 2261 | + cpus_read_unlock(); | |
| 2262 | 2262 | |
| 2263 | 2263 | if (running) |
| 2264 | 2264 | start_per_cpu_kthreads(); |
| @@ -2345,16 +2345,16 @@ osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, · 80 unmodified lines | ||
| 2345 | 2345 | if (running) |
| 2346 | 2346 | stop_per_cpu_kthreads(); |
| 2347 | 2347 | |
| 2348 | − mutex_lock(&interface_lock); | |
| 2349 | 2348 | /* |
| 2350 | 2349 | * osnoise_cpumask is read by CPU hotplug operations. |
| 2351 | 2350 | */ |
| 2352 | 2351 | cpus_read_lock(); |
| 2352 | + mutex_lock(&interface_lock); | |
| 2353 | 2353 | |
| 2354 | 2354 | cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new); |
| 2355 | 2355 | |
| 2356 | − cpus_read_unlock(); | |
| 2357 | 2356 | mutex_unlock(&interface_lock); |
| 2357 | + cpus_read_unlock(); | |
| 2358 | 2358 | |
| 2359 | 2359 | if (running) |
| 2360 | 2360 | start_per_cpu_kthreads(); |
| @@ -173,10 +173,8 @@ struct bug_entry *find_bug(unsigned long bugaddr) · 172 unmodified lines | ||
| 173 | 173 | return module_find_bug(bugaddr); |
| 174 | 174 | } |
| 175 | 175 | |
| 176 | −__diag_push(); | |
| 177 | −__diag_ignore(GCC, all, "-Wsuggest-attribute=format", | |
| 178 | − "Not a valid __printf() conversion candidate."); | |
| 179 | −static void __warn_printf(const char *fmt, struct pt_regs *regs) | |
| 176 | +static __printf(1, 0) | |
| 177 | +void __warn_printf(const char *fmt, struct pt_regs *regs) | |
| 180 | 178 | { |
| 181 | 179 | if (!fmt) |
| 182 | 180 | return; |
| @@ -195,7 +193,6 @@ static void __warn_printf(const char *fmt, struct pt_regs *regs) · 12 unmodified lines | ||
| 195 | 193 | |
| 196 | 194 | printk("%s", fmt); |
| 197 | 195 | } |
| 198 | −__diag_pop(); | |
| 199 | 196 | |
| 200 | 197 | static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs) |
| 201 | 198 | { |
| @@ -1252,6 +1252,7 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) · 1251 unmodified lines | ||
| 1252 | 1252 | { |
| 1253 | 1253 | int err; |
| 1254 | 1254 | |
| 1255 | + dst->maybe_corrupted = true; | |
| 1255 | 1256 | if (!is_power_of_2(src->min_region_sz)) |
| 1256 | 1257 | return -EINVAL; |
| 1257 | 1258 | |
| @@ -1277,6 +1278,7 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) · 19 unmodified lines | ||
| 1277 | 1278 | dst->addr_unit = src->addr_unit; |
| 1278 | 1279 | dst->min_region_sz = src->min_region_sz; |
| 1279 | 1280 | |
| 1281 | + dst->maybe_corrupted = false; | |
| 1280 | 1282 | return 0; |
| 1281 | 1283 | } |
| 1282 | 1284 | |
| @@ -2678,6 +2680,8 @@ static void kdamond_call(struct damon_ctx *ctx, bool cancel) · 1395 unmodified lines | ||
| 2678 | 2680 | complete(&control->completion); |
| 2679 | 2681 | else if (control->canceled && control->dealloc_on_cancel) |
| 2680 | 2682 | kfree(control); |
| 2683 | + if (!cancel && ctx->maybe_corrupted) | |
| 2684 | + break; | |
| 2681 | 2685 | } |
| 2682 | 2686 | |
| 2683 | 2687 | mutex_lock(&ctx->call_controls_lock); |
| @@ -2707,6 +2711,8 @@ static int kdamond_wait_activation(struct damon_ctx *ctx) · 23 unmodified lines | ||
| 2707 | 2711 | kdamond_usleep(min_wait_time); |
| 2708 | 2712 | |
| 2709 | 2713 | kdamond_call(ctx, false); |
| 2714 | + if (ctx->maybe_corrupted) | |
| 2715 | + return -EINVAL; | |
| 2710 | 2716 | damos_walk_cancel(ctx); |
| 2711 | 2717 | } |
| 2712 | 2718 | return -EBUSY; |
| @@ -2790,6 +2796,8 @@ static int kdamond_fn(void *data) · 77 unmodified lines | ||
| 2790 | 2796 | * kdamond_merge_regions() if possible, to reduce overhead |
| 2791 | 2797 | */ |
| 2792 | 2798 | kdamond_call(ctx, false); |
| 2799 | + if (ctx->maybe_corrupted) | |
| 2800 | + break; | |
| 2793 | 2801 | if (!list_empty(&ctx->schemes)) |
| 2794 | 2802 | kdamond_apply_schemes(ctx); |
| 2795 | 2803 | else |
| @@ -145,12 +145,59 @@ static int damon_stat_damon_call_fn(void *data) · 144 unmodified lines | ||
| 145 | 145 | return 0; |
| 146 | 146 | } |
| 147 | 147 | |
| 148 | +struct damon_stat_system_ram_range_walk_arg { | |
| 149 | + bool walked; | |
| 150 | + struct resource res; | |
| 151 | +}; | |
| 152 | + | |
| 153 | +static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg) | |
| 154 | +{ | |
| 155 | + struct damon_stat_system_ram_range_walk_arg *a = arg; | |
| 156 | + | |
| 157 | + if (!a->walked) { | |
| 158 | + a->walked = true; | |
| 159 | + a->res.start = res->start; | |
| 160 | + } | |
| 161 | + a->res.end = res->end; | |
| 162 | + return 0; | |
| 163 | +} | |
| 164 | + | |
| 165 | +static unsigned long damon_stat_res_to_core_addr(resource_size_t ra, | |
| 166 | + unsigned long addr_unit) | |
| 167 | +{ | |
| 168 | + /* | |
| 169 | + * Use div_u64() for avoiding linking errors related with __udivdi3, | |
| 170 | + * __aeabi_uldivmod, or similar problems. This should also improve the | |
| 171 | + * performance optimization (read div_u64() comment for the detail). | |
| 172 | + */ | |
| 173 | + if (sizeof(ra) == 8 && sizeof(addr_unit) == 4) | |
| 174 | + return div_u64(ra, addr_unit); | |
| 175 | + return ra / addr_unit; | |
| 176 | +} | |
| 177 | + | |
| 178 | +static int damon_stat_set_monitoring_region(struct damon_target *t, | |
| 179 | + unsigned long addr_unit, unsigned long min_region_sz) | |
| 180 | +{ | |
| 181 | + struct damon_addr_range addr_range; | |
| 182 | + struct damon_stat_system_ram_range_walk_arg arg = {}; | |
| 183 | + | |
| 184 | + walk_system_ram_res(0, -1, &arg, damon_stat_system_ram_walk_fn); | |
| 185 | + if (!arg.walked) | |
| 186 | + return -EINVAL; | |
| 187 | + addr_range.start = damon_stat_res_to_core_addr( | |
| 188 | + arg.res.start, addr_unit); | |
| 189 | + addr_range.end = damon_stat_res_to_core_addr( | |
| 190 | + arg.res.end + 1, addr_unit); | |
| 191 | + if (addr_range.end <= addr_range.start) | |
| 192 | + return -EINVAL; | |
| 193 | + return damon_set_regions(t, &addr_range, 1, min_region_sz); | |
| 194 | +} | |
| 195 | + | |
| 148 | 196 | static struct damon_ctx *damon_stat_build_ctx(void) |
| 149 | 197 | { |
| 150 | 198 | struct damon_ctx *ctx; |
| 151 | 199 | struct damon_attrs attrs; |
| 152 | 200 | struct damon_target *target; |
| 153 | − unsigned long start = 0, end = 0; | |
| 154 | 201 | |
| 155 | 202 | ctx = damon_new_ctx(); |
| 156 | 203 | if (!ctx) |
| @@ -180,8 +227,8 @@ static struct damon_ctx *damon_stat_build_ctx(void) · 23 unmodified lines | ||
| 180 | 227 | if (!target) |
| 181 | 228 | goto free_out; |
| 182 | 229 | damon_add_target(ctx, target); |
| 183 | − if (damon_set_region_biggest_system_ram_default(target, &start, &end, | |
| 184 | − ctx->min_region_sz)) | |
| 230 | + if (damon_stat_set_monitoring_region(target, ctx->addr_unit, | |
| 231 | + ctx->min_region_sz)) | |
| 185 | 232 | goto free_out; |
| 186 | 233 | return ctx; |
| 187 | 234 | free_out: |
| @@ -1524,8 +1524,10 @@ static int damon_sysfs_commit_input(void *data) · 1523 unmodified lines | ||
| 1524 | 1524 | if (IS_ERR(param_ctx)) |
| 1525 | 1525 | return PTR_ERR(param_ctx); |
| 1526 | 1526 | test_ctx = damon_sysfs_new_test_ctx(kdamond->damon_ctx); |
| 1527 | − if (!test_ctx) | |
| 1527 | + if (!test_ctx) { | |
| 1528 | + damon_destroy_ctx(param_ctx); | |
| 1528 | 1529 | return -ENOMEM; |
| 1530 | + } | |
| 1529 | 1531 | err = damon_commit_ctx(test_ctx, param_ctx); |
| 1530 | 1532 | if (err) |
| 1531 | 1533 | goto out; |
| @@ -1618,9 +1620,12 @@ static int damon_sysfs_repeat_call_fn(void *data) · 86 unmodified lines | ||
| 1618 | 1620 | |
| 1619 | 1621 | if (!mutex_trylock(&damon_sysfs_lock)) |
| 1620 | 1622 | return 0; |
| 1623 | + if (sysfs_kdamond->contexts->nr != 1) | |
| 1624 | + goto out; | |
| 1621 | 1625 | damon_sysfs_upd_tuned_intervals(sysfs_kdamond); |
| 1622 | 1626 | damon_sysfs_upd_schemes_stats(sysfs_kdamond); |
| 1623 | 1627 | damon_sysfs_upd_schemes_effective_quotas(sysfs_kdamond); |
| 1628 | +out: | |
| 1624 | 1629 | mutex_unlock(&damon_sysfs_lock); |
| 1625 | 1630 | return 0; |
| 1626 | 1631 | } |
| @@ -1747,6 +1752,9 @@ static int damon_sysfs_update_schemes_tried_regions( · 120 unmodified lines | ||
| 1747 | 1752 | static int damon_sysfs_handle_cmd(enum damon_sysfs_cmd cmd, |
| 1748 | 1753 | struct damon_sysfs_kdamond *kdamond) |
| 1749 | 1754 | { |
| 1755 | + if (cmd != DAMON_SYSFS_CMD_OFF && kdamond->contexts->nr != 1) | |
| 1756 | + return -EINVAL; | |
| 1757 | + | |
| 1750 | 1758 | switch (cmd) { |
| 1751 | 1759 | case DAMON_SYSFS_CMD_ON: |
| 1752 | 1760 | return damon_sysfs_turn_damon_on(kdamond); |
| @@ -778,7 +778,7 @@ dma_addr_t hmm_dma_map_pfn(struct device *dev, struct hmm_dma_map *map, · 777 unmodified lines | ||
| 778 | 778 | struct page *page = hmm_pfn_to_page(pfns[idx]); |
| 779 | 779 | phys_addr_t paddr = hmm_pfn_to_phys(pfns[idx]); |
| 780 | 780 | size_t offset = idx * map->dma_entry_size; |
| 781 | − unsigned long attrs = 0; | |
| 781 | + unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT; | |
| 782 | 782 | dma_addr_t dma_addr; |
| 783 | 783 | int ret; |
| 784 | 784 | |
| @@ -871,7 +871,7 @@ bool hmm_dma_unmap_pfn(struct device *dev, struct hmm_dma_map *map, size_t idx) · 86 unmodified lines | ||
| 871 | 871 | struct dma_iova_state *state = &map->state; |
| 872 | 872 | dma_addr_t *dma_addrs = map->dma_list; |
| 873 | 873 | unsigned long *pfns = map->pfn_list; |
| 874 | − unsigned long attrs = 0; | |
| 874 | + unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT; | |
| 875 | 875 | |
| 876 | 876 | if ((pfns[idx] & valid_dma) != valid_dma) |
| 877 | 877 | return false; |
| @@ -6815,11 +6815,16 @@ retry: · 6814 unmodified lines | ||
| 6815 | 6815 | |
| 6816 | 6816 | pudp = pud_offset(p4dp, address); |
| 6817 | 6817 | pud = pudp_get(pudp); |
| 6818 | − if (pud_none(pud)) | |
| 6818 | + if (!pud_present(pud)) | |
| 6819 | 6819 | goto out; |
| 6820 | 6820 | if (pud_leaf(pud)) { |
| 6821 | 6821 | lock = pud_lock(mm, pudp); |
| 6822 | − if (!unlikely(pud_leaf(pud))) { | |
| 6822 | + pud = pudp_get(pudp); | |
| 6823 | + | |
| 6824 | + if (unlikely(!pud_present(pud))) { | |
| 6825 | + spin_unlock(lock); | |
| 6826 | + goto out; | |
| 6827 | + } else if (unlikely(!pud_leaf(pud))) { | |
| 6823 | 6828 | spin_unlock(lock); |
| 6824 | 6829 | goto retry; |
| 6825 | 6830 | } |
| @@ -6831,9 +6836,16 @@ retry: · 5 unmodified lines | ||
| 6831 | 6836 | |
| 6832 | 6837 | pmdp = pmd_offset(pudp, address); |
| 6833 | 6838 | pmd = pmdp_get_lockless(pmdp); |
| 6839 | + if (!pmd_present(pmd)) | |
| 6840 | + goto out; | |
| 6834 | 6841 | if (pmd_leaf(pmd)) { |
| 6835 | 6842 | lock = pmd_lock(mm, pmdp); |
| 6836 | − if (!unlikely(pmd_leaf(pmd))) { | |
| 6843 | + pmd = pmdp_get(pmdp); | |
| 6844 | + | |
| 6845 | + if (unlikely(!pmd_present(pmd))) { | |
| 6846 | + spin_unlock(lock); | |
| 6847 | + goto out; | |
| 6848 | + } else if (unlikely(!pmd_leaf(pmd))) { | |
| 6837 | 6849 | spin_unlock(lock); |
| 6838 | 6850 | goto retry; |
| 6839 | 6851 | } |
| @@ -487,7 +487,13 @@ void __mpol_put(struct mempolicy *pol) · 486 unmodified lines | ||
| 487 | 487 | { |
| 488 | 488 | if (!atomic_dec_and_test(&pol->refcnt)) |
| 489 | 489 | return; |
| 490 | − kmem_cache_free(policy_cache, pol); | |
| 490 | + /* | |
| 491 | + * Required to allow mmap_lock_speculative*() access, see for example | |
| 492 | + * futex_key_to_node_opt(). All accesses are serialized by mmap_lock, | |
| 493 | + * however the speculative lock section unbound by the normal lock | |
| 494 | + * boundaries, requiring RCU freeing. | |
| 495 | + */ | |
| 496 | + kfree_rcu(pol, rcu); | |
| 491 | 497 | } |
| 492 | 498 | EXPORT_SYMBOL_FOR_MODULES(__mpol_put, "kvm"); |
| 493 | 499 | |
| @@ -1020,7 +1026,7 @@ static int vma_replace_policy(struct vm_area_struct *vma, · 526 unmodified lines | ||
| 1020 | 1026 | } |
| 1021 | 1027 | |
| 1022 | 1028 | old = vma->vm_policy; |
| 1023 | − vma->vm_policy = new; /* protected by mmap_lock */ | |
| 1029 | + WRITE_ONCE(vma->vm_policy, new); /* protected by mmap_lock */ | |
| 1024 | 1030 | mpol_put(old); |
| 1025 | 1031 | |
| 1026 | 1032 | return 0; |
| @@ -56,7 +56,6 @@ static int mseal_apply(struct mm_struct *mm, · 55 unmodified lines | ||
| 56 | 56 | unsigned long start, unsigned long end) |
| 57 | 57 | { |
| 58 | 58 | struct vm_area_struct *vma, *prev; |
| 59 | − unsigned long curr_start = start; | |
| 60 | 59 | VMA_ITERATOR(vmi, mm, start); |
| 61 | 60 | |
| 62 | 61 | /* We know there are no gaps so this will be non-NULL. */ |
| @@ -66,6 +65,7 @@ static int mseal_apply(struct mm_struct *mm, · 3 unmodified lines | ||
| 66 | 65 | prev = vma; |
| 67 | 66 | |
| 68 | 67 | for_each_vma_range(vmi, vma, end) { |
| 68 | + const unsigned long curr_start = MAX(vma->vm_start, start); | |
| 69 | 69 | const unsigned long curr_end = MIN(vma->vm_end, end); |
| 70 | 70 | |
| 71 | 71 | if (!(vma->vm_flags & VM_SEALED)) { |
| @@ -79,7 +79,6 @@ static int mseal_apply(struct mm_struct *mm, · 7 unmodified lines | ||
| 79 | 79 | } |
| 80 | 80 | |
| 81 | 81 | prev = vma; |
| 82 | − curr_start = curr_end; | |
| 83 | 82 | } |
| 84 | 83 | |
| 85 | 84 | return 0; |
| @@ -97,6 +97,7 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, · 96 unmodified lines | ||
| 97 | 97 | static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, |
| 98 | 98 | struct mm_walk *walk) |
| 99 | 99 | { |
| 100 | + pud_t pudval = pudp_get(pud); | |
| 100 | 101 | pmd_t *pmd; |
| 101 | 102 | unsigned long next; |
| 102 | 103 | const struct mm_walk_ops *ops = walk->ops; |
| @@ -105,6 +106,24 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, · 2 unmodified lines | ||
| 105 | 106 | int err = 0; |
| 106 | 107 | int depth = real_depth(3); |
| 107 | 108 | |
| 109 | + /* | |
| 110 | + * For PTE handling, pte_offset_map_lock() takes care of checking | |
| 111 | + * whether there actually is a page table. But it also has to be | |
| 112 | + * very careful about concurrent page table reclaim. | |
| 113 | + * | |
| 114 | + * Similarly, we have to be careful here - a PUD entry that points | |
| 115 | + * to a PMD table cannot go away, so we can just walk it. But if | |
| 116 | + * it's something else, we need to ensure we didn't race something, | |
| 117 | + * so need to retry. | |
| 118 | + * | |
| 119 | + * A pertinent example of this is a PUD refault after PUD split - | |
| 120 | + * we will need to split again or risk accessing invalid memory. | |
| 121 | + */ | |
| 122 | + if (!pud_present(pudval) || pud_leaf(pudval)) { | |
| 123 | + walk->action = ACTION_AGAIN; | |
| 124 | + return 0; | |
| 125 | + } | |
| 126 | + | |
| 108 | 127 | pmd = pmd_offset(pud, addr); |
| 109 | 128 | do { |
| 110 | 129 | again: |
| @@ -218,12 +237,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, · 107 unmodified lines | ||
| 218 | 237 | else if (pud_leaf(*pud) || !pud_present(*pud)) |
| 219 | 238 | continue; /* Nothing to do. */ |
| 220 | 239 | |
| 221 | − if (pud_none(*pud)) | |
| 222 | − goto again; | |
| 223 | − | |
| 224 | 240 | err = walk_pmd_range(pud, addr, next, walk); |
| 225 | 241 | if (err) |
| 226 | 242 | break; |
| 243 | + | |
| 244 | + if (walk->action == ACTION_AGAIN) | |
| 245 | + goto again; | |
| 227 | 246 | } while (pud++, addr = next, addr != end); |
| 228 | 247 | |
| 229 | 248 | return err; |
| @@ -457,6 +457,13 @@ static void cleanup_partial_anon_vmas(struct vm_area_struct *vma) · 456 unmodified lines | ||
| 457 | 457 | list_del(&avc->same_vma); |
| 458 | 458 | anon_vma_chain_free(avc); |
| 459 | 459 | } |
| 460 | + | |
| 461 | + /* | |
| 462 | + * The anon_vma assigned to this VMA is no longer valid, as we were not | |
| 463 | + * able to correctly clone AVC state. Avoid inconsistent anon_vma tree | |
| 464 | + * state by resetting. | |
| 465 | + */ | |
| 466 | + vma->anon_vma = NULL; | |
| 460 | 467 | } |
| 461 | 468 | |
| 462 | 469 | /** |
| @@ -494,6 +494,10 @@ static struct folio *__swap_cache_prepare_and_add(swp_entry_t entry, · 493 unmodified lines | ||
| 494 | 494 | |
| 495 | 495 | __folio_set_locked(folio); |
| 496 | 496 | __folio_set_swapbacked(folio); |
| 497 | + | |
| 498 | + if (!charged && mem_cgroup_swapin_charge_folio(folio, NULL, gfp, entry)) | |
| 499 | + goto failed; | |
| 500 | + | |
| 497 | 501 | for (;;) { |
| 498 | 502 | ret = swap_cache_add_folio(folio, entry, &shadow); |
| 499 | 503 | if (!ret) |
| @@ -514,11 +518,6 @@ static struct folio *__swap_cache_prepare_and_add(swp_entry_t entry, · 14 unmodified lines | ||
| 514 | 518 | goto failed; |
| 515 | 519 | } |
| 516 | 520 | |
| 517 | − if (!charged && mem_cgroup_swapin_charge_folio(folio, NULL, gfp, entry)) { | |
| 518 | − swap_cache_del_folio(folio); | |
| 519 | − goto failed; | |
| 520 | − } | |
| 521 | − | |
| 522 | 521 | memcg1_swapin(entry, folio_nr_pages(folio)); |
| 523 | 522 | if (shadow) |
| 524 | 523 | workingset_refault(folio, shadow); |
| @@ -942,9 +942,15 @@ static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio) · 941 unmodified lines | ||
| 942 | 942 | |
| 943 | 943 | /* zswap entries of length PAGE_SIZE are not compressed. */ |
| 944 | 944 | if (entry->length == PAGE_SIZE) { |
| 945 | + void *dst; | |
| 946 | + | |
| 945 | 947 | WARN_ON_ONCE(input->length != PAGE_SIZE); |
| 946 | − memcpy_from_sglist(kmap_local_folio(folio, 0), input, 0, PAGE_SIZE); | |
| 948 | + | |
| 949 | + dst = kmap_local_folio(folio, 0); | |
| 950 | + memcpy_from_sglist(dst, input, 0, PAGE_SIZE); | |
| 947 | 951 | dlen = PAGE_SIZE; |
| 952 | + kunmap_local(dst); | |
| 953 | + flush_dcache_folio(folio); | |
| 948 | 954 | } else { |
| 949 | 955 | sg_init_table(&output, 1); |
| 950 | 956 | sg_set_folio(&output, folio, PAGE_SIZE, 0); |
| @@ -3095,7 +3095,7 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) · 3094 unmodified lines | ||
| 3095 | 3095 | * hci_connect_le serializes the connection attempts so only one |
| 3096 | 3096 | * connection can be in BT_CONNECT at time. |
| 3097 | 3097 | */ |
| 3098 | − if (conn->state == BT_CONNECT && hdev->req_status == HCI_REQ_PEND) { | |
| 3098 | + if (conn->state == BT_CONNECT && READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { | |
| 3099 | 3099 | switch (hci_skb_event(hdev->sent_cmd)) { |
| 3100 | 3100 | case HCI_EV_CONN_COMPLETE: |
| 3101 | 3101 | case HCI_EV_LE_CONN_COMPLETE: |
| @@ -4126,7 +4126,7 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) · 4125 unmodified lines | ||
| 4126 | 4126 | kfree_skb(skb); |
| 4127 | 4127 | } |
| 4128 | 4128 | |
| 4129 | − if (hdev->req_status == HCI_REQ_PEND && | |
| 4129 | + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND && | |
| 4130 | 4130 | !hci_dev_test_and_set_flag(hdev, HCI_CMD_PENDING)) { |
| 4131 | 4131 | kfree_skb(hdev->req_skb); |
| 4132 | 4132 | hdev->req_skb = skb_clone(hdev->sent_cmd, GFP_KERNEL); |
| @@ -25,11 +25,11 @@ static void hci_cmd_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, · 24 unmodified lines | ||
| 25 | 25 | { |
| 26 | 26 | bt_dev_dbg(hdev, "result 0x%2.2x", result); |
| 27 | 27 | |
| 28 | − if (hdev->req_status != HCI_REQ_PEND) | |
| 28 | + if (READ_ONCE(hdev->req_status) != HCI_REQ_PEND) | |
| 29 | 29 | return; |
| 30 | 30 | |
| 31 | 31 | hdev->req_result = result; |
| 32 | − hdev->req_status = HCI_REQ_DONE; | |
| 32 | + WRITE_ONCE(hdev->req_status, HCI_REQ_DONE); | |
| 33 | 33 | |
| 34 | 34 | /* Free the request command so it is not used as response */ |
| 35 | 35 | kfree_skb(hdev->req_skb); |
| @@ -167,20 +167,20 @@ struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, · 131 unmodified lines | ||
| 167 | 167 | |
| 168 | 168 | hci_cmd_sync_add(&req, opcode, plen, param, event, sk); |
| 169 | 169 | |
| 170 | − hdev->req_status = HCI_REQ_PEND; | |
| 170 | + WRITE_ONCE(hdev->req_status, HCI_REQ_PEND); | |
| 171 | 171 | |
| 172 | 172 | err = hci_req_sync_run(&req); |
| 173 | 173 | if (err < 0) |
| 174 | 174 | return ERR_PTR(err); |
| 175 | 175 | |
| 176 | 176 | err = wait_event_interruptible_timeout(hdev->req_wait_q, |
| 177 | − hdev->req_status != HCI_REQ_PEND, | |
| 177 | + READ_ONCE(hdev->req_status) != HCI_REQ_PEND, | |
| 178 | 178 | timeout); |
| 179 | 179 | |
| 180 | 180 | if (err == -ERESTARTSYS) |
| 181 | 181 | return ERR_PTR(-EINTR); |
| 182 | 182 | |
| 183 | − switch (hdev->req_status) { | |
| 183 | + switch (READ_ONCE(hdev->req_status)) { | |
| 184 | 184 | case HCI_REQ_DONE: |
| 185 | 185 | err = -bt_to_errno(hdev->req_result); |
| 186 | 186 | break; |
| @@ -194,7 +194,7 @@ struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, · 7 unmodified lines | ||
| 194 | 194 | break; |
| 195 | 195 | } |
| 196 | 196 | |
| 197 | − hdev->req_status = 0; | |
| 197 | + WRITE_ONCE(hdev->req_status, 0); | |
| 198 | 198 | hdev->req_result = 0; |
| 199 | 199 | skb = hdev->req_rsp; |
| 200 | 200 | hdev->req_rsp = NULL; |
| @@ -665,9 +665,9 @@ void hci_cmd_sync_cancel(struct hci_dev *hdev, int err) · 464 unmodified lines | ||
| 665 | 665 | { |
| 666 | 666 | bt_dev_dbg(hdev, "err 0x%2.2x", err); |
| 667 | 667 | |
| 668 | − if (hdev->req_status == HCI_REQ_PEND) { | |
| 668 | + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { | |
| 669 | 669 | hdev->req_result = err; |
| 670 | − hdev->req_status = HCI_REQ_CANCELED; | |
| 670 | + WRITE_ONCE(hdev->req_status, HCI_REQ_CANCELED); | |
| 671 | 671 | |
| 672 | 672 | queue_work(hdev->workqueue, &hdev->cmd_sync_cancel_work); |
| 673 | 673 | } |
| @@ -683,12 +683,12 @@ void hci_cmd_sync_cancel_sync(struct hci_dev *hdev, int err) · 9 unmodified lines | ||
| 683 | 683 | { |
| 684 | 684 | bt_dev_dbg(hdev, "err 0x%2.2x", err); |
| 685 | 685 | |
| 686 | − if (hdev->req_status == HCI_REQ_PEND) { | |
| 686 | + if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { | |
| 687 | 687 | /* req_result is __u32 so error must be positive to be properly |
| 688 | 688 | * propagated. |
| 689 | 689 | */ |
| 690 | 690 | hdev->req_result = err < 0 ? -err : err; |
| 691 | − hdev->req_status = HCI_REQ_CANCELED; | |
| 691 | + WRITE_ONCE(hdev->req_status, HCI_REQ_CANCELED); | |
| 692 | 692 | |
| 693 | 693 | wake_up_interruptible(&hdev->req_wait_q); |
| 694 | 694 | } |
| @@ -926,16 +926,39 @@ int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator) · 925 unmodified lines | ||
| 926 | 926 | |
| 927 | 927 | static int l2cap_get_ident(struct l2cap_conn *conn) |
| 928 | 928 | { |
| 929 | + u8 max; | |
| 930 | + int ident; | |
| 931 | + | |
| 929 | 932 | /* LE link does not support tools like l2ping so use the full range */ |
| 930 | 933 | if (conn->hcon->type == LE_LINK) |
| 931 | − return ida_alloc_range(&conn->tx_ida, 1, 255, GFP_ATOMIC); | |
| 932 | − | |
| 934 | + max = 255; | |
| 933 | 935 | /* Get next available identificator. |
| 934 | 936 | * 1 - 128 are used by kernel. |
| 935 | 937 | * 129 - 199 are reserved. |
| 936 | 938 | * 200 - 254 are used by utilities like l2ping, etc. |
| 937 | 939 | */ |
| 938 | − return ida_alloc_range(&conn->tx_ida, 1, 128, GFP_ATOMIC); | |
| 940 | + else | |
| 941 | + max = 128; | |
| 942 | + | |
| 943 | + /* Allocate ident using min as last used + 1 (cyclic) */ | |
| 944 | + ident = ida_alloc_range(&conn->tx_ida, READ_ONCE(conn->tx_ident) + 1, | |
| 945 | + max, GFP_ATOMIC); | |
| 946 | + /* Force min 1 to start over */ | |
| 947 | + if (ident <= 0) { | |
| 948 | + ident = ida_alloc_range(&conn->tx_ida, 1, max, GFP_ATOMIC); | |
| 949 | + if (ident <= 0) { | |
| 950 | + /* If all idents are in use, log an error, this is | |
| 951 | + * extremely unlikely to happen and would indicate a bug | |
| 952 | + * in the code that idents are not being freed properly. | |
| 953 | + */ | |
| 954 | + BT_ERR("Unable to allocate ident: %d", ident); | |
| 955 | + return 0; | |
| 956 | + } | |
| 957 | + } | |
| 958 | + | |
| 959 | + WRITE_ONCE(conn->tx_ident, ident); | |
| 960 | + | |
| 961 | + return ident; | |
| 939 | 962 | } |
| 940 | 963 | |
| 941 | 964 | static void l2cap_send_acl(struct l2cap_conn *conn, struct sk_buff *skb, |
| @@ -1748,6 +1771,9 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) · 806 unmodified lines | ||
| 1748 | 1771 | |
| 1749 | 1772 | BT_DBG("hcon %p conn %p, err %d", hcon, conn, err); |
| 1750 | 1773 | |
| 1774 | + disable_delayed_work_sync(&conn->info_timer); | |
| 1775 | + disable_delayed_work_sync(&conn->id_addr_timer); | |
| 1776 | + | |
| 1751 | 1777 | mutex_lock(&conn->lock); |
| 1752 | 1778 | |
| 1753 | 1779 | kfree_skb(conn->rx_skb); |
| @@ -1763,8 +1789,6 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) · 9 unmodified lines | ||
| 1763 | 1789 | |
| 1764 | 1790 | ida_destroy(&conn->tx_ida); |
| 1765 | 1791 | |
| 1766 | − cancel_delayed_work_sync(&conn->id_addr_timer); | |
| 1767 | − | |
| 1768 | 1792 | l2cap_unregister_all_users(conn); |
| 1769 | 1793 | |
| 1770 | 1794 | /* Force the connection to be immediately dropped */ |
| @@ -1783,9 +1807,6 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) · 12 unmodified lines | ||
| 1783 | 1807 | l2cap_chan_put(chan); |
| 1784 | 1808 | } |
| 1785 | 1809 | |
| 1786 | − if (conn->info_state & L2CAP_INFO_FEAT_MASK_REQ_SENT) | |
| 1787 | − cancel_delayed_work_sync(&conn->info_timer); | |
| 1788 | − | |
| 1789 | 1810 | hci_chan_del(conn->hchan); |
| 1790 | 1811 | conn->hchan = NULL; |
| 1791 | 1812 | |
| @@ -2377,6 +2398,9 @@ static int l2cap_segment_sdu(struct l2cap_chan *chan, · 585 unmodified lines | ||
| 2377 | 2398 | /* Remote device may have requested smaller PDUs */ |
| 2378 | 2399 | pdu_len = min_t(size_t, pdu_len, chan->remote_mps); |
| 2379 | 2400 | |
| 2401 | + if (!pdu_len) | |
| 2402 | + return -EINVAL; | |
| 2403 | + | |
| 2380 | 2404 | if (len <= pdu_len) { |
| 2381 | 2405 | sar = L2CAP_SAR_UNSEGMENTED; |
| 2382 | 2406 | sdu_len = 0; |
| @@ -4312,14 +4336,16 @@ static inline int l2cap_config_req(struct l2cap_conn *conn, · 1929 unmodified lines | ||
| 4312 | 4336 | if (test_bit(CONF_INPUT_DONE, &chan->conf_state)) { |
| 4313 | 4337 | set_default_fcs(chan); |
| 4314 | 4338 | |
| 4315 | − if (chan->mode == L2CAP_MODE_ERTM || | |
| 4316 | − chan->mode == L2CAP_MODE_STREAMING) | |
| 4317 | − err = l2cap_ertm_init(chan); | |
| 4339 | + if (chan->state != BT_CONNECTED) { | |
| 4340 | + if (chan->mode == L2CAP_MODE_ERTM || | |
| 4341 | + chan->mode == L2CAP_MODE_STREAMING) | |
| 4342 | + err = l2cap_ertm_init(chan); | |
| 4318 | 4343 | |
| 4319 | − if (err < 0) | |
| 4320 | − l2cap_send_disconn_req(chan, -err); | |
| 4321 | − else | |
| 4322 | − l2cap_chan_ready(chan); | |
| 4344 | + if (err < 0) | |
| 4345 | + l2cap_send_disconn_req(chan, -err); | |
| 4346 | + else | |
| 4347 | + l2cap_chan_ready(chan); | |
| 4348 | + } | |
| 4323 | 4349 | |
| 4324 | 4350 | goto unlock; |
| 4325 | 4351 | } |
| @@ -5081,14 +5107,14 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, · 755 unmodified lines | ||
| 5081 | 5107 | cmd_len -= sizeof(*req); |
| 5082 | 5108 | num_scid = cmd_len / sizeof(u16); |
| 5083 | 5109 | |
| 5084 | − /* Always respond with the same number of scids as in the request */ | |
| 5085 | − rsp_len = cmd_len; | |
| 5086 | − | |
| 5087 | 5110 | if (num_scid > L2CAP_ECRED_MAX_CID) { |
| 5088 | 5111 | result = L2CAP_CR_LE_INVALID_PARAMS; |
| 5089 | 5112 | goto response; |
| 5090 | 5113 | } |
| 5091 | 5114 | |
| 5115 | + /* Always respond with the same number of scids as in the request */ | |
| 5116 | + rsp_len = cmd_len; | |
| 5117 | + | |
| 5092 | 5118 | mtu = __le16_to_cpu(req->mtu); |
| 5093 | 5119 | mps = __le16_to_cpu(req->mps); |
| 5094 | 5120 | |
| @@ -6607,6 +6633,10 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) · 1512 unmodified lines | ||
| 6607 | 6633 | struct l2cap_le_credits pkt; |
| 6608 | 6634 | u16 return_credits = l2cap_le_rx_credits(chan); |
| 6609 | 6635 | |
| 6636 | + if (chan->mode != L2CAP_MODE_LE_FLOWCTL && | |
| 6637 | + chan->mode != L2CAP_MODE_EXT_FLOWCTL) | |
| 6638 | + return; | |
| 6639 | + | |
| 6610 | 6640 | if (chan->rx_credits >= return_credits) |
| 6611 | 6641 | return; |
| 6612 | 6642 | |
| @@ -6690,6 +6720,11 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) · 77 unmodified lines | ||
| 6690 | 6720 | if (!chan->sdu) { |
| 6691 | 6721 | u16 sdu_len; |
| 6692 | 6722 | |
| 6723 | + if (!pskb_may_pull(skb, L2CAP_SDULEN_SIZE)) { | |
| 6724 | + err = -EINVAL; | |
| 6725 | + goto failed; | |
| 6726 | + } | |
| 6727 | + | |
| 6693 | 6728 | sdu_len = get_unaligned_le16(skb->data); |
| 6694 | 6729 | skb_pull(skb, L2CAP_SDULEN_SIZE); |
| 6695 | 6730 | |
| @@ -1698,6 +1698,9 @@ static void l2cap_sock_ready_cb(struct l2cap_chan *chan) · 1697 unmodified lines | ||
| 1698 | 1698 | struct sock *sk = chan->data; |
| 1699 | 1699 | struct sock *parent; |
| 1700 | 1700 | |
| 1701 | + if (!sk) | |
| 1702 | + return; | |
| 1703 | + | |
| 1701 | 1704 | lock_sock(sk); |
| 1702 | 1705 | |
| 1703 | 1706 | parent = bt_sk(sk)->parent; |
| @@ -5355,7 +5355,7 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, · 5354 unmodified lines | ||
| 5355 | 5355 | * hci_adv_monitors_clear is about to be called which will take care of |
| 5356 | 5356 | * freeing the adv_monitor instances. |
| 5357 | 5357 | */ |
| 5358 | − if (status == -ECANCELED && !mgmt_pending_valid(hdev, cmd)) | |
| 5358 | + if (status == -ECANCELED || !mgmt_pending_valid(hdev, cmd)) | |
| 5359 | 5359 | return; |
| 5360 | 5360 | |
| 5361 | 5361 | monitor = cmd->user_data; |
| @@ -401,7 +401,7 @@ static void sco_recv_frame(struct sco_conn *conn, struct sk_buff *skb) · 400 unmodified lines | ||
| 401 | 401 | struct sock *sk; |
| 402 | 402 | |
| 403 | 403 | sco_conn_lock(conn); |
| 404 | − sk = conn->sk; | |
| 404 | + sk = sco_sock_hold(conn); | |
| 405 | 405 | sco_conn_unlock(conn); |
| 406 | 406 | |
| 407 | 407 | if (!sk) |
| @@ -410,11 +410,15 @@ static void sco_recv_frame(struct sco_conn *conn, struct sk_buff *skb) · 2 unmodified lines | ||
| 410 | 410 | BT_DBG("sk %p len %u", sk, skb->len); |
| 411 | 411 | |
| 412 | 412 | if (sk->sk_state != BT_CONNECTED) |
| 413 | − goto drop; | |
| 413 | + goto drop_put; | |
| 414 | 414 | |
| 415 | − if (!sock_queue_rcv_skb(sk, skb)) | |
| 415 | + if (!sock_queue_rcv_skb(sk, skb)) { | |
| 416 | + sock_put(sk); | |
| 416 | 417 | return; |
| 418 | + } | |
| 417 | 419 | |
| 420 | +drop_put: | |
| 421 | + sock_put(sk); | |
| 418 | 422 | drop: |
| 419 | 423 | kfree_skb(skb); |
| 420 | 424 | } |
| @@ -469,7 +469,7 @@ int can_rx_register(struct net *net, struct net_device *dev, canid_t can_id, · 468 unmodified lines | ||
| 469 | 469 | |
| 470 | 470 | rcv->can_id = can_id; |
| 471 | 471 | rcv->mask = mask; |
| 472 | − rcv->matches = 0; | |
| 472 | + atomic_long_set(&rcv->matches, 0); | |
| 473 | 473 | rcv->func = func; |
| 474 | 474 | rcv->data = data; |
| 475 | 475 | rcv->ident = ident; |
| @@ -573,7 +573,7 @@ EXPORT_SYMBOL(can_rx_unregister); · 97 unmodified lines | ||
| 573 | 573 | static inline void deliver(struct sk_buff *skb, struct receiver *rcv) |
| 574 | 574 | { |
| 575 | 575 | rcv->func(skb, rcv->data); |
| 576 | − rcv->matches++; | |
| 576 | + atomic_long_inc(&rcv->matches); | |
| 577 | 577 | } |
| 578 | 578 | |
| 579 | 579 | static int can_rcv_filter(struct can_dev_rcv_lists *dev_rcv_lists, struct sk_buff *skb) |
| @@ -52,7 +52,7 @@ struct receiver { · 51 unmodified lines | ||
| 52 | 52 | struct hlist_node list; |
| 53 | 53 | canid_t can_id; |
| 54 | 54 | canid_t mask; |
| 55 | − unsigned long matches; | |
| 55 | + atomic_long_t matches; | |
| 56 | 56 | void (*func)(struct sk_buff *skb, void *data); |
| 57 | 57 | void *data; |
| 58 | 58 | char *ident; |
| @@ -375,10 +375,10 @@ static void cgw_csum_crc8_rel(struct canfd_frame *cf, · 374 unmodified lines | ||
| 375 | 375 | return; |
| 376 | 376 | |
| 377 | 377 | if (from <= to) { |
| 378 | − for (i = crc8->from_idx; i <= crc8->to_idx; i++) | |
| 378 | + for (i = from; i <= to; i++) | |
| 379 | 379 | crc = crc8->crctab[crc ^ cf->data[i]]; |
| 380 | 380 | } else { |
| 381 | − for (i = crc8->from_idx; i >= crc8->to_idx; i--) | |
| 381 | + for (i = from; i >= to; i--) | |
| 382 | 382 | crc = crc8->crctab[crc ^ cf->data[i]]; |
| 383 | 383 | } |
| 384 | 384 | |
| @@ -397,7 +397,7 @@ static void cgw_csum_crc8_rel(struct canfd_frame *cf, · 12 unmodified lines | ||
| 397 | 397 | break; |
| 398 | 398 | } |
| 399 | 399 | |
| 400 | − cf->data[crc8->result_idx] = crc ^ crc8->final_xor_val; | |
| 400 | + cf->data[res] = crc ^ crc8->final_xor_val; | |
| 401 | 401 | } |
| 402 | 402 | |
| 403 | 403 | static void cgw_csum_crc8_pos(struct canfd_frame *cf, |
| @@ -1248,12 +1248,6 @@ static int isotp_release(struct socket *sock) · 1247 unmodified lines | ||
| 1248 | 1248 | so->ifindex = 0; |
| 1249 | 1249 | so->bound = 0; |
| 1250 | 1250 | |
| 1251 | − if (so->rx.buf != so->rx.sbuf) | |
| 1252 | − kfree(so->rx.buf); | |
| 1253 | − | |
| 1254 | − if (so->tx.buf != so->tx.sbuf) | |
| 1255 | − kfree(so->tx.buf); | |
| 1256 | − | |
| 1257 | 1251 | sock_orphan(sk); |
| 1258 | 1252 | sock->sk = NULL; |
| 1259 | 1253 | |
| @@ -1622,6 +1616,21 @@ static int isotp_notifier(struct notifier_block *nb, unsigned long msg, · 362 unmodified lines | ||
| 1622 | 1616 | return NOTIFY_DONE; |
| 1623 | 1617 | } |
| 1624 | 1618 | |
| 1619 | +static void isotp_sock_destruct(struct sock *sk) | |
| 1620 | +{ | |
| 1621 | + struct isotp_sock *so = isotp_sk(sk); | |
| 1622 | + | |
| 1623 | + /* do the standard CAN sock destruct work */ | |
| 1624 | + can_sock_destruct(sk); | |
| 1625 | + | |
| 1626 | + /* free potential extended PDU buffers */ | |
| 1627 | + if (so->rx.buf != so->rx.sbuf) | |
| 1628 | + kfree(so->rx.buf); | |
| 1629 | + | |
| 1630 | + if (so->tx.buf != so->tx.sbuf) | |
| 1631 | + kfree(so->tx.buf); | |
| 1632 | +} | |
| 1633 | + | |
| 1625 | 1634 | static int isotp_init(struct sock *sk) |
| 1626 | 1635 | { |
| 1627 | 1636 | struct isotp_sock *so = isotp_sk(sk); |
| @@ -1666,6 +1675,9 @@ static int isotp_init(struct sock *sk) · 38 unmodified lines | ||
| 1666 | 1675 | list_add_tail(&so->notifier, &isotp_notifier_list); |
| 1667 | 1676 | spin_unlock(&isotp_notifier_lock); |
| 1668 | 1677 | |
| 1678 | + /* re-assign default can_sock_destruct() reference */ | |
| 1679 | + sk->sk_destruct = isotp_sock_destruct; | |
| 1680 | + | |
| 1669 | 1681 | return 0; |
| 1670 | 1682 | } |
| 1671 | 1683 | |
| @@ -196,7 +196,8 @@ static void can_print_rcvlist(struct seq_file *m, struct hlist_head *rx_list, · 195 unmodified lines | ||
| 196 | 196 | " %-5s %03x %08x %pK %pK %8ld %s\n"; |
| 197 | 197 | |
| 198 | 198 | seq_printf(m, fmt, DNAME(dev), r->can_id, r->mask, |
| 199 | − r->func, r->data, r->matches, r->ident); | |
| 199 | + r->func, r->data, atomic_long_read(&r->matches), | |
| 200 | + r->ident); | |
| 200 | 201 | } |
| 201 | 202 | } |
| 202 | 203 | |
| @@ -3769,6 +3769,22 @@ static netdev_features_t dflt_features_check(struct sk_buff *skb, · 3768 unmodified lines | ||
| 3769 | 3769 | return vlan_features_check(skb, features); |
| 3770 | 3770 | } |
| 3771 | 3771 | |
| 3772 | +static bool skb_gso_has_extension_hdr(const struct sk_buff *skb) | |
| 3773 | +{ | |
| 3774 | + if (!skb->encapsulation) | |
| 3775 | + return ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || | |
| 3776 | + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && | |
| 3777 | + vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && | |
| 3778 | + skb_transport_header_was_set(skb) && | |
| 3779 | + skb_network_header_len(skb) != sizeof(struct ipv6hdr)); | |
| 3780 | + else | |
| 3781 | + return (!skb_inner_network_header_was_set(skb) || | |
| 3782 | + ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || | |
| 3783 | + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && | |
| 3784 | + inner_ip_hdr(skb)->version == 6)) && | |
| 3785 | + skb_inner_network_header_len(skb) != sizeof(struct ipv6hdr))); | |
| 3786 | +} | |
| 3787 | + | |
| 3772 | 3788 | static netdev_features_t gso_features_check(const struct sk_buff *skb, |
| 3773 | 3789 | struct net_device *dev, |
| 3774 | 3790 | netdev_features_t features) |
| @@ -3816,11 +3832,7 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, · 41 unmodified lines | ||
| 3816 | 3832 | * so neither does TSO that depends on it. |
| 3817 | 3833 | */ |
| 3818 | 3834 | if (features & NETIF_F_IPV6_CSUM && |
| 3819 | − (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || | |
| 3820 | − (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && | |
| 3821 | − vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && | |
| 3822 | − skb_transport_header_was_set(skb) && | |
| 3823 | − skb_network_header_len(skb) != sizeof(struct ipv6hdr)) | |
| 3835 | + skb_gso_has_extension_hdr(skb)) | |
| 3824 | 3836 | features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4); |
| 3825 | 3837 | |
| 3826 | 3838 | return features; |
| @@ -629,6 +629,9 @@ int rtnl_link_register(struct rtnl_link_ops *ops) · 628 unmodified lines | ||
| 629 | 629 | unlock: |
| 630 | 630 | mutex_unlock(&link_ops_mutex); |
| 631 | 631 | |
| 632 | + if (err) | |
| 633 | + cleanup_srcu_struct(&ops->srcu); | |
| 634 | + | |
| 632 | 635 | return err; |
| 633 | 636 | } |
| 634 | 637 | EXPORT_SYMBOL_GPL(rtnl_link_register); |
| @@ -707,11 +710,14 @@ static size_t rtnl_link_get_slave_info_data_size(const struct net_device *dev) · 72 unmodified lines | ||
| 707 | 710 | goto out; |
| 708 | 711 | |
| 709 | 712 | ops = master_dev->rtnl_link_ops; |
| 710 | − if (!ops || !ops->get_slave_size) | |
| 713 | + if (!ops) | |
| 714 | + goto out; | |
| 715 | + size += nla_total_size(strlen(ops->kind) + 1); /* IFLA_INFO_SLAVE_KIND */ | |
| 716 | + if (!ops->get_slave_size) | |
| 711 | 717 | goto out; |
| 712 | 718 | /* IFLA_INFO_SLAVE_DATA + nested data */ |
| 713 | − size = nla_total_size(sizeof(struct nlattr)) + | |
| 714 | − ops->get_slave_size(master_dev, dev); | |
| 719 | + size += nla_total_size(sizeof(struct nlattr)) + | |
| 720 | + ops->get_slave_size(master_dev, dev); | |
| 715 | 721 | |
| 716 | 722 | out: |
| 717 | 723 | rcu_read_unlock(); |
| @@ -1267,6 +1273,21 @@ static size_t rtnl_dpll_pin_size(const struct net_device *dev) · 549 unmodified lines | ||
| 1267 | 1273 | return size; |
| 1268 | 1274 | } |
| 1269 | 1275 | |
| 1276 | +static size_t rtnl_dev_parent_size(const struct net_device *dev) | |
| 1277 | +{ | |
| 1278 | + size_t size = 0; | |
| 1279 | + | |
| 1280 | + /* IFLA_PARENT_DEV_NAME */ | |
| 1281 | + if (dev->dev.parent) | |
| 1282 | + size += nla_total_size(strlen(dev_name(dev->dev.parent)) + 1); | |
| 1283 | + | |
| 1284 | + /* IFLA_PARENT_DEV_BUS_NAME */ | |
| 1285 | + if (dev->dev.parent && dev->dev.parent->bus) | |
| 1286 | + size += nla_total_size(strlen(dev->dev.parent->bus->name) + 1); | |
| 1287 | + | |
| 1288 | + return size; | |
| 1289 | +} | |
| 1290 | + | |
| 1270 | 1291 | static noinline size_t if_nlmsg_size(const struct net_device *dev, |
| 1271 | 1292 | u32 ext_filter_mask) |
| 1272 | 1293 | { |
| @@ -1328,6 +1349,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, · 55 unmodified lines | ||
| 1328 | 1349 | + nla_total_size(8) /* IFLA_MAX_PACING_OFFLOAD_HORIZON */ |
| 1329 | 1350 | + nla_total_size(2) /* IFLA_HEADROOM */ |
| 1330 | 1351 | + nla_total_size(2) /* IFLA_TAILROOM */ |
| 1352 | + + rtnl_dev_parent_size(dev) | |
| 1331 | 1353 | + 0; |
| 1332 | 1354 | |
| 1333 | 1355 | if (!(ext_filter_mask & RTEXT_FILTER_SKIP_STATS)) |
| @@ -235,10 +235,13 @@ static void esp_output_done(void *data, int err) · 234 unmodified lines | ||
| 235 | 235 | xfrm_dev_resume(skb); |
| 236 | 236 | } else { |
| 237 | 237 | if (!err && |
| 238 | − x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) | |
| 239 | − esp_output_tail_tcp(x, skb); | |
| 240 | − else | |
| 238 | + x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) { | |
| 239 | + err = esp_output_tail_tcp(x, skb); | |
| 240 | + if (err != -EINPROGRESS) | |
| 241 | + kfree_skb(skb); | |
| 242 | + } else { | |
| 241 | 243 | xfrm_output_resume(skb_to_full_sk(skb), skb, err); |
| 244 | + } | |
| 242 | 245 | } |
| 243 | 246 | } |
| 244 | 247 | |
| @@ -154,20 +154,6 @@ bool inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high) · 153 unmodified lines | ||
| 154 | 154 | } |
| 155 | 155 | EXPORT_SYMBOL(inet_sk_get_local_port_range); |
| 156 | 156 | |
| 157 | −static bool inet_use_bhash2_on_bind(const struct sock *sk) | |
| 158 | −{ | |
| 159 | −#if IS_ENABLED(CONFIG_IPV6) | |
| 160 | − if (sk->sk_family == AF_INET6) { | |
| 161 | − if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) | |
| 162 | − return false; | |
| 163 | − | |
| 164 | − if (!ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) | |
| 165 | − return true; | |
| 166 | − } | |
| 167 | −#endif | |
| 168 | − return sk->sk_rcv_saddr != htonl(INADDR_ANY); | |
| 169 | −} | |
| 170 | − | |
| 171 | 157 | static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2, |
| 172 | 158 | kuid_t uid, bool relax, |
| 173 | 159 | bool reuseport_cb_ok, bool reuseport_ok) |
| @@ -259,7 +245,7 @@ static int inet_csk_bind_conflict(const struct sock *sk, · 85 unmodified lines | ||
| 259 | 245 | * checks separately because their spinlocks have to be acquired/released |
| 260 | 246 | * independently of each other, to prevent possible deadlocks |
| 261 | 247 | */ |
| 262 | − if (inet_use_bhash2_on_bind(sk)) | |
| 248 | + if (inet_use_hash2_on_bind(sk)) | |
| 263 | 249 | return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, |
| 264 | 250 | reuseport_cb_ok, reuseport_ok); |
| 265 | 251 | |
| @@ -376,7 +362,7 @@ other_parity_scan: · 110 unmodified lines | ||
| 376 | 362 | head = &hinfo->bhash[inet_bhashfn(net, port, |
| 377 | 363 | hinfo->bhash_size)]; |
| 378 | 364 | spin_lock_bh(&head->lock); |
| 379 | − if (inet_use_bhash2_on_bind(sk)) { | |
| 365 | + if (inet_use_hash2_on_bind(sk)) { | |
| 380 | 366 | if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false)) |
| 381 | 367 | goto next_port; |
| 382 | 368 | } |
| @@ -562,7 +548,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum) · 179 unmodified lines | ||
| 562 | 548 | check_bind_conflict = false; |
| 563 | 549 | } |
| 564 | 550 | |
| 565 | − if (check_bind_conflict && inet_use_bhash2_on_bind(sk)) { | |
| 551 | + if (check_bind_conflict && inet_use_hash2_on_bind(sk)) { | |
| 566 | 552 | if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true)) |
| 567 | 553 | goto fail_unlock; |
| 568 | 554 | } |
| @@ -287,7 +287,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, · 286 unmodified lines | ||
| 287 | 287 | } else { |
| 288 | 288 | hslot = udp_hashslot(udptable, net, snum); |
| 289 | 289 | spin_lock_bh(&hslot->lock); |
| 290 | − if (hslot->count > 10) { | |
| 290 | + if (inet_use_hash2_on_bind(sk) && hslot->count > 10) { | |
| 291 | 291 | int exist; |
| 292 | 292 | unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; |
| 293 | 293 | |
| @@ -2862,7 +2862,7 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao) · 2861 unmodified lines | ||
| 2862 | 2862 | fib6_add_gc_list(rt); |
| 2863 | 2863 | } else { |
| 2864 | 2864 | fib6_clean_expires(rt); |
| 2865 | − fib6_remove_gc_list(rt); | |
| 2865 | + fib6_may_remove_gc_list(net, rt); | |
| 2866 | 2866 | } |
| 2867 | 2867 | |
| 2868 | 2868 | spin_unlock_bh(&table->tb6_lock); |
| @@ -4840,7 +4840,7 @@ static int modify_prefix_route(struct net *net, struct inet6_ifaddr *ifp, · 1971 unmodified lines | ||
| 4840 | 4840 | |
| 4841 | 4841 | if (!(flags & RTF_EXPIRES)) { |
| 4842 | 4842 | fib6_clean_expires(f6i); |
| 4843 | − fib6_remove_gc_list(f6i); | |
| 4843 | + fib6_may_remove_gc_list(net, f6i); | |
| 4844 | 4844 | } else { |
| 4845 | 4845 | fib6_set_expires(f6i, expires); |
| 4846 | 4846 | fib6_add_gc_list(f6i); |
| @@ -271,10 +271,13 @@ static void esp_output_done(void *data, int err) · 270 unmodified lines | ||
| 271 | 271 | xfrm_dev_resume(skb); |
| 272 | 272 | } else { |
| 273 | 273 | if (!err && |
| 274 | − x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) | |
| 275 | − esp_output_tail_tcp(x, skb); | |
| 276 | − else | |
| 274 | + x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) { | |
| 275 | + err = esp_output_tail_tcp(x, skb); | |
| 276 | + if (err != -EINPROGRESS) | |
| 277 | + kfree_skb(skb); | |
| 278 | + } else { | |
| 277 | 279 | xfrm_output_resume(skb_to_full_sk(skb), skb, err); |
| 280 | + } | |
| 278 | 281 | } |
| 279 | 282 | } |
| 280 | 283 | |
| @@ -1133,7 +1133,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, · 1132 unmodified lines | ||
| 1133 | 1133 | return -EEXIST; |
| 1134 | 1134 | if (!(rt->fib6_flags & RTF_EXPIRES)) { |
| 1135 | 1135 | fib6_clean_expires(iter); |
| 1136 | − fib6_remove_gc_list(iter); | |
| 1136 | + fib6_may_remove_gc_list(info->nl_net, iter); | |
| 1137 | 1137 | } else { |
| 1138 | 1138 | fib6_set_expires(iter, rt->expires); |
| 1139 | 1139 | fib6_add_gc_list(iter); |
| @@ -2348,6 +2348,17 @@ static void fib6_flush_trees(struct net *net) · 1208 unmodified lines | ||
| 2348 | 2348 | /* |
| 2349 | 2349 | * Garbage collection |
| 2350 | 2350 | */ |
| 2351 | +void fib6_age_exceptions(struct fib6_info *rt, struct fib6_gc_args *gc_args, | |
| 2352 | + unsigned long now) | |
| 2353 | +{ | |
| 2354 | + bool may_expire = rt->fib6_flags & RTF_EXPIRES && rt->expires; | |
| 2355 | + int old_more = gc_args->more; | |
| 2356 | + | |
| 2357 | + rt6_age_exceptions(rt, gc_args, now); | |
| 2358 | + | |
| 2359 | + if (!may_expire && old_more == gc_args->more) | |
| 2360 | + fib6_remove_gc_list(rt); | |
| 2361 | +} | |
| 2351 | 2362 | |
| 2352 | 2363 | static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args) |
| 2353 | 2364 | { |
| @@ -2370,7 +2381,7 @@ static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args) · 16 unmodified lines | ||
| 2370 | 2381 | * Note, that clones are aged out |
| 2371 | 2382 | * only if they are not in use now. |
| 2372 | 2383 | */ |
| 2373 | − rt6_age_exceptions(rt, gc_args, now); | |
| 2384 | + fib6_age_exceptions(rt, gc_args, now); | |
| 2374 | 2385 | |
| 2375 | 2386 | return 0; |
| 2376 | 2387 | } |
| @@ -157,6 +157,10 @@ static int rt_mt6_check(const struct xt_mtchk_param *par) · 156 unmodified lines | ||
| 157 | 157 | pr_debug("unknown flags %X\n", rtinfo->invflags); |
| 158 | 158 | return -EINVAL; |
| 159 | 159 | } |
| 160 | + if (rtinfo->addrnr > IP6T_RT_HOPS) { | |
| 161 | + pr_debug("too many addresses specified\n"); | |
| 162 | + return -EINVAL; | |
| 163 | + } | |
| 160 | 164 | if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) && |
| 161 | 165 | (!(rtinfo->flags & IP6T_RT_TYP) || |
| 162 | 166 | (rtinfo->rt_type != 0) || |
| @@ -1033,7 +1033,7 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, · 1032 unmodified lines | ||
| 1033 | 1033 | |
| 1034 | 1034 | if (!addrconf_finite_timeout(lifetime)) { |
| 1035 | 1035 | fib6_clean_expires(rt); |
| 1036 | − fib6_remove_gc_list(rt); | |
| 1036 | + fib6_may_remove_gc_list(net, rt); | |
| 1037 | 1037 | } else { |
| 1038 | 1038 | fib6_set_expires(rt, jiffies + HZ * lifetime); |
| 1039 | 1039 | fib6_add_gc_list(rt); |
| @@ -3518,7 +3518,7 @@ static int set_sadb_kmaddress(struct sk_buff *skb, const struct xfrm_kmaddress * · 3517 unmodified lines | ||
| 3518 | 3518 | |
| 3519 | 3519 | static int set_ipsecrequest(struct sk_buff *skb, |
| 3520 | 3520 | uint8_t proto, uint8_t mode, int level, |
| 3521 | − uint32_t reqid, uint8_t family, | |
| 3521 | + uint32_t reqid, sa_family_t family, | |
| 3522 | 3522 | const xfrm_address_t *src, const xfrm_address_t *dst) |
| 3523 | 3523 | { |
| 3524 | 3524 | struct sadb_x_ipsecrequest *rq; |
| @@ -3583,12 +3583,17 @@ static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, · 58 unmodified lines | ||
| 3583 | 3583 | |
| 3584 | 3584 | /* ipsecrequests */ |
| 3585 | 3585 | for (i = 0, mp = m; i < num_bundles; i++, mp++) { |
| 3586 | − /* old locator pair */ | |
| 3587 | − size_pol += sizeof(struct sadb_x_ipsecrequest) + | |
| 3588 | − pfkey_sockaddr_pair_size(mp->old_family); | |
| 3589 | − /* new locator pair */ | |
| 3590 | − size_pol += sizeof(struct sadb_x_ipsecrequest) + | |
| 3591 | − pfkey_sockaddr_pair_size(mp->new_family); | |
| 3586 | + int pair_size; | |
| 3587 | + | |
| 3588 | + pair_size = pfkey_sockaddr_pair_size(mp->old_family); | |
| 3589 | + if (!pair_size) | |
| 3590 | + return -EINVAL; | |
| 3591 | + size_pol += sizeof(struct sadb_x_ipsecrequest) + pair_size; | |
| 3592 | + | |
| 3593 | + pair_size = pfkey_sockaddr_pair_size(mp->new_family); | |
| 3594 | + if (!pair_size) | |
| 3595 | + return -EINVAL; | |
| 3596 | + size_pol += sizeof(struct sadb_x_ipsecrequest) + pair_size; | |
| 3592 | 3597 | } |
| 3593 | 3598 | |
| 3594 | 3599 | size += sizeof(struct sadb_msg) + size_pol; |
| @@ -21,6 +21,7 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, · 20 unmodified lines | ||
| 21 | 21 | unsigned int timeout) |
| 22 | 22 | { |
| 23 | 23 | const struct nf_conntrack_helper *helper; |
| 24 | + struct net *net = read_pnet(&ct->ct_net); | |
| 24 | 25 | struct nf_conntrack_expect *exp; |
| 25 | 26 | struct iphdr *iph = ip_hdr(skb); |
| 26 | 27 | struct rtable *rt = skb_rtable(skb); |
| @@ -70,8 +71,11 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, · 43 unmodified lines | ||
| 70 | 71 | exp->expectfn = NULL; |
| 71 | 72 | exp->flags = NF_CT_EXPECT_PERMANENT; |
| 72 | 73 | exp->class = NF_CT_EXPECT_CLASS_DEFAULT; |
| 73 | − exp->helper = NULL; | |
| 74 | − | |
| 74 | + rcu_assign_pointer(exp->helper, helper); | |
| 75 | + write_pnet(&exp->net, net); | |
| 76 | +#ifdef CONFIG_NF_CONNTRACK_ZONES | |
| 77 | + exp->zone = ct->zone; | |
| 78 | +#endif | |
| 75 | 79 | nf_ct_expect_related(exp, 0); |
| 76 | 80 | nf_ct_expect_put(exp); |
| 77 | 81 | |
| @@ -247,6 +247,8 @@ void nf_ct_expect_event_report(enum ip_conntrack_expect_events event, · 246 unmodified lines | ||
| 247 | 247 | struct nf_ct_event_notifier *notify; |
| 248 | 248 | struct nf_conntrack_ecache *e; |
| 249 | 249 | |
| 250 | + lockdep_nfct_expect_lock_held(); | |
| 251 | + | |
| 250 | 252 | rcu_read_lock(); |
| 251 | 253 | notify = rcu_dereference(net->ct.nf_conntrack_event_cb); |
| 252 | 254 | if (!notify) |
| @@ -51,6 +51,7 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, · 50 unmodified lines | ||
| 51 | 51 | struct net *net = nf_ct_exp_net(exp); |
| 52 | 52 | struct nf_conntrack_net *cnet; |
| 53 | 53 | |
| 54 | + lockdep_nfct_expect_lock_held(); | |
| 54 | 55 | WARN_ON(!master_help); |
| 55 | 56 | WARN_ON(timer_pending(&exp->timeout)); |
| 56 | 57 | |
| @@ -112,12 +113,14 @@ nf_ct_exp_equal(const struct nf_conntrack_tuple *tuple, · 55 unmodified lines | ||
| 112 | 113 | const struct net *net) |
| 113 | 114 | { |
| 114 | 115 | return nf_ct_tuple_mask_cmp(tuple, &i->tuple, &i->mask) && |
| 115 | − net_eq(net, nf_ct_net(i->master)) && | |
| 116 | − nf_ct_zone_equal_any(i->master, zone); | |
| 116 | + net_eq(net, read_pnet(&i->net)) && | |
| 117 | + nf_ct_exp_zone_equal_any(i, zone); | |
| 117 | 118 | } |
| 118 | 119 | |
| 119 | 120 | bool nf_ct_remove_expect(struct nf_conntrack_expect *exp) |
| 120 | 121 | { |
| 122 | + lockdep_nfct_expect_lock_held(); | |
| 123 | + | |
| 121 | 124 | if (timer_delete(&exp->timeout)) { |
| 122 | 125 | nf_ct_unlink_expect(exp); |
| 123 | 126 | nf_ct_expect_put(exp); |
| @@ -177,6 +180,8 @@ nf_ct_find_expectation(struct net *net, · 53 unmodified lines | ||
| 177 | 180 | struct nf_conntrack_expect *i, *exp = NULL; |
| 178 | 181 | unsigned int h; |
| 179 | 182 | |
| 183 | + lockdep_nfct_expect_lock_held(); | |
| 184 | + | |
| 180 | 185 | if (!cnet->expect_count) |
| 181 | 186 | return NULL; |
| 182 | 187 | |
| @@ -309,12 +314,20 @@ struct nf_conntrack_expect *nf_ct_expect_alloc(struct nf_conn *me) · 126 unmodified lines | ||
| 309 | 314 | } |
| 310 | 315 | EXPORT_SYMBOL_GPL(nf_ct_expect_alloc); |
| 311 | 316 | |
| 317 | +/* This function can only be used from packet path, where accessing | |
| 318 | + * master's helper is safe, because the packet holds a reference on | |
| 319 | + * the conntrack object. Never use it from control plane. | |
| 320 | + */ | |
| 312 | 321 | void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, |
| 313 | 322 | u_int8_t family, |
| 314 | 323 | const union nf_inet_addr *saddr, |
| 315 | 324 | const union nf_inet_addr *daddr, |
| 316 | 325 | u_int8_t proto, const __be16 *src, const __be16 *dst) |
| 317 | 326 | { |
| 327 | + struct nf_conntrack_helper *helper = NULL; | |
| 328 | + struct nf_conn *ct = exp->master; | |
| 329 | + struct net *net = read_pnet(&ct->ct_net); | |
| 330 | + struct nf_conn_help *help; | |
| 318 | 331 | int len; |
| 319 | 332 | |
| 320 | 333 | if (family == AF_INET) |
| @@ -325,7 +338,16 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, · 4 unmodified lines | ||
| 325 | 338 | exp->flags = 0; |
| 326 | 339 | exp->class = class; |
| 327 | 340 | exp->expectfn = NULL; |
| 328 | − exp->helper = NULL; | |
| 341 | + | |
| 342 | + help = nfct_help(ct); | |
| 343 | + if (help) | |
| 344 | + helper = rcu_dereference(help->helper); | |
| 345 | + | |
| 346 | + rcu_assign_pointer(exp->helper, helper); | |
| 347 | + write_pnet(&exp->net, net); | |
| 348 | +#ifdef CONFIG_NF_CONNTRACK_ZONES | |
| 349 | + exp->zone = ct->zone; | |
| 350 | +#endif | |
| 329 | 351 | exp->tuple.src.l3num = family; |
| 330 | 352 | exp->tuple.dst.protonum = proto; |
| 331 | 353 | |
| @@ -442,6 +464,8 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, · 110 unmodified lines | ||
| 442 | 464 | unsigned int h; |
| 443 | 465 | int ret = 0; |
| 444 | 466 | |
| 467 | + lockdep_nfct_expect_lock_held(); | |
| 468 | + | |
| 445 | 469 | if (!master_help) { |
| 446 | 470 | ret = -ESHUTDOWN; |
| 447 | 471 | goto out; |
| @@ -498,8 +522,9 @@ int nf_ct_expect_related_report(struct nf_conntrack_expect *expect, · 50 unmodified lines | ||
| 498 | 522 | |
| 499 | 523 | nf_ct_expect_insert(expect); |
| 500 | 524 | |
| 501 | − spin_unlock_bh(&nf_conntrack_expect_lock); | |
| 502 | 525 | nf_ct_expect_event_report(IPEXP_NEW, expect, portid, report); |
| 526 | + spin_unlock_bh(&nf_conntrack_expect_lock); | |
| 527 | + | |
| 503 | 528 | return 0; |
| 504 | 529 | out: |
| 505 | 530 | spin_unlock_bh(&nf_conntrack_expect_lock); |
| @@ -627,11 +652,15 @@ static int exp_seq_show(struct seq_file *s, void *v) · 121 unmodified lines | ||
| 627 | 652 | { |
| 628 | 653 | struct nf_conntrack_expect *expect; |
| 629 | 654 | struct nf_conntrack_helper *helper; |
| 655 | + struct net *net = seq_file_net(s); | |
| 630 | 656 | struct hlist_node *n = v; |
| 631 | 657 | char *delim = ""; |
| 632 | 658 | |
| 633 | 659 | expect = hlist_entry(n, struct nf_conntrack_expect, hnode); |
| 634 | 660 | |
| 661 | + if (!net_eq(nf_ct_exp_net(expect), net)) | |
| 662 | + return 0; | |
| 663 | + | |
| 635 | 664 | if (expect->timeout.function) |
| 636 | 665 | seq_printf(s, "%ld ", timer_pending(&expect->timeout) |
| 637 | 666 | ? (long)(expect->timeout.expires - jiffies)/HZ : 0); |
| @@ -654,7 +683,7 @@ static int exp_seq_show(struct seq_file *s, void *v) · 16 unmodified lines | ||
| 654 | 683 | if (expect->flags & NF_CT_EXPECT_USERSPACE) |
| 655 | 684 | seq_printf(s, "%sUSERSPACE", delim); |
| 656 | 685 | |
| 657 | − helper = rcu_dereference(nfct_help(expect->master)->helper); | |
| 686 | + helper = rcu_dereference(expect->helper); | |
| 658 | 687 | if (helper) { |
| 659 | 688 | seq_printf(s, "%s%s", expect->flags ? " " : "", helper->name); |
| 660 | 689 | if (helper->expect_policy[expect->class].name[0]) |
| @@ -643,7 +643,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, · 642 unmodified lines | ||
| 643 | 643 | &ct->tuplehash[!dir].tuple.src.u3, |
| 644 | 644 | &ct->tuplehash[!dir].tuple.dst.u3, |
| 645 | 645 | IPPROTO_TCP, NULL, &port); |
| 646 | − exp->helper = &nf_conntrack_helper_h245; | |
| 646 | + rcu_assign_pointer(exp->helper, &nf_conntrack_helper_h245); | |
| 647 | 647 | |
| 648 | 648 | nathook = rcu_dereference(nfct_h323_nat_hook); |
| 649 | 649 | if (memcmp(&ct->tuplehash[dir].tuple.src.u3, |
| @@ -767,7 +767,7 @@ static int expect_callforwarding(struct sk_buff *skb, · 117 unmodified lines | ||
| 767 | 767 | nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), |
| 768 | 768 | &ct->tuplehash[!dir].tuple.src.u3, &addr, |
| 769 | 769 | IPPROTO_TCP, NULL, &port); |
| 770 | − exp->helper = nf_conntrack_helper_q931; | |
| 770 | + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); | |
| 771 | 771 | |
| 772 | 772 | nathook = rcu_dereference(nfct_h323_nat_hook); |
| 773 | 773 | if (memcmp(&ct->tuplehash[dir].tuple.src.u3, |
| @@ -1234,7 +1234,7 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, · 460 unmodified lines | ||
| 1234 | 1234 | &ct->tuplehash[!dir].tuple.src.u3 : NULL, |
| 1235 | 1235 | &ct->tuplehash[!dir].tuple.dst.u3, |
| 1236 | 1236 | IPPROTO_TCP, NULL, &port); |
| 1237 | − exp->helper = nf_conntrack_helper_q931; | |
| 1237 | + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); | |
| 1238 | 1238 | exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */ |
| 1239 | 1239 | |
| 1240 | 1240 | nathook = rcu_dereference(nfct_h323_nat_hook); |
| @@ -1306,7 +1306,7 @@ static int process_gcf(struct sk_buff *skb, struct nf_conn *ct, · 65 unmodified lines | ||
| 1306 | 1306 | nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), |
| 1307 | 1307 | &ct->tuplehash[!dir].tuple.src.u3, &addr, |
| 1308 | 1308 | IPPROTO_UDP, NULL, &port); |
| 1309 | − exp->helper = nf_conntrack_helper_ras; | |
| 1309 | + rcu_assign_pointer(exp->helper, nf_conntrack_helper_ras); | |
| 1310 | 1310 | |
| 1311 | 1311 | if (nf_ct_expect_related(exp, 0) == 0) { |
| 1312 | 1312 | pr_debug("nf_ct_ras: expect RAS "); |
| @@ -1523,7 +1523,7 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, · 210 unmodified lines | ||
| 1523 | 1523 | &ct->tuplehash[!dir].tuple.src.u3, &addr, |
| 1524 | 1524 | IPPROTO_TCP, NULL, &port); |
| 1525 | 1525 | exp->flags = NF_CT_EXPECT_PERMANENT; |
| 1526 | − exp->helper = nf_conntrack_helper_q931; | |
| 1526 | + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); | |
| 1527 | 1527 | |
| 1528 | 1528 | if (nf_ct_expect_related(exp, 0) == 0) { |
| 1529 | 1529 | pr_debug("nf_ct_ras: expect Q.931 "); |
| @@ -1577,7 +1577,7 @@ static int process_lcf(struct sk_buff *skb, struct nf_conn *ct, · 47 unmodified lines | ||
| 1577 | 1577 | &ct->tuplehash[!dir].tuple.src.u3, &addr, |
| 1578 | 1578 | IPPROTO_TCP, NULL, &port); |
| 1579 | 1579 | exp->flags = NF_CT_EXPECT_PERMANENT; |
| 1580 | − exp->helper = nf_conntrack_helper_q931; | |
| 1580 | + rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); | |
| 1581 | 1581 | |
| 1582 | 1582 | if (nf_ct_expect_related(exp, 0) == 0) { |
| 1583 | 1583 | pr_debug("nf_ct_ras: expect Q.931 "); |
| @@ -395,14 +395,10 @@ EXPORT_SYMBOL_GPL(nf_conntrack_helper_register); · 394 unmodified lines | ||
| 395 | 395 | |
| 396 | 396 | static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data) |
| 397 | 397 | { |
| 398 | − struct nf_conn_help *help = nfct_help(exp->master); | |
| 399 | 398 | const struct nf_conntrack_helper *me = data; |
| 400 | 399 | const struct nf_conntrack_helper *this; |
| 401 | 400 | |
| 402 | − if (exp->helper == me) | |
| 403 | − return true; | |
| 404 | − | |
| 405 | − this = rcu_dereference_protected(help->helper, | |
| 401 | + this = rcu_dereference_protected(exp->helper, | |
| 406 | 402 | lockdep_is_held(&nf_conntrack_expect_lock)); |
| 407 | 403 | return this == me; |
| 408 | 404 | } |
| @@ -421,6 +417,11 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) · 12 unmodified lines | ||
| 421 | 417 | |
| 422 | 418 | nf_ct_expect_iterate_destroy(expect_iter_me, NULL); |
| 423 | 419 | nf_ct_iterate_destroy(unhelp, me); |
| 420 | + | |
| 421 | + /* nf_ct_iterate_destroy() does an unconditional synchronize_rcu() as | |
| 422 | + * last step, this ensures rcu readers of exp->helper are done. | |
| 423 | + * No need for another synchronize_rcu() here. | |
| 424 | + */ | |
| 424 | 425 | } |
| 425 | 426 | EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister); |
| 426 | 427 | |
| @@ -910,8 +910,8 @@ struct ctnetlink_filter { · 909 unmodified lines | ||
| 910 | 910 | }; |
| 911 | 911 | |
| 912 | 912 | static const struct nla_policy cta_filter_nla_policy[CTA_FILTER_MAX + 1] = { |
| 913 | − [CTA_FILTER_ORIG_FLAGS] = { .type = NLA_U32 }, | |
| 914 | − [CTA_FILTER_REPLY_FLAGS] = { .type = NLA_U32 }, | |
| 913 | + [CTA_FILTER_ORIG_FLAGS] = NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL), | |
| 914 | + [CTA_FILTER_REPLY_FLAGS] = NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL), | |
| 915 | 915 | }; |
| 916 | 916 | |
| 917 | 917 | static int ctnetlink_parse_filter(const struct nlattr *attr, |
| @@ -925,17 +925,11 @@ static int ctnetlink_parse_filter(const struct nlattr *attr, · 7 unmodified lines | ||
| 925 | 925 | if (ret) |
| 926 | 926 | return ret; |
| 927 | 927 | |
| 928 | − if (tb[CTA_FILTER_ORIG_FLAGS]) { | |
| 928 | + if (tb[CTA_FILTER_ORIG_FLAGS]) | |
| 929 | 929 | filter->orig_flags = nla_get_u32(tb[CTA_FILTER_ORIG_FLAGS]); |
| 930 | − if (filter->orig_flags & ~CTA_FILTER_F_ALL) | |
| 931 | − return -EOPNOTSUPP; | |
| 932 | − } | |
| 933 | 930 | |
| 934 | − if (tb[CTA_FILTER_REPLY_FLAGS]) { | |
| 931 | + if (tb[CTA_FILTER_REPLY_FLAGS]) | |
| 935 | 932 | filter->reply_flags = nla_get_u32(tb[CTA_FILTER_REPLY_FLAGS]); |
| 936 | − if (filter->reply_flags & ~CTA_FILTER_F_ALL) | |
| 937 | − return -EOPNOTSUPP; | |
| 938 | − } | |
| 939 | 933 | |
| 940 | 934 | return 0; |
| 941 | 935 | } |
| @@ -2634,7 +2628,7 @@ static const struct nla_policy exp_nla_policy[CTA_EXPECT_MAX+1] = { · 1692 unmodified lines | ||
| 2634 | 2628 | [CTA_EXPECT_HELP_NAME] = { .type = NLA_NUL_STRING, |
| 2635 | 2629 | .len = NF_CT_HELPER_NAME_LEN - 1 }, |
| 2636 | 2630 | [CTA_EXPECT_ZONE] = { .type = NLA_U16 }, |
| 2637 | − [CTA_EXPECT_FLAGS] = { .type = NLA_U32 }, | |
| 2631 | + [CTA_EXPECT_FLAGS] = NLA_POLICY_MASK(NLA_BE32, NF_CT_EXPECT_MASK), | |
| 2638 | 2632 | [CTA_EXPECT_CLASS] = { .type = NLA_U32 }, |
| 2639 | 2633 | [CTA_EXPECT_NAT] = { .type = NLA_NESTED }, |
| 2640 | 2634 | [CTA_EXPECT_FN] = { .type = NLA_NUL_STRING }, |
| @@ -3012,7 +3006,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, · 371 unmodified lines | ||
| 3012 | 3006 | { |
| 3013 | 3007 | struct nf_conn *master = exp->master; |
| 3014 | 3008 | long timeout = ((long)exp->timeout.expires - (long)jiffies) / HZ; |
| 3015 | − struct nf_conn_help *help; | |
| 3009 | + struct nf_conntrack_helper *helper; | |
| 3016 | 3010 | #if IS_ENABLED(CONFIG_NF_NAT) |
| 3017 | 3011 | struct nlattr *nest_parms; |
| 3018 | 3012 | struct nf_conntrack_tuple nat_tuple = {}; |
| @@ -3057,15 +3051,12 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, · 38 unmodified lines | ||
| 3057 | 3051 | nla_put_be32(skb, CTA_EXPECT_FLAGS, htonl(exp->flags)) || |
| 3058 | 3052 | nla_put_be32(skb, CTA_EXPECT_CLASS, htonl(exp->class))) |
| 3059 | 3053 | goto nla_put_failure; |
| 3060 | − help = nfct_help(master); | |
| 3061 | − if (help) { | |
| 3062 | − struct nf_conntrack_helper *helper; | |
| 3063 | 3054 | |
| 3064 | − helper = rcu_dereference(help->helper); | |
| 3065 | − if (helper && | |
| 3066 | − nla_put_string(skb, CTA_EXPECT_HELP_NAME, helper->name)) | |
| 3067 | − goto nla_put_failure; | |
| 3068 | − } | |
| 3055 | + helper = rcu_dereference(exp->helper); | |
| 3056 | + if (helper && | |
| 3057 | + nla_put_string(skb, CTA_EXPECT_HELP_NAME, helper->name)) | |
| 3058 | + goto nla_put_failure; | |
| 3059 | + | |
| 3069 | 3060 | expfn = nf_ct_helper_expectfn_find_by_symbol(exp->expectfn); |
| 3070 | 3061 | if (expfn != NULL && |
| 3071 | 3062 | nla_put_string(skb, CTA_EXPECT_FN, expfn->name)) |
| @@ -3358,31 +3349,37 @@ static int ctnetlink_get_expect(struct sk_buff *skb, · 286 unmodified lines | ||
| 3358 | 3349 | if (err < 0) |
| 3359 | 3350 | return err; |
| 3360 | 3351 | |
| 3352 | + skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); | |
| 3353 | + if (!skb2) | |
| 3354 | + return -ENOMEM; | |
| 3355 | + | |
| 3356 | + spin_lock_bh(&nf_conntrack_expect_lock); | |
| 3361 | 3357 | exp = nf_ct_expect_find_get(info->net, &zone, &tuple); |
| 3362 | − if (!exp) | |
| 3358 | + if (!exp) { | |
| 3359 | + spin_unlock_bh(&nf_conntrack_expect_lock); | |
| 3360 | + kfree_skb(skb2); | |
| 3363 | 3361 | return -ENOENT; |
| 3362 | + } | |
| 3364 | 3363 | |
| 3365 | 3364 | if (cda[CTA_EXPECT_ID]) { |
| 3366 | 3365 | __be32 id = nla_get_be32(cda[CTA_EXPECT_ID]); |
| 3367 | 3366 | |
| 3368 | 3367 | if (id != nf_expect_get_id(exp)) { |
| 3369 | 3368 | nf_ct_expect_put(exp); |
| 3369 | + spin_unlock_bh(&nf_conntrack_expect_lock); | |
| 3370 | + kfree_skb(skb2); | |
| 3370 | 3371 | return -ENOENT; |
| 3371 | 3372 | } |
| 3372 | 3373 | } |
| 3373 | 3374 | |
| 3374 | − skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); | |
| 3375 | − if (!skb2) { | |
| 3376 | − nf_ct_expect_put(exp); | |
| 3377 | − return -ENOMEM; | |
| 3378 | − } | |
| 3379 | − | |
| 3380 | 3375 | rcu_read_lock(); |
| 3381 | 3376 | err = ctnetlink_exp_fill_info(skb2, NETLINK_CB(skb).portid, |
| 3382 | 3377 | info->nlh->nlmsg_seq, IPCTNL_MSG_EXP_NEW, |
| 3383 | 3378 | exp); |
| 3384 | 3379 | rcu_read_unlock(); |
| 3385 | 3380 | nf_ct_expect_put(exp); |
| 3381 | + spin_unlock_bh(&nf_conntrack_expect_lock); | |
| 3382 | + | |
| 3386 | 3383 | if (err <= 0) { |
| 3387 | 3384 | kfree_skb(skb2); |
| 3388 | 3385 | return -ENOMEM; |
| @@ -3394,12 +3391,9 @@ static int ctnetlink_get_expect(struct sk_buff *skb, · 5 unmodified lines | ||
| 3394 | 3391 | static bool expect_iter_name(struct nf_conntrack_expect *exp, void *data) |
| 3395 | 3392 | { |
| 3396 | 3393 | struct nf_conntrack_helper *helper; |
| 3397 | − const struct nf_conn_help *m_help; | |
| 3398 | 3394 | const char *name = data; |
| 3399 | 3395 | |
| 3400 | − m_help = nfct_help(exp->master); | |
| 3401 | − | |
| 3402 | − helper = rcu_dereference(m_help->helper); | |
| 3396 | + helper = rcu_dereference(exp->helper); | |
| 3403 | 3397 | if (!helper) |
| 3404 | 3398 | return false; |
| 3405 | 3399 | |
| @@ -3432,22 +3426,26 @@ static int ctnetlink_del_expect(struct sk_buff *skb, · 26 unmodified lines | ||
| 3432 | 3426 | if (err < 0) |
| 3433 | 3427 | return err; |
| 3434 | 3428 | |
| 3429 | + spin_lock_bh(&nf_conntrack_expect_lock); | |
| 3430 | + | |
| 3435 | 3431 | /* bump usage count to 2 */ |
| 3436 | 3432 | exp = nf_ct_expect_find_get(info->net, &zone, &tuple); |
| 3437 | − if (!exp) | |
| 3433 | + if (!exp) { | |
| 3434 | + spin_unlock_bh(&nf_conntrack_expect_lock); | |
| 3438 | 3435 | return -ENOENT; |
| 3436 | + } | |
| 3439 | 3437 | |
| 3440 | 3438 | if (cda[CTA_EXPECT_ID]) { |
| 3441 | 3439 | __be32 id = nla_get_be32(cda[CTA_EXPECT_ID]); |
| 3442 | 3440 | |
| 3443 | 3441 | if (id != nf_expect_get_id(exp)) { |
| 3444 | 3442 | nf_ct_expect_put(exp); |
| 3443 | + spin_unlock_bh(&nf_conntrack_expect_lock); | |
| 3445 | 3444 | return -ENOENT; |
| 3446 | 3445 | } |
| 3447 | 3446 | } |
| 3448 | 3447 | |
| 3449 | 3448 | /* after list removal, usage count == 1 */ |
| 3450 | − spin_lock_bh(&nf_conntrack_expect_lock); | |
| 3451 | 3449 | if (timer_delete(&exp->timeout)) { |
| 3452 | 3450 | nf_ct_unlink_expect_report(exp, NETLINK_CB(skb).portid, |
| 3453 | 3451 | nlmsg_report(info->nlh)); |
| @@ -3534,9 +3532,10 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, · 80 unmodified lines | ||
| 3534 | 3532 | struct nf_conntrack_tuple *tuple, |
| 3535 | 3533 | struct nf_conntrack_tuple *mask) |
| 3536 | 3534 | { |
| 3537 | − u_int32_t class = 0; | |
| 3535 | + struct net *net = read_pnet(&ct->ct_net); | |
| 3538 | 3536 | struct nf_conntrack_expect *exp; |
| 3539 | 3537 | struct nf_conn_help *help; |
| 3538 | + u32 class = 0; | |
| 3540 | 3539 | int err; |
| 3541 | 3540 | |
| 3542 | 3541 | help = nfct_help(ct); |
| @@ -3573,7 +3572,13 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, · 30 unmodified lines | ||
| 3573 | 3572 | |
| 3574 | 3573 | exp->class = class; |
| 3575 | 3574 | exp->master = ct; |
| 3576 | − exp->helper = helper; | |
| 3575 | + write_pnet(&exp->net, net); | |
| 3576 | +#ifdef CONFIG_NF_CONNTRACK_ZONES | |
| 3577 | + exp->zone = ct->zone; | |
| 3578 | +#endif | |
| 3579 | + if (!helper) | |
| 3580 | + helper = rcu_dereference(help->helper); | |
| 3581 | + rcu_assign_pointer(exp->helper, helper); | |
| 3577 | 3582 | exp->tuple = *tuple; |
| 3578 | 3583 | exp->mask.src.u3 = mask->src.u3; |
| 3579 | 3584 | exp->mask.src.u.all = mask->src.u.all; |
| @@ -1385,9 +1385,9 @@ nla_put_failure: · 1384 unmodified lines | ||
| 1385 | 1385 | } |
| 1386 | 1386 | |
| 1387 | 1387 | static const struct nla_policy tcp_nla_policy[CTA_PROTOINFO_TCP_MAX+1] = { |
| 1388 | − [CTA_PROTOINFO_TCP_STATE] = { .type = NLA_U8 }, | |
| 1389 | − [CTA_PROTOINFO_TCP_WSCALE_ORIGINAL] = { .type = NLA_U8 }, | |
| 1390 | − [CTA_PROTOINFO_TCP_WSCALE_REPLY] = { .type = NLA_U8 }, | |
| 1388 | + [CTA_PROTOINFO_TCP_STATE] = NLA_POLICY_MAX(NLA_U8, TCP_CONNTRACK_SYN_SENT2), | |
| 1389 | + [CTA_PROTOINFO_TCP_WSCALE_ORIGINAL] = NLA_POLICY_MAX(NLA_U8, TCP_MAX_WSCALE), | |
| 1390 | + [CTA_PROTOINFO_TCP_WSCALE_REPLY] = NLA_POLICY_MAX(NLA_U8, TCP_MAX_WSCALE), | |
| 1391 | 1391 | [CTA_PROTOINFO_TCP_FLAGS_ORIGINAL] = { .len = sizeof(struct nf_ct_tcp_flags) }, |
| 1392 | 1392 | [CTA_PROTOINFO_TCP_FLAGS_REPLY] = { .len = sizeof(struct nf_ct_tcp_flags) }, |
| 1393 | 1393 | }; |
| @@ -1414,10 +1414,6 @@ static int nlattr_to_tcp(struct nlattr *cda[], struct nf_conn *ct) · 20 unmodified lines | ||
| 1414 | 1414 | if (err < 0) |
| 1415 | 1415 | return err; |
| 1416 | 1416 | |
| 1417 | − if (tb[CTA_PROTOINFO_TCP_STATE] && | |
| 1418 | − nla_get_u8(tb[CTA_PROTOINFO_TCP_STATE]) >= TCP_CONNTRACK_MAX) | |
| 1419 | − return -EINVAL; | |
| 1420 | − | |
| 1421 | 1417 | spin_lock_bh(&ct->lock); |
| 1422 | 1418 | if (tb[CTA_PROTOINFO_TCP_STATE]) |
| 1423 | 1419 | ct->proto.tcp.state = nla_get_u8(tb[CTA_PROTOINFO_TCP_STATE]); |
| @@ -924,7 +924,7 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, · 923 unmodified lines | ||
| 924 | 924 | exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); |
| 925 | 925 | |
| 926 | 926 | if (!exp || exp->master == ct || |
| 927 | − nfct_help(exp->master)->helper != nfct_help(ct)->helper || | |
| 927 | + exp->helper != nfct_help(ct)->helper || | |
| 928 | 928 | exp->class != class) |
| 929 | 929 | break; |
| 930 | 930 | #if IS_ENABLED(CONFIG_NF_NAT) |
| @@ -1040,6 +1040,7 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, · 109 unmodified lines | ||
| 1040 | 1040 | unsigned int port; |
| 1041 | 1041 | const struct sdp_media_type *t; |
| 1042 | 1042 | int ret = NF_ACCEPT; |
| 1043 | + bool have_rtp_addr = false; | |
| 1043 | 1044 | |
| 1044 | 1045 | hooks = rcu_dereference(nf_nat_sip_hooks); |
| 1045 | 1046 | |
| @@ -1056,8 +1057,11 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, · 10 unmodified lines | ||
| 1056 | 1057 | caddr_len = 0; |
| 1057 | 1058 | if (ct_sip_parse_sdp_addr(ct, *dptr, sdpoff, *datalen, |
| 1058 | 1059 | SDP_HDR_CONNECTION, SDP_HDR_MEDIA, |
| 1059 | − &matchoff, &matchlen, &caddr) > 0) | |
| 1060 | + &matchoff, &matchlen, &caddr) > 0) { | |
| 1060 | 1061 | caddr_len = matchlen; |
| 1062 | + memcpy(&rtp_addr, &caddr, sizeof(rtp_addr)); | |
| 1063 | + have_rtp_addr = true; | |
| 1064 | + } | |
| 1061 | 1065 | |
| 1062 | 1066 | mediaoff = sdpoff; |
| 1063 | 1067 | for (i = 0; i < ARRAY_SIZE(sdp_media_types); ) { |
| @@ -1091,9 +1095,11 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, · 27 unmodified lines | ||
| 1091 | 1095 | &matchoff, &matchlen, &maddr) > 0) { |
| 1092 | 1096 | maddr_len = matchlen; |
| 1093 | 1097 | memcpy(&rtp_addr, &maddr, sizeof(rtp_addr)); |
| 1094 | − } else if (caddr_len) | |
| 1098 | + have_rtp_addr = true; | |
| 1099 | + } else if (caddr_len) { | |
| 1095 | 1100 | memcpy(&rtp_addr, &caddr, sizeof(rtp_addr)); |
| 1096 | − else { | |
| 1101 | + have_rtp_addr = true; | |
| 1102 | + } else { | |
| 1097 | 1103 | nf_ct_helper_log(skb, ct, "cannot parse SDP message"); |
| 1098 | 1104 | return NF_DROP; |
| 1099 | 1105 | } |
| @@ -1125,7 +1131,7 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, · 25 unmodified lines | ||
| 1125 | 1131 | |
| 1126 | 1132 | /* Update session connection and owner addresses */ |
| 1127 | 1133 | hooks = rcu_dereference(nf_nat_sip_hooks); |
| 1128 | − if (hooks && ct->status & IPS_NAT_MASK) | |
| 1134 | + if (hooks && ct->status & IPS_NAT_MASK && have_rtp_addr) | |
| 1129 | 1135 | ret = hooks->sdp_session(skb, protoff, dataoff, |
| 1130 | 1136 | dptr, datalen, sdpoff, |
| 1131 | 1137 | &rtp_addr); |
| @@ -1297,7 +1303,7 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, · 165 unmodified lines | ||
| 1297 | 1303 | nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), |
| 1298 | 1304 | saddr, &daddr, proto, NULL, &port); |
| 1299 | 1305 | exp->timeout.expires = sip_timeout * HZ; |
| 1300 | − exp->helper = helper; | |
| 1306 | + rcu_assign_pointer(exp->helper, helper); | |
| 1301 | 1307 | exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE; |
| 1302 | 1308 | |
| 1303 | 1309 | hooks = rcu_dereference(nf_nat_sip_hooks); |
| @@ -647,15 +647,11 @@ __build_packet_message(struct nfnl_log_net *log, · 646 unmodified lines | ||
| 647 | 647 | |
| 648 | 648 | if (data_len) { |
| 649 | 649 | struct nlattr *nla; |
| 650 | − int size = nla_attr_size(data_len); | |
| 651 | 650 | |
| 652 | − if (skb_tailroom(inst->skb) < nla_total_size(data_len)) | |
| 651 | + nla = nla_reserve(inst->skb, NFULA_PAYLOAD, data_len); | |
| 652 | + if (!nla) | |
| 653 | 653 | goto nla_put_failure; |
| 654 | 654 | |
| 655 | − nla = skb_put(inst->skb, nla_total_size(data_len)); | |
| 656 | − nla->nla_type = NFULA_PAYLOAD; | |
| 657 | − nla->nla_len = size; | |
| 658 | − | |
| 659 | 655 | if (skb_copy_bits(skb, 0, nla_data(nla), data_len)) |
| 660 | 656 | BUG(); |
| 661 | 657 | } |
| @@ -242,7 +242,7 @@ static int nft_pipapo_avx2_lookup_4b_2(unsigned long *map, unsigned long *fill, · 241 unmodified lines | ||
| 242 | 242 | |
| 243 | 243 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 244 | 244 | if (last) |
| 245 | − return b; | |
| 245 | + ret = b; | |
| 246 | 246 | |
| 247 | 247 | if (unlikely(ret == -1)) |
| 248 | 248 | ret = b / XSAVE_YMM_SIZE; |
| @@ -319,7 +319,7 @@ static int nft_pipapo_avx2_lookup_4b_4(unsigned long *map, unsigned long *fill, · 70 unmodified lines | ||
| 319 | 319 | |
| 320 | 320 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 321 | 321 | if (last) |
| 322 | − return b; | |
| 322 | + ret = b; | |
| 323 | 323 | |
| 324 | 324 | if (unlikely(ret == -1)) |
| 325 | 325 | ret = b / XSAVE_YMM_SIZE; |
| @@ -414,7 +414,7 @@ static int nft_pipapo_avx2_lookup_4b_8(unsigned long *map, unsigned long *fill, · 88 unmodified lines | ||
| 414 | 414 | |
| 415 | 415 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 416 | 416 | if (last) |
| 417 | − return b; | |
| 417 | + ret = b; | |
| 418 | 418 | |
| 419 | 419 | if (unlikely(ret == -1)) |
| 420 | 420 | ret = b / XSAVE_YMM_SIZE; |
| @@ -505,7 +505,7 @@ static int nft_pipapo_avx2_lookup_4b_12(unsigned long *map, unsigned long *fill, · 84 unmodified lines | ||
| 505 | 505 | |
| 506 | 506 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 507 | 507 | if (last) |
| 508 | − return b; | |
| 508 | + ret = b; | |
| 509 | 509 | |
| 510 | 510 | if (unlikely(ret == -1)) |
| 511 | 511 | ret = b / XSAVE_YMM_SIZE; |
| @@ -641,7 +641,7 @@ static int nft_pipapo_avx2_lookup_4b_32(unsigned long *map, unsigned long *fill, · 129 unmodified lines | ||
| 641 | 641 | |
| 642 | 642 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 643 | 643 | if (last) |
| 644 | − return b; | |
| 644 | + ret = b; | |
| 645 | 645 | |
| 646 | 646 | if (unlikely(ret == -1)) |
| 647 | 647 | ret = b / XSAVE_YMM_SIZE; |
| @@ -699,7 +699,7 @@ static int nft_pipapo_avx2_lookup_8b_1(unsigned long *map, unsigned long *fill, · 51 unmodified lines | ||
| 699 | 699 | |
| 700 | 700 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 701 | 701 | if (last) |
| 702 | − return b; | |
| 702 | + ret = b; | |
| 703 | 703 | |
| 704 | 704 | if (unlikely(ret == -1)) |
| 705 | 705 | ret = b / XSAVE_YMM_SIZE; |
| @@ -764,7 +764,7 @@ static int nft_pipapo_avx2_lookup_8b_2(unsigned long *map, unsigned long *fill, · 58 unmodified lines | ||
| 764 | 764 | |
| 765 | 765 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 766 | 766 | if (last) |
| 767 | − return b; | |
| 767 | + ret = b; | |
| 768 | 768 | |
| 769 | 769 | if (unlikely(ret == -1)) |
| 770 | 770 | ret = b / XSAVE_YMM_SIZE; |
| @@ -839,7 +839,7 @@ static int nft_pipapo_avx2_lookup_8b_4(unsigned long *map, unsigned long *fill, · 68 unmodified lines | ||
| 839 | 839 | |
| 840 | 840 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 841 | 841 | if (last) |
| 842 | − return b; | |
| 842 | + ret = b; | |
| 843 | 843 | |
| 844 | 844 | if (unlikely(ret == -1)) |
| 845 | 845 | ret = b / XSAVE_YMM_SIZE; |
| @@ -925,7 +925,7 @@ static int nft_pipapo_avx2_lookup_8b_6(unsigned long *map, unsigned long *fill, · 79 unmodified lines | ||
| 925 | 925 | |
| 926 | 926 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 927 | 927 | if (last) |
| 928 | − return b; | |
| 928 | + ret = b; | |
| 929 | 929 | |
| 930 | 930 | if (unlikely(ret == -1)) |
| 931 | 931 | ret = b / XSAVE_YMM_SIZE; |
| @@ -1019,7 +1019,7 @@ static int nft_pipapo_avx2_lookup_8b_16(unsigned long *map, unsigned long *fill, · 87 unmodified lines | ||
| 1019 | 1019 | |
| 1020 | 1020 | b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last); |
| 1021 | 1021 | if (last) |
| 1022 | − return b; | |
| 1022 | + ret = b; | |
| 1023 | 1023 | |
| 1024 | 1024 | if (unlikely(ret == -1)) |
| 1025 | 1025 | ret = b / XSAVE_YMM_SIZE; |
| @@ -572,14 +572,12 @@ static struct nft_array *nft_array_alloc(u32 max_intervals) · 571 unmodified lines | ||
| 572 | 572 | return array; |
| 573 | 573 | } |
| 574 | 574 | |
| 575 | −#define NFT_ARRAY_EXTRA_SIZE 10240 | |
| 576 | − | |
| 577 | 575 | /* Similar to nft_rbtree_{u,k}size to hide details to userspace, but consider |
| 578 | 576 | * packed representation coming from userspace for anonymous sets too. |
| 579 | 577 | */ |
| 580 | 578 | static u32 nft_array_elems(const struct nft_set *set) |
| 581 | 579 | { |
| 582 | − u32 nelems = atomic_read(&set->nelems); | |
| 580 | + u32 nelems = atomic_read(&set->nelems) - set->ndeact; | |
| 583 | 581 | |
| 584 | 582 | /* Adjacent intervals are represented with a single start element in |
| 585 | 583 | * anonymous sets, use the current element counter as is. |
| @@ -595,27 +593,87 @@ static u32 nft_array_elems(const struct nft_set *set) · 9 unmodified lines | ||
| 595 | 593 | return (nelems / 2) + 2; |
| 596 | 594 | } |
| 597 | 595 | |
| 598 | −static int nft_array_may_resize(const struct nft_set *set) | |
| 596 | +#define NFT_ARRAY_INITIAL_SIZE 1024 | |
| 597 | +#define NFT_ARRAY_INITIAL_ANON_SIZE 16 | |
| 598 | +#define NFT_ARRAY_INITIAL_ANON_THRESH (8192U / sizeof(struct nft_array_interval)) | |
| 599 | + | |
| 600 | +static int nft_array_may_resize(const struct nft_set *set, bool flush) | |
| 599 | 601 | { |
| 600 | − u32 nelems = nft_array_elems(set), new_max_intervals; | |
| 602 | + u32 initial_intervals, max_intervals, new_max_intervals, delta; | |
| 603 | + u32 shrinked_max_intervals, nelems = nft_array_elems(set); | |
| 601 | 604 | struct nft_rbtree *priv = nft_set_priv(set); |
| 602 | 605 | struct nft_array *array; |
| 603 | 606 | |
| 604 | − if (!priv->array_next) { | |
| 605 | − array = nft_array_alloc(nelems + NFT_ARRAY_EXTRA_SIZE); | |
| 606 | − if (!array) | |
| 607 | − return -ENOMEM; | |
| 607 | + if (nft_set_is_anonymous(set)) | |
| 608 | + initial_intervals = NFT_ARRAY_INITIAL_ANON_SIZE; | |
| 609 | + else | |
| 610 | + initial_intervals = NFT_ARRAY_INITIAL_SIZE; | |
| 611 | + | |
| 612 | + if (priv->array_next) { | |
| 613 | + max_intervals = priv->array_next->max_intervals; | |
| 614 | + new_max_intervals = priv->array_next->max_intervals; | |
| 615 | + } else { | |
| 616 | + if (priv->array) { | |
| 617 | + max_intervals = priv->array->max_intervals; | |
| 618 | + new_max_intervals = priv->array->max_intervals; | |
| 619 | + } else { | |
| 620 | + max_intervals = 0; | |
| 621 | + new_max_intervals = initial_intervals; | |
| 622 | + } | |
| 623 | + } | |
| 608 | 624 | |
| 609 | − priv->array_next = array; | |
| 625 | + if (nft_set_is_anonymous(set)) | |
| 626 | + goto maybe_grow; | |
| 627 | + | |
| 628 | + if (flush) { | |
| 629 | + /* Set flush just started, nelems still report elements.*/ | |
| 630 | + nelems = 0; | |
| 631 | + new_max_intervals = NFT_ARRAY_INITIAL_SIZE; | |
| 632 | + goto realloc_array; | |
| 610 | 633 | } |
| 611 | 634 | |
| 612 | − if (nelems < priv->array_next->max_intervals) | |
| 613 | − return 0; | |
| 635 | + if (check_add_overflow(new_max_intervals, new_max_intervals, | |
| 636 | + &shrinked_max_intervals)) | |
| 637 | + return -EOVERFLOW; | |
| 638 | + | |
| 639 | + shrinked_max_intervals = DIV_ROUND_UP(shrinked_max_intervals, 3); | |
| 614 | 640 | |
| 615 | − new_max_intervals = priv->array_next->max_intervals + NFT_ARRAY_EXTRA_SIZE; | |
| 616 | − if (nft_array_intervals_alloc(priv->array_next, new_max_intervals) < 0) | |
| 641 | + if (shrinked_max_intervals > NFT_ARRAY_INITIAL_SIZE && | |
| 642 | + nelems < shrinked_max_intervals) { | |
| 643 | + new_max_intervals = shrinked_max_intervals; | |
| 644 | + goto realloc_array; | |
| 645 | + } | |
| 646 | +maybe_grow: | |
| 647 | + if (nelems > new_max_intervals) { | |
| 648 | + if (nft_set_is_anonymous(set) && | |
| 649 | + new_max_intervals < NFT_ARRAY_INITIAL_ANON_THRESH) { | |
| 650 | + new_max_intervals <<= 1; | |
| 651 | + } else { | |
| 652 | + delta = new_max_intervals >> 1; | |
| 653 | + if (check_add_overflow(new_max_intervals, delta, | |
| 654 | + &new_max_intervals)) | |
| 655 | + return -EOVERFLOW; | |
| 656 | + } | |
| 657 | + } | |
| 658 | + | |
| 659 | +realloc_array: | |
| 660 | + if (WARN_ON_ONCE(nelems > new_max_intervals)) | |
| 617 | 661 | return -ENOMEM; |
| 618 | 662 | |
| 663 | + if (priv->array_next) { | |
| 664 | + if (max_intervals == new_max_intervals) | |
| 665 | + return 0; | |
| 666 | + | |
| 667 | + if (nft_array_intervals_alloc(priv->array_next, new_max_intervals) < 0) | |
| 668 | + return -ENOMEM; | |
| 669 | + } else { | |
| 670 | + array = nft_array_alloc(new_max_intervals); | |
| 671 | + if (!array) | |
| 672 | + return -ENOMEM; | |
| 673 | + | |
| 674 | + priv->array_next = array; | |
| 675 | + } | |
| 676 | + | |
| 619 | 677 | return 0; |
| 620 | 678 | } |
| 621 | 679 | |
| @@ -630,7 +688,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, · 8 unmodified lines | ||
| 630 | 688 | |
| 631 | 689 | nft_rbtree_maybe_reset_start_cookie(priv, tstamp); |
| 632 | 690 | |
| 633 | − if (nft_array_may_resize(set) < 0) | |
| 691 | + if (nft_array_may_resize(set, false) < 0) | |
| 634 | 692 | return -ENOMEM; |
| 635 | 693 | |
| 636 | 694 | do { |
| @@ -741,7 +799,7 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, · 104 unmodified lines | ||
| 741 | 799 | nft_rbtree_interval_null(set, this)) |
| 742 | 800 | priv->start_rbe_cookie = 0; |
| 743 | 801 | |
| 744 | − if (nft_array_may_resize(set) < 0) | |
| 802 | + if (nft_array_may_resize(set, false) < 0) | |
| 745 | 803 | return NULL; |
| 746 | 804 | |
| 747 | 805 | while (parent != NULL) { |
| @@ -811,7 +869,7 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, · 63 unmodified lines | ||
| 811 | 869 | |
| 812 | 870 | switch (iter->type) { |
| 813 | 871 | case NFT_ITER_UPDATE_CLONE: |
| 814 | − if (nft_array_may_resize(set) < 0) { | |
| 872 | + if (nft_array_may_resize(set, true) < 0) { | |
| 815 | 873 | iter->err = -ENOMEM; |
| 816 | 874 | break; |
| 817 | 875 | } |
| @@ -579,8 +579,7 @@ static int nci_close_device(struct nci_dev *ndev) · 578 unmodified lines | ||
| 579 | 579 | skb_queue_purge(&ndev->rx_q); |
| 580 | 580 | skb_queue_purge(&ndev->tx_q); |
| 581 | 581 | |
| 582 | − /* Flush RX and TX wq */ | |
| 583 | − flush_workqueue(ndev->rx_wq); | |
| 582 | + /* Flush TX wq, RX wq flush can't be under the lock */ | |
| 584 | 583 | flush_workqueue(ndev->tx_wq); |
| 585 | 584 | |
| 586 | 585 | /* Reset device */ |
| @@ -592,13 +591,13 @@ static int nci_close_device(struct nci_dev *ndev) · 5 unmodified lines | ||
| 592 | 591 | msecs_to_jiffies(NCI_RESET_TIMEOUT)); |
| 593 | 592 | |
| 594 | 593 | /* After this point our queues are empty |
| 595 | − * and no works are scheduled. | |
| 594 | + * rx work may be running but will see that NCI_UP was cleared | |
| 596 | 595 | */ |
| 597 | 596 | ndev->ops->close(ndev); |
| 598 | 597 | |
| 599 | 598 | clear_bit(NCI_INIT, &ndev->flags); |
| 600 | 599 | |
| 601 | − /* Flush cmd wq */ | |
| 600 | + /* Flush cmd and tx wq */ | |
| 602 | 601 | flush_workqueue(ndev->cmd_wq); |
| 603 | 602 | |
| 604 | 603 | timer_delete_sync(&ndev->cmd_timer); |
| @@ -613,6 +612,9 @@ static int nci_close_device(struct nci_dev *ndev) · 8 unmodified lines | ||
| 613 | 612 | |
| 614 | 613 | mutex_unlock(&ndev->req_lock); |
| 615 | 614 | |
| 615 | + /* rx_work may take req_lock via nci_deactivate_target */ | |
| 616 | + flush_workqueue(ndev->rx_wq); | |
| 617 | + | |
| 616 | 618 | return 0; |
| 617 | 619 | } |
| 618 | 620 | |
| @@ -2953,6 +2953,8 @@ static int validate_set(const struct nlattr *a, · 2952 unmodified lines | ||
| 2953 | 2953 | case OVS_KEY_ATTR_MPLS: |
| 2954 | 2954 | if (!eth_p_mpls(eth_type)) |
| 2955 | 2955 | return -EINVAL; |
| 2956 | + if (key_len != sizeof(struct ovs_key_mpls)) | |
| 2957 | + return -EINVAL; | |
| 2956 | 2958 | break; |
| 2957 | 2959 | |
| 2958 | 2960 | case OVS_KEY_ATTR_SCTP: |
| @@ -151,11 +151,15 @@ static void vport_netdev_free(struct rcu_head *rcu) · 150 unmodified lines | ||
| 151 | 151 | void ovs_netdev_detach_dev(struct vport *vport) |
| 152 | 152 | { |
| 153 | 153 | ASSERT_RTNL(); |
| 154 | − vport->dev->priv_flags &= ~IFF_OVS_DATAPATH; | |
| 155 | 154 | netdev_rx_handler_unregister(vport->dev); |
| 156 | 155 | netdev_upper_dev_unlink(vport->dev, |
| 157 | 156 | netdev_master_upper_dev_get(vport->dev)); |
| 158 | 157 | dev_set_promiscuity(vport->dev, -1); |
| 158 | + | |
| 159 | + /* paired with smp_mb() in netdev_destroy() */ | |
| 160 | + smp_wmb(); | |
| 161 | + | |
| 162 | + vport->dev->priv_flags &= ~IFF_OVS_DATAPATH; | |
| 159 | 163 | } |
| 160 | 164 | |
| 161 | 165 | static void netdev_destroy(struct vport *vport) |
| @@ -174,6 +178,9 @@ static void netdev_destroy(struct vport *vport) · 12 unmodified lines | ||
| 174 | 178 | rtnl_unlock(); |
| 175 | 179 | } |
| 176 | 180 | |
| 181 | + /* paired with smp_wmb() in ovs_netdev_detach_dev() */ | |
| 182 | + smp_mb(); | |
| 183 | + | |
| 177 | 184 | call_rcu(&vport->rcu, vport_netdev_free); |
| 178 | 185 | } |
| 179 | 186 | |
| @@ -189,8 +196,6 @@ void ovs_netdev_tunnel_destroy(struct vport *vport) · 9 unmodified lines | ||
| 189 | 196 | */ |
| 190 | 197 | if (vport->dev->reg_state == NETREG_REGISTERED) |
| 191 | 198 | rtnl_delete_link(vport->dev, 0, NULL); |
| 192 | − netdev_put(vport->dev, &vport->dev_tracker); | |
| 193 | − vport->dev = NULL; | |
| 194 | 199 | rtnl_unlock(); |
| 195 | 200 | |
| 196 | 201 | call_rcu(&vport->rcu, vport_netdev_free); |
| @@ -3135,6 +3135,7 @@ static int packet_release(struct socket *sock) · 3134 unmodified lines | ||
| 3135 | 3135 | |
| 3136 | 3136 | spin_lock(&po->bind_lock); |
| 3137 | 3137 | unregister_prot_hook(sk, false); |
| 3138 | + WRITE_ONCE(po->num, 0); | |
| 3138 | 3139 | packet_cached_dev_reset(po); |
| 3139 | 3140 | |
| 3140 | 3141 | if (po->prot_hook.dev) { |
| @@ -135,9 +135,16 @@ out: · 134 unmodified lines | ||
| 135 | 135 | sock_put(sk); |
| 136 | 136 | } |
| 137 | 137 | |
| 138 | +static bool smc_rx_pipe_buf_get(struct pipe_inode_info *pipe, | |
| 139 | + struct pipe_buffer *buf) | |
| 140 | +{ | |
| 141 | + /* smc_spd_priv in buf->private is not shareable; disallow cloning. */ | |
| 142 | + return false; | |
| 143 | +} | |
| 144 | + | |
| 138 | 145 | static const struct pipe_buf_operations smc_pipe_ops = { |
| 139 | 146 | .release = smc_rx_pipe_buf_release, |
| 140 | − .get = generic_pipe_buf_get | |
| 147 | + .get = smc_rx_pipe_buf_get, | |
| 141 | 148 | }; |
| 142 | 149 | |
| 143 | 150 | static void smc_rx_spd_release(struct splice_pipe_desc *spd, |
| @@ -246,6 +246,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) · 245 unmodified lines | ||
| 246 | 246 | crypto_wait_req(-EINPROGRESS, &ctx->async_wait); |
| 247 | 247 | atomic_inc(&ctx->decrypt_pending); |
| 248 | 248 | |
| 249 | + __skb_queue_purge(&ctx->async_hold); | |
| 249 | 250 | return ctx->async_wait.err; |
| 250 | 251 | } |
| 251 | 252 | |
| @@ -2225,7 +2226,6 @@ recv_end: · 1973 unmodified lines | ||
| 2225 | 2226 | |
| 2226 | 2227 | /* Wait for all previously submitted records to be decrypted */ |
| 2227 | 2228 | ret = tls_decrypt_async_wait(ctx); |
| 2228 | − __skb_queue_purge(&ctx->async_hold); | |
| 2229 | 2229 | |
| 2230 | 2230 | if (ret) { |
| 2231 | 2231 | if (err >= 0 || err == -EINPROGRESS) |
| @@ -75,7 +75,10 @@ int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo) · 74 unmodified lines | ||
| 75 | 75 | |
| 76 | 76 | spin_lock_bh(&xfrm_input_afinfo_lock); |
| 77 | 77 | if (likely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) { |
| 78 | − if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family] != afinfo)) | |
| 78 | + const struct xfrm_input_afinfo *cur; | |
| 79 | + | |
| 80 | + cur = rcu_access_pointer(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family]); | |
| 81 | + if (unlikely(cur != afinfo)) | |
| 79 | 82 | err = -EINVAL; |
| 80 | 83 | else |
| 81 | 84 | RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], NULL); |
| @@ -901,6 +901,12 @@ static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq, · 900 unmodified lines | ||
| 901 | 901 | iptfs_skb_can_add_frags(newskb, fragwalk, data, copylen)) { |
| 902 | 902 | iptfs_skb_add_frags(newskb, fragwalk, data, copylen); |
| 903 | 903 | } else { |
| 904 | + if (skb_linearize(newskb)) { | |
| 905 | + XFRM_INC_STATS(xs_net(xtfs->x), | |
| 906 | + LINUX_MIB_XFRMINBUFFERERROR); | |
| 907 | + goto abandon; | |
| 908 | + } | |
| 909 | + | |
| 904 | 910 | /* copy fragment data into newskb */ |
| 905 | 911 | if (skb_copy_seq_read(st, data, skb_put(newskb, copylen), |
| 906 | 912 | copylen)) { |
| @@ -991,6 +997,11 @@ static bool __input_process_payload(struct xfrm_state *x, u32 data, · 84 unmodified lines | ||
| 991 | 997 | |
| 992 | 998 | iplen = be16_to_cpu(iph->tot_len); |
| 993 | 999 | iphlen = iph->ihl << 2; |
| 1000 | + if (iplen < iphlen || iphlen < sizeof(*iph)) { | |
| 1001 | + XFRM_INC_STATS(net, | |
| 1002 | + LINUX_MIB_XFRMINHDRERROR); | |
| 1003 | + goto done; | |
| 1004 | + } | |
| 994 | 1005 | protocol = cpu_to_be16(ETH_P_IP); |
| 995 | 1006 | XFRM_MODE_SKB_CB(skbseq->root_skb)->tos = iph->tos; |
| 996 | 1007 | } else if (iph->version == 0x6) { |
| @@ -2653,9 +2664,6 @@ static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig) · 1656 unmodified lines | ||
| 2653 | 2664 | if (!xtfs) |
| 2654 | 2665 | return -ENOMEM; |
| 2655 | 2666 | |
| 2656 | − x->mode_data = xtfs; | |
| 2657 | − xtfs->x = x; | |
| 2658 | − | |
| 2659 | 2667 | xtfs->ra_newskb = NULL; |
| 2660 | 2668 | if (xtfs->cfg.reorder_win_size) { |
| 2661 | 2669 | xtfs->w_saved = kzalloc_objs(*xtfs->w_saved, |
| @@ -2666,6 +2674,9 @@ static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig) · 4 unmodified lines | ||
| 2666 | 2674 | } |
| 2667 | 2675 | } |
| 2668 | 2676 | |
| 2677 | + x->mode_data = xtfs; | |
| 2678 | + xtfs->x = x; | |
| 2679 | + | |
| 2669 | 2680 | return 0; |
| 2670 | 2681 | } |
| 2671 | 2682 | |
| @@ -261,7 +261,7 @@ int __net_init xfrm_nat_keepalive_net_init(struct net *net) · 260 unmodified lines | ||
| 261 | 261 | |
| 262 | 262 | int xfrm_nat_keepalive_net_fini(struct net *net) |
| 263 | 263 | { |
| 264 | − cancel_delayed_work_sync(&net->xfrm.nat_keepalive_work); | |
| 264 | + disable_delayed_work_sync(&net->xfrm.nat_keepalive_work); | |
| 265 | 265 | return 0; |
| 266 | 266 | } |
| 267 | 267 | |
| @@ -4156,7 +4156,7 @@ void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo) · 4155 unmodified lines | ||
| 4156 | 4156 | int i; |
| 4157 | 4157 | |
| 4158 | 4158 | for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) { |
| 4159 | − if (xfrm_policy_afinfo[i] != afinfo) | |
| 4159 | + if (rcu_access_pointer(xfrm_policy_afinfo[i]) != afinfo) | |
| 4160 | 4160 | continue; |
| 4161 | 4161 | RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL); |
| 4162 | 4162 | break; |
| @@ -4242,7 +4242,7 @@ static int __net_init xfrm_policy_init(struct net *net) · 79 unmodified lines | ||
| 4242 | 4242 | net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0; |
| 4243 | 4243 | |
| 4244 | 4244 | htab = &net->xfrm.policy_bydst[dir]; |
| 4245 | − htab->table = xfrm_hash_alloc(sz); | |
| 4245 | + rcu_assign_pointer(htab->table, xfrm_hash_alloc(sz)); | |
| 4246 | 4246 | if (!htab->table) |
| 4247 | 4247 | goto out_bydst; |
| 4248 | 4248 | htab->hmask = hmask; |
| @@ -4269,7 +4269,7 @@ out_bydst: · 20 unmodified lines | ||
| 4269 | 4269 | struct xfrm_policy_hash *htab; |
| 4270 | 4270 | |
| 4271 | 4271 | htab = &net->xfrm.policy_bydst[dir]; |
| 4272 | − xfrm_hash_free(htab->table, sz); | |
| 4272 | + xfrm_hash_free(rcu_dereference_protected(htab->table, true), sz); | |
| 4273 | 4273 | } |
| 4274 | 4274 | xfrm_hash_free(net->xfrm.policy_byidx, sz); |
| 4275 | 4275 | out_byidx: |
| @@ -4282,6 +4282,8 @@ static void xfrm_policy_fini(struct net *net) · 6 unmodified lines | ||
| 4282 | 4282 | unsigned int sz; |
| 4283 | 4283 | int dir; |
| 4284 | 4284 | |
| 4285 | + disable_work_sync(&net->xfrm.policy_hthresh.work); | |
| 4286 | + | |
| 4285 | 4287 | flush_work(&net->xfrm.policy_hash_work); |
| 4286 | 4288 | #ifdef CONFIG_XFRM_SUB_POLICY |
| 4287 | 4289 | xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false); |
| @@ -4295,8 +4297,8 @@ static void xfrm_policy_fini(struct net *net) · 7 unmodified lines | ||
| 4295 | 4297 | |
| 4296 | 4298 | htab = &net->xfrm.policy_bydst[dir]; |
| 4297 | 4299 | sz = (htab->hmask + 1) * sizeof(struct hlist_head); |
| 4298 | − WARN_ON(!hlist_empty(htab->table)); | |
| 4299 | − xfrm_hash_free(htab->table, sz); | |
| 4300 | + WARN_ON(!hlist_empty(rcu_dereference_protected(htab->table, true))); | |
| 4301 | + xfrm_hash_free(rcu_dereference_protected(htab->table, true), sz); | |
| 4300 | 4302 | } |
| 4301 | 4303 | |
| 4302 | 4304 | sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); |
| @@ -53,7 +53,7 @@ static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task); · 52 unmodified lines | ||
| 53 | 53 | static HLIST_HEAD(xfrm_state_gc_list); |
| 54 | 54 | static HLIST_HEAD(xfrm_state_dev_gc_list); |
| 55 | 55 | |
| 56 | −static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x) | |
| 56 | +static inline bool xfrm_state_hold_rcu(struct xfrm_state *x) | |
| 57 | 57 | { |
| 58 | 58 | return refcount_inc_not_zero(&x->refcnt); |
| 59 | 59 | } |
| @@ -870,7 +870,7 @@ xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid) · 810 unmodified lines | ||
| 870 | 870 | for (i = 0; i <= net->xfrm.state_hmask; i++) { |
| 871 | 871 | struct xfrm_state *x; |
| 872 | 872 | |
| 873 | − hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { | |
| 873 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { | |
| 874 | 874 | if (xfrm_id_proto_match(x->id.proto, proto) && |
| 875 | 875 | (err = security_xfrm_state_delete(x)) != 0) { |
| 876 | 876 | xfrm_audit_state_delete(x, 0, task_valid); |
| @@ -891,7 +891,7 @@ xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool · 14 unmodified lines | ||
| 891 | 891 | struct xfrm_state *x; |
| 892 | 892 | struct xfrm_dev_offload *xso; |
| 893 | 893 | |
| 894 | − hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { | |
| 894 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { | |
| 895 | 895 | xso = &x->xso; |
| 896 | 896 | |
| 897 | 897 | if (xso->dev == dev && |
| @@ -931,7 +931,7 @@ int xfrm_state_flush(struct net *net, u8 proto, bool task_valid) · 33 unmodified lines | ||
| 931 | 931 | for (i = 0; i <= net->xfrm.state_hmask; i++) { |
| 932 | 932 | struct xfrm_state *x; |
| 933 | 933 | restart: |
| 934 | − hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { | |
| 934 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { | |
| 935 | 935 | if (!xfrm_state_kern(x) && |
| 936 | 936 | xfrm_id_proto_match(x->id.proto, proto)) { |
| 937 | 937 | xfrm_state_hold(x); |
| @@ -973,7 +973,7 @@ int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_vali · 35 unmodified lines | ||
| 973 | 973 | err = -ESRCH; |
| 974 | 974 | for (i = 0; i <= net->xfrm.state_hmask; i++) { |
| 975 | 975 | restart: |
| 976 | − hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) { | |
| 976 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) { | |
| 977 | 977 | xso = &x->xso; |
| 978 | 978 | |
| 979 | 979 | if (!xfrm_state_kern(x) && xso->dev == dev) { |
| @@ -1563,23 +1563,23 @@ found: · 583 unmodified lines | ||
| 1563 | 1563 | list_add(&x->km.all, &net->xfrm.state_all); |
| 1564 | 1564 | h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family); |
| 1565 | 1565 | XFRM_STATE_INSERT(bydst, &x->bydst, |
| 1566 | − net->xfrm.state_bydst + h, | |
| 1566 | + xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, | |
| 1567 | 1567 | x->xso.type); |
| 1568 | 1568 | h = xfrm_src_hash(net, daddr, saddr, encap_family); |
| 1569 | 1569 | XFRM_STATE_INSERT(bysrc, &x->bysrc, |
| 1570 | − net->xfrm.state_bysrc + h, | |
| 1570 | + xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, | |
| 1571 | 1571 | x->xso.type); |
| 1572 | 1572 | INIT_HLIST_NODE(&x->state_cache); |
| 1573 | 1573 | if (x->id.spi) { |
| 1574 | 1574 | h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family); |
| 1575 | 1575 | XFRM_STATE_INSERT(byspi, &x->byspi, |
| 1576 | − net->xfrm.state_byspi + h, | |
| 1576 | + xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h, | |
| 1577 | 1577 | x->xso.type); |
| 1578 | 1578 | } |
| 1579 | 1579 | if (x->km.seq) { |
| 1580 | 1580 | h = xfrm_seq_hash(net, x->km.seq); |
| 1581 | 1581 | XFRM_STATE_INSERT(byseq, &x->byseq, |
| 1582 | − net->xfrm.state_byseq + h, | |
| 1582 | + xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, | |
| 1583 | 1583 | x->xso.type); |
| 1584 | 1584 | } |
| 1585 | 1585 | x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires; |
| @@ -1652,7 +1652,7 @@ xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id, · 66 unmodified lines | ||
| 1652 | 1652 | |
| 1653 | 1653 | spin_lock_bh(&net->xfrm.xfrm_state_lock); |
| 1654 | 1654 | h = xfrm_dst_hash(net, daddr, saddr, reqid, family); |
| 1655 | − hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { | |
| 1655 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { | |
| 1656 | 1656 | if (x->props.family == family && |
| 1657 | 1657 | x->props.reqid == reqid && |
| 1658 | 1658 | (mark & x->mark.m) == x->mark.v && |
| @@ -1703,18 +1703,12 @@ static struct xfrm_state *xfrm_state_lookup_spi_proto(struct net *net, __be32 sp · 44 unmodified lines | ||
| 1703 | 1703 | struct xfrm_state *x; |
| 1704 | 1704 | unsigned int i; |
| 1705 | 1705 | |
| 1706 | − rcu_read_lock(); | |
| 1707 | 1706 | for (i = 0; i <= net->xfrm.state_hmask; i++) { |
| 1708 | − hlist_for_each_entry_rcu(x, &net->xfrm.state_byspi[i], byspi) { | |
| 1709 | − if (x->id.spi == spi && x->id.proto == proto) { | |
| 1710 | − if (!xfrm_state_hold_rcu(x)) | |
| 1711 | − continue; | |
| 1712 | − rcu_read_unlock(); | |
| 1707 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byspi, net) + i, byspi) { | |
| 1708 | + if (x->id.spi == spi && x->id.proto == proto) | |
| 1713 | 1709 | return x; |
| 1714 | − } | |
| 1715 | 1710 | } |
| 1716 | 1711 | } |
| 1717 | − rcu_read_unlock(); | |
| 1718 | 1712 | return NULL; |
| 1719 | 1713 | } |
| 1720 | 1714 | |
| @@ -1730,25 +1724,29 @@ static void __xfrm_state_insert(struct xfrm_state *x) · 9 unmodified lines | ||
| 1730 | 1724 | |
| 1731 | 1725 | h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr, |
| 1732 | 1726 | x->props.reqid, x->props.family); |
| 1733 | − XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, | |
| 1727 | + XFRM_STATE_INSERT(bydst, &x->bydst, | |
| 1728 | + xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, | |
| 1734 | 1729 | x->xso.type); |
| 1735 | 1730 | |
| 1736 | 1731 | h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family); |
| 1737 | − XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, | |
| 1732 | + XFRM_STATE_INSERT(bysrc, &x->bysrc, | |
| 1733 | + xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, | |
| 1738 | 1734 | x->xso.type); |
| 1739 | 1735 | |
| 1740 | 1736 | if (x->id.spi) { |
| 1741 | 1737 | h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, |
| 1742 | 1738 | x->props.family); |
| 1743 | 1739 | |
| 1744 | − XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, | |
| 1740 | + XFRM_STATE_INSERT(byspi, &x->byspi, | |
| 1741 | + xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h, | |
| 1745 | 1742 | x->xso.type); |
| 1746 | 1743 | } |
| 1747 | 1744 | |
| 1748 | 1745 | if (x->km.seq) { |
| 1749 | 1746 | h = xfrm_seq_hash(net, x->km.seq); |
| 1750 | 1747 | |
| 1751 | − XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h, | |
| 1748 | + XFRM_STATE_INSERT(byseq, &x->byseq, | |
| 1749 | + xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, | |
| 1752 | 1750 | x->xso.type); |
| 1753 | 1751 | } |
| 1754 | 1752 | |
| @@ -1775,7 +1773,7 @@ static void __xfrm_state_bump_genids(struct xfrm_state *xnew) · 20 unmodified lines | ||
| 1775 | 1773 | u32 cpu_id = xnew->pcpu_num; |
| 1776 | 1774 | |
| 1777 | 1775 | h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family); |
| 1778 | − hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { | |
| 1776 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { | |
| 1779 | 1777 | if (x->props.family == family && |
| 1780 | 1778 | x->props.reqid == reqid && |
| 1781 | 1779 | x->if_id == if_id && |
| @@ -1811,7 +1809,7 @@ static struct xfrm_state *__find_acq_core(struct net *net, · 29 unmodified lines | ||
| 1811 | 1809 | struct xfrm_state *x; |
| 1812 | 1810 | u32 mark = m->v & m->m; |
| 1813 | 1811 | |
| 1814 | − hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { | |
| 1812 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { | |
| 1815 | 1813 | if (x->props.reqid != reqid || |
| 1816 | 1814 | x->props.mode != mode || |
| 1817 | 1815 | x->props.family != family || |
| @@ -1868,10 +1866,12 @@ static struct xfrm_state *__find_acq_core(struct net *net, · 50 unmodified lines | ||
| 1868 | 1866 | ktime_set(net->xfrm.sysctl_acq_expires, 0), |
| 1869 | 1867 | HRTIMER_MODE_REL_SOFT); |
| 1870 | 1868 | list_add(&x->km.all, &net->xfrm.state_all); |
| 1871 | − XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h, | |
| 1869 | + XFRM_STATE_INSERT(bydst, &x->bydst, | |
| 1870 | + xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, | |
| 1872 | 1871 | x->xso.type); |
| 1873 | 1872 | h = xfrm_src_hash(net, daddr, saddr, family); |
| 1874 | − XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h, | |
| 1873 | + XFRM_STATE_INSERT(bysrc, &x->bysrc, | |
| 1874 | + xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, | |
| 1875 | 1875 | x->xso.type); |
| 1876 | 1876 | |
| 1877 | 1877 | net->xfrm.state_num++; |
| @@ -2091,7 +2091,7 @@ struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *n · 213 unmodified lines | ||
| 2091 | 2091 | if (m->reqid) { |
| 2092 | 2092 | h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr, |
| 2093 | 2093 | m->reqid, m->old_family); |
| 2094 | − hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) { | |
| 2094 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) { | |
| 2095 | 2095 | if (x->props.mode != m->mode || |
| 2096 | 2096 | x->id.proto != m->proto) |
| 2097 | 2097 | continue; |
| @@ -2110,7 +2110,7 @@ struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *n · 12 unmodified lines | ||
| 2110 | 2110 | } else { |
| 2111 | 2111 | h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr, |
| 2112 | 2112 | m->old_family); |
| 2113 | − hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) { | |
| 2113 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, bysrc) { | |
| 2114 | 2114 | if (x->props.mode != m->mode || |
| 2115 | 2115 | x->id.proto != m->proto) |
| 2116 | 2116 | continue; |
| @@ -2264,6 +2264,7 @@ out: · 147 unmodified lines | ||
| 2264 | 2264 | |
| 2265 | 2265 | err = 0; |
| 2266 | 2266 | x->km.state = XFRM_STATE_DEAD; |
| 2267 | + xfrm_dev_state_delete(x); | |
| 2267 | 2268 | __xfrm_state_put(x); |
| 2268 | 2269 | } |
| 2269 | 2270 | |
| @@ -2312,7 +2313,7 @@ void xfrm_state_update_stats(struct net *net) · 42 unmodified lines | ||
| 2312 | 2313 | |
| 2313 | 2314 | spin_lock_bh(&net->xfrm.xfrm_state_lock); |
| 2314 | 2315 | for (i = 0; i <= net->xfrm.state_hmask; i++) { |
| 2315 | − hlist_for_each_entry(x, net->xfrm.state_bydst + i, bydst) | |
| 2316 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) | |
| 2316 | 2317 | xfrm_dev_state_update_stats(x); |
| 2317 | 2318 | } |
| 2318 | 2319 | spin_unlock_bh(&net->xfrm.xfrm_state_lock); |
| @@ -2503,7 +2504,7 @@ static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 s · 184 unmodified lines | ||
| 2503 | 2504 | unsigned int h = xfrm_seq_hash(net, seq); |
| 2504 | 2505 | struct xfrm_state *x; |
| 2505 | 2506 | |
| 2506 | − hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) { | |
| 2507 | + hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, byseq) { | |
| 2507 | 2508 | if (x->km.seq == seq && |
| 2508 | 2509 | (mark & x->mark.m) == x->mark.v && |
| 2509 | 2510 | x->pcpu_num == pcpu_num && |
| @@ -2602,12 +2603,13 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high, · 92 unmodified lines | ||
| 2602 | 2603 | if (!x0) { |
| 2603 | 2604 | x->id.spi = newspi; |
| 2604 | 2605 | h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family); |
| 2605 | − XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, x->xso.type); | |
| 2606 | + XFRM_STATE_INSERT(byspi, &x->byspi, | |
| 2607 | + xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h, | |
| 2608 | + x->xso.type); | |
| 2606 | 2609 | spin_unlock_bh(&net->xfrm.xfrm_state_lock); |
| 2607 | 2610 | err = 0; |
| 2608 | 2611 | goto unlock; |
| 2609 | 2612 | } |
| 2610 | − xfrm_state_put(x0); | |
| 2611 | 2613 | spin_unlock_bh(&net->xfrm.xfrm_state_lock); |
| 2612 | 2614 | |
| 2613 | 2615 | next: |
| @@ -3258,6 +3260,7 @@ EXPORT_SYMBOL(xfrm_init_state); · 644 unmodified lines | ||
| 3258 | 3260 | |
| 3259 | 3261 | int __net_init xfrm_state_init(struct net *net) |
| 3260 | 3262 | { |
| 3263 | + struct hlist_head *ndst, *nsrc, *nspi, *nseq; | |
| 3261 | 3264 | unsigned int sz; |
| 3262 | 3265 | |
| 3263 | 3266 | if (net_eq(net, &init_net)) |
| @@ -3268,18 +3271,25 @@ int __net_init xfrm_state_init(struct net *net) · 4 unmodified lines | ||
| 3268 | 3271 | |
| 3269 | 3272 | sz = sizeof(struct hlist_head) * 8; |
| 3270 | 3273 | |
| 3271 | − net->xfrm.state_bydst = xfrm_hash_alloc(sz); | |
| 3272 | − if (!net->xfrm.state_bydst) | |
| 3274 | + ndst = xfrm_hash_alloc(sz); | |
| 3275 | + if (!ndst) | |
| 3273 | 3276 | goto out_bydst; |
| 3274 | − net->xfrm.state_bysrc = xfrm_hash_alloc(sz); | |
| 3275 | − if (!net->xfrm.state_bysrc) | |
| 3277 | + rcu_assign_pointer(net->xfrm.state_bydst, ndst); | |
| 3278 | + | |
| 3279 | + nsrc = xfrm_hash_alloc(sz); | |
| 3280 | + if (!nsrc) | |
| 3276 | 3281 | goto out_bysrc; |
| 3277 | − net->xfrm.state_byspi = xfrm_hash_alloc(sz); | |
| 3278 | − if (!net->xfrm.state_byspi) | |
| 3282 | + rcu_assign_pointer(net->xfrm.state_bysrc, nsrc); | |
| 3283 | + | |
| 3284 | + nspi = xfrm_hash_alloc(sz); | |
| 3285 | + if (!nspi) | |
| 3279 | 3286 | goto out_byspi; |
| 3280 | − net->xfrm.state_byseq = xfrm_hash_alloc(sz); | |
| 3281 | − if (!net->xfrm.state_byseq) | |
| 3287 | + rcu_assign_pointer(net->xfrm.state_byspi, nspi); | |
| 3288 | + | |
| 3289 | + nseq = xfrm_hash_alloc(sz); | |
| 3290 | + if (!nseq) | |
| 3282 | 3291 | goto out_byseq; |
| 3292 | + rcu_assign_pointer(net->xfrm.state_byseq, nseq); | |
| 3283 | 3293 | |
| 3284 | 3294 | net->xfrm.state_cache_input = alloc_percpu(struct hlist_head); |
| 3285 | 3295 | if (!net->xfrm.state_cache_input) |
| @@ -3295,17 +3305,19 @@ int __net_init xfrm_state_init(struct net *net) · 9 unmodified lines | ||
| 3295 | 3305 | return 0; |
| 3296 | 3306 | |
| 3297 | 3307 | out_state_cache_input: |
| 3298 | − xfrm_hash_free(net->xfrm.state_byseq, sz); | |
| 3308 | + xfrm_hash_free(nseq, sz); | |
| 3299 | 3309 | out_byseq: |
| 3300 | − xfrm_hash_free(net->xfrm.state_byspi, sz); | |
| 3310 | + xfrm_hash_free(nspi, sz); | |
| 3301 | 3311 | out_byspi: |
| 3302 | − xfrm_hash_free(net->xfrm.state_bysrc, sz); | |
| 3312 | + xfrm_hash_free(nsrc, sz); | |
| 3303 | 3313 | out_bysrc: |
| 3304 | − xfrm_hash_free(net->xfrm.state_bydst, sz); | |
| 3314 | + xfrm_hash_free(ndst, sz); | |
| 3305 | 3315 | out_bydst: |
| 3306 | 3316 | return -ENOMEM; |
| 3307 | 3317 | } |
| 3308 | 3318 | |
| 3319 | +#define xfrm_state_deref_netexit(table) \ | |
| 3320 | + rcu_dereference_protected((table), true /* netns is going away */) | |
| 3309 | 3321 | void xfrm_state_fini(struct net *net) |
| 3310 | 3322 | { |
| 3311 | 3323 | unsigned int sz; |
| @@ -3318,17 +3330,17 @@ void xfrm_state_fini(struct net *net) · 6 unmodified lines | ||
| 3318 | 3330 | WARN_ON(!list_empty(&net->xfrm.state_all)); |
| 3319 | 3331 | |
| 3320 | 3332 | for (i = 0; i <= net->xfrm.state_hmask; i++) { |
| 3321 | − WARN_ON(!hlist_empty(net->xfrm.state_byseq + i)); | |
| 3322 | − WARN_ON(!hlist_empty(net->xfrm.state_byspi + i)); | |
| 3323 | − WARN_ON(!hlist_empty(net->xfrm.state_bysrc + i)); | |
| 3324 | − WARN_ON(!hlist_empty(net->xfrm.state_bydst + i)); | |
| 3333 | + WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byseq) + i)); | |
| 3334 | + WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byspi) + i)); | |
| 3335 | + WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bysrc) + i)); | |
| 3336 | + WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bydst) + i)); | |
| 3325 | 3337 | } |
| 3326 | 3338 | |
| 3327 | 3339 | sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head); |
| 3328 | − xfrm_hash_free(net->xfrm.state_byseq, sz); | |
| 3329 | − xfrm_hash_free(net->xfrm.state_byspi, sz); | |
| 3330 | − xfrm_hash_free(net->xfrm.state_bysrc, sz); | |
| 3331 | − xfrm_hash_free(net->xfrm.state_bydst, sz); | |
| 3340 | + xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byseq), sz); | |
| 3341 | + xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byspi), sz); | |
| 3342 | + xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bysrc), sz); | |
| 3343 | + xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bydst), sz); | |
| 3332 | 3344 | free_percpu(net->xfrm.state_cache_input); |
| 3333 | 3345 | } |
| 3334 | 3346 | |
| @@ -35,6 +35,15 @@ · 34 unmodified lines | ||
| 35 | 35 | #endif |
| 36 | 36 | #include <linux/unaligned.h> |
| 37 | 37 | |
| 38 | +static struct sock *xfrm_net_nlsk(const struct net *net, const struct sk_buff *skb) | |
| 39 | +{ | |
| 40 | + /* get the source of this request, see netlink_unicast_kernel */ | |
| 41 | + const struct sock *sk = NETLINK_CB(skb).sk; | |
| 42 | + | |
| 43 | + /* sk is refcounted, the netns stays alive and nlsk with it */ | |
| 44 | + return rcu_dereference_protected(net->xfrm.nlsk, sk->sk_net_refcnt); | |
| 45 | +} | |
| 46 | + | |
| 38 | 47 | static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type, |
| 39 | 48 | struct netlink_ext_ack *extack) |
| 40 | 49 | { |
| @@ -1727,7 +1736,7 @@ static int xfrm_get_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh, · 1686 unmodified lines | ||
| 1727 | 1736 | err = build_spdinfo(r_skb, net, sportid, seq, *flags); |
| 1728 | 1737 | BUG_ON(err < 0); |
| 1729 | 1738 | |
| 1730 | − return nlmsg_unicast(net->xfrm.nlsk, r_skb, sportid); | |
| 1739 | + return nlmsg_unicast(xfrm_net_nlsk(net, skb), r_skb, sportid); | |
| 1731 | 1740 | } |
| 1732 | 1741 | |
| 1733 | 1742 | static inline unsigned int xfrm_sadinfo_msgsize(void) |
| @@ -1787,7 +1796,7 @@ static int xfrm_get_sadinfo(struct sk_buff *skb, struct nlmsghdr *nlh, · 53 unmodified lines | ||
| 1787 | 1796 | err = build_sadinfo(r_skb, net, sportid, seq, *flags); |
| 1788 | 1797 | BUG_ON(err < 0); |
| 1789 | 1798 | |
| 1790 | − return nlmsg_unicast(net->xfrm.nlsk, r_skb, sportid); | |
| 1799 | + return nlmsg_unicast(xfrm_net_nlsk(net, skb), r_skb, sportid); | |
| 1791 | 1800 | } |
| 1792 | 1801 | |
| 1793 | 1802 | static int xfrm_get_sa(struct sk_buff *skb, struct nlmsghdr *nlh, |
| @@ -1807,7 +1816,7 @@ static int xfrm_get_sa(struct sk_buff *skb, struct nlmsghdr *nlh, · 13 unmodified lines | ||
| 1807 | 1816 | if (IS_ERR(resp_skb)) { |
| 1808 | 1817 | err = PTR_ERR(resp_skb); |
| 1809 | 1818 | } else { |
| 1810 | − err = nlmsg_unicast(net->xfrm.nlsk, resp_skb, NETLINK_CB(skb).portid); | |
| 1819 | + err = nlmsg_unicast(xfrm_net_nlsk(net, skb), resp_skb, NETLINK_CB(skb).portid); | |
| 1811 | 1820 | } |
| 1812 | 1821 | xfrm_state_put(x); |
| 1813 | 1822 | out_noput: |
| @@ -1850,6 +1859,7 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, · 36 unmodified lines | ||
| 1850 | 1859 | pcpu_num = nla_get_u32(attrs[XFRMA_SA_PCPU]); |
| 1851 | 1860 | if (pcpu_num >= num_possible_cpus()) { |
| 1852 | 1861 | err = -EINVAL; |
| 1862 | + NL_SET_ERR_MSG(extack, "pCPU number too big"); | |
| 1853 | 1863 | goto out_noput; |
| 1854 | 1864 | } |
| 1855 | 1865 | } |
| @@ -1897,7 +1907,7 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, · 41 unmodified lines | ||
| 1897 | 1907 | } |
| 1898 | 1908 | } |
| 1899 | 1909 | |
| 1900 | − err = nlmsg_unicast(net->xfrm.nlsk, resp_skb, NETLINK_CB(skb).portid); | |
| 1910 | + err = nlmsg_unicast(xfrm_net_nlsk(net, skb), resp_skb, NETLINK_CB(skb).portid); | |
| 1901 | 1911 | |
| 1902 | 1912 | out: |
| 1903 | 1913 | xfrm_state_put(x); |
| @@ -2542,7 +2552,7 @@ static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh, · 638 unmodified lines | ||
| 2542 | 2552 | r_up->out = net->xfrm.policy_default[XFRM_POLICY_OUT]; |
| 2543 | 2553 | nlmsg_end(r_skb, r_nlh); |
| 2544 | 2554 | |
| 2545 | − return nlmsg_unicast(net->xfrm.nlsk, r_skb, portid); | |
| 2555 | + return nlmsg_unicast(xfrm_net_nlsk(net, skb), r_skb, portid); | |
| 2546 | 2556 | } |
| 2547 | 2557 | |
| 2548 | 2558 | static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh, |
| @@ -2608,7 +2618,7 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh, · 59 unmodified lines | ||
| 2608 | 2618 | if (IS_ERR(resp_skb)) { |
| 2609 | 2619 | err = PTR_ERR(resp_skb); |
| 2610 | 2620 | } else { |
| 2611 | − err = nlmsg_unicast(net->xfrm.nlsk, resp_skb, | |
| 2621 | + err = nlmsg_unicast(xfrm_net_nlsk(net, skb), resp_skb, | |
| 2612 | 2622 | NETLINK_CB(skb).portid); |
| 2613 | 2623 | } |
| 2614 | 2624 | } else { |
| @@ -2781,7 +2791,7 @@ static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh, · 166 unmodified lines | ||
| 2781 | 2791 | err = build_aevent(r_skb, x, &c); |
| 2782 | 2792 | BUG_ON(err < 0); |
| 2783 | 2793 | |
| 2784 | − err = nlmsg_unicast(net->xfrm.nlsk, r_skb, NETLINK_CB(skb).portid); | |
| 2794 | + err = nlmsg_unicast(xfrm_net_nlsk(net, skb), r_skb, NETLINK_CB(skb).portid); | |
| 2785 | 2795 | spin_unlock_bh(&x->lock); |
| 2786 | 2796 | xfrm_state_put(x); |
| 2787 | 2797 | return err; |
| @@ -3001,8 +3011,10 @@ static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh, · 213 unmodified lines | ||
| 3001 | 3011 | if (attrs[XFRMA_SA_PCPU]) { |
| 3002 | 3012 | x->pcpu_num = nla_get_u32(attrs[XFRMA_SA_PCPU]); |
| 3003 | 3013 | err = -EINVAL; |
| 3004 | − if (x->pcpu_num >= num_possible_cpus()) | |
| 3014 | + if (x->pcpu_num >= num_possible_cpus()) { | |
| 3015 | + NL_SET_ERR_MSG(extack, "pCPU number too big"); | |
| 3005 | 3016 | goto free_state; |
| 3017 | + } | |
| 3006 | 3018 | } |
| 3007 | 3019 | |
| 3008 | 3020 | err = verify_newpolicy_info(&ua->policy, extack); |
| @@ -3483,7 +3495,7 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, · 474 unmodified lines | ||
| 3483 | 3495 | goto err; |
| 3484 | 3496 | } |
| 3485 | 3497 | |
| 3486 | − err = netlink_dump_start(net->xfrm.nlsk, skb, nlh, &c); | |
| 3498 | + err = netlink_dump_start(xfrm_net_nlsk(net, skb), skb, nlh, &c); | |
| 3487 | 3499 | goto err; |
| 3488 | 3500 | } |
| 3489 | 3501 | |
| @@ -3673,7 +3685,7 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) · 183 unmodified lines | ||
| 3673 | 3685 | } |
| 3674 | 3686 | if (x->if_id) |
| 3675 | 3687 | l += nla_total_size(sizeof(x->if_id)); |
| 3676 | − if (x->pcpu_num) | |
| 3688 | + if (x->pcpu_num != UINT_MAX) | |
| 3677 | 3689 | l += nla_total_size(sizeof(x->pcpu_num)); |
| 3678 | 3690 | |
| 3679 | 3691 | /* Must count x->lastused as it may become non-zero behind our back. */ |
| @@ -23,7 +23,10 @@ use crate::{ · 22 unmodified lines | ||
| 23 | 23 | prelude::*, |
| 24 | 24 | }; |
| 25 | 25 | |
| 26 | −use core::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull}; | |
| 26 | +use core::{ | |
| 27 | + marker::PhantomData, | |
| 28 | + mem::ManuallyDrop, // | |
| 29 | +}; | |
| 27 | 30 | |
| 28 | 31 | mod private { |
| 29 | 32 | pub trait Sealed {} |
| @@ -229,15 +232,17 @@ pub fn devm_enable_optional(dev: &Device<Bound>, name: &CStr) -> Result { · 199 unmodified lines | ||
| 229 | 232 | /// |
| 230 | 233 | /// # Invariants |
| 231 | 234 | /// |
| 232 | −/// - `inner` is a non-null wrapper over a pointer to a `struct | |
| 233 | −/// regulator` obtained from [`regulator_get()`]. | |
| 235 | +/// - `inner` is a pointer obtained from a successful call to | |
| 236 | +/// [`regulator_get()`]. It is treated as an opaque token that may only be | |
| 237 | +/// accessed using C API methods (e.g., it may be `NULL` if the C API returns | |
| 238 | +/// `NULL`). | |
| 234 | 239 | /// |
| 235 | 240 | /// [`regulator_get()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_get |
| 236 | 241 | pub struct Regulator<State> |
| 237 | 242 | where |
| 238 | 243 | State: RegulatorState, |
| 239 | 244 | { |
| 240 | − inner: NonNull<bindings::regulator>, | |
| 245 | + inner: *mut bindings::regulator, | |
| 241 | 246 | _phantom: PhantomData<State>, |
| 242 | 247 | } |
| 243 | 248 | |
| @@ -249,7 +254,7 @@ impl<T: RegulatorState> Regulator<T> { · 5 unmodified lines | ||
| 249 | 254 | // SAFETY: Safe as per the type invariants of `Regulator`. |
| 250 | 255 | to_result(unsafe { |
| 251 | 256 | bindings::regulator_set_voltage( |
| 252 | − self.inner.as_ptr(), | |
| 257 | + self.inner, | |
| 253 | 258 | min_voltage.as_microvolts(), |
| 254 | 259 | max_voltage.as_microvolts(), |
| 255 | 260 | ) |
| @@ -259,7 +264,7 @@ impl<T: RegulatorState> Regulator<T> { · 3 unmodified lines | ||
| 259 | 264 | /// Gets the current voltage of the regulator. |
| 260 | 265 | pub fn get_voltage(&self) -> Result<Voltage> { |
| 261 | 266 | // SAFETY: Safe as per the type invariants of `Regulator`. |
| 262 | − let voltage = unsafe { bindings::regulator_get_voltage(self.inner.as_ptr()) }; | |
| 267 | + let voltage = unsafe { bindings::regulator_get_voltage(self.inner) }; | |
| 263 | 268 | |
| 264 | 269 | to_result(voltage).map(|()| Voltage::from_microvolts(voltage)) |
| 265 | 270 | } |
| @@ -270,10 +275,8 @@ impl<T: RegulatorState> Regulator<T> { · 4 unmodified lines | ||
| 270 | 275 | // received from the C code. |
| 271 | 276 | from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_char_ptr()) })?; |
| 272 | 277 | |
| 273 | − // SAFETY: We can safely trust `inner` to be a pointer to a valid | |
| 274 | − // regulator if `ERR_PTR` was not returned. | |
| 275 | − let inner = unsafe { NonNull::new_unchecked(inner) }; | |
| 276 | − | |
| 278 | + // INVARIANT: `inner` is a pointer obtained from `regulator_get()`, and | |
| 279 | + // the call was successful. | |
| 277 | 280 | Ok(Self { |
| 278 | 281 | inner, |
| 279 | 282 | _phantom: PhantomData, |
| @@ -282,12 +285,12 @@ impl<T: RegulatorState> Regulator<T> { · 2 unmodified lines | ||
| 282 | 285 | |
| 283 | 286 | fn enable_internal(&self) -> Result { |
| 284 | 287 | // SAFETY: Safe as per the type invariants of `Regulator`. |
| 285 | − to_result(unsafe { bindings::regulator_enable(self.inner.as_ptr()) }) | |
| 288 | + to_result(unsafe { bindings::regulator_enable(self.inner) }) | |
| 286 | 289 | } |
| 287 | 290 | |
| 288 | 291 | fn disable_internal(&self) -> Result { |
| 289 | 292 | // SAFETY: Safe as per the type invariants of `Regulator`. |
| 290 | − to_result(unsafe { bindings::regulator_disable(self.inner.as_ptr()) }) | |
| 293 | + to_result(unsafe { bindings::regulator_disable(self.inner) }) | |
| 291 | 294 | } |
| 292 | 295 | } |
| 293 | 296 | |
| @@ -349,7 +352,7 @@ impl<T: IsEnabled> Regulator<T> { · 55 unmodified lines | ||
| 349 | 352 | /// Checks if the regulator is enabled. |
| 350 | 353 | pub fn is_enabled(&self) -> bool { |
| 351 | 354 | // SAFETY: Safe as per the type invariants of `Regulator`. |
| 352 | − unsafe { bindings::regulator_is_enabled(self.inner.as_ptr()) != 0 } | |
| 355 | + unsafe { bindings::regulator_is_enabled(self.inner) != 0 } | |
| 353 | 356 | } |
| 354 | 357 | } |
| 355 | 358 | |
| @@ -359,11 +362,11 @@ impl<T: RegulatorState> Drop for Regulator<T> { · 3 unmodified lines | ||
| 359 | 362 | // SAFETY: By the type invariants, we know that `self` owns a |
| 360 | 363 | // reference on the enabled refcount, so it is safe to relinquish it |
| 361 | 364 | // now. |
| 362 | − unsafe { bindings::regulator_disable(self.inner.as_ptr()) }; | |
| 365 | + unsafe { bindings::regulator_disable(self.inner) }; | |
| 363 | 366 | } |
| 364 | 367 | // SAFETY: By the type invariants, we know that `self` owns a reference, |
| 365 | 368 | // so it is safe to relinquish it now. |
| 366 | − unsafe { bindings::regulator_put(self.inner.as_ptr()) }; | |
| 369 | + unsafe { bindings::regulator_put(self.inner) }; | |
| 367 | 370 | } |
| 368 | 371 | } |
| 369 | 372 | |
| @@ -299,7 +299,7 @@ out_unset: · 298 unmodified lines | ||
| 299 | 299 | |
| 300 | 300 | /* clang-format on */ |
| 301 | 301 | |
| 302 | −#define LANDLOCK_ABI_LAST 7 | |
| 302 | +#define LANDLOCK_ABI_LAST 8 | |
| 303 | 303 | |
| 304 | 304 | #define XSTR(s) #s |
| 305 | 305 | #define STR(s) XSTR(s) |
| @@ -436,7 +436,8 @@ int main(const int argc, char *const argv[], char *const *const envp) · 130 unmodified lines | ||
| 436 | 436 | /* Removes LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON for ABI < 7 */ |
| 437 | 437 | supported_restrict_flags &= |
| 438 | 438 | ~LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON; |
| 439 | − | |
| 439 | + __attribute__((fallthrough)); | |
| 440 | + case 7: | |
| 440 | 441 | /* Must be printed for any ABI < LANDLOCK_ABI_LAST. */ |
| 441 | 442 | fprintf(stderr, |
| 442 | 443 | "Hint: You should update the running kernel " |
| @@ -122,3 +122,14 @@ fresh identifier ALLOC_OBJS = script:python(ALLOC_ARRAY) { alloc_array(ALLOC_ARR · 121 unmodified lines | ||
| 122 | 122 | - ALLOC(struct_size_t(TYPE, FLEX, COUNT), GFP) |
| 123 | 123 | + ALLOC_FLEX(TYPE, FLEX, COUNT, GFP) |
| 124 | 124 | ) |
| 125 | + | |
| 126 | +@drop_gfp_kernel depends on patch && !(file in "tools") && !(file in "samples")@ | |
| 127 | +identifier ALLOC = {kmalloc_obj,kmalloc_objs,kmalloc_flex, | |
| 128 | + kzalloc_obj,kzalloc_objs,kzalloc_flex, | |
| 129 | + kvmalloc_obj,kvmalloc_objs,kvmalloc_flex, | |
| 130 | + kvzalloc_obj,kvzalloc_objs,kvzalloc_flex}; | |
| 131 | +@@ | |
| 132 | + | |
| 133 | + ALLOC(... | |
| 134 | +- , GFP_KERNEL | |
| 135 | + ) | |
| @@ -151,6 +151,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do · 150 unmodified lines | ||
| 151 | 151 | if ! "$AWK" -v prefix="$CONFIG_PREFIX" \ |
| 152 | 152 | -v warnoverride="$WARNOVERRIDE" \ |
| 153 | 153 | -v strict="$STRICT" \ |
| 154 | + -v outfile="$TMP_FILE.new" \ | |
| 154 | 155 | -v builtin="$BUILTIN" \ |
| 155 | 156 | -v warnredun="$WARNREDUN" ' |
| 156 | 157 | BEGIN { |
| @@ -195,7 +196,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do · 38 unmodified lines | ||
| 195 | 196 | |
| 196 | 197 | # First pass: read merge file, store all lines and index |
| 197 | 198 | FILENAME == ARGV[1] { |
| 198 | − mergefile = FILENAME | |
| 199 | + mergefile = FILENAME | |
| 199 | 200 | merge_lines[FNR] = $0 |
| 200 | 201 | merge_total = FNR |
| 201 | 202 | cfg = get_cfg($0) |
| @@ -212,17 +213,17 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do · 10 unmodified lines | ||
| 212 | 213 | |
| 213 | 214 | # Not a config or not in merge file - keep it |
| 214 | 215 | if (cfg == "" || !(cfg in merge_cfg)) { |
| 215 | − print $0 >> ARGV[3] | |
| 216 | + print $0 >> outfile | |
| 216 | 217 | next |
| 217 | 218 | } |
| 218 | 219 | |
| 219 | − prev_val = $0 | |
| 220 | + prev_val = $0 | |
| 220 | 221 | new_val = merge_cfg[cfg] |
| 221 | 222 | |
| 222 | 223 | # BUILTIN: do not demote y to m |
| 223 | 224 | if (builtin == "true" && new_val ~ /=m$/ && prev_val ~ /=y$/) { |
| 224 | 225 | warn_builtin(cfg, prev_val, new_val) |
| 225 | − print $0 >> ARGV[3] | |
| 226 | + print $0 >> outfile | |
| 226 | 227 | skip_merge[merge_cfg_line[cfg]] = 1 |
| 227 | 228 | next |
| 228 | 229 | } |
| @@ -235,7 +236,7 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do · 6 unmodified lines | ||
| 235 | 236 | |
| 236 | 237 | # "=n" is the same as "is not set" |
| 237 | 238 | if (prev_val ~ /=n$/ && new_val ~ / is not set$/) { |
| 238 | − print $0 >> ARGV[3] | |
| 239 | + print $0 >> outfile | |
| 239 | 240 | next |
| 240 | 241 | } |
| 241 | 242 | |
| @@ -246,25 +247,20 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do · 4 unmodified lines | ||
| 246 | 247 | } |
| 247 | 248 | } |
| 248 | 249 | |
| 249 | − # output file, skip all lines | |
| 250 | − FILENAME == ARGV[3] { | |
| 251 | − nextfile | |
| 252 | − } | |
| 253 | − | |
| 254 | 250 | END { |
| 255 | 251 | # Newline in case base file lacks trailing newline |
| 256 | − print "" >> ARGV[3] | |
| 252 | + print "" >> outfile | |
| 257 | 253 | # Append merge file, skipping lines marked for builtin preservation |
| 258 | 254 | for (i = 1; i <= merge_total; i++) { |
| 259 | 255 | if (!(i in skip_merge)) { |
| 260 | − print merge_lines[i] >> ARGV[3] | |
| 256 | + print merge_lines[i] >> outfile | |
| 261 | 257 | } |
| 262 | 258 | } |
| 263 | 259 | if (strict_violated) { |
| 264 | 260 | exit 1 |
| 265 | 261 | } |
| 266 | 262 | }' \ |
| 267 | − "$ORIG_MERGE_FILE" "$TMP_FILE" "$TMP_FILE.new"; then | |
| 263 | + "$ORIG_MERGE_FILE" "$TMP_FILE"; then | |
| 268 | 264 | # awk exited non-zero, strict mode was violated |
| 269 | 265 | STRICT_MODE_VIOLATED=true |
| 270 | 266 | fi |
| @@ -381,7 +377,7 @@ END { · 110 unmodified lines | ||
| 381 | 377 | STRICT_MODE_VIOLATED=true |
| 382 | 378 | fi |
| 383 | 379 | |
| 384 | −if [ "$STRICT" == "true" ] && [ "$STRICT_MODE_VIOLATED" == "true" ]; then | |
| 380 | +if [ "$STRICT" = "true" ] && [ "$STRICT_MODE_VIOLATED" = "true" ]; then | |
| 385 | 381 | echo "Requested and effective config differ" |
| 386 | 382 | exit 1 |
| 387 | 383 | fi |
| @@ -94,8 +94,7 @@ static struct landlock_details *get_current_details(void) · 93 unmodified lines | ||
| 94 | 94 | * allocate with GFP_KERNEL_ACCOUNT because it is independent from the |
| 95 | 95 | * caller. |
| 96 | 96 | */ |
| 97 | − details = | |
| 98 | − kzalloc_flex(*details, exe_path, path_size); | |
| 97 | + details = kzalloc_flex(*details, exe_path, path_size); | |
| 99 | 98 | if (!details) |
| 100 | 99 | return ERR_PTR(-ENOMEM); |
| 101 | 100 | |
| @@ -32,9 +32,8 @@ static struct landlock_ruleset *create_ruleset(const u32 num_layers) · 31 unmodified lines | ||
| 32 | 32 | { |
| 33 | 33 | struct landlock_ruleset *new_ruleset; |
| 34 | 34 | |
| 35 | − new_ruleset = | |
| 36 | − kzalloc_flex(*new_ruleset, access_masks, num_layers, | |
| 37 | − GFP_KERNEL_ACCOUNT); | |
| 35 | + new_ruleset = kzalloc_flex(*new_ruleset, access_masks, num_layers, | |
| 36 | + GFP_KERNEL_ACCOUNT); | |
| 38 | 37 | if (!new_ruleset) |
| 39 | 38 | return ERR_PTR(-ENOMEM); |
| 40 | 39 | refcount_set(&new_ruleset->usage, 1); |
| @@ -559,8 +558,8 @@ landlock_merge_ruleset(struct landlock_ruleset *const parent, · 518 unmodified lines | ||
| 559 | 558 | if (IS_ERR(new_dom)) |
| 560 | 559 | return new_dom; |
| 561 | 560 | |
| 562 | − new_dom->hierarchy = kzalloc_obj(*new_dom->hierarchy, | |
| 563 | − GFP_KERNEL_ACCOUNT); | |
| 561 | + new_dom->hierarchy = | |
| 562 | + kzalloc_obj(*new_dom->hierarchy, GFP_KERNEL_ACCOUNT); | |
| 564 | 563 | if (!new_dom->hierarchy) |
| 565 | 564 | return ERR_PTR(-ENOMEM); |
| 566 | 565 | |
| @@ -203,6 +203,40 @@ static struct tsync_work *tsync_works_provide(struct tsync_works *s, · 202 unmodified lines | ||
| 203 | 203 | return ctx; |
| 204 | 204 | } |
| 205 | 205 | |
| 206 | +/** | |
| 207 | + * tsync_works_trim - Put the last tsync_work element | |
| 208 | + * | |
| 209 | + * @s: TSYNC works to trim. | |
| 210 | + * | |
| 211 | + * Put the last task and decrement the size of @s. | |
| 212 | + * | |
| 213 | + * This helper does not cancel a running task, but just reset the last element | |
| 214 | + * to zero. | |
| 215 | + */ | |
| 216 | +static void tsync_works_trim(struct tsync_works *s) | |
| 217 | +{ | |
| 218 | + struct tsync_work *ctx; | |
| 219 | + | |
| 220 | + if (WARN_ON_ONCE(s->size <= 0)) | |
| 221 | + return; | |
| 222 | + | |
| 223 | + ctx = s->works[s->size - 1]; | |
| 224 | + | |
| 225 | + /* | |
| 226 | + * For consistency, remove the task from ctx so that it does not look like | |
| 227 | + * we handed it a task_work. | |
| 228 | + */ | |
| 229 | + put_task_struct(ctx->task); | |
| 230 | + *ctx = (typeof(*ctx)){}; | |
| 231 | + | |
| 232 | + /* | |
| 233 | + * Cancel the tsync_works_provide() change to recycle the reserved memory | |
| 234 | + * for the next thread, if any. This also ensures that cancel_tsync_works() | |
| 235 | + * and tsync_works_release() do not see any NULL task pointers. | |
| 236 | + */ | |
| 237 | + s->size--; | |
| 238 | +} | |
| 239 | + | |
| 206 | 240 | /* |
| 207 | 241 | * tsync_works_grow_by - preallocates space for n more contexts in s |
| 208 | 242 | * |
| @@ -256,13 +290,14 @@ static int tsync_works_grow_by(struct tsync_works *s, size_t n, gfp_t flags) · 47 unmodified lines | ||
| 256 | 290 | * tsync_works_contains - checks for presence of task in s |
| 257 | 291 | */ |
| 258 | 292 | static bool tsync_works_contains_task(const struct tsync_works *s, |
| 259 | − struct task_struct *task) | |
| 293 | + const struct task_struct *task) | |
| 260 | 294 | { |
| 261 | 295 | size_t i; |
| 262 | 296 | |
| 263 | 297 | for (i = 0; i < s->size; i++) |
| 264 | 298 | if (s->works[i]->task == task) |
| 265 | 299 | return true; |
| 300 | + | |
| 266 | 301 | return false; |
| 267 | 302 | } |
| 268 | 303 | |
| @@ -276,7 +311,7 @@ static void tsync_works_release(struct tsync_works *s) · 7 unmodified lines | ||
| 276 | 311 | size_t i; |
| 277 | 312 | |
| 278 | 313 | for (i = 0; i < s->size; i++) { |
| 279 | − if (!s->works[i]->task) | |
| 314 | + if (WARN_ON_ONCE(!s->works[i]->task)) | |
| 280 | 315 | continue; |
| 281 | 316 | |
| 282 | 317 | put_task_struct(s->works[i]->task); |
| @@ -284,6 +319,7 @@ static void tsync_works_release(struct tsync_works *s) · 1 unmodified line | ||
| 284 | 319 | |
| 285 | 320 | for (i = 0; i < s->capacity; i++) |
| 286 | 321 | kfree(s->works[i]); |
| 322 | + | |
| 287 | 323 | kfree(s->works); |
| 288 | 324 | s->works = NULL; |
| 289 | 325 | s->size = 0; |
| @@ -295,7 +331,7 @@ static void tsync_works_release(struct tsync_works *s) · 5 unmodified lines | ||
| 295 | 331 | */ |
| 296 | 332 | static size_t count_additional_threads(const struct tsync_works *works) |
| 297 | 333 | { |
| 298 | − struct task_struct *thread, *caller; | |
| 334 | + const struct task_struct *caller, *thread; | |
| 299 | 335 | size_t n = 0; |
| 300 | 336 | |
| 301 | 337 | caller = current; |
| @@ -334,7 +370,8 @@ static bool schedule_task_work(struct tsync_works *works, · 32 unmodified lines | ||
| 334 | 370 | struct tsync_shared_context *shared_ctx) |
| 335 | 371 | { |
| 336 | 372 | int err; |
| 337 | − struct task_struct *thread, *caller; | |
| 373 | + const struct task_struct *caller; | |
| 374 | + struct task_struct *thread; | |
| 338 | 375 | struct tsync_work *ctx; |
| 339 | 376 | bool found_more_threads = false; |
| 340 | 377 | |
| @@ -379,16 +416,14 @@ static bool schedule_task_work(struct tsync_works *works, · 38 unmodified lines | ||
| 379 | 416 | |
| 380 | 417 | init_task_work(&ctx->work, restrict_one_thread_callback); |
| 381 | 418 | err = task_work_add(thread, &ctx->work, TWA_SIGNAL); |
| 382 | − if (err) { | |
| 419 | + if (unlikely(err)) { | |
| 383 | 420 | /* |
| 384 | 421 | * task_work_add() only fails if the task is about to exit. We |
| 385 | 422 | * checked that earlier, but it can happen as a race. Resume |
| 386 | 423 | * without setting an error, as the task is probably gone in the |
| 387 | − * next loop iteration. For consistency, remove the task from ctx | |
| 388 | − * so that it does not look like we handed it a task_work. | |
| 424 | + * next loop iteration. | |
| 389 | 425 | */ |
| 390 | − put_task_struct(ctx->task); | |
| 391 | − ctx->task = NULL; | |
| 426 | + tsync_works_trim(works); | |
| 392 | 427 | |
| 393 | 428 | atomic_dec(&shared_ctx->num_preparing); |
| 394 | 429 | atomic_dec(&shared_ctx->num_unfinished); |
| @@ -406,12 +441,15 @@ static bool schedule_task_work(struct tsync_works *works, · 11 unmodified lines | ||
| 406 | 441 | * shared_ctx->num_preparing and shared_ctx->num_unfished and mark the two |
| 407 | 442 | * completions if needed, as if the task was never scheduled. |
| 408 | 443 | */ |
| 409 | −static void cancel_tsync_works(struct tsync_works *works, | |
| 444 | +static void cancel_tsync_works(const struct tsync_works *works, | |
| 410 | 445 | struct tsync_shared_context *shared_ctx) |
| 411 | 446 | { |
| 412 | − int i; | |
| 447 | + size_t i; | |
| 413 | 448 | |
| 414 | 449 | for (i = 0; i < works->size; i++) { |
| 450 | + if (WARN_ON_ONCE(!works->works[i]->task)) | |
| 451 | + continue; | |
| 452 | + | |
| 415 | 453 | if (!task_work_cancel(works->works[i]->task, |
| 416 | 454 | &works->works[i]->work)) |
| 417 | 455 | continue; |
| @@ -447,6 +485,16 @@ int landlock_restrict_sibling_threads(const struct cred *old_cred, · 29 unmodified lines | ||
| 447 | 485 | shared_ctx.new_cred = new_cred; |
| 448 | 486 | shared_ctx.set_no_new_privs = task_no_new_privs(current); |
| 449 | 487 | |
| 488 | + /* | |
| 489 | + * Serialize concurrent TSYNC operations to prevent deadlocks when | |
| 490 | + * multiple threads call landlock_restrict_self() simultaneously. | |
| 491 | + * If the lock is already held, we gracefully yield by restarting the | |
| 492 | + * syscall. This allows the current thread to process pending | |
| 493 | + * task_works before retrying. | |
| 494 | + */ | |
| 495 | + if (!down_write_trylock(¤t->signal->exec_update_lock)) | |
| 496 | + return restart_syscall(); | |
| 497 | + | |
| 450 | 498 | /* |
| 451 | 499 | * We schedule a pseudo-signal task_work for each of the calling task's |
| 452 | 500 | * sibling threads. In the task work, each thread: |
| @@ -527,24 +575,30 @@ int landlock_restrict_sibling_threads(const struct cred *old_cred, · 74 unmodified lines | ||
| 527 | 575 | -ERESTARTNOINTR); |
| 528 | 576 | |
| 529 | 577 | /* |
| 530 | − * Cancel task works for tasks that did not start running yet, | |
| 531 | − * and decrement all_prepared and num_unfinished accordingly. | |
| 578 | + * Opportunistic improvement: try to cancel task | |
| 579 | + * works for tasks that did not start running | |
| 580 | + * yet. We do not have a guarantee that it | |
| 581 | + * cancels any of the enqueued task works | |
| 582 | + * because task_work_run() might already have | |
| 583 | + * dequeued them. | |
| 532 | 584 | */ |
| 533 | 585 | cancel_tsync_works(&works, &shared_ctx); |
| 534 | 586 | |
| 535 | 587 | /* |
| 536 | − * The remaining task works have started running, so waiting for | |
| 537 | − * their completion will finish. | |
| 588 | + * Break the loop with error. The cleanup code | |
| 589 | + * after the loop unblocks the remaining | |
| 590 | + * task_works. | |
| 538 | 591 | */ |
| 539 | − wait_for_completion(&shared_ctx.all_prepared); | |
| 592 | + break; | |
| 540 | 593 | } |
| 541 | 594 | } |
| 542 | 595 | } while (found_more_threads && |
| 543 | 596 | !atomic_read(&shared_ctx.preparation_error)); |
| 544 | 597 | |
| 545 | 598 | /* |
| 546 | − * We now have all sibling threads blocking and in "prepared" state in the | |
| 547 | − * task work. Ask all threads to commit. | |
| 599 | + * We now have either (a) all sibling threads blocking and in "prepared" | |
| 600 | + * state in the task work, or (b) the preparation error is set. Ask all | |
| 601 | + * threads to commit (or abort). | |
| 548 | 602 | */ |
| 549 | 603 | complete_all(&shared_ctx.ready_to_commit); |
| 550 | 604 | |
| @@ -556,6 +610,6 @@ int landlock_restrict_sibling_threads(const struct cred *old_cred, · 5 unmodified lines | ||
| 556 | 610 | wait_for_completion(&shared_ctx.all_finished); |
| 557 | 611 | |
| 558 | 612 | tsync_works_release(&works); |
| 559 | − | |
| 613 | + up_write(¤t->signal->exec_update_lock); | |
| 560 | 614 | return atomic_read(&shared_ctx.preparation_error); |
| 561 | 615 | } |
| @@ -61,6 +61,7 @@ const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX + 1] = { · 60 unmodified lines | ||
| 61 | 61 | [LOCKDOWN_BPF_WRITE_USER] = "use of bpf to write user RAM", |
| 62 | 62 | [LOCKDOWN_DBG_WRITE_KERNEL] = "use of kgdb/kdb to write kernel RAM", |
| 63 | 63 | [LOCKDOWN_RTAS_ERROR_INJECTION] = "RTAS error injection", |
| 64 | + [LOCKDOWN_XEN_USER_ACTIONS] = "Xen guest user action", | |
| 64 | 65 | [LOCKDOWN_INTEGRITY_MAX] = "integrity", |
| 65 | 66 | [LOCKDOWN_KCORE] = "/proc/kcore access", |
| 66 | 67 | [LOCKDOWN_KPROBES] = "use of kprobes", |
| @@ -1164,7 +1164,7 @@ static void process_rx_packets(struct fw_iso_context *context, u32 tstamp, size_ · 1163 unmodified lines | ||
| 1164 | 1164 | struct pkt_desc *desc = s->packet_descs_cursor; |
| 1165 | 1165 | unsigned int pkt_header_length; |
| 1166 | 1166 | unsigned int packets; |
| 1167 | − u32 curr_cycle_time; | |
| 1167 | + u32 curr_cycle_time = 0; | |
| 1168 | 1168 | bool need_hw_irq; |
| 1169 | 1169 | int i; |
| 1170 | 1170 | |
| @@ -1017,12 +1017,30 @@ static int alc269_resume(struct hda_codec *codec) · 1016 unmodified lines | ||
| 1017 | 1017 | return 0; |
| 1018 | 1018 | } |
| 1019 | 1019 | |
| 1020 | −#define STARLABS_STARFIGHTER_SHUTUP_DELAY_MS 30 | |
| 1020 | +#define ALC233_STARFIGHTER_SPK_PIN 0x1b | |
| 1021 | +#define ALC233_STARFIGHTER_GPIO2 0x04 | |
| 1021 | 1022 | |
| 1022 | −static void starlabs_starfighter_shutup(struct hda_codec *codec) | |
| 1023 | +static void alc233_starfighter_update_amp(struct hda_codec *codec, bool on) | |
| 1023 | 1024 | { |
| 1024 | − if (snd_hda_gen_shutup_speakers(codec)) | |
| 1025 | − msleep(STARLABS_STARFIGHTER_SHUTUP_DELAY_MS); | |
| 1025 | + snd_hda_codec_write(codec, ALC233_STARFIGHTER_SPK_PIN, 0, | |
| 1026 | + AC_VERB_SET_EAPD_BTLENABLE, | |
| 1027 | + on ? AC_EAPDBTL_EAPD : 0); | |
| 1028 | + alc_update_gpio_data(codec, ALC233_STARFIGHTER_GPIO2, on); | |
| 1029 | +} | |
| 1030 | + | |
| 1031 | +static void alc233_starfighter_pcm_hook(struct hda_pcm_stream *hinfo, | |
| 1032 | + struct hda_codec *codec, | |
| 1033 | + struct snd_pcm_substream *substream, | |
| 1034 | + int action) | |
| 1035 | +{ | |
| 1036 | + switch (action) { | |
| 1037 | + case HDA_GEN_PCM_ACT_PREPARE: | |
| 1038 | + alc233_starfighter_update_amp(codec, true); | |
| 1039 | + break; | |
| 1040 | + case HDA_GEN_PCM_ACT_CLEANUP: | |
| 1041 | + alc233_starfighter_update_amp(codec, false); | |
| 1042 | + break; | |
| 1043 | + } | |
| 1026 | 1044 | } |
| 1027 | 1045 | |
| 1028 | 1046 | static void alc233_fixup_starlabs_starfighter(struct hda_codec *codec, |
| @@ -1031,8 +1049,16 @@ static void alc233_fixup_starlabs_starfighter(struct hda_codec *codec, · 2 unmodified lines | ||
| 1031 | 1049 | { |
| 1032 | 1050 | struct alc_spec *spec = codec->spec; |
| 1033 | 1051 | |
| 1034 | − if (action == HDA_FIXUP_ACT_PRE_PROBE) | |
| 1035 | − spec->shutup = starlabs_starfighter_shutup; | |
| 1052 | + switch (action) { | |
| 1053 | + case HDA_FIXUP_ACT_PRE_PROBE: | |
| 1054 | + spec->gpio_mask |= ALC233_STARFIGHTER_GPIO2; | |
| 1055 | + spec->gpio_dir |= ALC233_STARFIGHTER_GPIO2; | |
| 1056 | + spec->gpio_data &= ~ALC233_STARFIGHTER_GPIO2; | |
| 1057 | + break; | |
| 1058 | + case HDA_FIXUP_ACT_PROBE: | |
| 1059 | + spec->gen.pcm_playback_hook = alc233_starfighter_pcm_hook; | |
| 1060 | + break; | |
| 1061 | + } | |
| 1036 | 1062 | } |
| 1037 | 1063 | |
| 1038 | 1064 | static void alc269_fixup_pincfg_no_hp_to_lineout(struct hda_codec *codec, |
| @@ -3699,22 +3725,42 @@ static void alc245_tas2781_spi_hp_fixup_muteled(struct hda_codec *codec, · 2660 unmodified lines | ||
| 3699 | 3725 | alc_fixup_hp_gpio_led(codec, action, 0x04, 0x0); |
| 3700 | 3726 | alc285_fixup_hp_coef_micmute_led(codec, fix, action); |
| 3701 | 3727 | } |
| 3728 | + | |
| 3729 | +static void alc245_hp_spk_mute_led_update(void *private_data, int enabled) | |
| 3730 | +{ | |
| 3731 | + struct hda_codec *codec = private_data; | |
| 3732 | + unsigned int val; | |
| 3733 | + | |
| 3734 | + val = enabled ? 0x08 : 0x04; /* 0x08 led on, 0x04 led off */ | |
| 3735 | + alc_update_coef_idx(codec, 0x0b, 0x0c, val); | |
| 3736 | +} | |
| 3737 | + | |
| 3702 | 3738 | /* JD2: mute led GPIO3: micmute led */ |
| 3703 | 3739 | static void alc245_tas2781_i2c_hp_fixup_muteled(struct hda_codec *codec, |
| 3704 | 3740 | const struct hda_fixup *fix, int action) |
| 3705 | 3741 | { |
| 3706 | 3742 | struct alc_spec *spec = codec->spec; |
| 3743 | + hda_nid_t hp_pin = alc_get_hp_pin(spec); | |
| 3707 | 3744 | static const hda_nid_t conn[] = { 0x02 }; |
| 3708 | 3745 | |
| 3709 | 3746 | switch (action) { |
| 3710 | 3747 | case HDA_FIXUP_ACT_PRE_PROBE: |
| 3748 | + if (!hp_pin) { | |
| 3749 | + spec->gen.vmaster_mute.hook = alc245_hp_spk_mute_led_update; | |
| 3750 | + spec->gen.vmaster_mute_led = 1; | |
| 3751 | + } | |
| 3711 | 3752 | spec->gen.auto_mute_via_amp = 1; |
| 3712 | 3753 | snd_hda_override_conn_list(codec, 0x17, ARRAY_SIZE(conn), conn); |
| 3713 | 3754 | break; |
| 3755 | + case HDA_FIXUP_ACT_INIT: | |
| 3756 | + if (!hp_pin) | |
| 3757 | + alc245_hp_spk_mute_led_update(codec, !spec->gen.master_mute); | |
| 3758 | + break; | |
| 3714 | 3759 | } |
| 3715 | 3760 | |
| 3716 | 3761 | tas2781_fixup_txnw_i2c(codec, fix, action); |
| 3717 | − alc245_fixup_hp_mute_led_coefbit(codec, fix, action); | |
| 3762 | + if (hp_pin) | |
| 3763 | + alc245_fixup_hp_mute_led_coefbit(codec, fix, action); | |
| 3718 | 3764 | alc285_fixup_hp_coef_micmute_led(codec, fix, action); |
| 3719 | 3765 | } |
| 3720 | 3766 | /* |
| @@ -6854,6 +6900,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { · 3133 unmodified lines | ||
| 6854 | 6900 | SND_PCI_QUIRK(0x103c, 0x8730, "HP ProBook 445 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), |
| 6855 | 6901 | SND_PCI_QUIRK(0x103c, 0x8735, "HP ProBook 435 G7", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), |
| 6856 | 6902 | SND_PCI_QUIRK(0x103c, 0x8736, "HP", ALC285_FIXUP_HP_GPIO_AMP_INIT), |
| 6903 | + SND_PCI_QUIRK(0x103c, 0x8756, "HP ENVY Laptop 13-ba0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), | |
| 6857 | 6904 | SND_PCI_QUIRK(0x103c, 0x8760, "HP EliteBook 8{4,5}5 G7", ALC285_FIXUP_HP_BEEP_MICMUTE_LED), |
| 6858 | 6905 | SND_PCI_QUIRK(0x103c, 0x876e, "HP ENVY x360 Convertible 13-ay0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), |
| 6859 | 6906 | SND_PCI_QUIRK(0x103c, 0x877a, "HP", ALC285_FIXUP_HP_MUTE_LED), |
| @@ -6867,6 +6914,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { · 7 unmodified lines | ||
| 6867 | 6914 | SND_PCI_QUIRK(0x103c, 0x8788, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED), |
| 6868 | 6915 | SND_PCI_QUIRK(0x103c, 0x87b7, "HP Laptop 14-fq0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), |
| 6869 | 6916 | SND_PCI_QUIRK(0x103c, 0x87c8, "HP", ALC287_FIXUP_HP_GPIO_LED), |
| 6917 | + SND_PCI_QUIRK(0x103c, 0x87cb, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), | |
| 6870 | 6918 | SND_PCI_QUIRK(0x103c, 0x87cc, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), |
| 6871 | 6919 | SND_PCI_QUIRK(0x103c, 0x87d3, "HP Laptop 15-gw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), |
| 6872 | 6920 | SND_PCI_QUIRK(0x103c, 0x87df, "HP ProBook 430 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), |
| @@ -7098,6 +7146,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { · 225 unmodified lines | ||
| 7098 | 7146 | SND_PCI_QUIRK(0x103c, 0x8da7, "HP 14 Enstrom OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), |
| 7099 | 7147 | SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), |
| 7100 | 7148 | SND_PCI_QUIRK(0x103c, 0x8dd4, "HP EliteStudio 8 AIO", ALC274_FIXUP_HP_AIO_BIND_DACS), |
| 7149 | + SND_PCI_QUIRK(0x103c, 0x8dd7, "HP Laptop 15-fd0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), | |
| 7101 | 7150 | SND_PCI_QUIRK(0x103c, 0x8de8, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2), |
| 7102 | 7151 | SND_PCI_QUIRK(0x103c, 0x8de9, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2), |
| 7103 | 7152 | SND_PCI_QUIRK(0x103c, 0x8dec, "HP EliteBook 640 G12", ALC236_FIXUP_HP_GPIO_LED), |
| @@ -7177,6 +7226,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { · 73 unmodified lines | ||
| 7177 | 7226 | SND_PCI_QUIRK(0x1043, 0x115d, "Asus 1015E", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), |
| 7178 | 7227 | SND_PCI_QUIRK(0x1043, 0x1194, "ASUS UM3406KA", ALC287_FIXUP_CS35L41_I2C_2), |
| 7179 | 7228 | SND_PCI_QUIRK(0x1043, 0x11c0, "ASUS X556UR", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), |
| 7229 | + HDA_CODEC_QUIRK(0x1043, 0x1204, "ASUS Strix G16 G615JMR", ALC287_FIXUP_TXNW2781_I2C_ASUS), | |
| 7180 | 7230 | SND_PCI_QUIRK(0x1043, 0x1204, "ASUS Strix G615JHR_JMR_JPR", ALC287_FIXUP_TAS2781_I2C), |
| 7181 | 7231 | SND_PCI_QUIRK(0x1043, 0x1214, "ASUS Strix G615LH_LM_LP", ALC287_FIXUP_TAS2781_I2C), |
| 7182 | 7232 | SND_PCI_QUIRK(0x1043, 0x125e, "ASUS Q524UQK", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), |
| @@ -7206,6 +7256,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { · 23 unmodified lines | ||
| 7206 | 7256 | SND_PCI_QUIRK(0x1043, 0x14e3, "ASUS G513PI/PU/PV", ALC287_FIXUP_CS35L41_I2C_2), |
| 7207 | 7257 | SND_PCI_QUIRK(0x1043, 0x14f2, "ASUS VivoBook X515JA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), |
| 7208 | 7258 | SND_PCI_QUIRK(0x1043, 0x1503, "ASUS G733PY/PZ/PZV/PYV", ALC287_FIXUP_CS35L41_I2C_2), |
| 7259 | + SND_PCI_QUIRK(0x1043, 0x1514, "ASUS ROG Flow Z13 GZ302EAC", ALC287_FIXUP_CS35L41_I2C_2), | |
| 7209 | 7260 | SND_PCI_QUIRK(0x1043, 0x1517, "Asus Zenbook UX31A", ALC269VB_FIXUP_ASUS_ZENBOOK_UX31A), |
| 7210 | 7261 | SND_PCI_QUIRK(0x1043, 0x1533, "ASUS GV302XA/XJ/XQ/XU/XV/XI", ALC287_FIXUP_CS35L41_I2C_2), |
| 7211 | 7262 | SND_PCI_QUIRK(0x1043, 0x1573, "ASUS GZ301VV/VQ/VU/VJ/VA/VC/VE/VVC/VQC/VUC/VJC/VEC/VCC", ALC285_FIXUP_ASUS_HEADSET_MIC), |
| @@ -7575,6 +7626,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { · 363 unmodified lines | ||
| 7575 | 7626 | SND_PCI_QUIRK(0x17aa, 0x38ab, "Thinkbook 16P", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), |
| 7576 | 7627 | SND_PCI_QUIRK(0x17aa, 0x38b4, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), |
| 7577 | 7628 | HDA_CODEC_QUIRK(0x17aa, 0x391c, "Lenovo Yoga 7 2-in-1 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), |
| 7629 | + HDA_CODEC_QUIRK(0x17aa, 0x391d, "Lenovo Yoga 7 2-in-1 16AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), | |
| 7578 | 7630 | SND_PCI_QUIRK(0x17aa, 0x38b5, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), |
| 7579 | 7631 | SND_PCI_QUIRK(0x17aa, 0x38b6, "Legion Slim 7 16APH8", ALC287_FIXUP_CS35L41_I2C_2), |
| 7580 | 7632 | SND_PCI_QUIRK(0x17aa, 0x38b7, "Legion Slim 7 16APH8", ALC287_FIXUP_CS35L41_I2C_2), |
| @@ -2077,7 +2077,6 @@ static const struct pci_device_id driver_denylist[] = { · 2076 unmodified lines | ||
| 2077 | 2077 | { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1043, 0x874f) }, /* ASUS ROG Zenith II / Strix */ |
| 2078 | 2078 | { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1462, 0xcb59) }, /* MSI TRX40 Creator */ |
| 2079 | 2079 | { PCI_DEVICE_SUB(0x1022, 0x1487, 0x1462, 0xcb60) }, /* MSI TRX40 */ |
| 2080 | − { PCI_DEVICE_SUB(0x1022, 0x15e3, 0x1462, 0xee59) }, /* MSI X870E Tomahawk WiFi */ | |
| 2081 | 2080 | {} |
| 2082 | 2081 | }; |
| 2083 | 2082 | |
| @@ -581,8 +581,10 @@ static u16 adapter_prepare(u16 adapter) · 580 unmodified lines | ||
| 581 | 581 | HPI_ADAPTER_OPEN); |
| 582 | 582 | hm.adapter_index = adapter; |
| 583 | 583 | hw_entry_point(&hm, &hr); |
| 584 | − memcpy(&rESP_HPI_ADAPTER_OPEN[adapter], &hr, | |
| 585 | − sizeof(rESP_HPI_ADAPTER_OPEN[0])); | |
| 584 | + memcpy(&rESP_HPI_ADAPTER_OPEN[adapter].h, &hr, | |
| 585 | + sizeof(rESP_HPI_ADAPTER_OPEN[adapter].h)); | |
| 586 | + memcpy(&rESP_HPI_ADAPTER_OPEN[adapter].a, &hr.u.ax.info, | |
| 587 | + sizeof(rESP_HPI_ADAPTER_OPEN[adapter].a)); | |
| 586 | 588 | if (hr.error) |
| 587 | 589 | return hr.error; |
| 588 | 590 | |
| @@ -52,6 +52,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { · 51 unmodified lines | ||
| 52 | 52 | DMI_MATCH(DMI_PRODUCT_NAME, "Bellator N176"), |
| 53 | 53 | } |
| 54 | 54 | }, |
| 55 | + { | |
| 56 | + .driver_data = &acp6x_card, | |
| 57 | + .matches = { | |
| 58 | + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), | |
| 59 | + DMI_MATCH(DMI_PRODUCT_NAME, "HP Laptop 15-fc0xxx"), | |
| 60 | + } | |
| 61 | + }, | |
| 55 | 62 | { |
| 56 | 63 | .driver_data = &acp6x_card, |
| 57 | 64 | .matches = { |
| @@ -752,6 +759,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { · 694 unmodified lines | ||
| 752 | 759 | DMI_MATCH(DMI_PRODUCT_NAME, "Thin A15 B7VE"), |
| 753 | 760 | } |
| 754 | 761 | }, |
| 762 | + { | |
| 763 | + .driver_data = &acp6x_card, | |
| 764 | + .matches = { | |
| 765 | + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), | |
| 766 | + DMI_MATCH(DMI_PRODUCT_NAME, "M7601RM"), | |
| 767 | + } | |
| 768 | + }, | |
| 755 | 769 | {} |
| 756 | 770 | }; |
| 757 | 771 | |
| @@ -762,7 +762,7 @@ static int adau1372_startup(struct snd_pcm_substream *substream, struct snd_soc_ · 761 unmodified lines | ||
| 762 | 762 | return 0; |
| 763 | 763 | } |
| 764 | 764 | |
| 765 | −static void adau1372_enable_pll(struct adau1372 *adau1372) | |
| 765 | +static int adau1372_enable_pll(struct adau1372 *adau1372) | |
| 766 | 766 | { |
| 767 | 767 | unsigned int val, timeout = 0; |
| 768 | 768 | int ret; |
| @@ -778,19 +778,26 @@ static void adau1372_enable_pll(struct adau1372 *adau1372) · 9 unmodified lines | ||
| 778 | 778 | timeout++; |
| 779 | 779 | } while (!(val & 1) && timeout < 3); |
| 780 | 780 | |
| 781 | − if (ret < 0 || !(val & 1)) | |
| 781 | + if (ret < 0 || !(val & 1)) { | |
| 782 | 782 | dev_err(adau1372->dev, "Failed to lock PLL\n"); |
| 783 | + return ret < 0 ? ret : -ETIMEDOUT; | |
| 784 | + } | |
| 785 | + | |
| 786 | + return 0; | |
| 783 | 787 | } |
| 784 | 788 | |
| 785 | −static void adau1372_set_power(struct adau1372 *adau1372, bool enable) | |
| 789 | +static int adau1372_set_power(struct adau1372 *adau1372, bool enable) | |
| 786 | 790 | { |
| 787 | 791 | if (adau1372->enabled == enable) |
| 788 | − return; | |
| 792 | + return 0; | |
| 789 | 793 | |
| 790 | 794 | if (enable) { |
| 791 | 795 | unsigned int clk_ctrl = ADAU1372_CLK_CTRL_MCLK_EN; |
| 796 | + int ret; | |
| 792 | 797 | |
| 793 | − clk_prepare_enable(adau1372->mclk); | |
| 798 | + ret = clk_prepare_enable(adau1372->mclk); | |
| 799 | + if (ret) | |
| 800 | + return ret; | |
| 794 | 801 | if (adau1372->pd_gpio) |
| 795 | 802 | gpiod_set_value(adau1372->pd_gpio, 0); |
| 796 | 803 | |
| @@ -804,7 +811,14 @@ static void adau1372_set_power(struct adau1372 *adau1372, bool enable) · 7 unmodified lines | ||
| 804 | 811 | * accessed. |
| 805 | 812 | */ |
| 806 | 813 | if (adau1372->use_pll) { |
| 807 | − adau1372_enable_pll(adau1372); | |
| 814 | + ret = adau1372_enable_pll(adau1372); | |
| 815 | + if (ret) { | |
| 816 | + regcache_cache_only(adau1372->regmap, true); | |
| 817 | + if (adau1372->pd_gpio) | |
| 818 | + gpiod_set_value(adau1372->pd_gpio, 1); | |
| 819 | + clk_disable_unprepare(adau1372->mclk); | |
| 820 | + return ret; | |
| 821 | + } | |
| 808 | 822 | clk_ctrl |= ADAU1372_CLK_CTRL_CLKSRC; |
| 809 | 823 | } |
| 810 | 824 | |
| @@ -829,6 +843,8 @@ static void adau1372_set_power(struct adau1372 *adau1372, bool enable) · 18 unmodified lines | ||
| 829 | 843 | } |
| 830 | 844 | |
| 831 | 845 | adau1372->enabled = enable; |
| 846 | + | |
| 847 | + return 0; | |
| 832 | 848 | } |
| 833 | 849 | |
| 834 | 850 | static int adau1372_set_bias_level(struct snd_soc_component *component, |
| @@ -842,11 +858,9 @@ static int adau1372_set_bias_level(struct snd_soc_component *component, · 7 unmodified lines | ||
| 842 | 858 | case SND_SOC_BIAS_PREPARE: |
| 843 | 859 | break; |
| 844 | 860 | case SND_SOC_BIAS_STANDBY: |
| 845 | − adau1372_set_power(adau1372, true); | |
| 846 | − break; | |
| 861 | + return adau1372_set_power(adau1372, true); | |
| 847 | 862 | case SND_SOC_BIAS_OFF: |
| 848 | − adau1372_set_power(adau1372, false); | |
| 849 | − break; | |
| 863 | + return adau1372_set_power(adau1372, false); | |
| 850 | 864 | } |
| 851 | 865 | |
| 852 | 866 | return 0; |
| @@ -216,9 +216,6 @@ static int find_sdca_init_table(struct device *dev, · 215 unmodified lines | ||
| 216 | 216 | } else if (num_init_writes % sizeof(*raw) != 0) { |
| 217 | 217 | dev_err(dev, "%pfwP: init table size invalid\n", function_node); |
| 218 | 218 | return -EINVAL; |
| 219 | − } else if ((num_init_writes / sizeof(*raw)) > SDCA_MAX_INIT_COUNT) { | |
| 220 | − dev_err(dev, "%pfwP: maximum init table size exceeded\n", function_node); | |
| 221 | − return -EINVAL; | |
| 222 | 219 | } |
| 223 | 220 | |
| 224 | 221 | raw = kzalloc(num_init_writes, GFP_KERNEL); |
| @@ -1604,10 +1601,19 @@ static int find_sdca_entities(struct device *dev, struct sdw_slave *sdw, · 1379 unmodified lines | ||
| 1604 | 1601 | static struct sdca_entity *find_sdca_entity_by_label(struct sdca_function_data *function, |
| 1605 | 1602 | const char *entity_label) |
| 1606 | 1603 | { |
| 1604 | + struct sdca_entity *entity = NULL; | |
| 1607 | 1605 | int i; |
| 1608 | 1606 | |
| 1609 | 1607 | for (i = 0; i < function->num_entities; i++) { |
| 1610 | − struct sdca_entity *entity = &function->entities[i]; | |
| 1608 | + entity = &function->entities[i]; | |
| 1609 | + | |
| 1610 | + /* check whole string first*/ | |
| 1611 | + if (!strcmp(entity->label, entity_label)) | |
| 1612 | + return entity; | |
| 1613 | + } | |
| 1614 | + | |
| 1615 | + for (i = 0; i < function->num_entities; i++) { | |
| 1616 | + entity = &function->entities[i]; | |
| 1611 | 1617 | |
| 1612 | 1618 | if (!strncmp(entity->label, entity_label, strlen(entity_label))) |
| 1613 | 1619 | return entity; |
| @@ -2951,7 +2951,7 @@ static int sof_ipc4_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_ · 2950 unmodified lines | ||
| 2951 | 2951 | return -EINVAL; |
| 2952 | 2952 | } |
| 2953 | 2953 | |
| 2954 | − if (scontrol->priv_size < sizeof(struct sof_abi_hdr)) { | |
| 2954 | + if (scontrol->priv_size && scontrol->priv_size < sizeof(struct sof_abi_hdr)) { | |
| 2955 | 2955 | dev_err(sdev->dev, |
| 2956 | 2956 | "bytes control %s initial data size %zu is insufficient.\n", |
| 2957 | 2957 | scontrol->name, scontrol->priv_size); |
| @@ -192,6 +192,7 @@ config SND_USB_AUDIO_QMI · 191 unmodified lines | ||
| 192 | 192 | tristate "Qualcomm Audio Offload driver" |
| 193 | 193 | depends on QCOM_QMI_HELPERS && SND_USB_AUDIO && SND_SOC_USB |
| 194 | 194 | depends on USB_XHCI_HCD && USB_XHCI_SIDEBAND |
| 195 | + select AUXILIARY_BUS | |
| 195 | 196 | help |
| 196 | 197 | Say Y here to enable the Qualcomm USB audio offloading feature. |
| 197 | 198 | |
| @@ -1,6 +1,6 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | 2 | /* |
| 3 | − * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved. | |
| 3 | + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. | |
| 4 | 4 | */ |
| 5 | 5 | |
| 6 | 6 | #include <linux/auxiliary_bus.h> |
| @@ -2148,6 +2148,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { · 2147 unmodified lines | ||
| 2148 | 2148 | /* Device matches */ |
| 2149 | 2149 | DEVICE_FLG(0x001f, 0x0b21, /* AB13X USB Audio */ |
| 2150 | 2150 | QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), |
| 2151 | + DEVICE_FLG(0x001f, 0x0b23, /* AB17X USB Audio */ | |
| 2152 | + QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), | |
| 2151 | 2153 | DEVICE_FLG(0x0020, 0x0b21, /* GHW-123P */ |
| 2152 | 2154 | QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), |
| 2153 | 2155 | DEVICE_FLG(0x03f0, 0x654a, /* HP 320 FHD Webcam */ |
| @@ -2429,6 +2431,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { · 275 unmodified lines | ||
| 2429 | 2431 | QUIRK_FLAG_CTL_MSG_DELAY | QUIRK_FLAG_IFACE_DELAY), |
| 2430 | 2432 | VENDOR_FLG(0x07fd, /* MOTU */ |
| 2431 | 2433 | QUIRK_FLAG_VALIDATE_RATES), |
| 2434 | + DEVICE_FLG(0x1235, 0x8006, 0), /* Focusrite Scarlett 2i2 1st Gen */ | |
| 2435 | + DEVICE_FLG(0x1235, 0x800a, 0), /* Focusrite Scarlett 2i4 1st Gen */ | |
| 2432 | 2436 | VENDOR_FLG(0x1235, /* Focusrite Novation */ |
| 2433 | 2437 | QUIRK_FLAG_SKIP_CLOCK_SELECTOR | |
| 2434 | 2438 | QUIRK_FLAG_SKIP_IFACE_SETUP), |
| @@ -740,7 +740,10 @@ · 739 unmodified lines | ||
| 740 | 740 | #define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT) |
| 741 | 741 | #define MSR_AMD64_SNP_SECURE_AVIC_BIT 18 |
| 742 | 742 | #define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT) |
| 743 | −#define MSR_AMD64_SNP_RESV_BIT 19 | |
| 743 | +#define MSR_AMD64_SNP_RESERVED_BITS19_22 GENMASK_ULL(22, 19) | |
| 744 | +#define MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT 23 | |
| 745 | +#define MSR_AMD64_SNP_IBPB_ON_ENTRY BIT_ULL(MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT) | |
| 746 | +#define MSR_AMD64_SNP_RESV_BIT 24 | |
| 744 | 747 | #define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT) |
| 745 | 748 | #define MSR_AMD64_SAVIC_CONTROL 0xc0010138 |
| 746 | 749 | #define MSR_AMD64_SAVIC_EN_BIT 0 |
| @@ -476,6 +476,7 @@ struct kvm_sync_regs { · 475 unmodified lines | ||
| 476 | 476 | #define KVM_X86_QUIRK_SLOT_ZAP_ALL (1 << 7) |
| 477 | 477 | #define KVM_X86_QUIRK_STUFF_FEATURE_MSRS (1 << 8) |
| 478 | 478 | #define KVM_X86_QUIRK_IGNORE_GUEST_PAT (1 << 9) |
| 479 | +#define KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM (1 << 10) | |
| 479 | 480 | |
| 480 | 481 | #define KVM_STATE_NESTED_FORMAT_VMX 0 |
| 481 | 482 | #define KVM_STATE_NESTED_FORMAT_SVM 1 |
| @@ -32,7 +32,8 @@ · 31 unmodified lines | ||
| 32 | 32 | /** |
| 33 | 33 | * BUILD_BUG_ON_MSG - break compile if a condition is true & emit supplied |
| 34 | 34 | * error message. |
| 35 | − * @condition: the condition which the compiler should know is false. | |
| 35 | + * @cond: the condition which the compiler should know is false. | |
| 36 | + * @msg: build-time error message | |
| 36 | 37 | * |
| 37 | 38 | * See BUILD_BUG_ON for description. |
| 38 | 39 | */ |
| @@ -60,6 +61,7 @@ · 21 unmodified lines | ||
| 60 | 61 | |
| 61 | 62 | /** |
| 62 | 63 | * static_assert - check integer constant expression at build time |
| 64 | + * @expr: expression to be checked | |
| 63 | 65 | * |
| 64 | 66 | * static_assert() is a wrapper for the C11 _Static_assert, with a |
| 65 | 67 | * little macro magic to make the message optional (defaulting to the |
| @@ -14,6 +14,10 @@ · 13 unmodified lines | ||
| 14 | 14 | #include <linux/ioctl.h> |
| 15 | 15 | #include <asm/kvm.h> |
| 16 | 16 | |
| 17 | +#ifdef __KERNEL__ | |
| 18 | +#include <linux/kvm_types.h> | |
| 19 | +#endif | |
| 20 | + | |
| 17 | 21 | #define KVM_API_VERSION 12 |
| 18 | 22 | |
| 19 | 23 | /* |
| @@ -1601,7 +1605,11 @@ struct kvm_stats_desc { · 1581 unmodified lines | ||
| 1601 | 1605 | __u16 size; |
| 1602 | 1606 | __u32 offset; |
| 1603 | 1607 | __u32 bucket_size; |
| 1608 | +#ifdef __KERNEL__ | |
| 1609 | + char name[KVM_STATS_NAME_SIZE]; | |
| 1610 | +#else | |
| 1604 | 1611 | char name[]; |
| 1612 | +#endif | |
| 1605 | 1613 | }; |
| 1606 | 1614 | |
| 1607 | 1615 | #define KVM_GET_STATS_FD _IO(KVMIO, 0xce) |
| @@ -187,7 +187,6 @@ done · 186 unmodified lines | ||
| 187 | 187 | check arch/x86/lib/memcpy_64.S '-I "^EXPORT_SYMBOL" -I "^#include <asm/export.h>" -I"^SYM_FUNC_START\(_LOCAL\)*(memcpy_\(erms\|orig\))" -I"^#include <linux/cfi_types.h>"' |
| 188 | 188 | check arch/x86/lib/memset_64.S '-I "^EXPORT_SYMBOL" -I "^#include <asm/export.h>" -I"^SYM_FUNC_START\(_LOCAL\)*(memset_\(erms\|orig\))"' |
| 189 | 189 | check arch/x86/include/asm/amd/ibs.h '-I "^#include .*/msr-index.h"' |
| 190 | −check arch/arm64/include/asm/cputype.h '-I "^#include [<\"]\(asm/\)*sysreg.h"' | |
| 191 | 190 | check include/linux/unaligned.h '-I "^#include <linux/unaligned/packed_struct.h>" -I "^#include <asm/byteorder.h>" -I "^#pragma GCC diagnostic"' |
| 192 | 191 | check include/uapi/asm-generic/mman.h '-I "^#include <\(uapi/\)*asm-generic/mman-common\(-tools\)*.h>"' |
| 193 | 192 | check include/uapi/linux/mman.h '-I "^#include <\(uapi/\)*asm/mman.h>"' |
| @@ -4,9 +4,9 @@ · 3 unmodified lines | ||
| 4 | 4 | #include "../kvm-stat.h" |
| 5 | 5 | #include "../evsel.h" |
| 6 | 6 | #include "../env.h" |
| 7 | −#include "../../arch/x86/include/uapi/asm/svm.h" | |
| 8 | −#include "../../arch/x86/include/uapi/asm/vmx.h" | |
| 9 | −#include "../../arch/x86/include/uapi/asm/kvm.h" | |
| 7 | +#include "../../../arch/x86/include/uapi/asm/svm.h" | |
| 8 | +#include "../../../arch/x86/include/uapi/asm/vmx.h" | |
| 9 | +#include "../../../arch/x86/include/uapi/asm/kvm.h" | |
| 10 | 10 | #include <subcmd/parse-options.h> |
| 11 | 11 | |
| 12 | 12 | define_exit_reasons_table(vmx_exit_reasons, VMX_EXIT_REASONS); |
| @@ -1605,9 +1605,9 @@ bool metricgroup__has_metric_or_groups(const char *pmu, const char *metric_or_gr · 1604 unmodified lines | ||
| 1605 | 1605 | .metric_or_groups = metric_or_groups, |
| 1606 | 1606 | }; |
| 1607 | 1607 | |
| 1608 | − return pmu_metrics_table__for_each_metric(table, | |
| 1609 | − metricgroup__has_metric_or_groups_callback, | |
| 1610 | − &data) | |
| 1608 | + return metricgroup__for_each_metric(table, | |
| 1609 | + metricgroup__has_metric_or_groups_callback, | |
| 1610 | + &data) | |
| 1611 | 1611 | ? true : false; |
| 1612 | 1612 | } |
| 1613 | 1613 | |
| @@ -1117,7 +1117,7 @@ static int config_attr(struct perf_event_attr *attr, · 1116 unmodified lines | ||
| 1117 | 1117 | |
| 1118 | 1118 | static struct evsel_config_term *add_config_term(enum evsel_term_type type, |
| 1119 | 1119 | struct list_head *head_terms, |
| 1120 | − bool weak) | |
| 1120 | + bool weak, char *str, u64 val) | |
| 1121 | 1121 | { |
| 1122 | 1122 | struct evsel_config_term *t; |
| 1123 | 1123 | |
| @@ -1128,8 +1128,62 @@ static struct evsel_config_term *add_config_term(enum evsel_term_type type, · 4 unmodified lines | ||
| 1128 | 1128 | INIT_LIST_HEAD(&t->list); |
| 1129 | 1129 | t->type = type; |
| 1130 | 1130 | t->weak = weak; |
| 1131 | − list_add_tail(&t->list, head_terms); | |
| 1132 | 1131 | |
| 1132 | + switch (type) { | |
| 1133 | + case EVSEL__CONFIG_TERM_PERIOD: | |
| 1134 | + case EVSEL__CONFIG_TERM_FREQ: | |
| 1135 | + case EVSEL__CONFIG_TERM_STACK_USER: | |
| 1136 | + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG: | |
| 1137 | + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG1: | |
| 1138 | + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG2: | |
| 1139 | + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG3: | |
| 1140 | + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG4: | |
| 1141 | + t->val.val = val; | |
| 1142 | + break; | |
| 1143 | + case EVSEL__CONFIG_TERM_TIME: | |
| 1144 | + t->val.time = val; | |
| 1145 | + break; | |
| 1146 | + case EVSEL__CONFIG_TERM_INHERIT: | |
| 1147 | + t->val.inherit = val; | |
| 1148 | + break; | |
| 1149 | + case EVSEL__CONFIG_TERM_OVERWRITE: | |
| 1150 | + t->val.overwrite = val; | |
| 1151 | + break; | |
| 1152 | + case EVSEL__CONFIG_TERM_MAX_STACK: | |
| 1153 | + t->val.max_stack = val; | |
| 1154 | + break; | |
| 1155 | + case EVSEL__CONFIG_TERM_MAX_EVENTS: | |
| 1156 | + t->val.max_events = val; | |
| 1157 | + break; | |
| 1158 | + case EVSEL__CONFIG_TERM_PERCORE: | |
| 1159 | + t->val.percore = val; | |
| 1160 | + break; | |
| 1161 | + case EVSEL__CONFIG_TERM_AUX_OUTPUT: | |
| 1162 | + t->val.aux_output = val; | |
| 1163 | + break; | |
| 1164 | + case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE: | |
| 1165 | + t->val.aux_sample_size = val; | |
| 1166 | + break; | |
| 1167 | + case EVSEL__CONFIG_TERM_CALLGRAPH: | |
| 1168 | + case EVSEL__CONFIG_TERM_BRANCH: | |
| 1169 | + case EVSEL__CONFIG_TERM_DRV_CFG: | |
| 1170 | + case EVSEL__CONFIG_TERM_RATIO_TO_PREV: | |
| 1171 | + case EVSEL__CONFIG_TERM_AUX_ACTION: | |
| 1172 | + if (str) { | |
| 1173 | + t->val.str = strdup(str); | |
| 1174 | + if (!t->val.str) { | |
| 1175 | + zfree(&t); | |
| 1176 | + return NULL; | |
| 1177 | + } | |
| 1178 | + t->free_str = true; | |
| 1179 | + } | |
| 1180 | + break; | |
| 1181 | + default: | |
| 1182 | + t->val.val = val; | |
| 1183 | + break; | |
| 1184 | + } | |
| 1185 | + | |
| 1186 | + list_add_tail(&t->list, head_terms); | |
| 1133 | 1187 | return t; |
| 1134 | 1188 | } |
| 1135 | 1189 | |
| @@ -1142,7 +1196,7 @@ static int get_config_terms(const struct parse_events_terms *head_config, · 6 unmodified lines | ||
| 1142 | 1196 | struct evsel_config_term *new_term; |
| 1143 | 1197 | enum evsel_term_type new_type; |
| 1144 | 1198 | bool str_type = false; |
| 1145 | − u64 val; | |
| 1199 | + u64 val = 0; | |
| 1146 | 1200 | |
| 1147 | 1201 | switch (term->type_term) { |
| 1148 | 1202 | case PARSE_EVENTS__TERM_TYPE_SAMPLE_PERIOD: |
| @@ -1234,20 +1288,15 @@ static int get_config_terms(const struct parse_events_terms *head_config, · 85 unmodified lines | ||
| 1234 | 1288 | continue; |
| 1235 | 1289 | } |
| 1236 | 1290 | |
| 1237 | − new_term = add_config_term(new_type, head_terms, term->weak); | |
| 1291 | + /* | |
| 1292 | + * Note: Members evsel_config_term::val and | |
| 1293 | + * parse_events_term::val are unions and endianness needs | |
| 1294 | + * to be taken into account when changing such union members. | |
| 1295 | + */ | |
| 1296 | + new_term = add_config_term(new_type, head_terms, term->weak, | |
| 1297 | + str_type ? term->val.str : NULL, val); | |
| 1238 | 1298 | if (!new_term) |
| 1239 | 1299 | return -ENOMEM; |
| 1240 | − | |
| 1241 | − if (str_type) { | |
| 1242 | − new_term->val.str = strdup(term->val.str); | |
| 1243 | − if (!new_term->val.str) { | |
| 1244 | − zfree(&new_term); | |
| 1245 | − return -ENOMEM; | |
| 1246 | − } | |
| 1247 | − new_term->free_str = true; | |
| 1248 | − } else { | |
| 1249 | − new_term->val.val = val; | |
| 1250 | − } | |
| 1251 | 1300 | } |
| 1252 | 1301 | return 0; |
| 1253 | 1302 | } |
| @@ -1277,10 +1326,9 @@ static int add_cfg_chg(const struct perf_pmu *pmu, · 23 unmodified lines | ||
| 1277 | 1326 | if (bits) { |
| 1278 | 1327 | struct evsel_config_term *new_term; |
| 1279 | 1328 | |
| 1280 | − new_term = add_config_term(new_term_type, head_terms, false); | |
| 1329 | + new_term = add_config_term(new_term_type, head_terms, false, NULL, bits); | |
| 1281 | 1330 | if (!new_term) |
| 1282 | 1331 | return -ENOMEM; |
| 1283 | − new_term->val.cfg_chg = bits; | |
| 1284 | 1332 | } |
| 1285 | 1333 | |
| 1286 | 1334 | return 0; |
| @@ -3,6 +3,7 @@ · 2 unmodified lines | ||
| 3 | 3 | |
| 4 | 4 | TEST_PROGS := \ |
| 5 | 5 | dev_addr_lists.sh \ |
| 6 | + non_ether_header_ops.sh \ | |
| 6 | 7 | options.sh \ |
| 7 | 8 | propagation.sh \ |
| 8 | 9 | refleak.sh \ |
| @@ -1,7 +1,9 @@ | ||
| 1 | +CONFIG_BONDING=y | |
| 1 | 2 | CONFIG_DUMMY=y |
| 2 | 3 | CONFIG_IPV6=y |
| 3 | 4 | CONFIG_MACVLAN=y |
| 4 | 5 | CONFIG_NETDEVSIM=m |
| 6 | +CONFIG_NET_IPGRE=y | |
| 5 | 7 | CONFIG_NET_TEAM=y |
| 6 | 8 | CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=y |
| 7 | 9 | CONFIG_NET_TEAM_MODE_LOADBALANCE=y |
| @@ -0,0 +1,41 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# SPDX-License-Identifier: GPL-2.0 | |
| 3 | +# shellcheck disable=SC2154 | |
| 4 | +# | |
| 5 | +# Reproduce the non-Ethernet header_ops confusion scenario with: | |
| 6 | +# g0 (gre) -> b0 (bond) -> t0 (team) | |
| 7 | +# | |
| 8 | +# Before the fix, direct header_ops inheritance in this stack could call | |
| 9 | +# callbacks with the wrong net_device context and crash. | |
| 10 | + | |
| 11 | +lib_dir=$(dirname "$0") | |
| 12 | +source "$lib_dir"/../../../net/lib.sh | |
| 13 | + | |
| 14 | +trap cleanup_all_ns EXIT | |
| 15 | + | |
| 16 | +setup_ns ns1 | |
| 17 | + | |
| 18 | +ip -n "$ns1" link add d0 type dummy | |
| 19 | +ip -n "$ns1" addr add 10.10.10.1/24 dev d0 | |
| 20 | +ip -n "$ns1" link set d0 up | |
| 21 | + | |
| 22 | +ip -n "$ns1" link add g0 type gre local 10.10.10.1 | |
| 23 | +ip -n "$ns1" link add b0 type bond mode active-backup | |
| 24 | +ip -n "$ns1" link add t0 type team | |
| 25 | + | |
| 26 | +ip -n "$ns1" link set g0 master b0 | |
| 27 | +ip -n "$ns1" link set b0 master t0 | |
| 28 | + | |
| 29 | +ip -n "$ns1" link set g0 up | |
| 30 | +ip -n "$ns1" link set b0 up | |
| 31 | +ip -n "$ns1" link set t0 up | |
| 32 | + | |
| 33 | +# IPv6 address assignment triggers MLD join reports that call | |
| 34 | +# dev_hard_header() on t0, exercising the inherited header_ops path. | |
| 35 | +ip -n "$ns1" -6 addr add 2001:db8:1::1/64 dev t0 nodad | |
| 36 | +for i in $(seq 1 20); do | |
| 37 | + ip netns exec "$ns1" ping -6 -I t0 ff02::1 -c1 -W1 &>/dev/null || true | |
| 38 | +done | |
| 39 | + | |
| 40 | +echo "PASS: non-Ethernet header_ops stacking did not crash" | |
| 41 | +exit "$EXIT_STATUS" | |
| @@ -206,6 +206,7 @@ TEST_GEN_PROGS_s390 += s390/ucontrol_test · 205 unmodified lines | ||
| 206 | 206 | TEST_GEN_PROGS_s390 += s390/user_operexec |
| 207 | 207 | TEST_GEN_PROGS_s390 += s390/keyop |
| 208 | 208 | TEST_GEN_PROGS_s390 += rseq_test |
| 209 | +TEST_GEN_PROGS_s390 += s390/irq_routing | |
| 209 | 210 | |
| 210 | 211 | TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) |
| 211 | 212 | TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test |
| @@ -0,0 +1,75 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* | |
| 3 | + * IRQ routing offset tests. | |
| 4 | + * | |
| 5 | + * Copyright IBM Corp. 2026 | |
| 6 | + * | |
| 7 | + * Authors: | |
| 8 | + * Janosch Frank <frankja@linux.ibm.com> | |
| 9 | + */ | |
| 10 | +#include <stdio.h> | |
| 11 | +#include <stdlib.h> | |
| 12 | +#include <string.h> | |
| 13 | +#include <sys/ioctl.h> | |
| 14 | + | |
| 15 | +#include "test_util.h" | |
| 16 | +#include "kvm_util.h" | |
| 17 | +#include "kselftest.h" | |
| 18 | +#include "ucall_common.h" | |
| 19 | + | |
| 20 | +extern char guest_code[]; | |
| 21 | +asm("guest_code:\n" | |
| 22 | + "diag %r0,%r0,0\n" | |
| 23 | + "j .\n"); | |
| 24 | + | |
| 25 | +static void test(void) | |
| 26 | +{ | |
| 27 | + struct kvm_irq_routing *routing; | |
| 28 | + struct kvm_vcpu *vcpu; | |
| 29 | + struct kvm_vm *vm; | |
| 30 | + vm_paddr_t mem; | |
| 31 | + int ret; | |
| 32 | + | |
| 33 | + struct kvm_irq_routing_entry ue = { | |
| 34 | + .type = KVM_IRQ_ROUTING_S390_ADAPTER, | |
| 35 | + .gsi = 1, | |
| 36 | + }; | |
| 37 | + | |
| 38 | + vm = vm_create_with_one_vcpu(&vcpu, guest_code); | |
| 39 | + mem = vm_phy_pages_alloc(vm, 2, 4096 * 42, 0); | |
| 40 | + | |
| 41 | + routing = kvm_gsi_routing_create(); | |
| 42 | + routing->nr = 1; | |
| 43 | + routing->entries[0] = ue; | |
| 44 | + routing->entries[0].u.adapter.summary_addr = (uintptr_t)mem; | |
| 45 | + routing->entries[0].u.adapter.ind_addr = (uintptr_t)mem; | |
| 46 | + | |
| 47 | + routing->entries[0].u.adapter.summary_offset = 4096 * 8; | |
| 48 | + ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing); | |
| 49 | + ksft_test_result(ret == -1 && errno == EINVAL, "summary offset outside of page\n"); | |
| 50 | + | |
| 51 | + routing->entries[0].u.adapter.summary_offset -= 4; | |
| 52 | + ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing); | |
| 53 | + ksft_test_result(ret == 0, "summary offset inside of page\n"); | |
| 54 | + | |
| 55 | + routing->entries[0].u.adapter.ind_offset = 4096 * 8; | |
| 56 | + ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing); | |
| 57 | + ksft_test_result(ret == -1 && errno == EINVAL, "ind offset outside of page\n"); | |
| 58 | + | |
| 59 | + routing->entries[0].u.adapter.ind_offset -= 4; | |
| 60 | + ret = __vm_ioctl(vm, KVM_SET_GSI_ROUTING, routing); | |
| 61 | + ksft_test_result(ret == 0, "ind offset inside of page\n"); | |
| 62 | + | |
| 63 | + kvm_vm_free(vm); | |
| 64 | +} | |
| 65 | + | |
| 66 | +int main(int argc, char *argv[]) | |
| 67 | +{ | |
| 68 | + TEST_REQUIRE(kvm_has_cap(KVM_CAP_IRQ_ROUTING)); | |
| 69 | + | |
| 70 | + ksft_print_header(); | |
| 71 | + ksft_set_plan(4); | |
| 72 | + test(); | |
| 73 | + | |
| 74 | + ksft_finished(); /* Print results and exit() accordingly */ | |
| 75 | +} | |
| @@ -6,9 +6,10 @@ · 5 unmodified lines | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | #define _GNU_SOURCE |
| 9 | +#include <linux/landlock.h> | |
| 9 | 10 | #include <pthread.h> |
| 11 | +#include <signal.h> | |
| 10 | 12 | #include <sys/prctl.h> |
| 11 | −#include <linux/landlock.h> | |
| 12 | 13 | |
| 13 | 14 | #include "common.h" |
| 14 | 15 | |
| @@ -158,4 +159,92 @@ TEST(competing_enablement) · 143 unmodified lines | ||
| 158 | 159 | EXPECT_EQ(0, close(ruleset_fd)); |
| 159 | 160 | } |
| 160 | 161 | |
| 162 | +static void signal_nop_handler(int sig) | |
| 163 | +{ | |
| 164 | +} | |
| 165 | + | |
| 166 | +struct signaler_data { | |
| 167 | + pthread_t target; | |
| 168 | + volatile bool stop; | |
| 169 | +}; | |
| 170 | + | |
| 171 | +static void *signaler_thread(void *data) | |
| 172 | +{ | |
| 173 | + struct signaler_data *sd = data; | |
| 174 | + | |
| 175 | + while (!sd->stop) | |
| 176 | + pthread_kill(sd->target, SIGUSR1); | |
| 177 | + | |
| 178 | + return NULL; | |
| 179 | +} | |
| 180 | + | |
| 181 | +/* | |
| 182 | + * Number of idle sibling threads. This must be large enough that even on | |
| 183 | + * machines with many cores, the sibling threads cannot all complete their | |
| 184 | + * credential preparation in a single parallel wave, otherwise the signaler | |
| 185 | + * thread has no window to interrupt wait_for_completion_interruptible(). | |
| 186 | + * 200 threads on a 64-core machine yields ~3 serialized waves, giving the | |
| 187 | + * tight signal loop enough time to land an interruption. | |
| 188 | + */ | |
| 189 | +#define NUM_IDLE_THREADS 200 | |
| 190 | + | |
| 191 | +/* | |
| 192 | + * Exercises the tsync interruption and cancellation paths in tsync.c. | |
| 193 | + * | |
| 194 | + * When a signal interrupts the calling thread while it waits for sibling | |
| 195 | + * threads to finish their credential preparation | |
| 196 | + * (wait_for_completion_interruptible in landlock_restrict_sibling_threads), | |
| 197 | + * the kernel sets ERESTARTNOINTR, cancels queued task works that have not | |
| 198 | + * started yet (cancel_tsync_works), then waits for the remaining works to | |
| 199 | + * finish. On the error return, syscalls.c aborts the prepared credentials. | |
| 200 | + * The kernel automatically restarts the syscall, so userspace sees success. | |
| 201 | + */ | |
| 202 | +TEST(tsync_interrupt) | |
| 203 | +{ | |
| 204 | + size_t i; | |
| 205 | + pthread_t threads[NUM_IDLE_THREADS]; | |
| 206 | + pthread_t signaler; | |
| 207 | + struct signaler_data sd; | |
| 208 | + struct sigaction sa = {}; | |
| 209 | + const int ruleset_fd = create_ruleset(_metadata); | |
| 210 | + | |
| 211 | + disable_caps(_metadata); | |
| 212 | + | |
| 213 | + /* Install a no-op SIGUSR1 handler so the signal does not kill us. */ | |
| 214 | + sa.sa_handler = signal_nop_handler; | |
| 215 | + sigemptyset(&sa.sa_mask); | |
| 216 | + ASSERT_EQ(0, sigaction(SIGUSR1, &sa, NULL)); | |
| 217 | + | |
| 218 | + ASSERT_EQ(0, prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)); | |
| 219 | + | |
| 220 | + for (i = 0; i < NUM_IDLE_THREADS; i++) | |
| 221 | + ASSERT_EQ(0, pthread_create(&threads[i], NULL, idle, NULL)); | |
| 222 | + | |
| 223 | + /* | |
| 224 | + * Start a signaler thread that continuously sends SIGUSR1 to the | |
| 225 | + * calling thread. This maximizes the chance of interrupting | |
| 226 | + * wait_for_completion_interruptible() in the kernel's tsync path. | |
| 227 | + */ | |
| 228 | + sd.target = pthread_self(); | |
| 229 | + sd.stop = false; | |
| 230 | + ASSERT_EQ(0, pthread_create(&signaler, NULL, signaler_thread, &sd)); | |
| 231 | + | |
| 232 | + /* | |
| 233 | + * The syscall may be interrupted and transparently restarted by the | |
| 234 | + * kernel (ERESTARTNOINTR). From userspace, it should always succeed. | |
| 235 | + */ | |
| 236 | + EXPECT_EQ(0, landlock_restrict_self(ruleset_fd, | |
| 237 | + LANDLOCK_RESTRICT_SELF_TSYNC)); | |
| 238 | + | |
| 239 | + sd.stop = true; | |
| 240 | + ASSERT_EQ(0, pthread_join(signaler, NULL)); | |
| 241 | + | |
| 242 | + for (i = 0; i < NUM_IDLE_THREADS; i++) { | |
| 243 | + ASSERT_EQ(0, pthread_cancel(threads[i])); | |
| 244 | + ASSERT_EQ(0, pthread_join(threads[i], NULL)); | |
| 245 | + } | |
| 246 | + | |
| 247 | + EXPECT_EQ(0, close(ruleset_fd)); | |
| 248 | +} | |
| 249 | + | |
| 161 | 250 | TEST_HARNESS_MAIN |
| @@ -1020,7 +1020,7 @@ FIXTURE_SETUP(mount_setattr_idmapped) · 1019 unmodified lines | ||
| 1020 | 1020 | "size=100000,mode=700"), 0); |
| 1021 | 1021 | |
| 1022 | 1022 | ASSERT_EQ(mount("testing", "/mnt", "tmpfs", MS_NOATIME | MS_NODEV, |
| 1023 | − "size=2m,mode=700"), 0); | |
| 1023 | + "size=256m,mode=700"), 0); | |
| 1024 | 1024 | |
| 1025 | 1025 | ASSERT_EQ(mkdir("/mnt/A", 0777), 0); |
| 1026 | 1026 | |
| @@ -868,6 +868,64 @@ fib6_gc_test() · 867 unmodified lines | ||
| 868 | 868 | check_rt_num 5 $($IP -6 route list |grep -v expires|grep 2001:20::|wc -l) |
| 869 | 869 | log_test $ret 0 "ipv6 route garbage collection (replace with permanent)" |
| 870 | 870 | |
| 871 | + # Delete dummy_10 and remove all routes | |
| 872 | + $IP link del dev dummy_10 | |
| 873 | + | |
| 874 | + # rd6 is required for the next test. (ipv6toolkit) | |
| 875 | + if [ ! -x "$(command -v rd6)" ]; then | |
| 876 | + echo "SKIP: rd6 not found." | |
| 877 | + set +e | |
| 878 | + cleanup &> /dev/null | |
| 879 | + return | |
| 880 | + fi | |
| 881 | + | |
| 882 | + setup_ns ns2 | |
| 883 | + $IP link add veth1 type veth peer veth2 netns $ns2 | |
| 884 | + $IP link set veth1 up | |
| 885 | + ip -netns $ns2 link set veth2 up | |
| 886 | + $IP addr add fe80:dead::1/64 dev veth1 | |
| 887 | + ip -netns $ns2 addr add fe80:dead::2/64 dev veth2 | |
| 888 | + | |
| 889 | + # Add NTF_ROUTER neighbour to prevent rt6_age_examine_exception() | |
| 890 | + # from removing not-yet-expired exceptions. | |
| 891 | + ip -netns $ns2 link set veth2 address 00:11:22:33:44:55 | |
| 892 | + $IP neigh add fe80:dead::3 lladdr 00:11:22:33:44:55 dev veth1 router | |
| 893 | + | |
| 894 | + $NS_EXEC sysctl -wq net.ipv6.conf.veth1.accept_redirects=1 | |
| 895 | + $NS_EXEC sysctl -wq net.ipv6.conf.veth1.forwarding=0 | |
| 896 | + | |
| 897 | + # Temporary routes | |
| 898 | + for i in $(seq 1 5); do | |
| 899 | + # Expire route after $EXPIRE seconds | |
| 900 | + $IP -6 route add 2001:10::$i \ | |
| 901 | + via fe80:dead::2 dev veth1 expires $EXPIRE | |
| 902 | + | |
| 903 | + ip netns exec $ns2 rd6 -i veth2 \ | |
| 904 | + -s fe80:dead::2 -d fe80:dead::1 \ | |
| 905 | + -r 2001:10::$i -t fe80:dead::3 -p ICMP6 | |
| 906 | + done | |
| 907 | + | |
| 908 | + check_rt_num 5 $($IP -6 route list | grep expires | grep 2001:10:: | wc -l) | |
| 909 | + | |
| 910 | + # Promote to permanent routes by "prepend" (w/o NLM_F_EXCL and NLM_F_REPLACE) | |
| 911 | + for i in $(seq 1 5); do | |
| 912 | + # -EEXIST, but the temporary route becomes the permanent route. | |
| 913 | + $IP -6 route append 2001:10::$i \ | |
| 914 | + via fe80:dead::2 dev veth1 2>/dev/null || true | |
| 915 | + done | |
| 916 | + | |
| 917 | + check_rt_num 5 $($IP -6 route list | grep -v expires | grep 2001:10:: | wc -l) | |
| 918 | + check_rt_num 5 $($IP -6 route list cache | grep 2001:10:: | wc -l) | |
| 919 | + | |
| 920 | + # Trigger GC instead of waiting $GC_WAIT_TIME. | |
| 921 | + # rt6_nh_dump_exceptions() just skips expired exceptions. | |
| 922 | + $NS_EXEC sysctl -wq net.ipv6.route.flush=1 | |
| 923 | + check_rt_num 0 $($IP -6 route list cache | grep 2001:10:: | wc -l) | |
| 924 | + log_test $ret 0 "ipv6 route garbage collection (promote to permanent routes)" | |
| 925 | + | |
| 926 | + $IP neigh del fe80:dead::3 lladdr 00:11:22:33:44:55 dev veth1 router | |
| 927 | + $IP link del veth1 | |
| 928 | + | |
| 871 | 929 | # ra6 is required for the next test. (ipv6toolkit) |
| 872 | 930 | if [ ! -x "$(command -v ra6)" ]; then |
| 873 | 931 | echo "SKIP: ra6 not found." |
| @@ -876,9 +934,6 @@ fib6_gc_test() · 2 unmodified lines | ||
| 876 | 934 | return |
| 877 | 935 | fi |
| 878 | 936 | |
| 879 | − # Delete dummy_10 and remove all routes | |
| 880 | − $IP link del dev dummy_10 | |
| 881 | − | |
| 882 | 937 | # Create a pair of veth devices to send a RA message from one |
| 883 | 938 | # device to another. |
| 884 | 939 | $IP link add veth1 type veth peer name veth2 |
| @@ -29,7 +29,8 @@ TYPES="net_port port_net net6_port port_proto net6_port_mac net6_port_mac_proto · 28 unmodified lines | ||
| 29 | 29 | net6_port_net6_port net_port_mac_proto_net" |
| 30 | 30 | |
| 31 | 31 | # Reported bugs, also described by TYPE_ variables below |
| 32 | −BUGS="flush_remove_add reload net_port_proto_match avx2_mismatch doublecreate insert_overlap" | |
| 32 | +BUGS="flush_remove_add reload net_port_proto_match avx2_mismatch doublecreate | |
| 33 | + insert_overlap load_flush_load4 load_flush_load8" | |
| 33 | 34 | |
| 34 | 35 | # List of possible paths to pktgen script from kernel tree for performance tests |
| 35 | 36 | PKTGEN_SCRIPT_PATHS=" |
| @@ -432,6 +433,30 @@ race_repeat 0 · 396 unmodified lines | ||
| 432 | 433 | perf_duration 0 |
| 433 | 434 | " |
| 434 | 435 | |
| 436 | +TYPE_load_flush_load4=" | |
| 437 | +display reload with flush, 4bit groups | |
| 438 | +type_spec ipv4_addr . ipv4_addr | |
| 439 | +chain_spec ip saddr . ip daddr | |
| 440 | +dst addr4 | |
| 441 | +proto icmp | |
| 442 | + | |
| 443 | +race_repeat 0 | |
| 444 | + | |
| 445 | +perf_duration 0 | |
| 446 | +" | |
| 447 | + | |
| 448 | +TYPE_load_flush_load8=" | |
| 449 | +display reload with flush, 8bit groups | |
| 450 | +type_spec ipv4_addr . ipv4_addr | |
| 451 | +chain_spec ip saddr . ip daddr | |
| 452 | +dst addr4 | |
| 453 | +proto icmp | |
| 454 | + | |
| 455 | +race_repeat 0 | |
| 456 | + | |
| 457 | +perf_duration 0 | |
| 458 | +" | |
| 459 | + | |
| 435 | 460 | # Set template for all tests, types and rules are filled in depending on test |
| 436 | 461 | set_template=' |
| 437 | 462 | flush ruleset |
| @@ -1997,6 +2022,49 @@ test_bug_insert_overlap() · 1559 unmodified lines | ||
| 1997 | 2022 | return 0 |
| 1998 | 2023 | } |
| 1999 | 2024 | |
| 2025 | +test_bug_load_flush_load4() | |
| 2026 | +{ | |
| 2027 | + local i | |
| 2028 | + | |
| 2029 | + setup veth send_"${proto}" set || return ${ksft_skip} | |
| 2030 | + | |
| 2031 | + for i in $(seq 0 255); do | |
| 2032 | + local addelem="add element inet filter test" | |
| 2033 | + local j | |
| 2034 | + | |
| 2035 | + for j in $(seq 0 20); do | |
| 2036 | + echo "$addelem { 10.$j.0.$i . 10.$j.1.$i }" | |
| 2037 | + echo "$addelem { 10.$j.0.$i . 10.$j.2.$i }" | |
| 2038 | + done | |
| 2039 | + done > "$tmp" | |
| 2040 | + | |
| 2041 | + nft -f "$tmp" || return 1 | |
| 2042 | + | |
| 2043 | + ( echo "flush set inet filter test";cat "$tmp") | nft -f - | |
| 2044 | + [ $? -eq 0 ] || return 1 | |
| 2045 | + | |
| 2046 | + return 0 | |
| 2047 | +} | |
| 2048 | + | |
| 2049 | +test_bug_load_flush_load8() | |
| 2050 | +{ | |
| 2051 | + local i | |
| 2052 | + | |
| 2053 | + setup veth send_"${proto}" set || return ${ksft_skip} | |
| 2054 | + | |
| 2055 | + for i in $(seq 1 100); do | |
| 2056 | + echo "add element inet filter test { 10.0.0.$i . 10.0.1.$i }" | |
| 2057 | + echo "add element inet filter test { 10.0.0.$i . 10.0.2.$i }" | |
| 2058 | + done > "$tmp" | |
| 2059 | + | |
| 2060 | + nft -f "$tmp" || return 1 | |
| 2061 | + | |
| 2062 | + ( echo "flush set inet filter test";cat "$tmp") | nft -f - | |
| 2063 | + [ $? -eq 0 ] || return 1 | |
| 2064 | + | |
| 2065 | + return 0 | |
| 2066 | +} | |
| 2067 | + | |
| 2000 | 2068 | test_reported_issues() { |
| 2001 | 2069 | eval test_bug_"${subtest}" |
| 2002 | 2070 | } |