History

Merge tag 'dmaengine-5.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine

Pull dmaengine updates from Vinod Koul: "Nothing special, this includes a couple of new device support and new driver support and bunch of driver updates. New support: - Tegra gpcdma driver support - Qualcomm SM8350, Sm8450 and SC7280 device support - Renesas RZN1 dma and platform support Updates: - stm32 device pause/resume support and updates - DMA memset ops Documentation and usage clarification - deprecate '#dma-channels' & '#dma-requests' bindings - driver updates for stm32, ptdma idsx etc" * tag 'dmaengine-5.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (87 commits) dmaengine: idxd: make idxd_wq_enable() return 0 if wq is already enabled dmaengine: sun6i: Add support for the D1 variant dmaengine: sun6i: Add support for 34-bit physical addresses dmaengine: sun6i: Do not use virt_to_phys dt-bindings: dma: sun50i-a64: Add compatible for D1 dmaengine: tegra: Remove unused switch case dmaengine: tegra: Fix uninitialized variable usage dmaengine: stm32-dma: add device_pause/device_resume support dmaengine: stm32-dma: rename pm ops before dma pause/resume introduction dmaengine: stm32-dma: pass DMA_SxSCR value to stm32_dma_handle_chan_done() dmaengine: stm32-dma: introduce stm32_dma_sg_inc to manage chan->next_sg dmaengine: stm32-dmamux: avoid reset of dmamux if used by coprocessor dmaengine: qcom: gpi: Add support for sc7280 dt-bindings: dma: pl330: Add power-domains dmaengine: stm32-mdma: use dev_dbg on non-busy channel spurious it dmaengine: stm32-mdma: fix chan initialization in stm32_mdma_irq_handler() dmaengine: stm32-mdma: remove GISR1 register dmaengine: ti: deprecate '#dma-channels' dmaengine: mmp: deprecate '#dma-channels' dmaengine: pxa: deprecate '#dma-channels' and '#dma-requests' ...

Linus Torvalds committed 4 years ago

Commit b00ed482 parents c3a9a3c d1a285964 files · +2744 −349Browse files

Unified · Split

64 files
  1. Documentation/devicetree/bindings/clock/renesas,r9a06g032-sysctrl.yaml +11 −0
  2. Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml +6 −3
  3. Documentation/devicetree/bindings/dma/altr,msgdma.yaml +1 −1
  4. Documentation/devicetree/bindings/dma/arm,pl330.yaml +3 −0
  5. Documentation/devicetree/bindings/dma/mmp-dma.txt +6 −4
  6. Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml +110 −0
  7. Documentation/devicetree/bindings/dma/qcom,gpi.yaml +3 −0
  8. Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml +4 −6
  9. Documentation/devicetree/bindings/dma/renesas,rzn1-dmamux.yaml +51 −0
  10. Documentation/devicetree/bindings/dma/sifive,fu540-c000-pdma.yaml +17 −2
  11. Documentation/devicetree/bindings/dma/snps,dma-spear1340.yaml +7 −1
  12. Documentation/devicetree/bindings/dma/sprd-dma.txt +5 −2
  13. Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt +5 −1
  14. Documentation/driver-api/dmaengine/provider.rst +7 −1
  15. MAINTAINERS +2 −1
  16. drivers/clk/renesas/r9a06g032-clocks.c +39 −1
  17. drivers/dma/Kconfig +13 −1
  18. drivers/dma/Makefile +1 −0
  19. drivers/dma/amba-pl08x.c +0 −11
  20. drivers/dma/at_hdmac.c +9 −1
  21. drivers/dma/at_xdmac.c +8 −1
  22. drivers/dma/bestcomm/bestcomm.c +2 −0
  23. drivers/dma/dma-jz4780.c +9 −0
  24. drivers/dma/dmaengine.c +0 −7
  25. drivers/dma/dmatest.c +10 −3
  26. drivers/dma/dw/Kconfig +9 −0
  27. drivers/dma/dw/Makefile +2 −0
  28. drivers/dma/dw/platform.c +1 −0
  29. drivers/dma/dw/rzn1-dmamux.c +155 −0
  30. drivers/dma/ep93xx_dma.c +1 −1
  31. drivers/dma/idxd/cdev.c +12 −6
  32. drivers/dma/idxd/device.c +105 −46
  33. drivers/dma/idxd/dma.c +28 −37
  34. drivers/dma/idxd/idxd.h +14 −6
  35. drivers/dma/idxd/init.c +15 −15
  36. drivers/dma/idxd/registers.h +1 −0
  37. drivers/dma/idxd/sysfs.c +9 −3
  38. drivers/dma/mediatek/mtk-cqdma.c +4 −8
  39. drivers/dma/mediatek/mtk-hsdma.c +5 −8
  40. drivers/dma/mmp_pdma.c +9 −5
  41. drivers/dma/mv_xor_v2.c +2 −2
  42. drivers/dma/nbpfaxi.c +6 −8
  43. drivers/dma/plx_dma.c +2 −2
  44. drivers/dma/ptdma/ptdma-dev.c +19 −17
  45. drivers/dma/ptdma/ptdma-dmaengine.c +15 −1
  46. drivers/dma/ptdma/ptdma.h +13 −0
  47. drivers/dma/pxa_dma.c +10 −3
  48. drivers/dma/qcom/gpi.c +16 −5
  49. drivers/dma/qcom/hidma.c +12 −1
  50. drivers/dma/sf-pdma/sf-pdma.c +16 −8
  51. drivers/dma/sf-pdma/sf-pdma.h +2 −6
  52. drivers/dma/sh/Kconfig +1 −1
  53. drivers/dma/sprd-dma.c +5 −1
  54. drivers/dma/stm32-dma.c +268 −43
  55. drivers/dma/stm32-dmamux.c +1 −1
  56. drivers/dma/stm32-mdma.c +32 −21
  57. drivers/dma/sun6i-dma.c +58 −34
  58. drivers/dma/tegra186-gpc-dma.c +1498 −0
  59. drivers/dma/ti/cppi41.c +5 −1
  60. drivers/dma/ti/k3-psil-am62.c +4 −4
  61. drivers/dma/xilinx/zynqmp_dma.c +13 −4
  62. include/linux/dmaengine.h +8 −1
  63. include/linux/soc/renesas/r9a06g032-sysctrl.h +11 −0
  64. include/uapi/linux/idxd.h +28 −3
Documentation/devicetree/bindings/clock/renesas,r9a06g032-sysctrl.yaml+11−0
@@ -39,6 +39,17 @@ properties: · 38 unmodified lines
3939 '#power-domain-cells':
4040 const: 0
4141
42+ '#address-cells':
43+ const: 1
44+
45+ '#size-cells':
46+ const: 1
47+
48+patternProperties:
49+ "^dma-router@[a-f0-9]+$":
50+ type: object
51+ $ref: "../dma/renesas,rzn1-dmamux.yaml#"
52+
4253 required:
4354 - compatible
4455 - reg
Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml+6−3
@@ -20,9 +20,11 @@ properties: · 19 unmodified lines
2020
2121 compatible:
2222 oneOf:
23− - const: allwinner,sun50i-a64-dma
24− - const: allwinner,sun50i-a100-dma
25− - const: allwinner,sun50i-h6-dma
23+ - enum:
24+ - allwinner,sun20i-d1-dma
25+ - allwinner,sun50i-a64-dma
26+ - allwinner,sun50i-a100-dma
27+ - allwinner,sun50i-h6-dma
2628 - items:
2729 - const: allwinner,sun8i-r40-dma
2830 - const: allwinner,sun50i-a64-dma
@@ -58,6 +60,7 @@ if: · 29 unmodified lines
5860 properties:
5961 compatible:
6062 enum:
63+ - allwinner,sun20i-d1-dma
6164 - allwinner,sun50i-a100-dma
6265 - allwinner,sun50i-h6-dma
6366
Documentation/devicetree/bindings/dma/altr,msgdma.yaml+1−1
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# · 6 unmodified lines
77 title: Altera mSGDMA IP core
88
99 maintainers:
10− - Olivier Dautricourt <olivier.dautricourt@orolia.com>
10+ - Olivier Dautricourt <olivierdautricourt@gmail.com>
1111
1212 description: |
1313 Altera / Intel modular Scatter-Gather Direct Memory Access (mSGDMA)
Documentation/devicetree/bindings/dma/arm,pl330.yaml+3−0
@@ -55,6 +55,9 @@ properties: · 54 unmodified lines
5555
5656 dma-coherent: true
5757
58+ power-domains:
59+ maxItems: 1
60+
5861 resets:
5962 minItems: 1
6063 maxItems: 2
Documentation/devicetree/bindings/dma/mmp-dma.txt+6−4
@@ -10,10 +10,12 @@ Required properties: · 9 unmodified lines
1010 or one irq for pdma device
1111
1212 Optional properties:
13−- #dma-channels: Number of DMA channels supported by the controller (defaults
13+- dma-channels: Number of DMA channels supported by the controller (defaults
1414 to 32 when not specified)
15−- #dma-requests: Number of DMA requestor lines supported by the controller
15+- #dma-channels: deprecated
16+- dma-requests: Number of DMA requestor lines supported by the controller
1617 (defaults to 32 when not specified)
18+- #dma-requests: deprecated
1719
1820 "marvell,pdma-1.0"
1921 Used platforms: pxa25x, pxa27x, pxa3xx, pxa93x, pxa168, pxa910, pxa688.
@@ -33,7 +35,7 @@ pdma: dma-controller@d4000000 { · 13 unmodified lines
3335 reg = <0xd4000000 0x10000>;
3436 interrupts = <0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15>;
3537 interrupt-parent = <&intcmux32>;
36− #dma-channels = <16>;
38+ dma-channels = <16>;
3739 };
3840
3941 /*
@@ -45,7 +47,7 @@ pdma: dma-controller@d4000000 { · 5 unmodified lines
4547 compatible = "marvell,pdma-1.0";
4648 reg = <0xd4000000 0x10000>;
4749 interrupts = <47>;
48− #dma-channels = <16>;
50+ dma-channels = <16>;
4951 };
5052
5153
Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml+110−0
@@ -0,0 +1,110 @@
1+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2+%YAML 1.2
3+---
4+$id: http://devicetree.org/schemas/dma/nvidia,tegra186-gpc-dma.yaml#
5+$schema: http://devicetree.org/meta-schemas/core.yaml#
6+
7+title: NVIDIA Tegra GPC DMA Controller Device Tree Bindings
8+
9+description: |
10+ The Tegra General Purpose Central (GPC) DMA controller is used for faster
11+ data transfers between memory to memory, memory to device and device to
12+ memory.
13+
14+maintainers:
15+ - Jon Hunter <jonathanh@nvidia.com>
16+ - Rajesh Gumasta <rgumasta@nvidia.com>
17+
18+allOf:
19+ - $ref: "dma-controller.yaml#"
20+
21+properties:
22+ compatible:
23+ oneOf:
24+ - const: nvidia,tegra186-gpcdma
25+ - items:
26+ - const: nvidia,tegra194-gpcdma
27+ - const: nvidia,tegra186-gpcdma
28+
29+ "#dma-cells":
30+ const: 1
31+
32+ reg:
33+ maxItems: 1
34+
35+ interrupts:
36+ description:
37+ Should contain all of the per-channel DMA interrupts in
38+ ascending order with respect to the DMA channel index.
39+ minItems: 1
40+ maxItems: 31
41+
42+ resets:
43+ maxItems: 1
44+
45+ reset-names:
46+ const: gpcdma
47+
48+ iommus:
49+ maxItems: 1
50+
51+ dma-coherent: true
52+
53+required:
54+ - compatible
55+ - reg
56+ - interrupts
57+ - resets
58+ - reset-names
59+ - "#dma-cells"
60+ - iommus
61+
62+additionalProperties: false
63+
64+examples:
65+ - |
66+ #include <dt-bindings/interrupt-controller/arm-gic.h>
67+ #include <dt-bindings/memory/tegra186-mc.h>
68+ #include <dt-bindings/reset/tegra186-reset.h>
69+
70+ dma-controller@2600000 {
71+ compatible = "nvidia,tegra186-gpcdma";
72+ reg = <0x2600000 0x210000>;
73+ resets = <&bpmp TEGRA186_RESET_GPCDMA>;
74+ reset-names = "gpcdma";
75+ interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>,
76+ <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>,
77+ <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>,
78+ <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>,
79+ <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>,
80+ <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>,
81+ <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>,
82+ <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>,
83+ <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
84+ <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
85+ <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>,
86+ <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>,
87+ <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
88+ <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>,
89+ <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>,
90+ <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>,
91+ <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>,
92+ <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>,
93+ <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>,
94+ <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
95+ <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,
96+ <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
97+ <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
98+ <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
99+ <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
100+ <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
101+ <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
102+ <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>,
103+ <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
104+ <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
105+ <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
106+ #dma-cells = <1>;
107+ iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
108+ dma-coherent;
109+ };
110+...
Documentation/devicetree/bindings/dma/qcom,gpi.yaml+3−0
@@ -19,9 +19,12 @@ allOf: · 18 unmodified lines
1919 properties:
2020 compatible:
2121 enum:
22+ - qcom,sc7280-gpi-dma
2223 - qcom,sdm845-gpi-dma
2324 - qcom,sm8150-gpi-dma
2425 - qcom,sm8250-gpi-dma
26+ - qcom,sm8350-gpi-dma
27+ - qcom,sm8450-gpi-dma
2528
2629 reg:
2730 maxItems: 1
Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml+4−6
@@ -42,11 +42,10 @@ properties: · 41 unmodified lines
4242 - const: renesas,rcar-dmac
4343
4444 - items:
45− - const: renesas,dmac-r8a779a0 # R-Car V3U
46−
47− - items:
48− - const: renesas,dmac-r8a779f0 # R-Car S4-8
49− - const: renesas,rcar-gen4-dmac
45+ - enum:
46+ - renesas,dmac-r8a779a0 # R-Car V3U
47+ - renesas,dmac-r8a779f0 # R-Car S4-8
48+ - const: renesas,rcar-gen4-dmac # R-Car Gen4
5049
5150 reg: true
5251
@@ -121,7 +120,6 @@ if: · 68 unmodified lines
121120 compatible:
122121 contains:
123122 enum:
124− - renesas,dmac-r8a779a0
125123 - renesas,rcar-gen4-dmac
126124 then:
127125 properties:
Documentation/devicetree/bindings/dma/renesas,rzn1-dmamux.yaml+51−0
@@ -0,0 +1,51 @@
1+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2+%YAML 1.2
3+---
4+$id: http://devicetree.org/schemas/dma/renesas,rzn1-dmamux.yaml#
5+$schema: http://devicetree.org/meta-schemas/core.yaml#
6+
7+title: Renesas RZ/N1 DMA mux
8+
9+maintainers:
10+ - Miquel Raynal <miquel.raynal@bootlin.com>
11+
12+allOf:
13+ - $ref: "dma-router.yaml#"
14+
15+properties:
16+ compatible:
17+ const: renesas,rzn1-dmamux
18+
19+ reg:
20+ maxItems: 1
21+ description: DMA mux first register offset within the system control parent.
22+
23+ '#dma-cells':
24+ const: 6
25+ description:
26+ The first four cells are dedicated to the master DMA controller. The fifth
27+ cell gives the DMA mux bit index that must be set starting from 0. The
28+ sixth cell gives the binary value that must be written there, ie. 0 or 1.
29+
30+ dma-masters:
31+ minItems: 1
32+ maxItems: 2
33+
34+ dma-requests:
35+ const: 32
36+
37+required:
38+ - reg
39+ - dma-requests
40+
41+additionalProperties: false
42+
43+examples:
44+ - |
45+ dma-router@a0 {
46+ compatible = "renesas,rzn1-dmamux";
47+ reg = <0xa0 4>;
48+ #dma-cells = <6>;
49+ dma-masters = <&dma0 &dma1>;
50+ dma-requests = <32>;
51+ };
Documentation/devicetree/bindings/dma/sifive,fu540-c000-pdma.yaml+17−2
@@ -28,7 +28,15 @@ allOf: · 27 unmodified lines
2828 properties:
2929 compatible:
3030 items:
31− - const: sifive,fu540-c000-pdma
31+ - enum:
32+ - sifive,fu540-c000-pdma
33+ - const: sifive,pdma0
34+ description:
35+ Should be "sifive,<chip>-pdma" and "sifive,pdma<version>".
36+ Supported compatible strings are -
37+ "sifive,fu540-c000-pdma" for the SiFive PDMA v0 as integrated onto the
38+ SiFive FU540 chip resp and "sifive,pdma0" for the SiFive PDMA v0 IP block
39+ with no chip integration tweaks.
3240
3341 reg:
3442 maxItems: 1
@@ -37,6 +45,12 @@ properties: · 2 unmodified lines
3745 minItems: 1
3846 maxItems: 8
3947
48+ dma-channels:
49+ description: For backwards-compatibility, the default value is 4
50+ minimum: 1
51+ maximum: 4
52+ default: 4
53+
4054 '#dma-cells':
4155 const: 1
4256
@@ -50,8 +64,9 @@ unevaluatedProperties: false · 7 unmodified lines
5064 examples:
5165 - |
5266 dma-controller@3000000 {
53− compatible = "sifive,fu540-c000-pdma";
67+ compatible = "sifive,fu540-c000-pdma", "sifive,pdma0";
5468 reg = <0x3000000 0x8000>;
69+ dma-channels = <4>;
5570 interrupts = <23>, <24>, <25>, <26>, <27>, <28>, <29>, <30>;
5671 #dma-cells = <1>;
5772 };
Documentation/devicetree/bindings/dma/snps,dma-spear1340.yaml+7−1
@@ -15,7 +15,13 @@ allOf: · 14 unmodified lines
1515
1616 properties:
1717 compatible:
18− const: snps,dma-spear1340
18+ oneOf:
19+ - const: snps,dma-spear1340
20+ - items:
21+ - enum:
22+ - renesas,r9a06g032-dma
23+ - const: renesas,rzn1-dma
24+
1925
2026 "#dma-cells":
2127 minimum: 3
Documentation/devicetree/bindings/dma/sprd-dma.txt+5−2
@@ -8,10 +8,13 @@ Required properties: · 7 unmodified lines
88 - interrupts: Should contain one interrupt shared by all channel.
99 - #dma-cells: must be <1>. Used to represent the number of integer
1010 cells in the dmas property of client device.
11−- #dma-channels : Number of DMA channels supported. Should be 32.
11+- dma-channels : Number of DMA channels supported. Should be 32.
1212 - clock-names: Should contain the clock of the DMA controller.
1313 - clocks: Should contain a clock specifier for each entry in clock-names.
1414
15+Deprecated properties:
16+- #dma-channels : Number of DMA channels supported. Should be 32.
17+
1518 Example:
1619
1720 Controller:
@@ -20,7 +23,7 @@ apdma: dma-controller@20100000 { · 2 unmodified lines
2023 reg = <0x20100000 0x4000>;
2124 interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
2225 #dma-cells = <1>;
23− #dma-channels = <32>;
26+ dma-channels = <32>;
2427 clock-names = "enable";
2528 clocks = <&clk_ap_ahb_gates 5>;
2629 };
Documentation/devicetree/bindings/dma/xilinx/xilinx_dma.txt+5−1
@@ -110,7 +110,11 @@ axi_vdma_0: axivdma@40030000 { · 109 unmodified lines
110110 Required properties:
111111 - dmas: a list of <[Video DMA device phandle] [Channel ID]> pairs,
112112 where Channel ID is '0' for write/tx and '1' for read/rx
113− channel.
113+ channel. For MCMDA, MM2S channel(write/tx) ID start from
114+ '0' and is in [0-15] range. S2MM channel(read/rx) ID start
115+ from '16' and is in [16-31] range. These channels ID are
116+ fixed irrespective of IP configuration.
117+
114118 - dma-names: a list of DMA channel names, one per "dmas" entry
115119
116120 Example:
Documentation/driver-api/dmaengine/provider.rst+7−1
@@ -206,6 +206,12 @@ Currently, the types available are: · 205 unmodified lines
206206 - The device is able to perform parity check using RAID6 P+Q
207207 algorithm against a memory buffer.
208208
209+- DMA_MEMSET
210+
211+ - The device is able to fill memory with the provided pattern
212+
213+ - The pattern is treated as a single byte signed value.
214+
209215 - DMA_INTERRUPT
210216
211217 - The device is able to trigger a dummy transfer that will
@@ -457,7 +463,7 @@ supported. · 245 unmodified lines
457463 - Should use dma_set_residue to report it
458464
459465 - In the case of a cyclic transfer, it should only take into
460− account the current period.
466+ account the total size of the cyclic buffer.
461467
462468 - Should return DMA_OUT_OF_ORDER if the device does not support in order
463469 completion and is completing the operation out of order.
MAINTAINERS+2−1
@@ -820,7 +820,7 @@ S: Maintained · 819 unmodified lines
820820 F: drivers/mailbox/mailbox-altera.c
821821
822822 ALTERA MSGDMA IP CORE DRIVER
823−M: Olivier Dautricourt <olivier.dautricourt@orolia.com>
823+M: Olivier Dautricourt <olivierdautricourt@gmail.com>
824824 R: Stefan Roese <sr@denx.de>
825825 L: dmaengine@vger.kernel.org
826826 S: Odd Fixes
@@ -19202,6 +19202,7 @@ SYNOPSYS DESIGNWARE DMAC DRIVER · 18375 unmodified lines
1920219202 M: Viresh Kumar <vireshk@kernel.org>
1920319203 R: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
1920419204 S: Maintained
19205+F: Documentation/devicetree/bindings/dma/renesas,rzn1-dmamux.yaml
1920519206 F: Documentation/devicetree/bindings/dma/snps,dma-spear1340.yaml
1920619207 F: drivers/dma/dw/
1920719208 F: include/dt-bindings/dma/dw-dmac.h
drivers/clk/renesas/r9a06g032-clocks.c+39−1
@@ -16,13 +16,17 @@ · 15 unmodified lines
1616 #include <linux/math64.h>
1717 #include <linux/of.h>
1818 #include <linux/of_address.h>
19+#include <linux/of_platform.h>
1920 #include <linux/platform_device.h>
2021 #include <linux/pm_clock.h>
2122 #include <linux/pm_domain.h>
2223 #include <linux/slab.h>
24+#include <linux/soc/renesas/r9a06g032-sysctrl.h>
2325 #include <linux/spinlock.h>
2426 #include <dt-bindings/clock/r9a06g032-sysctrl.h>
2527
28+#define R9A06G032_SYSCTRL_DMAMUX 0xA0
29+
2630 struct r9a06g032_gate {
2731 u16 gate, reset, ready, midle,
2832 scon, mirack, mistat;
@@ -315,6 +319,30 @@ struct r9a06g032_priv { · 286 unmodified lines
315319 void __iomem *reg;
316320 };
317321
322+static struct r9a06g032_priv *sysctrl_priv;
323+
324+/* Exported helper to access the DMAMUX register */
325+int r9a06g032_sysctrl_set_dmamux(u32 mask, u32 val)
326+{
327+ unsigned long flags;
328+ u32 dmamux;
329+
330+ if (!sysctrl_priv)
331+ return -EPROBE_DEFER;
332+
333+ spin_lock_irqsave(&sysctrl_priv->lock, flags);
334+
335+ dmamux = readl(sysctrl_priv->reg + R9A06G032_SYSCTRL_DMAMUX);
336+ dmamux &= ~mask;
337+ dmamux |= val & mask;
338+ writel(dmamux, sysctrl_priv->reg + R9A06G032_SYSCTRL_DMAMUX);
339+
340+ spin_unlock_irqrestore(&sysctrl_priv->lock, flags);
341+
342+ return 0;
343+}
344+EXPORT_SYMBOL_GPL(r9a06g032_sysctrl_set_dmamux);
345+
318346 /* register/bit pairs are encoded as an uint16_t */
319347 static void
320348 clk_rdesc_set(struct r9a06g032_priv *clocks,
@@ -963,7 +991,17 @@ static int __init r9a06g032_clocks_probe(struct platform_device *pdev) · 642 unmodified lines
963991 if (error)
964992 return error;
965993
966− return r9a06g032_add_clk_domain(dev);
994+ error = r9a06g032_add_clk_domain(dev);
995+ if (error)
996+ return error;
997+
998+ sysctrl_priv = clocks;
999+
1000+ error = of_platform_populate(np, NULL, NULL, dev);
1001+ if (error)
1002+ dev_err(dev, "Failed to populate children (%d)\n", error);
1003+
1004+ return 0;
9671005 }
9681006
9691007 static const struct of_device_id r9a06g032_match[] = {
drivers/dma/Kconfig+13−1
@@ -163,7 +163,7 @@ config DMA_SUN4I · 162 unmodified lines
163163
164164 config DMA_SUN6I
165165 tristate "Allwinner A31 SoCs DMA support"
166− depends on MACH_SUN6I || MACH_SUN8I || (ARM64 && ARCH_SUNXI) || COMPILE_TEST
166+ depends on ARCH_SUNXI || COMPILE_TEST
167167 depends on RESET_CONTROLLER
168168 select DMA_ENGINE
169169 select DMA_VIRTUAL_CHANNELS
@@ -629,6 +629,18 @@ config TXX9_DMAC · 459 unmodified lines
629629 Support the TXx9 SoC internal DMA controller. This can be
630630 integrated in chips such as the Toshiba TX4927/38/39.
631631
632+config TEGRA186_GPC_DMA
633+ tristate "NVIDIA Tegra GPC DMA support"
634+ depends on (ARCH_TEGRA || COMPILE_TEST) && ARCH_DMA_ADDR_T_64BIT
635+ depends on IOMMU_API
636+ select DMA_ENGINE
637+ help
638+ Support for the NVIDIA Tegra General Purpose Central DMA controller.
639+ The DMA controller has multiple DMA channels which can be configured
640+ for different peripherals like UART, SPI, etc which are on APB bus.
641+ This DMA controller transfers data from memory to peripheral FIFO
642+ or vice versa. It also supports memory to memory data transfer.
643+
632644 config TEGRA20_APB_DMA
633645 tristate "NVIDIA Tegra20 APB DMA support"
634646 depends on ARCH_TEGRA || COMPILE_TEST
drivers/dma/Makefile+1−0
@@ -72,6 +72,7 @@ obj-$(CONFIG_STM32_MDMA) += stm32-mdma.o · 71 unmodified lines
7272 obj-$(CONFIG_SPRD_DMA) += sprd-dma.o
7373 obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o
7474 obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
75+obj-$(CONFIG_TEGRA186_GPC_DMA) += tegra186-gpc-dma.o
7576 obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o
7677 obj-$(CONFIG_TEGRA210_ADMA) += tegra210-adma.o
7778 obj-$(CONFIG_TIMB_DMA) += timb_dma.o
drivers/dma/amba-pl08x.c+0−11
@@ -1535,14 +1535,6 @@ static void pl08x_free_chan_resources(struct dma_chan *chan) · 1534 unmodified lines
15351535 vchan_free_chan_resources(to_virt_chan(chan));
15361536 }
15371537
1538−static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
1539− struct dma_chan *chan, unsigned long flags)
1540−{
1541− struct dma_async_tx_descriptor *retval = NULL;
1542−
1543− return retval;
1544−}
1545−
15461538 /*
15471539 * Code accessing dma_async_is_complete() in a tight loop may give problems.
15481540 * If slaves are relying on interrupts to signal completion this function
@@ -2760,7 +2752,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) · 1211 unmodified lines
27602752 pl08x->memcpy.dev = &adev->dev;
27612753 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
27622754 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
2763− pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
27642755 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
27652756 pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
27662757 pl08x->memcpy.device_config = pl08x_config;
@@ -2787,8 +2778,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) · 20 unmodified lines
27872778 pl08x->slave.dev = &adev->dev;
27882779 pl08x->slave.device_free_chan_resources =
27892780 pl08x_free_chan_resources;
2790− pl08x->slave.device_prep_dma_interrupt =
2791− pl08x_prep_dma_interrupt;
27922781 pl08x->slave.device_tx_status = pl08x_dma_tx_status;
27932782 pl08x->slave.device_issue_pending = pl08x_issue_pending;
27942783 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
drivers/dma/at_hdmac.c+9−1
@@ -942,6 +942,7 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, · 941 unmodified lines
942942 struct at_desc *desc;
943943 void __iomem *vaddr;
944944 dma_addr_t paddr;
945+ char fill_pattern;
945946
946947 dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__,
947948 &dest, value, len, flags);
@@ -963,7 +964,14 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, · 15 unmodified lines
963964 __func__);
964965 return NULL;
965966 }
966− *(u32*)vaddr = value;
967+
968+ /* Only the first byte of value is to be used according to dmaengine */
969+ fill_pattern = (char)value;
970+
971+ *(u32*)vaddr = (fill_pattern << 24) |
972+ (fill_pattern << 16) |
973+ (fill_pattern << 8) |
974+ fill_pattern;
967975
968976 desc = atc_create_memset_desc(chan, paddr, dest, len);
969977 if (!desc) {
drivers/dma/at_xdmac.c+8−1
@@ -1202,6 +1202,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, · 1201 unmodified lines
12021202 unsigned long flags;
12031203 size_t ublen;
12041204 u32 dwidth;
1205+ char pattern;
12051206 /*
12061207 * WARNING: The channel configuration is set here since there is no
12071208 * dmaengine_slave_config call in this case. Moreover we don't know the
@@ -1244,10 +1245,16 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, · 36 unmodified lines
12441245
12451246 chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
12461247
1248+ /* Only the first byte of value is to be used according to dmaengine */
1249+ pattern = (char)value;
1250+
12471251 ublen = len >> dwidth;
12481252
12491253 desc->lld.mbr_da = dst_addr;
1250− desc->lld.mbr_ds = value;
1254+ desc->lld.mbr_ds = (pattern << 24) |
1255+ (pattern << 16) |
1256+ (pattern << 8) |
1257+ pattern;
12511258 desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3
12521259 | AT_XDMAC_MBR_UBC_NDEN
12531260 | AT_XDMAC_MBR_UBC_NSEN
drivers/dma/bestcomm/bestcomm.c+2−0
@@ -17,7 +17,9 @@ · 16 unmodified lines
1717 #include <linux/kernel.h>
1818 #include <linux/slab.h>
1919 #include <linux/of.h>
20+#include <linux/of_address.h>
2021 #include <linux/of_device.h>
22+#include <linux/of_irq.h>
2123 #include <linux/of_platform.h>
2224 #include <asm/io.h>
2325 #include <asm/irq.h>
drivers/dma/dma-jz4780.c+9−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88
99 #include <linux/clk.h>
1010 #include <linux/dmapool.h>
11+#include <linux/dma-mapping.h>
1112 #include <linux/init.h>
1213 #include <linux/interrupt.h>
1314 #include <linux/module.h>
@@ -911,6 +912,14 @@ static int jz4780_dma_probe(struct platform_device *pdev) · 897 unmodified lines
911912
912913 dd = &jzdma->dma_device;
913914
915+ /*
916+ * The real segment size limit is dependent on the size unit selected
917+ * for the transfer. Because the size unit is selected automatically
918+ * and may be as small as 1 byte, use a safe limit of 2^24-1 bytes to
919+ * ensure the 24-bit transfer count in the descriptor cannot overflow.
920+ */
921+ dma_set_max_seg_size(dev, 0xffffff);
922+
914923 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
915924 dma_cap_set(DMA_SLAVE, dd->cap_mask);
916925 dma_cap_set(DMA_CYCLIC, dd->cap_mask);
drivers/dma/dmaengine.c+0−7
@@ -1053,9 +1053,7 @@ static int __dma_async_device_channel_register(struct dma_device *device, · 1052 unmodified lines
10531053 * When the chan_id is a negative value, we are dynamically adding
10541054 * the channel. Otherwise we are static enumerating.
10551055 */
1056− mutex_lock(&device->chan_mutex);
10571056 chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);
1058− mutex_unlock(&device->chan_mutex);
10591057 if (chan->chan_id < 0) {
10601058 pr_err("%s: unable to alloc ida for chan: %d\n",
10611059 __func__, chan->chan_id);
@@ -1078,9 +1076,7 @@ static int __dma_async_device_channel_register(struct dma_device *device, · 16 unmodified lines
10781076 return 0;
10791077
10801078 err_out_ida:
1081− mutex_lock(&device->chan_mutex);
10821079 ida_free(&device->chan_ida, chan->chan_id);
1083− mutex_unlock(&device->chan_mutex);
10841080 err_free_dev:
10851081 kfree(chan->dev);
10861082 err_free_local:
@@ -1113,9 +1109,7 @@ static void __dma_async_device_channel_unregister(struct dma_device *device, · 26 unmodified lines
11131109 device->chancnt--;
11141110 chan->dev->chan = NULL;
11151111 mutex_unlock(&dma_list_mutex);
1116− mutex_lock(&device->chan_mutex);
11171112 ida_free(&device->chan_ida, chan->chan_id);
1118− mutex_unlock(&device->chan_mutex);
11191113 device_unregister(&chan->dev->device);
11201114 free_percpu(chan->local);
11211115 }
@@ -1250,7 +1244,6 @@ int dma_async_device_register(struct dma_device *device) · 128 unmodified lines
12501244 if (rc != 0)
12511245 return rc;
12521246
1253− mutex_init(&device->chan_mutex);
12541247 ida_init(&device->chan_ida);
12551248
12561249 /* represent channels in sysfs. Probably want devs too */
drivers/dma/dmatest.c+10−3
@@ -675,10 +675,16 @@ static int dmatest_func(void *data) · 674 unmodified lines
675675 /*
676676 * src and dst buffers are freed by ourselves below
677677 */
678− if (params->polled)
678+ if (params->polled) {
679679 flags = DMA_CTRL_ACK;
680− else
681− flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
680+ } else {
681+ if (dma_has_cap(DMA_INTERRUPT, dev->cap_mask)) {
682+ flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
683+ } else {
684+ pr_err("Channel does not support interrupt!\n");
685+ goto err_pq_array;
686+ }
687+ }
682688
683689 ktime = ktime_get();
684690 while (!(kthread_should_stop() ||
@@ -906,6 +912,7 @@ error_unmap_continue: · 221 unmodified lines
906912 runtime = ktime_to_us(ktime);
907913
908914 ret = 0;
915+err_pq_array:
909916 kfree(dma_pq);
910917 err_srcs_array:
911918 kfree(srcs);
drivers/dma/dw/Kconfig+9−0
@@ -16,6 +16,15 @@ config DW_DMAC · 15 unmodified lines
1616 Support the Synopsys DesignWare AHB DMA controller. This
1717 can be integrated in chips such as the Intel Cherrytrail.
1818
19+config RZN1_DMAMUX
20+ tristate "Renesas RZ/N1 DMAMUX driver"
21+ depends on DW_DMAC
22+ depends on ARCH_RZN1 || COMPILE_TEST
23+ help
24+ Support the Renesas RZ/N1 DMAMUX which is located in front of
25+ the Synopsys DesignWare AHB DMA controller located on Renesas
26+ SoCs.
27+
1928 config DW_DMAC_PCI
2029 tristate "Synopsys DesignWare AHB DMA PCI driver"
2130 depends on PCI
drivers/dma/dw/Makefile+2−0
@@ -9,3 +9,5 @@ dw_dmac-$(CONFIG_OF) += of.o · 8 unmodified lines
99
1010 obj-$(CONFIG_DW_DMAC_PCI) += dw_dmac_pci.o
1111 dw_dmac_pci-y := pci.o
12+
13+obj-$(CONFIG_RZN1_DMAMUX) += rzn1-dmamux.o
drivers/dma/dw/platform.c+1−0
@@ -137,6 +137,7 @@ static void dw_shutdown(struct platform_device *pdev) · 136 unmodified lines
137137 #ifdef CONFIG_OF
138138 static const struct of_device_id dw_dma_of_id_table[] = {
139139 { .compatible = "snps,dma-spear1340", .data = &dw_dma_chip_pdata },
140+ { .compatible = "renesas,rzn1-dma", .data = &dw_dma_chip_pdata },
140141 {}
141142 };
142143 MODULE_DEVICE_TABLE(of, dw_dma_of_id_table);
drivers/dma/dw/rzn1-dmamux.c+155−0
@@ -0,0 +1,155 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (C) 2022 Schneider-Electric
4+ * Author: Miquel Raynal <miquel.raynal@bootlin.com
5+ * Based on TI crossbar driver written by Peter Ujfalusi <peter.ujfalusi@ti.com>
6+ */
7+#include <linux/bitops.h>
8+#include <linux/of_device.h>
9+#include <linux/of_dma.h>
10+#include <linux/slab.h>
11+#include <linux/soc/renesas/r9a06g032-sysctrl.h>
12+#include <linux/types.h>
13+
14+#define RNZ1_DMAMUX_NCELLS 6
15+#define RZN1_DMAMUX_MAX_LINES 64
16+#define RZN1_DMAMUX_LINES_PER_CTLR 16
17+
18+struct rzn1_dmamux_data {
19+ struct dma_router dmarouter;
20+ DECLARE_BITMAP(used_chans, 2 * RZN1_DMAMUX_LINES_PER_CTLR);
21+};
22+
23+struct rzn1_dmamux_map {
24+ unsigned int req_idx;
25+};
26+
27+static void rzn1_dmamux_free(struct device *dev, void *route_data)
28+{
29+ struct rzn1_dmamux_data *dmamux = dev_get_drvdata(dev);
30+ struct rzn1_dmamux_map *map = route_data;
31+
32+ dev_dbg(dev, "Unmapping DMAMUX request %u\n", map->req_idx);
33+
34+ clear_bit(map->req_idx, dmamux->used_chans);
35+
36+ kfree(map);
37+}
38+
39+static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec,
40+ struct of_dma *ofdma)
41+{
42+ struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
43+ struct rzn1_dmamux_data *dmamux = platform_get_drvdata(pdev);
44+ struct rzn1_dmamux_map *map;
45+ unsigned int dmac_idx, chan, val;
46+ u32 mask;
47+ int ret;
48+
49+ if (dma_spec->args_count != RNZ1_DMAMUX_NCELLS)
50+ return ERR_PTR(-EINVAL);
51+
52+ map = kzalloc(sizeof(*map), GFP_KERNEL);
53+ if (!map)
54+ return ERR_PTR(-ENOMEM);
55+
56+ chan = dma_spec->args[0];
57+ map->req_idx = dma_spec->args[4];
58+ val = dma_spec->args[5];
59+ dma_spec->args_count -= 2;
60+
61+ if (chan >= RZN1_DMAMUX_LINES_PER_CTLR) {
62+ dev_err(&pdev->dev, "Invalid DMA request line: %u\n", chan);
63+ ret = -EINVAL;
64+ goto free_map;
65+ }
66+
67+ if (map->req_idx >= RZN1_DMAMUX_MAX_LINES ||
68+ (map->req_idx % RZN1_DMAMUX_LINES_PER_CTLR) != chan) {
69+ dev_err(&pdev->dev, "Invalid MUX request line: %u\n", map->req_idx);
70+ ret = -EINVAL;
71+ goto free_map;
72+ }
73+
74+ dmac_idx = map->req_idx >= RZN1_DMAMUX_LINES_PER_CTLR ? 1 : 0;
75+ dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", dmac_idx);
76+ if (!dma_spec->np) {
77+ dev_err(&pdev->dev, "Can't get DMA master\n");
78+ ret = -EINVAL;
79+ goto free_map;
80+ }
81+
82+ dev_dbg(&pdev->dev, "Mapping DMAMUX request %u to DMAC%u request %u\n",
83+ map->req_idx, dmac_idx, chan);
84+
85+ if (test_and_set_bit(map->req_idx, dmamux->used_chans)) {
86+ ret = -EBUSY;
87+ goto free_map;
88+ }
89+
90+ mask = BIT(map->req_idx);
91+ ret = r9a06g032_sysctrl_set_dmamux(mask, val ? mask : 0);
92+ if (ret)
93+ goto clear_bitmap;
94+
95+ return map;
96+
97+clear_bitmap:
98+ clear_bit(map->req_idx, dmamux->used_chans);
99+free_map:
100+ kfree(map);
101+
102+ return ERR_PTR(ret);
103+}
104+
105+static const struct of_device_id rzn1_dmac_match[] = {
106+ { .compatible = "renesas,rzn1-dma" },
107+ {}
108+};
109+
110+static int rzn1_dmamux_probe(struct platform_device *pdev)
111+{
112+ struct device_node *mux_node = pdev->dev.of_node;
113+ const struct of_device_id *match;
114+ struct device_node *dmac_node;
115+ struct rzn1_dmamux_data *dmamux;
116+
117+ dmamux = devm_kzalloc(&pdev->dev, sizeof(*dmamux), GFP_KERNEL);
118+ if (!dmamux)
119+ return -ENOMEM;
120+
121+ dmac_node = of_parse_phandle(mux_node, "dma-masters", 0);
122+ if (!dmac_node)
123+ return dev_err_probe(&pdev->dev, -ENODEV, "Can't get DMA master node\n");
124+
125+ match = of_match_node(rzn1_dmac_match, dmac_node);
126+ of_node_put(dmac_node);
127+ if (!match)
128+ return dev_err_probe(&pdev->dev, -EINVAL, "DMA master is not supported\n");
129+
130+ dmamux->dmarouter.dev = &pdev->dev;
131+ dmamux->dmarouter.route_free = rzn1_dmamux_free;
132+
133+ platform_set_drvdata(pdev, dmamux);
134+
135+ return of_dma_router_register(mux_node, rzn1_dmamux_route_allocate,
136+ &dmamux->dmarouter);
137+}
138+
139+static const struct of_device_id rzn1_dmamux_match[] = {
140+ { .compatible = "renesas,rzn1-dmamux" },
141+ {}
142+};
143+
144+static struct platform_driver rzn1_dmamux_driver = {
145+ .driver = {
146+ .name = "renesas,rzn1-dmamux",
147+ .of_match_table = rzn1_dmamux_match,
148+ },
149+ .probe = rzn1_dmamux_probe,
150+};
151+module_platform_driver(rzn1_dmamux_driver);
152+
153+MODULE_LICENSE("GPL");
154+MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com");
155+MODULE_DESCRIPTION("Renesas RZ/N1 DMAMUX driver");
drivers/dma/ep93xx_dma.c+1−1
@@ -132,7 +132,7 @@ struct ep93xx_dma_desc { · 131 unmodified lines
132132 /**
133133 * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel
134134 * @chan: dmaengine API channel
135− * @edma: pointer to to the engine device
135+ * @edma: pointer to the engine device
136136 * @regs: memory mapped registers
137137 * @irq: interrupt number of the channel
138138 * @clk: clock used by this channel
drivers/dma/idxd/cdev.c+12−6
@@ -99,7 +99,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) · 98 unmodified lines
9999 ctx->wq = wq;
100100 filp->private_data = ctx;
101101
102− if (device_pasid_enabled(idxd)) {
102+ if (device_user_pasid_enabled(idxd)) {
103103 sva = iommu_sva_bind_device(dev, current->mm, NULL);
104104 if (IS_ERR(sva)) {
105105 rc = PTR_ERR(sva);
@@ -152,7 +152,7 @@ static int idxd_cdev_release(struct inode *node, struct file *filep) · 46 unmodified lines
152152 if (wq_shared(wq)) {
153153 idxd_device_drain_pasid(idxd, ctx->pasid);
154154 } else {
155− if (device_pasid_enabled(idxd)) {
155+ if (device_user_pasid_enabled(idxd)) {
156156 /* The wq disable in the disable pasid function will drain the wq */
157157 rc = idxd_wq_disable_pasid(wq);
158158 if (rc < 0)
@@ -314,7 +314,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) · 155 unmodified lines
314314
315315 mutex_lock(&wq->wq_lock);
316316 wq->type = IDXD_WQT_USER;
317− rc = __drv_enable_wq(wq);
317+ rc = drv_enable_wq(wq);
318318 if (rc < 0)
319319 goto err;
320320
@@ -329,7 +329,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) · 8 unmodified lines
329329 return 0;
330330
331331 err_cdev:
332− __drv_disable_wq(wq);
332+ drv_disable_wq(wq);
333333 err:
334334 wq->type = IDXD_WQT_NONE;
335335 mutex_unlock(&wq->wq_lock);
@@ -342,7 +342,7 @@ static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) · 6 unmodified lines
342342
343343 mutex_lock(&wq->wq_lock);
344344 idxd_wq_del_cdev(wq);
345− __drv_disable_wq(wq);
345+ drv_disable_wq(wq);
346346 wq->type = IDXD_WQT_NONE;
347347 mutex_unlock(&wq->wq_lock);
348348 }
@@ -369,10 +369,16 @@ int idxd_cdev_register(void) · 20 unmodified lines
369369 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
370370 ictx[i].name);
371371 if (rc)
372− return rc;
372+ goto err_free_chrdev_region;
373373 }
374374
375375 return 0;
376+
377+err_free_chrdev_region:
378+ for (i--; i >= 0; i--)
379+ unregister_chrdev_region(ictx[i].devt, MINORMASK);
380+
381+ return rc;
376382 }
377383
378384 void idxd_cdev_remove(void)
drivers/dma/idxd/device.c+105−46
@@ -184,7 +184,7 @@ int idxd_wq_enable(struct idxd_wq *wq) · 183 unmodified lines
184184
185185 if (wq->state == IDXD_WQ_ENABLED) {
186186 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
187− return -ENXIO;
187+ return 0;
188188 }
189189
190190 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
@@ -299,24 +299,46 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd) · 108 unmodified lines
299299 }
300300 }
301301
302−int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
302+static void __idxd_wq_set_priv_locked(struct idxd_wq *wq, int priv)
303303 {
304304 struct idxd_device *idxd = wq->idxd;
305− int rc;
306305 union wqcfg wqcfg;
307306 unsigned int offset;
308307
309− rc = idxd_wq_disable(wq, false);
310− if (rc < 0)
311− return rc;
308+ offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PRIVL_IDX);
309+ spin_lock(&idxd->dev_lock);
310+ wqcfg.bits[WQCFG_PRIVL_IDX] = ioread32(idxd->reg_base + offset);
311+ wqcfg.priv = priv;
312+ wq->wqcfg->bits[WQCFG_PRIVL_IDX] = wqcfg.bits[WQCFG_PRIVL_IDX];
313+ iowrite32(wqcfg.bits[WQCFG_PRIVL_IDX], idxd->reg_base + offset);
314+ spin_unlock(&idxd->dev_lock);
315+}
316+
317+static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)
318+{
319+ struct idxd_device *idxd = wq->idxd;
320+ union wqcfg wqcfg;
321+ unsigned int offset;
312322
313323 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
314324 spin_lock(&idxd->dev_lock);
315325 wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
316326 wqcfg.pasid_en = 1;
317327 wqcfg.pasid = pasid;
328+ wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX];
318329 iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
319330 spin_unlock(&idxd->dev_lock);
331+}
332+
333+int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
334+{
335+ int rc;
336+
337+ rc = idxd_wq_disable(wq, false);
338+ if (rc < 0)
339+ return rc;
340+
341+ __idxd_wq_set_pasid_locked(wq, pasid);
320342
321343 rc = idxd_wq_enable(wq);
322344 if (rc < 0)
@@ -555,19 +577,15 @@ int idxd_device_disable(struct idxd_device *idxd) · 232 unmodified lines
555577 return -ENXIO;
556578 }
557579
558− spin_lock(&idxd->dev_lock);
559580 idxd_device_clear_state(idxd);
560− idxd->state = IDXD_DEV_DISABLED;
561− spin_unlock(&idxd->dev_lock);
562581 return 0;
563582 }
564583
565584 void idxd_device_reset(struct idxd_device *idxd)
566585 {
567586 idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
568− spin_lock(&idxd->dev_lock);
569587 idxd_device_clear_state(idxd);
570− idxd->state = IDXD_DEV_DISABLED;
588+ spin_lock(&idxd->dev_lock);
571589 idxd_unmask_error_interrupts(idxd);
572590 spin_unlock(&idxd->dev_lock);
573591 }
@@ -694,15 +712,16 @@ static void idxd_device_wqs_clear_state(struct idxd_device *idxd) · 120 unmodified lines
694712 {
695713 int i;
696714
697− lockdep_assert_held(&idxd->dev_lock);
698715 for (i = 0; i < idxd->max_wqs; i++) {
699716 struct idxd_wq *wq = idxd->wqs[i];
700717
718+ mutex_lock(&wq->wq_lock);
701719 if (wq->state == IDXD_WQ_ENABLED) {
702720 idxd_wq_disable_cleanup(wq);
703721 wq->state = IDXD_WQ_DISABLED;
704722 }
705723 idxd_wq_device_reset_cleanup(wq);
724+ mutex_unlock(&wq->wq_lock);
706725 }
707726 }
708727
@@ -711,9 +730,12 @@ void idxd_device_clear_state(struct idxd_device *idxd) · 2 unmodified lines
711730 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
712731 return;
713732
733+ idxd_device_wqs_clear_state(idxd);
734+ spin_lock(&idxd->dev_lock);
714735 idxd_groups_clear_state(idxd);
715736 idxd_engines_clear_state(idxd);
716− idxd_device_wqs_clear_state(idxd);
737+ idxd->state = IDXD_DEV_DISABLED;
738+ spin_unlock(&idxd->dev_lock);
717739 }
718740
719741 static void idxd_group_config_write(struct idxd_group *group)
@@ -799,7 +821,7 @@ static int idxd_wq_config_write(struct idxd_wq *wq) · 79 unmodified lines
799821 */
800822 for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
801823 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
802− wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset);
824+ wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset);
803825 }
804826
805827 if (wq->size == 0 && wq->type != IDXD_WQT_NONE)
@@ -815,14 +837,8 @@ static int idxd_wq_config_write(struct idxd_wq *wq) · 9 unmodified lines
815837 if (wq_dedicated(wq))
816838 wq->wqcfg->mode = 1;
817839
818− if (device_pasid_enabled(idxd)) {
819− wq->wqcfg->pasid_en = 1;
820− if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq))
821− wq->wqcfg->pasid = idxd->pasid;
822− }
823−
824840 /*
825− * Here the priv bit is set depending on the WQ type. priv = 1 if the
841+ * The WQ priv bit is set depending on the WQ type. priv = 1 if the
826842 * WQ type is kernel to indicate privileged access. This setting only
827843 * matters for dedicated WQ. According to the DSA spec:
828844 * If the WQ is in dedicated mode, WQ PASID Enable is 1, and the
@@ -832,7 +848,6 @@ static int idxd_wq_config_write(struct idxd_wq *wq) · 3 unmodified lines
832848 * In the case of a dedicated kernel WQ that is not able to support
833849 * the PASID cap, then the configuration will be rejected.
834850 */
835− wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
836851 if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&
837852 !idxd_device_pasid_priv_enabled(idxd) &&
838853 wq->type == IDXD_WQT_KERNEL) {
@@ -953,7 +968,7 @@ static int idxd_wqs_setup(struct idxd_device *idxd) · 114 unmodified lines
953968 if (!wq->group)
954969 continue;
955970
956− if (wq_shared(wq) && !device_swq_supported(idxd)) {
971+ if (wq_shared(wq) && !wq_shared_supported(wq)) {
957972 idxd->cmd_status = IDXD_SCMD_WQ_NO_SWQ_SUPPORT;
958973 dev_warn(dev, "No shared wq support but configured.\n");
959974 return -EINVAL;
@@ -1018,6 +1033,9 @@ static int idxd_wq_load_config(struct idxd_wq *wq) · 58 unmodified lines
10181033
10191034 wq->priority = wq->wqcfg->priority;
10201035
1036+ wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift;
1037+ wq->max_batch_size = 1ULL << wq->wqcfg->max_batch_shift;
1038+
10211039 for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
10221040 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);
10231041 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);
@@ -1161,7 +1179,9 @@ void idxd_wq_free_irq(struct idxd_wq *wq) · 137 unmodified lines
11611179 struct idxd_device *idxd = wq->idxd;
11621180 struct idxd_irq_entry *ie = &wq->ie;
11631181
1164− synchronize_irq(ie->vector);
1182+ if (wq->type != IDXD_WQT_KERNEL)
1183+ return;
1184+
11651185 free_irq(ie->vector, ie);
11661186 idxd_flush_pending_descs(ie);
11671187 if (idxd->request_int_handles)
@@ -1180,6 +1200,9 @@ int idxd_wq_request_irq(struct idxd_wq *wq) · 12 unmodified lines
11801200 struct idxd_irq_entry *ie;
11811201 int rc;
11821202
1203+ if (wq->type != IDXD_WQT_KERNEL)
1204+ return 0;
1205+
11831206 ie = &wq->ie;
11841207 ie->vector = pci_irq_vector(pdev, ie->id);
11851208 ie->pasid = device_pasid_enabled(idxd) ? idxd->pasid : INVALID_IOASID;
@@ -1211,7 +1234,7 @@ err_irq: · 25 unmodified lines
12111234 return rc;
12121235 }
12131236
1214−int __drv_enable_wq(struct idxd_wq *wq)
1237+int drv_enable_wq(struct idxd_wq *wq)
12151238 {
12161239 struct idxd_device *idxd = wq->idxd;
12171240 struct device *dev = &idxd->pdev->dev;
@@ -1245,7 +1268,7 @@ int __drv_enable_wq(struct idxd_wq *wq) · 27 unmodified lines
12451268
12461269 /* Shared WQ checks */
12471270 if (wq_shared(wq)) {
1248− if (!device_swq_supported(idxd)) {
1271+ if (!wq_shared_supported(wq)) {
12491272 idxd->cmd_status = IDXD_SCMD_WQ_NO_SVM;
12501273 dev_dbg(dev, "PASID not enabled and shared wq.\n");
12511274 goto err;
@@ -1265,6 +1288,29 @@ int __drv_enable_wq(struct idxd_wq *wq) · 13 unmodified lines
12651288 }
12661289 }
12671290
1291+ /*
1292+ * In the event that the WQ is configurable for pasid and priv bits.
1293+ * For kernel wq, the driver should setup the pasid, pasid_en, and priv bit.
1294+ * However, for non-kernel wq, the driver should only set the pasid_en bit for
1295+ * shared wq. A dedicated wq that is not 'kernel' type will configure pasid and
1296+ * pasid_en later on so there is no need to setup.
1297+ */
1298+ if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
1299+ int priv = 0;
1300+
1301+ if (wq_pasid_enabled(wq)) {
1302+ if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {
1303+ u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;
1304+
1305+ __idxd_wq_set_pasid_locked(wq, pasid);
1306+ }
1307+ }
1308+
1309+ if (is_idxd_wq_kernel(wq))
1310+ priv = 1;
1311+ __idxd_wq_set_priv_locked(wq, priv);
1312+ }
1313+
12681314 rc = 0;
12691315 spin_lock(&idxd->dev_lock);
12701316 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
@@ -1289,8 +1335,36 @@ int __drv_enable_wq(struct idxd_wq *wq) · 18 unmodified lines
12891335 }
12901336
12911337 wq->client_count = 0;
1338+
1339+ rc = idxd_wq_request_irq(wq);
1340+ if (rc < 0) {
1341+ idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR;
1342+ dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);
1343+ goto err_irq;
1344+ }
1345+
1346+ rc = idxd_wq_alloc_resources(wq);
1347+ if (rc < 0) {
1348+ idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR;
1349+ dev_dbg(dev, "WQ resource alloc failed\n");
1350+ goto err_res_alloc;
1351+ }
1352+
1353+ rc = idxd_wq_init_percpu_ref(wq);
1354+ if (rc < 0) {
1355+ idxd->cmd_status = IDXD_SCMD_PERCPU_ERR;
1356+ dev_dbg(dev, "percpu_ref setup failed\n");
1357+ goto err_ref;
1358+ }
1359+
12921360 return 0;
12931361
1362+err_ref:
1363+ idxd_wq_free_resources(wq);
1364+err_res_alloc:
1365+ idxd_wq_free_irq(wq);
1366+err_irq:
1367+ idxd_wq_unmap_portal(wq);
12941368 err_map_portal:
12951369 rc = idxd_wq_disable(wq, false);
12961370 if (rc < 0)
@@ -1299,17 +1373,7 @@ err: · 2 unmodified lines
12991373 return rc;
13001374 }
13011375
1302−int drv_enable_wq(struct idxd_wq *wq)
1303−{
1304− int rc;
1305−
1306− mutex_lock(&wq->wq_lock);
1307− rc = __drv_enable_wq(wq);
1308− mutex_unlock(&wq->wq_lock);
1309− return rc;
1310−}
1311−
1312−void __drv_disable_wq(struct idxd_wq *wq)
1376+void drv_disable_wq(struct idxd_wq *wq)
13131377 {
13141378 struct idxd_device *idxd = wq->idxd;
13151379 struct device *dev = &idxd->pdev->dev;
@@ -1320,21 +1384,16 @@ void __drv_disable_wq(struct idxd_wq *wq) · 4 unmodified lines
13201384 dev_warn(dev, "Clients has claim on wq %d: %d\n",
13211385 wq->id, idxd_wq_refcount(wq));
13221386
1387+ idxd_wq_free_resources(wq);
13231388 idxd_wq_unmap_portal(wq);
1324−
13251389 idxd_wq_drain(wq);
1390+ idxd_wq_free_irq(wq);
13261391 idxd_wq_reset(wq);
1327−
1392+ percpu_ref_exit(&wq->wq_active);
1393+ wq->type = IDXD_WQT_NONE;
13281394 wq->client_count = 0;
13291395 }
13301396
1331−void drv_disable_wq(struct idxd_wq *wq)
1332−{
1333− mutex_lock(&wq->wq_lock);
1334− __drv_disable_wq(wq);
1335− mutex_unlock(&wq->wq_lock);
1336−}
1337−
13381397 int idxd_device_drv_probe(struct idxd_dev *idxd_dev)
13391398 {
13401399 struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
drivers/dma/idxd/dma.c+28−37
@@ -87,6 +87,27 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq, · 86 unmodified lines
8787 hw->completion_addr = compl;
8888 }
8989
90+static struct dma_async_tx_descriptor *
91+idxd_dma_prep_interrupt(struct dma_chan *c, unsigned long flags)
92+{
93+ struct idxd_wq *wq = to_idxd_wq(c);
94+ u32 desc_flags;
95+ struct idxd_desc *desc;
96+
97+ if (wq->state != IDXD_WQ_ENABLED)
98+ return NULL;
99+
100+ op_flag_setup(flags, &desc_flags);
101+ desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
102+ if (IS_ERR(desc))
103+ return NULL;
104+
105+ idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_NOOP,
106+ 0, 0, 0, desc->compl_dma, desc_flags);
107+ desc->txd.flags = flags;
108+ return &desc->txd;
109+}
110+
90111 static struct dma_async_tx_descriptor *
91112 idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
92113 dma_addr_t dma_src, size_t len, unsigned long flags)
@@ -193,10 +214,12 @@ int idxd_register_dma_device(struct idxd_device *idxd) · 100 unmodified lines
193214 INIT_LIST_HEAD(&dma->channels);
194215 dma->dev = dev;
195216
217+ dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
196218 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
197219 dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
198220 dma->device_release = idxd_dma_release;
199221
222+ dma->device_prep_dma_interrupt = idxd_dma_prep_interrupt;
200223 if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
201224 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
202225 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
@@ -227,7 +250,7 @@ void idxd_unregister_dma_device(struct idxd_device *idxd) · 24 unmodified lines
227250 dma_async_device_unregister(&idxd->idxd_dma->dma);
228251 }
229252
230−int idxd_register_dma_channel(struct idxd_wq *wq)
253+static int idxd_register_dma_channel(struct idxd_wq *wq)
231254 {
232255 struct idxd_device *idxd = wq->idxd;
233256 struct dma_device *dma = &idxd->idxd_dma->dma;
@@ -264,7 +287,7 @@ int idxd_register_dma_channel(struct idxd_wq *wq) · 30 unmodified lines
264287 return 0;
265288 }
266289
267−void idxd_unregister_dma_channel(struct idxd_wq *wq)
290+static void idxd_unregister_dma_channel(struct idxd_wq *wq)
268291 {
269292 struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
270293 struct dma_chan *chan = &idxd_chan->chan;
@@ -290,34 +313,13 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) · 19 unmodified lines
290313 mutex_lock(&wq->wq_lock);
291314 wq->type = IDXD_WQT_KERNEL;
292315
293− rc = idxd_wq_request_irq(wq);
294− if (rc < 0) {
295− idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR;
296− dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);
297− goto err_irq;
298− }
299−
300− rc = __drv_enable_wq(wq);
316+ rc = drv_enable_wq(wq);
301317 if (rc < 0) {
302318 dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
303319 rc = -ENXIO;
304320 goto err;
305321 }
306322
307− rc = idxd_wq_alloc_resources(wq);
308− if (rc < 0) {
309− idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR;
310− dev_dbg(dev, "WQ resource alloc failed\n");
311− goto err_res_alloc;
312− }
313−
314− rc = idxd_wq_init_percpu_ref(wq);
315− if (rc < 0) {
316− idxd->cmd_status = IDXD_SCMD_PERCPU_ERR;
317− dev_dbg(dev, "percpu_ref setup failed\n");
318− goto err_ref;
319− }
320−
321323 rc = idxd_register_dma_channel(wq);
322324 if (rc < 0) {
323325 idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR;
@@ -330,15 +332,8 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) · 6 unmodified lines
330332 return 0;
331333
332334 err_dma:
333− __idxd_wq_quiesce(wq);
334− percpu_ref_exit(&wq->wq_active);
335−err_ref:
336− idxd_wq_free_resources(wq);
337−err_res_alloc:
338− __drv_disable_wq(wq);
335+ drv_disable_wq(wq);
339336 err:
340− idxd_wq_free_irq(wq);
341−err_irq:
342337 wq->type = IDXD_WQT_NONE;
343338 mutex_unlock(&wq->wq_lock);
344339 return rc;
@@ -351,11 +346,7 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) · 6 unmodified lines
351346 mutex_lock(&wq->wq_lock);
352347 __idxd_wq_quiesce(wq);
353348 idxd_unregister_dma_channel(wq);
354− idxd_wq_free_resources(wq);
355− __drv_disable_wq(wq);
356− percpu_ref_exit(&wq->wq_active);
357− idxd_wq_free_irq(wq);
358− wq->type = IDXD_WQT_NONE;
349+ drv_disable_wq(wq);
359350 mutex_unlock(&wq->wq_lock);
360351 }
361352
drivers/dma/idxd/idxd.h+14−6
@@ -239,6 +239,7 @@ enum idxd_device_flag { · 238 unmodified lines
239239 IDXD_FLAG_CONFIGURABLE = 0,
240240 IDXD_FLAG_CMD_RUNNING,
241241 IDXD_FLAG_PASID_ENABLED,
242+ IDXD_FLAG_USER_PASID_ENABLED,
242243 };
243244
244245 struct idxd_dma_dev {
@@ -469,9 +470,20 @@ static inline bool device_pasid_enabled(struct idxd_device *idxd) · 224 unmodified lines
469470 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
470471 }
471472
472−static inline bool device_swq_supported(struct idxd_device *idxd)
473+static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
473474 {
474− return (support_enqcmd && device_pasid_enabled(idxd));
475+ return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
476+}
477+
478+static inline bool wq_pasid_enabled(struct idxd_wq *wq)
479+{
480+ return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
481+ (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
482+}
483+
484+static inline bool wq_shared_supported(struct idxd_wq *wq)
485+{
486+ return (support_enqcmd && wq_pasid_enabled(wq));
475487 }
476488
477489 enum idxd_portal_prot {
@@ -559,9 +571,7 @@ void idxd_unregister_idxd_drv(void); · 81 unmodified lines
559571 int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
560572 void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
561573 int drv_enable_wq(struct idxd_wq *wq);
562−int __drv_enable_wq(struct idxd_wq *wq);
563574 void drv_disable_wq(struct idxd_wq *wq);
564−void __drv_disable_wq(struct idxd_wq *wq);
565575 int idxd_device_init_reset(struct idxd_device *idxd);
566576 int idxd_device_enable(struct idxd_device *idxd);
567577 int idxd_device_disable(struct idxd_device *idxd);
@@ -602,8 +612,6 @@ int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); · 34 unmodified lines
602612 /* dmaengine */
603613 int idxd_register_dma_device(struct idxd_device *idxd);
604614 void idxd_unregister_dma_device(struct idxd_device *idxd);
605−int idxd_register_dma_channel(struct idxd_wq *wq);
606−void idxd_unregister_dma_channel(struct idxd_wq *wq);
607615 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
608616 void idxd_dma_complete_txd(struct idxd_desc *desc,
609617 enum idxd_complete_type comp_type, bool free_desc);
drivers/dma/idxd/init.c+15−15
@@ -512,18 +512,15 @@ static int idxd_probe(struct idxd_device *idxd) · 511 unmodified lines
512512 dev_dbg(dev, "IDXD reset complete\n");
513513
514514 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
515− rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA);
516− if (rc == 0) {
517− rc = idxd_enable_system_pasid(idxd);
518− if (rc < 0) {
519− iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
520− dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
521− } else {
522− set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
523− }
524− } else {
525− dev_warn(dev, "Unable to turn on SVA feature.\n");
526− }
515+ if (iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA))
516+ dev_warn(dev, "Unable to turn on user SVA feature.\n");
517+ else
518+ set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
519+
520+ if (idxd_enable_system_pasid(idxd))
521+ dev_warn(dev, "No in-kernel DMA with PASID.\n");
522+ else
523+ set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
527524 } else if (!sva) {
528525 dev_warn(dev, "User forced SVA off via module param.\n");
529526 }
@@ -561,7 +558,8 @@ static int idxd_probe(struct idxd_device *idxd) · 31 unmodified lines
561558 err:
562559 if (device_pasid_enabled(idxd))
563560 idxd_disable_system_pasid(idxd);
564− iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
561+ if (device_user_pasid_enabled(idxd))
562+ iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
565563 return rc;
566564 }
567565
@@ -574,7 +572,8 @@ static void idxd_cleanup(struct idxd_device *idxd) · 6 unmodified lines
574572 idxd_cleanup_internals(idxd);
575573 if (device_pasid_enabled(idxd))
576574 idxd_disable_system_pasid(idxd);
577− iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
575+ if (device_user_pasid_enabled(idxd))
576+ iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
578577 }
579578
580579 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
@@ -691,7 +690,8 @@ static void idxd_remove(struct pci_dev *pdev) · 110 unmodified lines
691690 free_irq(irq_entry->vector, irq_entry);
692691 pci_free_irq_vectors(pdev);
693692 pci_iounmap(pdev, idxd->reg_base);
694− iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
693+ if (device_user_pasid_enabled(idxd))
694+ iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
695695 pci_disable_device(pdev);
696696 destroy_workqueue(idxd->wq);
697697 perfmon_pmu_remove(idxd);
drivers/dma/idxd/registers.h+1−0
@@ -353,6 +353,7 @@ union wqcfg { · 352 unmodified lines
353353 } __packed;
354354
355355 #define WQCFG_PASID_IDX 2
356+#define WQCFG_PRIVL_IDX 2
356357 #define WQCFG_OCCUP_IDX 6
357358
358359 #define WQCFG_OCCUP_MASK 0xffff
drivers/dma/idxd/sysfs.c+9−3
@@ -588,7 +588,7 @@ static ssize_t wq_mode_store(struct device *dev, · 587 unmodified lines
588588 if (sysfs_streq(buf, "dedicated")) {
589589 set_bit(WQ_FLAG_DEDICATED, &wq->flags);
590590 wq->threshold = 0;
591− } else if (sysfs_streq(buf, "shared") && device_swq_supported(idxd)) {
591+ } else if (sysfs_streq(buf, "shared")) {
592592 clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
593593 } else {
594594 return -EINVAL;
@@ -832,6 +832,7 @@ static ssize_t wq_name_store(struct device *dev, · 237 unmodified lines
832832 size_t count)
833833 {
834834 struct idxd_wq *wq = confdev_to_wq(dev);
835+ char *input, *pos;
835836
836837 if (wq->state != IDXD_WQ_DISABLED)
837838 return -EPERM;
@@ -846,9 +847,14 @@ static ssize_t wq_name_store(struct device *dev, · 8 unmodified lines
846847 if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd))
847848 return -EOPNOTSUPP;
848849
850+ input = kstrndup(buf, count, GFP_KERNEL);
851+ if (!input)
852+ return -ENOMEM;
853+
854+ pos = strim(input);
849855 memset(wq->name, 0, WQ_NAME_SIZE + 1);
850− strncpy(wq->name, buf, WQ_NAME_SIZE);
851− strreplace(wq->name, '\n', '\0');
856+ sprintf(wq->name, "%s", pos);
857+ kfree(input);
852858 return count;
853859 }
854860
drivers/dma/mediatek/mtk-cqdma.c+4−8
@@ -751,7 +751,6 @@ static int mtk_cqdma_probe(struct platform_device *pdev) · 750 unmodified lines
751751 struct mtk_cqdma_device *cqdma;
752752 struct mtk_cqdma_vchan *vc;
753753 struct dma_device *dd;
754− struct resource *res;
755754 int err;
756755 u32 i;
757756
@@ -824,13 +823,10 @@ static int mtk_cqdma_probe(struct platform_device *pdev) · 66 unmodified lines
824823 return PTR_ERR(cqdma->pc[i]->base);
825824
826825 /* allocate IRQ resource */
827− res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
828− if (!res) {
829− dev_err(&pdev->dev, "No irq resource for %s\n",
830− dev_name(&pdev->dev));
831− return -EINVAL;
832− }
833− cqdma->pc[i]->irq = res->start;
826+ err = platform_get_irq(pdev, i);
827+ if (err < 0)
828+ return err;
829+ cqdma->pc[i]->irq = err;
834830
835831 err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
836832 mtk_cqdma_irq, 0, dev_name(&pdev->dev),
drivers/dma/mediatek/mtk-hsdma.c+5−8
@@ -601,7 +601,7 @@ static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma) · 600 unmodified lines
601601 cb->flag = 0;
602602 }
603603
604− cb->vd = 0;
604+ cb->vd = NULL;
605605
606606 /*
607607 * Recycle the RXD with the helper WRITE_ONCE that can ensure
@@ -923,13 +923,10 @@ static int mtk_hsdma_probe(struct platform_device *pdev) · 315 unmodified lines
923923 return PTR_ERR(hsdma->clk);
924924 }
925925
926− res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
927− if (!res) {
928− dev_err(&pdev->dev, "No irq resource for %s\n",
929− dev_name(&pdev->dev));
930− return -EINVAL;
931− }
932− hsdma->irq = res->start;
926+ err = platform_get_irq(pdev, 0);
927+ if (err < 0)
928+ return err;
929+ hsdma->irq = err;
933930
934931 refcount_set(&hsdma->pc_refcnt, 0);
935932 spin_lock_init(&hsdma->lock);
drivers/dma/mmp_pdma.c+9−5
@@ -1043,13 +1043,17 @@ static int mmp_pdma_probe(struct platform_device *op) · 1042 unmodified lines
10431043 return PTR_ERR(pdev->base);
10441044
10451045 of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev);
1046− if (of_id)
1047− of_property_read_u32(pdev->dev->of_node, "#dma-channels",
1048− &dma_channels);
1049− else if (pdata && pdata->dma_channels)
1046+ if (of_id) {
1047+ /* Parse new and deprecated dma-channels properties */
1048+ if (of_property_read_u32(pdev->dev->of_node, "dma-channels",
1049+ &dma_channels))
1050+ of_property_read_u32(pdev->dev->of_node, "#dma-channels",
1051+ &dma_channels);
1052+ } else if (pdata && pdata->dma_channels) {
10501053 dma_channels = pdata->dma_channels;
1051− else
1054+ } else {
10521055 dma_channels = 32; /* default 32 channel */
1056+ }
10531057 pdev->dma_channels = dma_channels;
10541058
10551059 for (i = 0; i < dma_channels; i++) {
drivers/dma/mv_xor_v2.c+2−2
@@ -591,14 +591,14 @@ static void mv_xor_v2_tasklet(struct tasklet_struct *t) · 590 unmodified lines
591591 dma_run_dependencies(&next_pending_sw_desc->async_tx);
592592
593593 /* Lock the channel */
594− spin_lock_bh(&xor_dev->lock);
594+ spin_lock(&xor_dev->lock);
595595
596596 /* add the SW descriptor to the free descriptors list */
597597 list_add(&next_pending_sw_desc->free_list,
598598 &xor_dev->free_sw_desc);
599599
600600 /* Release the channel */
601− spin_unlock_bh(&xor_dev->lock);
601+ spin_unlock(&xor_dev->lock);
602602
603603 /* increment the next descriptor */
604604 pending_ptr++;
drivers/dma/nbpfaxi.c+6−8
@@ -1294,7 +1294,7 @@ static int nbpf_probe(struct platform_device *pdev) · 1293 unmodified lines
12941294 struct device_node *np = dev->of_node;
12951295 struct nbpf_device *nbpf;
12961296 struct dma_device *dma_dev;
1297− struct resource *iomem, *irq_res;
1297+ struct resource *iomem;
12981298 const struct nbpf_config *cfg;
12991299 int num_channels;
13001300 int ret, irq, eirq, i;
@@ -1335,13 +1335,11 @@ static int nbpf_probe(struct platform_device *pdev) · 34 unmodified lines
13351335 nbpf->config = cfg;
13361336
13371337 for (i = 0; irqs < ARRAY_SIZE(irqbuf); i++) {
1338− irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
1339− if (!irq_res)
1340− break;
1341−
1342− for (irq = irq_res->start; irq <= irq_res->end;
1343− irq++, irqs++)
1344− irqbuf[irqs] = irq;
1338+ irq = platform_get_irq_optional(pdev, i);
1339+ if (irq < 0 && irq != -ENXIO)
1340+ return irq;
1341+ if (irq > 0)
1342+ irqbuf[irqs++] = irq;
13451343 }
13461344
13471345 /*
drivers/dma/plx_dma.c+2−2
@@ -137,7 +137,7 @@ static void plx_dma_process_desc(struct plx_dma_dev *plxdev) · 136 unmodified lines
137137 struct plx_dma_desc *desc;
138138 u32 flags;
139139
140− spin_lock_bh(&plxdev->ring_lock);
140+ spin_lock(&plxdev->ring_lock);
141141
142142 while (plxdev->tail != plxdev->head) {
143143 desc = plx_dma_get_desc(plxdev, plxdev->tail);
@@ -165,7 +165,7 @@ static void plx_dma_process_desc(struct plx_dma_dev *plxdev) · 21 unmodified lines
165165 plxdev->tail++;
166166 }
167167
168− spin_unlock_bh(&plxdev->ring_lock);
168+ spin_unlock(&plxdev->ring_lock);
169169 }
170170
171171 static void plx_dma_abort_desc(struct plx_dma_dev *plxdev)
drivers/dma/ptdma/ptdma-dev.c+19−17
@@ -100,6 +100,7 @@ int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, · 99 unmodified lines
100100 struct pt_passthru_engine *pt_engine)
101101 {
102102 struct ptdma_desc desc;
103+ struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q);
103104
104105 cmd_q->cmd_error = 0;
105106 cmd_q->total_pt_ops++;
@@ -111,17 +112,12 @@ int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, · 5 unmodified lines
111112 desc.dst_lo = lower_32_bits(pt_engine->dst_dma);
112113 desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);
113114
114− return pt_core_execute_cmd(&desc, cmd_q);
115−}
116−
117−static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)
118−{
119− iowrite32(0, pt->cmd_q.reg_control + 0x000C);
120−}
115+ if (cmd_q->int_en)
116+ pt_core_enable_queue_interrupts(pt);
117+ else
118+ pt_core_disable_queue_interrupts(pt);
121119
122−static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)
123−{
124− iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);
120+ return pt_core_execute_cmd(&desc, cmd_q);
125121 }
126122
127123 static void pt_do_cmd_complete(unsigned long data)
@@ -144,14 +140,10 @@ static void pt_do_cmd_complete(unsigned long data) · 16 unmodified lines
144140 cmd->pt_cmd_callback(cmd->data, cmd->ret);
145141 }
146142
147−static irqreturn_t pt_core_irq_handler(int irq, void *data)
143+void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q)
148144 {
149− struct pt_device *pt = data;
150− struct pt_cmd_queue *cmd_q = &pt->cmd_q;
151145 u32 status;
152146
153− pt_core_disable_queue_interrupts(pt);
154− pt->total_interrupts++;
155147 status = ioread32(cmd_q->reg_control + 0x0010);
156148 if (status) {
157149 cmd_q->int_status = status;
@@ -162,11 +154,21 @@ static irqreturn_t pt_core_irq_handler(int irq, void *data) · 4 unmodified lines
162154 if ((status & INT_ERROR) && !cmd_q->cmd_error)
163155 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
164156
165− /* Acknowledge the interrupt */
157+ /* Acknowledge the completion */
166158 iowrite32(status, cmd_q->reg_control + 0x0010);
167− pt_core_enable_queue_interrupts(pt);
168159 pt_do_cmd_complete((ulong)&pt->tdata);
169160 }
161+}
162+
163+static irqreturn_t pt_core_irq_handler(int irq, void *data)
164+{
165+ struct pt_device *pt = data;
166+ struct pt_cmd_queue *cmd_q = &pt->cmd_q;
167+
168+ pt_core_disable_queue_interrupts(pt);
169+ pt->total_interrupts++;
170+ pt_check_status_trans(pt, cmd_q);
171+ pt_core_enable_queue_interrupts(pt);
170172 return IRQ_HANDLED;
171173 }
172174
drivers/dma/ptdma/ptdma-dmaengine.c+15−1
@@ -171,6 +171,7 @@ static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, · 170 unmodified lines
171171 vchan_tx_prep(&chan->vc, &desc->vd, flags);
172172
173173 desc->pt = chan->pt;
174+ desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT);
174175 desc->issued_to_hw = 0;
175176 desc->status = DMA_IN_PROGRESS;
176177
@@ -257,6 +258,17 @@ static void pt_issue_pending(struct dma_chan *dma_chan) · 80 unmodified lines
257258 pt_cmd_callback(desc, 0);
258259 }
259260
261+static enum dma_status
262+pt_tx_status(struct dma_chan *c, dma_cookie_t cookie,
263+ struct dma_tx_state *txstate)
264+{
265+ struct pt_device *pt = to_pt_chan(c)->pt;
266+ struct pt_cmd_queue *cmd_q = &pt->cmd_q;
267+
268+ pt_check_status_trans(pt, cmd_q);
269+ return dma_cookie_status(c, cookie, txstate);
270+}
271+
260272 static int pt_pause(struct dma_chan *dma_chan)
261273 {
262274 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
@@ -291,8 +303,10 @@ static int pt_terminate_all(struct dma_chan *dma_chan) · 28 unmodified lines
291303 {
292304 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
293305 unsigned long flags;
306+ struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q;
294307 LIST_HEAD(head);
295308
309+ iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
296310 spin_lock_irqsave(&chan->vc.lock, flags);
297311 vchan_get_all_descriptors(&chan->vc, &head);
298312 spin_unlock_irqrestore(&chan->vc.lock, flags);
@@ -362,7 +376,7 @@ int pt_dmaengine_register(struct pt_device *pt) · 63 unmodified lines
362376 dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy;
363377 dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt;
364378 dma_dev->device_issue_pending = pt_issue_pending;
365− dma_dev->device_tx_status = dma_cookie_status;
379+ dma_dev->device_tx_status = pt_tx_status;
366380 dma_dev->device_pause = pt_pause;
367381 dma_dev->device_resume = pt_resume;
368382 dma_dev->device_terminate_all = pt_terminate_all;
drivers/dma/ptdma/ptdma.h+13−0
@@ -206,6 +206,9 @@ struct pt_cmd_queue { · 205 unmodified lines
206206 unsigned int active;
207207 unsigned int suspended;
208208
209+ /* Interrupt flag */
210+ bool int_en;
211+
209212 /* Register addresses for queue */
210213 void __iomem *reg_control;
211214 u32 qcontrol; /* Cached control register */
@@ -318,7 +321,17 @@ void pt_core_destroy(struct pt_device *pt); · 106 unmodified lines
318321 int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
319322 struct pt_passthru_engine *pt_engine);
320323
324+void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q);
321325 void pt_start_queue(struct pt_cmd_queue *cmd_q);
322326 void pt_stop_queue(struct pt_cmd_queue *cmd_q);
323327
328+static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)
329+{
330+ iowrite32(0, pt->cmd_q.reg_control + 0x000C);
331+}
332+
333+static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)
334+{
335+ iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);
336+}
324337 #endif
drivers/dma/pxa_dma.c+10−3
@@ -1365,10 +1365,17 @@ static int pxad_probe(struct platform_device *op) · 1364 unmodified lines
13651365
13661366 of_id = of_match_device(pxad_dt_ids, &op->dev);
13671367 if (of_id) {
1368− of_property_read_u32(op->dev.of_node, "#dma-channels",
1369− &dma_channels);
1370− ret = of_property_read_u32(op->dev.of_node, "#dma-requests",
1368+ /* Parse new and deprecated dma-channels properties */
1369+ if (of_property_read_u32(op->dev.of_node, "dma-channels",
1370+ &dma_channels))
1371+ of_property_read_u32(op->dev.of_node, "#dma-channels",
1372+ &dma_channels);
1373+ /* Parse new and deprecated dma-requests properties */
1374+ ret = of_property_read_u32(op->dev.of_node, "dma-requests",
13711375 &nb_requestors);
1376+ if (ret)
1377+ ret = of_property_read_u32(op->dev.of_node, "#dma-requests",
1378+ &nb_requestors);
13721379 if (ret) {
13731380 dev_warn(pdev->slave.dev,
13741381 "#dma-requests set to default 32 as missing in OF: %d",
drivers/dma/qcom/gpi.c+16−5
@@ -1754,10 +1754,14 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, · 1753 unmodified lines
17541754 tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN);
17551755
17561756 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);
1757− if (spi->cmd == SPI_RX)
1757+ if (spi->cmd == SPI_RX) {
17581758 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB);
1759− else
1759+ } else if (spi->cmd == SPI_TX) {
1760+ tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1761+ } else { /* SPI_DUPLEX */
17601762 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1763+ tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1764+ }
17611765 }
17621766
17631767 /* create the dma tre */
@@ -2148,6 +2152,7 @@ static int gpi_probe(struct platform_device *pdev) · 384 unmodified lines
21482152 {
21492153 struct gpi_dev *gpi_dev;
21502154 unsigned int i;
2155+ u32 ee_offset;
21512156 int ret;
21522157
21532158 gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL);
@@ -2175,6 +2180,9 @@ static int gpi_probe(struct platform_device *pdev) · 21 unmodified lines
21752180 return ret;
21762181 }
21772182
2183+ ee_offset = (uintptr_t)device_get_match_data(gpi_dev->dev);
2184+ gpi_dev->ee_base = gpi_dev->ee_base - ee_offset;
2185+
21782186 gpi_dev->ev_factor = EV_FACTOR;
21792187
21802188 ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64));
@@ -2278,9 +2286,12 @@ static int gpi_probe(struct platform_device *pdev) · 97 unmodified lines
22782286 }
22792287
22802288 static const struct of_device_id gpi_of_match[] = {
2281− { .compatible = "qcom,sdm845-gpi-dma" },
2282− { .compatible = "qcom,sm8150-gpi-dma" },
2283− { .compatible = "qcom,sm8250-gpi-dma" },
2289+ { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 },
2290+ { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 },
2291+ { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 },
2292+ { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 },
2293+ { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },
2294+ { .compatible = "qcom,sm8450-gpi-dma", .data = (void *)0x10000 },
22842295 { },
22852296 };
22862297 MODULE_DEVICE_TABLE(of, gpi_of_match);
drivers/dma/qcom/hidma.c+12−1
@@ -431,6 +431,7 @@ hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value, · 430 unmodified lines
431431 struct hidma_desc *mdesc = NULL;
432432 struct hidma_dev *mdma = mchan->dmadev;
433433 unsigned long irqflags;
434+ u64 byte_pattern, fill_pattern;
434435
435436 /* Get free descriptor */
436437 spin_lock_irqsave(&mchan->lock, irqflags);
@@ -443,9 +444,19 @@ hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value, · 6 unmodified lines
443444 if (!mdesc)
444445 return NULL;
445446
447+ byte_pattern = (char)value;
448+ fill_pattern = (byte_pattern << 56) |
449+ (byte_pattern << 48) |
450+ (byte_pattern << 40) |
451+ (byte_pattern << 32) |
452+ (byte_pattern << 24) |
453+ (byte_pattern << 16) |
454+ (byte_pattern << 8) |
455+ byte_pattern;
456+
446457 mdesc->desc.flags = flags;
447458 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
448− value, dest, len, flags,
459+ fill_pattern, dest, len, flags,
449460 HIDMA_TRE_MEMSET);
450461
451462 /* Place descriptor in prepared list */
drivers/dma/sf-pdma/sf-pdma.c+16−8
@@ -482,23 +482,30 @@ static void sf_pdma_setup_chans(struct sf_pdma *pdma) · 481 unmodified lines
482482 static int sf_pdma_probe(struct platform_device *pdev)
483483 {
484484 struct sf_pdma *pdma;
485− struct sf_pdma_chan *chan;
486485 struct resource *res;
487− int len, chans;
488− int ret;
486+ int ret, n_chans;
489487 const enum dma_slave_buswidth widths =
490488 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
491489 DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES |
492490 DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES |
493491 DMA_SLAVE_BUSWIDTH_64_BYTES;
494492
495− chans = PDMA_NR_CH;
496− len = sizeof(*pdma) + sizeof(*chan) * chans;
497− pdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
493+ ret = of_property_read_u32(pdev->dev.of_node, "dma-channels", &n_chans);
494+ if (ret) {
495+ /* backwards-compatibility for no dma-channels property */
496+ dev_dbg(&pdev->dev, "set number of channels to default value: 4\n");
497+ n_chans = PDMA_MAX_NR_CH;
498+ } else if (n_chans > PDMA_MAX_NR_CH) {
499+ dev_err(&pdev->dev, "the number of channels exceeds the maximum\n");
500+ return -EINVAL;
501+ }
502+
503+ pdma = devm_kzalloc(&pdev->dev, struct_size(pdma, chans, n_chans),
504+ GFP_KERNEL);
498505 if (!pdma)
499506 return -ENOMEM;
500507
501− pdma->n_chans = chans;
508+ pdma->n_chans = n_chans;
502509
503510 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
504511 pdma->membase = devm_ioremap_resource(&pdev->dev, res);
@@ -556,7 +563,7 @@ static int sf_pdma_remove(struct platform_device *pdev) · 51 unmodified lines
556563 struct sf_pdma_chan *ch;
557564 int i;
558565
559− for (i = 0; i < PDMA_NR_CH; i++) {
566+ for (i = 0; i < pdma->n_chans; i++) {
560567 ch = &pdma->chans[i];
561568
562569 devm_free_irq(&pdev->dev, ch->txirq, ch);
@@ -574,6 +581,7 @@ static int sf_pdma_remove(struct platform_device *pdev) · 11 unmodified lines
574581
575582 static const struct of_device_id sf_pdma_dt_ids[] = {
576583 { .compatible = "sifive,fu540-c000-pdma" },
584+ { .compatible = "sifive,pdma0" },
577585 {},
578586 };
579587 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids);
drivers/dma/sf-pdma/sf-pdma.h+2−6
@@ -22,11 +22,7 @@ · 21 unmodified lines
2222 #include "../dmaengine.h"
2323 #include "../virt-dma.h"
2424
25−#define PDMA_NR_CH 4
26−
27−#if (PDMA_NR_CH != 4)
28−#error "Please define PDMA_NR_CH to 4"
29−#endif
25+#define PDMA_MAX_NR_CH 4
3026
3127 #define PDMA_BASE_ADDR 0x3000000
3228 #define PDMA_CHAN_OFFSET 0x1000
@@ -118,7 +114,7 @@ struct sf_pdma { · 85 unmodified lines
118114 void __iomem *membase;
119115 void __iomem *mappedbase;
120116 u32 n_chans;
121− struct sf_pdma_chan chans[PDMA_NR_CH];
117+ struct sf_pdma_chan chans[];
122118 };
123119
124120 #endif /* _SF_PDMA_H */
drivers/dma/sh/Kconfig+1−1
@@ -50,7 +50,7 @@ config RENESAS_USB_DMAC · 49 unmodified lines
5050
5151 config RZ_DMAC
5252 tristate "Renesas RZ/{G2L,V2L} DMA Controller"
53− depends on ARCH_R9A07G044 || ARCH_R9A07G054 || COMPILE_TEST
53+ depends on ARCH_RZG2L || COMPILE_TEST
5454 select RENESAS_DMA
5555 select DMA_VIRTUAL_CHANNELS
5656 help
drivers/dma/sprd-dma.c+5−1
@@ -1117,7 +1117,11 @@ static int sprd_dma_probe(struct platform_device *pdev) · 1116 unmodified lines
11171117 u32 chn_count;
11181118 int ret, i;
11191119
1120− ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count);
1120+ /* Parse new and deprecated dma-channels properties */
1121+ ret = device_property_read_u32(&pdev->dev, "dma-channels", &chn_count);
1122+ if (ret)
1123+ ret = device_property_read_u32(&pdev->dev, "#dma-channels",
1124+ &chn_count);
11211125 if (ret) {
11221126 dev_err(&pdev->dev, "get dma channels count failed\n");
11231127 return ret;
drivers/dma/stm32-dma.c+268−43
@@ -208,6 +208,7 @@ struct stm32_dma_chan { · 207 unmodified lines
208208 u32 threshold;
209209 u32 mem_burst;
210210 u32 mem_width;
211+ enum dma_status status;
211212 };
212213
213214 struct stm32_dma_device {
@@ -485,6 +486,7 @@ static void stm32_dma_stop(struct stm32_dma_chan *chan) · 271 unmodified lines
485486 }
486487
487488 chan->busy = false;
489+ chan->status = DMA_COMPLETE;
488490 }
489491
490492 static int stm32_dma_terminate_all(struct dma_chan *c)
@@ -535,6 +537,13 @@ static void stm32_dma_dump_reg(struct stm32_dma_chan *chan) · 44 unmodified lines
535537 dev_dbg(chan2dev(chan), "SFCR: 0x%08x\n", sfcr);
536538 }
537539
540+static void stm32_dma_sg_inc(struct stm32_dma_chan *chan)
541+{
542+ chan->next_sg++;
543+ if (chan->desc->cyclic && (chan->next_sg == chan->desc->num_sgs))
544+ chan->next_sg = 0;
545+}
546+
538547 static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan);
539548
540549 static void stm32_dma_start_transfer(struct stm32_dma_chan *chan)
@@ -575,7 +584,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) · 34 unmodified lines
575584 stm32_dma_write(dmadev, STM32_DMA_SM1AR(chan->id), reg->dma_sm1ar);
576585 stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), reg->dma_sndtr);
577586
578− chan->next_sg++;
587+ stm32_dma_sg_inc(chan);
579588
580589 /* Clear interrupt status if it is there */
581590 status = stm32_dma_irq_status(chan);
@@ -588,11 +597,11 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) · 6 unmodified lines
588597 stm32_dma_dump_reg(chan);
589598
590599 /* Start DMA */
600+ chan->busy = true;
601+ chan->status = DMA_IN_PROGRESS;
591602 reg->dma_scr |= STM32_DMA_SCR_EN;
592603 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr);
593604
594− chan->busy = true;
595−
596605 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
597606 }
598607
@@ -605,41 +614,131 @@ static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) · 6 unmodified lines
605614 id = chan->id;
606615 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));
607616
608− if (dma_scr & STM32_DMA_SCR_DBM) {
609− if (chan->next_sg == chan->desc->num_sgs)
610− chan->next_sg = 0;
617+ sg_req = &chan->desc->sg_req[chan->next_sg];
611618
612− sg_req = &chan->desc->sg_req[chan->next_sg];
619+ if (dma_scr & STM32_DMA_SCR_CT) {
620+ dma_sm0ar = sg_req->chan_reg.dma_sm0ar;
621+ stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar);
622+ dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n",
623+ stm32_dma_read(dmadev, STM32_DMA_SM0AR(id)));
624+ } else {
625+ dma_sm1ar = sg_req->chan_reg.dma_sm1ar;
626+ stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar);
627+ dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n",
628+ stm32_dma_read(dmadev, STM32_DMA_SM1AR(id)));
629+ }
630+}
613631
614− if (dma_scr & STM32_DMA_SCR_CT) {
615− dma_sm0ar = sg_req->chan_reg.dma_sm0ar;
616− stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar);
617− dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n",
618− stm32_dma_read(dmadev, STM32_DMA_SM0AR(id)));
619− } else {
620− dma_sm1ar = sg_req->chan_reg.dma_sm1ar;
621− stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar);
622− dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n",
623− stm32_dma_read(dmadev, STM32_DMA_SM1AR(id)));
624− }
632+static void stm32_dma_handle_chan_paused(struct stm32_dma_chan *chan)
633+{
634+ struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
635+ u32 dma_scr;
636+
637+ /*
638+ * Read and store current remaining data items and peripheral/memory addresses to be
639+ * updated on resume
640+ */
641+ dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));
642+ /*
643+ * Transfer can be paused while between a previous resume and reconfiguration on transfer
644+ * complete. If transfer is cyclic and CIRC and DBM have been deactivated for resume, need
645+ * to set it here in SCR backup to ensure a good reconfiguration on transfer complete.
646+ */
647+ if (chan->desc && chan->desc->cyclic) {
648+ if (chan->desc->num_sgs == 1)
649+ dma_scr |= STM32_DMA_SCR_CIRC;
650+ else
651+ dma_scr |= STM32_DMA_SCR_DBM;
652+ }
653+ chan->chan_reg.dma_scr = dma_scr;
654+
655+ /*
656+ * Need to temporarily deactivate CIRC/DBM until next Transfer Complete interrupt, otherwise
657+ * on resume NDTR autoreload value will be wrong (lower than the initial period length)
658+ */
659+ if (chan->desc && chan->desc->cyclic) {
660+ dma_scr &= ~(STM32_DMA_SCR_DBM | STM32_DMA_SCR_CIRC);
661+ stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);
662+ }
663+
664+ chan->chan_reg.dma_sndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id));
665+
666+ dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan);
667+}
668+
669+static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan)
670+{
671+ struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
672+ struct stm32_dma_sg_req *sg_req;
673+ u32 dma_scr, status, id;
674+
675+ id = chan->id;
676+ dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));
677+
678+ /* Clear interrupt status if it is there */
679+ status = stm32_dma_irq_status(chan);
680+ if (status)
681+ stm32_dma_irq_clear(chan, status);
682+
683+ if (!chan->next_sg)
684+ sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1];
685+ else
686+ sg_req = &chan->desc->sg_req[chan->next_sg - 1];
687+
688+ /* Reconfigure NDTR with the initial value */
689+ stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), sg_req->chan_reg.dma_sndtr);
690+
691+ /* Restore SPAR */
692+ stm32_dma_write(dmadev, STM32_DMA_SPAR(id), sg_req->chan_reg.dma_spar);
693+
694+ /* Restore SM0AR/SM1AR whatever DBM/CT as they may have been modified */
695+ stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), sg_req->chan_reg.dma_sm0ar);
696+ stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), sg_req->chan_reg.dma_sm1ar);
697+
698+ /* Reactivate CIRC/DBM if needed */
699+ if (chan->chan_reg.dma_scr & STM32_DMA_SCR_DBM) {
700+ dma_scr |= STM32_DMA_SCR_DBM;
701+ /* Restore CT */
702+ if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CT)
703+ dma_scr &= ~STM32_DMA_SCR_CT;
704+ else
705+ dma_scr |= STM32_DMA_SCR_CT;
706+ } else if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CIRC) {
707+ dma_scr |= STM32_DMA_SCR_CIRC;
625708 }
709+ stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);
710+
711+ stm32_dma_configure_next_sg(chan);
712+
713+ stm32_dma_dump_reg(chan);
714+
715+ dma_scr |= STM32_DMA_SCR_EN;
716+ stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);
717+
718+ dev_dbg(chan2dev(chan), "vchan %pK: reconfigured after pause/resume\n", &chan->vchan);
626719 }
627720
628−static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan)
721+static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr)
629722 {
630− if (chan->desc) {
631− if (chan->desc->cyclic) {
632− vchan_cyclic_callback(&chan->desc->vdesc);
633− chan->next_sg++;
723+ if (!chan->desc)
724+ return;
725+
726+ if (chan->desc->cyclic) {
727+ vchan_cyclic_callback(&chan->desc->vdesc);
728+ stm32_dma_sg_inc(chan);
729+ /* cyclic while CIRC/DBM disable => post resume reconfiguration needed */
730+ if (!(scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM)))
731+ stm32_dma_post_resume_reconfigure(chan);
732+ else if (scr & STM32_DMA_SCR_DBM)
634733 stm32_dma_configure_next_sg(chan);
635− } else {
636− chan->busy = false;
637− if (chan->next_sg == chan->desc->num_sgs) {
638− vchan_cookie_complete(&chan->desc->vdesc);
639− chan->desc = NULL;
640− }
641− stm32_dma_start_transfer(chan);
734+ } else {
735+ chan->busy = false;
736+ chan->status = DMA_COMPLETE;
737+ if (chan->next_sg == chan->desc->num_sgs) {
738+ vchan_cookie_complete(&chan->desc->vdesc);
739+ chan->desc = NULL;
642740 }
741+ stm32_dma_start_transfer(chan);
643742 }
644743 }
645744
@@ -675,8 +774,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) · 29 unmodified lines
675774
676775 if (status & STM32_DMA_TCI) {
677776 stm32_dma_irq_clear(chan, STM32_DMA_TCI);
678− if (scr & STM32_DMA_SCR_TCIE)
679− stm32_dma_handle_chan_done(chan);
777+ if (scr & STM32_DMA_SCR_TCIE) {
778+ if (chan->status == DMA_PAUSED && !(scr & STM32_DMA_SCR_EN))
779+ stm32_dma_handle_chan_paused(chan);
780+ else
781+ stm32_dma_handle_chan_done(chan, scr);
782+ }
680783 status &= ~STM32_DMA_TCI;
681784 }
682785
@@ -711,6 +814,107 @@ static void stm32_dma_issue_pending(struct dma_chan *c) · 28 unmodified lines
711814 spin_unlock_irqrestore(&chan->vchan.lock, flags);
712815 }
713816
817+static int stm32_dma_pause(struct dma_chan *c)
818+{
819+ struct stm32_dma_chan *chan = to_stm32_dma_chan(c);
820+ unsigned long flags;
821+ int ret;
822+
823+ if (chan->status != DMA_IN_PROGRESS)
824+ return -EPERM;
825+
826+ spin_lock_irqsave(&chan->vchan.lock, flags);
827+ ret = stm32_dma_disable_chan(chan);
828+ /*
829+ * A transfer complete flag is set to indicate the end of transfer due to the stream
830+ * interruption, so wait for interrupt
831+ */
832+ if (!ret)
833+ chan->status = DMA_PAUSED;
834+ spin_unlock_irqrestore(&chan->vchan.lock, flags);
835+
836+ return ret;
837+}
838+
839+static int stm32_dma_resume(struct dma_chan *c)
840+{
841+ struct stm32_dma_chan *chan = to_stm32_dma_chan(c);
842+ struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
843+ struct stm32_dma_chan_reg chan_reg = chan->chan_reg;
844+ u32 id = chan->id, scr, ndtr, offset, spar, sm0ar, sm1ar;
845+ struct stm32_dma_sg_req *sg_req;
846+ unsigned long flags;
847+
848+ if (chan->status != DMA_PAUSED)
849+ return -EPERM;
850+
851+ scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));
852+ if (WARN_ON(scr & STM32_DMA_SCR_EN))
853+ return -EPERM;
854+
855+ spin_lock_irqsave(&chan->vchan.lock, flags);
856+
857+ /* sg_reg[prev_sg] contains original ndtr, sm0ar and sm1ar before pausing the transfer */
858+ if (!chan->next_sg)
859+ sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1];
860+ else
861+ sg_req = &chan->desc->sg_req[chan->next_sg - 1];
862+
863+ ndtr = sg_req->chan_reg.dma_sndtr;
864+ offset = (ndtr - chan_reg.dma_sndtr) << STM32_DMA_SCR_PSIZE_GET(chan_reg.dma_scr);
865+ spar = sg_req->chan_reg.dma_spar;
866+ sm0ar = sg_req->chan_reg.dma_sm0ar;
867+ sm1ar = sg_req->chan_reg.dma_sm1ar;
868+
869+ /*
870+ * The peripheral and/or memory addresses have to be updated in order to adjust the
871+ * address pointers. Need to check increment.
872+ */
873+ if (chan_reg.dma_scr & STM32_DMA_SCR_PINC)
874+ stm32_dma_write(dmadev, STM32_DMA_SPAR(id), spar + offset);
875+ else
876+ stm32_dma_write(dmadev, STM32_DMA_SPAR(id), spar);
877+
878+ if (!(chan_reg.dma_scr & STM32_DMA_SCR_MINC))
879+ offset = 0;
880+
881+ /*
882+ * In case of DBM, the current target could be SM1AR.
883+ * Need to temporarily deactivate CIRC/DBM to finish the current transfer, so
884+ * SM0AR becomes the current target and must be updated with SM1AR + offset if CT=1.
885+ */
886+ if ((chan_reg.dma_scr & STM32_DMA_SCR_DBM) && (chan_reg.dma_scr & STM32_DMA_SCR_CT))
887+ stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), sm1ar + offset);
888+ else
889+ stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), sm0ar + offset);
890+
891+ /* NDTR must be restored otherwise internal HW counter won't be correctly reset */
892+ stm32_dma_write(dmadev, STM32_DMA_SNDTR(id), chan_reg.dma_sndtr);
893+
894+ /*
895+ * Need to temporarily deactivate CIRC/DBM until next Transfer Complete interrupt,
896+ * otherwise NDTR autoreload value will be wrong (lower than the initial period length)
897+ */
898+ if (chan_reg.dma_scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM))
899+ chan_reg.dma_scr &= ~(STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM);
900+
901+ if (chan_reg.dma_scr & STM32_DMA_SCR_DBM)
902+ stm32_dma_configure_next_sg(chan);
903+
904+ stm32_dma_dump_reg(chan);
905+
906+ /* The stream may then be re-enabled to restart transfer from the point it was stopped */
907+ chan->status = DMA_IN_PROGRESS;
908+ chan_reg.dma_scr |= STM32_DMA_SCR_EN;
909+ stm32_dma_write(dmadev, STM32_DMA_SCR(id), chan_reg.dma_scr);
910+
911+ spin_unlock_irqrestore(&chan->vchan.lock, flags);
912+
913+ dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan);
914+
915+ return 0;
916+}
917+
714918 static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
715919 enum dma_transfer_direction direction,
716920 enum dma_slave_buswidth *buswidth,
@@ -978,10 +1182,12 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( · 261 unmodified lines
9781182 }
9791183
9801184 /* Enable Circular mode or double buffer mode */
981− if (buf_len == period_len)
1185+ if (buf_len == period_len) {
9821186 chan->chan_reg.dma_scr |= STM32_DMA_SCR_CIRC;
983− else
1187+ } else {
9841188 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM;
1189+ chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT;
1190+ }
9851191
9861192 /* Clear periph ctrl if client set it */
9871193 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL;
@@ -1091,24 +1297,36 @@ static bool stm32_dma_is_current_sg(struct stm32_dma_chan *chan) · 103 unmodified lines
10911297 {
10921298 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
10931299 struct stm32_dma_sg_req *sg_req;
1094− u32 dma_scr, dma_smar, id;
1300+ u32 dma_scr, dma_smar, id, period_len;
10951301
10961302 id = chan->id;
10971303 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));
10981304
1305+ /* In cyclic CIRC but not DBM, CT is not used */
10991306 if (!(dma_scr & STM32_DMA_SCR_DBM))
11001307 return true;
11011308
11021309 sg_req = &chan->desc->sg_req[chan->next_sg];
1310+ period_len = sg_req->len;
11031311
1312+ /* DBM - take care of a previous pause/resume not yet post reconfigured */
11041313 if (dma_scr & STM32_DMA_SCR_CT) {
11051314 dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM0AR(id));
1106− return (dma_smar == sg_req->chan_reg.dma_sm0ar);
1315+ /*
1316+ * If transfer has been pause/resumed,
1317+ * SM0AR is in the range of [SM0AR:SM0AR+period_len]
1318+ */
1319+ return (dma_smar >= sg_req->chan_reg.dma_sm0ar &&
1320+ dma_smar < sg_req->chan_reg.dma_sm0ar + period_len);
11071321 }
11081322
11091323 dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(id));
1110−
1111− return (dma_smar == sg_req->chan_reg.dma_sm1ar);
1324+ /*
1325+ * If transfer has been pause/resumed,
1326+ * SM1AR is in the range of [SM1AR:SM1AR+period_len]
1327+ */
1328+ return (dma_smar >= sg_req->chan_reg.dma_sm1ar &&
1329+ dma_smar < sg_req->chan_reg.dma_sm1ar + period_len);
11121330 }
11131331
11141332 static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan,
@@ -1148,7 +1366,7 @@ static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, · 33 unmodified lines
11481366
11491367 residue = stm32_dma_get_remaining_bytes(chan);
11501368
1151− if (!stm32_dma_is_current_sg(chan)) {
1369+ if (chan->desc->cyclic && !stm32_dma_is_current_sg(chan)) {
11521370 n_sg++;
11531371 if (n_sg == chan->desc->num_sgs)
11541372 n_sg = 0;
@@ -1188,7 +1406,12 @@ static enum dma_status stm32_dma_tx_status(struct dma_chan *c, · 33 unmodified lines
11881406 u32 residue = 0;
11891407
11901408 status = dma_cookie_status(c, cookie, state);
1191− if (status == DMA_COMPLETE || !state)
1409+ if (status == DMA_COMPLETE)
1410+ return status;
1411+
1412+ status = chan->status;
1413+
1414+ if (!state)
11921415 return status;
11931416
11941417 spin_lock_irqsave(&chan->vchan.lock, flags);
@@ -1377,6 +1600,8 @@ static int stm32_dma_probe(struct platform_device *pdev) · 182 unmodified lines
13771600 dd->device_prep_slave_sg = stm32_dma_prep_slave_sg;
13781601 dd->device_prep_dma_cyclic = stm32_dma_prep_dma_cyclic;
13791602 dd->device_config = stm32_dma_slave_config;
1603+ dd->device_pause = stm32_dma_pause;
1604+ dd->device_resume = stm32_dma_resume;
13801605 dd->device_terminate_all = stm32_dma_terminate_all;
13811606 dd->device_synchronize = stm32_dma_synchronize;
13821607 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
@@ -1482,7 +1707,7 @@ static int stm32_dma_runtime_resume(struct device *dev) · 99 unmodified lines
14821707 #endif
14831708
14841709 #ifdef CONFIG_PM_SLEEP
1485−static int stm32_dma_suspend(struct device *dev)
1710+static int stm32_dma_pm_suspend(struct device *dev)
14861711 {
14871712 struct stm32_dma_device *dmadev = dev_get_drvdata(dev);
14881713 int id, ret, scr;
@@ -1506,14 +1731,14 @@ static int stm32_dma_suspend(struct device *dev) · 17 unmodified lines
15061731 return 0;
15071732 }
15081733
1509−static int stm32_dma_resume(struct device *dev)
1734+static int stm32_dma_pm_resume(struct device *dev)
15101735 {
15111736 return pm_runtime_force_resume(dev);
15121737 }
15131738 #endif
15141739
15151740 static const struct dev_pm_ops stm32_dma_pm_ops = {
1516− SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_suspend, stm32_dma_resume)
1741+ SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_pm_suspend, stm32_dma_pm_resume)
15171742 SET_RUNTIME_PM_OPS(stm32_dma_runtime_suspend,
15181743 stm32_dma_runtime_resume, NULL)
15191744 };
drivers/dma/stm32-dmamux.c+1−1
@@ -267,7 +267,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev) · 266 unmodified lines
267267 ret = PTR_ERR(rst);
268268 if (ret == -EPROBE_DEFER)
269269 goto err_clk;
270− } else {
270+ } else if (count > 1) { /* Don't reset if there is only one dma-master */
271271 reset_control_assert(rst);
272272 udelay(2);
273273 reset_control_deassert(rst);
drivers/dma/stm32-mdma.c+32−21
@@ -34,7 +34,6 @@ · 33 unmodified lines
3434 #include "virt-dma.h"
3535
3636 #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */
37−#define STM32_MDMA_GISR1 0x0004 /* MDMA Int Status Reg 2 */
3837
3938 /* MDMA Channel x interrupt/status register */
4039 #define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */
@@ -73,6 +72,7 @@ · 32 unmodified lines
7372 #define STM32_MDMA_CCR_WEX BIT(14)
7473 #define STM32_MDMA_CCR_HEX BIT(13)
7574 #define STM32_MDMA_CCR_BEX BIT(12)
75+#define STM32_MDMA_CCR_SM BIT(8)
7676 #define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6)
7777 #define STM32_MDMA_CCR_PL(n) FIELD_PREP(STM32_MDMA_CCR_PL_MASK, (n))
7878 #define STM32_MDMA_CCR_TCIE BIT(5)
@@ -168,7 +168,7 @@ · 89 unmodified lines
168168
169169 #define STM32_MDMA_MAX_BUF_LEN 128
170170 #define STM32_MDMA_MAX_BLOCK_LEN 65536
171−#define STM32_MDMA_MAX_CHANNELS 63
171+#define STM32_MDMA_MAX_CHANNELS 32
172172 #define STM32_MDMA_MAX_REQUESTS 256
173173 #define STM32_MDMA_MAX_BURST 128
174174 #define STM32_MDMA_VERY_HIGH_PRIORITY 0x3
@@ -248,6 +248,7 @@ struct stm32_mdma_device { · 73 unmodified lines
248248 u32 nr_channels;
249249 u32 nr_requests;
250250 u32 nr_ahb_addr_masks;
251+ u32 chan_reserved;
251252 struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS];
252253 u32 ahb_addr_masks[];
253254 };
@@ -1317,26 +1318,16 @@ static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan) · 1063 unmodified lines
13171318 static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
13181319 {
13191320 struct stm32_mdma_device *dmadev = devid;
1320− struct stm32_mdma_chan *chan = devid;
1321+ struct stm32_mdma_chan *chan;
13211322 u32 reg, id, ccr, ien, status;
13221323
13231324 /* Find out which channel generates the interrupt */
13241325 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);
1325− if (status) {
1326− id = __ffs(status);
1327− } else {
1328− status = readl_relaxed(dmadev->base + STM32_MDMA_GISR1);
1329− if (!status) {
1330− dev_dbg(mdma2dev(dmadev), "spurious it\n");
1331− return IRQ_NONE;
1332− }
1333− id = __ffs(status);
1334− /*
1335− * As GISR0 provides status for channel id from 0 to 31,
1336− * so GISR1 provides status for channel id from 32 to 62
1337− */
1338− id += 32;
1326+ if (!status) {
1327+ dev_dbg(mdma2dev(dmadev), "spurious it\n");
1328+ return IRQ_NONE;
13391329 }
1330+ id = __ffs(status);
13401331
13411332 chan = &dmadev->chan[id];
13421333 if (!chan) {
@@ -1354,9 +1345,12 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) · 11 unmodified lines
13541345
13551346 if (!(status & ien)) {
13561347 spin_unlock(&chan->vchan.lock);
1357− dev_warn(chan2dev(chan),
1358− "spurious it (status=0x%04x, ien=0x%04x)\n",
1359− status, ien);
1348+ if (chan->busy)
1349+ dev_warn(chan2dev(chan),
1350+ "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);
1351+ else
1352+ dev_dbg(chan2dev(chan),
1353+ "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);
13601354 return IRQ_NONE;
13611355 }
13621356
@@ -1456,10 +1450,23 @@ static void stm32_mdma_free_chan_resources(struct dma_chan *c) · 93 unmodified lines
14561450 chan->desc_pool = NULL;
14571451 }
14581452
1453+static bool stm32_mdma_filter_fn(struct dma_chan *c, void *fn_param)
1454+{
1455+ struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1456+ struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1457+
1458+ /* Check if chan is marked Secure */
1459+ if (dmadev->chan_reserved & BIT(chan->id))
1460+ return false;
1461+
1462+ return true;
1463+}
1464+
14591465 static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec,
14601466 struct of_dma *ofdma)
14611467 {
14621468 struct stm32_mdma_device *dmadev = ofdma->of_dma_data;
1469+ dma_cap_mask_t mask = dmadev->ddev.cap_mask;
14631470 struct stm32_mdma_chan *chan;
14641471 struct dma_chan *c;
14651472 struct stm32_mdma_chan_config config;
@@ -1485,7 +1492,7 @@ static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec, · 19 unmodified lines
14851492 return NULL;
14861493 }
14871494
1488− c = dma_get_any_slave_channel(&dmadev->ddev);
1495+ c = __dma_request_channel(&mask, stm32_mdma_filter_fn, &config, ofdma->of_node);
14891496 if (!c) {
14901497 dev_err(mdma2dev(dmadev), "No more channels available\n");
14911498 return NULL;
@@ -1615,6 +1622,10 @@ static int stm32_mdma_probe(struct platform_device *pdev) · 123 unmodified lines
16151622 for (i = 0; i < dmadev->nr_channels; i++) {
16161623 chan = &dmadev->chan[i];
16171624 chan->id = i;
1625+
1626+ if (stm32_mdma_read(dmadev, STM32_MDMA_CCR(i)) & STM32_MDMA_CCR_SM)
1627+ dmadev->chan_reserved |= BIT(i);
1628+
16181629 chan->vchan.desc_free = stm32_mdma_desc_free;
16191630 vchan_init(&chan->vchan, dd);
16201631 }
drivers/dma/sun6i-dma.c+58−34
@@ -90,6 +90,14 @@ · 89 unmodified lines
9090
9191 #define DMA_CHAN_CUR_PARA 0x1c
9292
93+/*
94+ * LLI address mangling
95+ *
96+ * The LLI link physical address is also mangled, but we avoid dealing
97+ * with that by allocating LLIs from the DMA32 zone.
98+ */
99+#define SRC_HIGH_ADDR(x) (((x) & 0x3U) << 16)
100+#define DST_HIGH_ADDR(x) (((x) & 0x3U) << 18)
93101
94102 /*
95103 * Various hardware related defines
@@ -132,6 +140,7 @@ struct sun6i_dma_config { · 36 unmodified lines
132140 u32 dst_burst_lengths;
133141 u32 src_addr_widths;
134142 u32 dst_addr_widths;
143+ bool has_high_addr;
135144 bool has_mbus_clk;
136145 };
137146
@@ -241,9 +250,7 @@ static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev) · 103 unmodified lines
241250 static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev,
242251 struct sun6i_pchan *pchan)
243252 {
244− phys_addr_t reg = virt_to_phys(pchan->base);
245−
246− dev_dbg(sdev->slave.dev, "Chan %d reg: %pa\n"
253+ dev_dbg(sdev->slave.dev, "Chan %d reg:\n"
247254 "\t___en(%04x): \t0x%08x\n"
248255 "\tpause(%04x): \t0x%08x\n"
249256 "\tstart(%04x): \t0x%08x\n"
@@ -252,7 +259,7 @@ static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, · 2 unmodified lines
252259 "\t__dst(%04x): \t0x%08x\n"
253260 "\tcount(%04x): \t0x%08x\n"
254261 "\t_para(%04x): \t0x%08x\n\n",
255− pchan->idx, &reg,
262+ pchan->idx,
256263 DMA_CHAN_ENABLE,
257264 readl(pchan->base + DMA_CHAN_ENABLE),
258265 DMA_CHAN_PAUSE,
@@ -385,17 +392,16 @@ static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev, · 126 unmodified lines
385392 }
386393
387394 static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan,
388− struct sun6i_dma_lli *lli)
395+ struct sun6i_dma_lli *v_lli,
396+ dma_addr_t p_lli)
389397 {
390− phys_addr_t p_lli = virt_to_phys(lli);
391−
392398 dev_dbg(chan2dev(&vchan->vc.chan),
393− "\n\tdesc: p - %pa v - 0x%p\n"
399+ "\n\tdesc:\tp - %pad v - 0x%p\n"
394400 "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n"
395401 "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n",
396− &p_lli, lli,
397− lli->cfg, lli->src, lli->dst,
398− lli->len, lli->para, lli->p_lli_next);
402+ &p_lli, v_lli,
403+ v_lli->cfg, v_lli->src, v_lli->dst,
404+ v_lli->len, v_lli->para, v_lli->p_lli_next);
399405 }
400406
401407 static void sun6i_dma_free_desc(struct virt_dma_desc *vd)
@@ -445,7 +451,7 @@ static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) · 43 unmodified lines
445451 pchan->desc = to_sun6i_desc(&desc->tx);
446452 pchan->done = NULL;
447453
448− sun6i_dma_dump_lli(vchan, pchan->desc->v_lli);
454+ sun6i_dma_dump_lli(vchan, pchan->desc->v_lli, pchan->desc->p_lli);
449455
450456 irq_reg = pchan->idx / DMA_IRQ_CHAN_NR;
451457 irq_offset = pchan->idx % DMA_IRQ_CHAN_NR;
@@ -626,6 +632,18 @@ static int set_config(struct sun6i_dma_dev *sdev, · 174 unmodified lines
626632 return 0;
627633 }
628634
635+static inline void sun6i_dma_set_addr(struct sun6i_dma_dev *sdev,
636+ struct sun6i_dma_lli *v_lli,
637+ dma_addr_t src, dma_addr_t dst)
638+{
639+ v_lli->src = lower_32_bits(src);
640+ v_lli->dst = lower_32_bits(dst);
641+
642+ if (sdev->cfg->has_high_addr)
643+ v_lli->para |= SRC_HIGH_ADDR(upper_32_bits(src)) |
644+ DST_HIGH_ADDR(upper_32_bits(dst));
645+}
646+
629647 static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy(
630648 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
631649 size_t len, unsigned long flags)
@@ -648,16 +666,15 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( · 16 unmodified lines
648666 if (!txd)
649667 return NULL;
650668
651− v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
669+ v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli);
652670 if (!v_lli) {
653671 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
654672 goto err_txd_free;
655673 }
656674
657− v_lli->src = src;
658− v_lli->dst = dest;
659675 v_lli->len = len;
660676 v_lli->para = NORMAL_WAIT;
677+ sun6i_dma_set_addr(sdev, v_lli, src, dest);
661678
662679 burst = convert_burst(8);
663680 width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES);
@@ -670,7 +687,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( · 6 unmodified lines
670687
671688 sun6i_dma_lli_add(NULL, v_lli, p_lli, txd);
672689
673− sun6i_dma_dump_lli(vchan, v_lli);
690+ sun6i_dma_dump_lli(vchan, v_lli, p_lli);
674691
675692 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
676693
@@ -708,7 +725,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( · 31 unmodified lines
708725 return NULL;
709726
710727 for_each_sg(sgl, sg, sg_len, i) {
711− v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
728+ v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli);
712729 if (!v_lli)
713730 goto err_lli_free;
714731
@@ -716,8 +733,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( · 1 unmodified line
716733 v_lli->para = NORMAL_WAIT;
717734
718735 if (dir == DMA_MEM_TO_DEV) {
719− v_lli->src = sg_dma_address(sg);
720− v_lli->dst = sconfig->dst_addr;
736+ sun6i_dma_set_addr(sdev, v_lli,
737+ sg_dma_address(sg),
738+ sconfig->dst_addr);
721739 v_lli->cfg = lli_cfg;
722740 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
723741 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
@@ -729,8 +747,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( · 5 unmodified lines
729747 sg_dma_len(sg), flags);
730748
731749 } else {
732− v_lli->src = sconfig->src_addr;
733− v_lli->dst = sg_dma_address(sg);
750+ sun6i_dma_set_addr(sdev, v_lli,
751+ sconfig->src_addr,
752+ sg_dma_address(sg));
734753 v_lli->cfg = lli_cfg;
735754 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
736755 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
@@ -746,14 +765,16 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( · 9 unmodified lines
746765 }
747766
748767 dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli);
749− for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
750− sun6i_dma_dump_lli(vchan, prev);
768+ for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
769+ p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
770+ sun6i_dma_dump_lli(vchan, v_lli, p_lli);
751771
752772 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
753773
754774 err_lli_free:
755− for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
756− dma_pool_free(sdev->pool, prev, virt_to_phys(prev));
775+ for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
776+ p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
777+ dma_pool_free(sdev->pool, v_lli, p_lli);
757778 kfree(txd);
758779 return NULL;
759780 }
@@ -787,7 +808,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( · 27 unmodified lines
787808 return NULL;
788809
789810 for (i = 0; i < periods; i++) {
790− v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli);
811+ v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli);
791812 if (!v_lli) {
792813 dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
793814 goto err_lli_free;
@@ -797,14 +818,16 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( · 3 unmodified lines
797818 v_lli->para = NORMAL_WAIT;
798819
799820 if (dir == DMA_MEM_TO_DEV) {
800− v_lli->src = buf_addr + period_len * i;
801− v_lli->dst = sconfig->dst_addr;
821+ sun6i_dma_set_addr(sdev, v_lli,
822+ buf_addr + period_len * i,
823+ sconfig->dst_addr);
802824 v_lli->cfg = lli_cfg;
803825 sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
804826 sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
805827 } else {
806− v_lli->src = sconfig->src_addr;
807− v_lli->dst = buf_addr + period_len * i;
828+ sun6i_dma_set_addr(sdev, v_lli,
829+ sconfig->src_addr,
830+ buf_addr + period_len * i);
808831 v_lli->cfg = lli_cfg;
809832 sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
810833 sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
@@ -820,8 +843,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( · 9 unmodified lines
820843 return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
821844
822845 err_lli_free:
823− for (prev = txd->v_lli; prev; prev = prev->v_lli_next)
824− dma_pool_free(sdev->pool, prev, virt_to_phys(prev));
846+ for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
847+ p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
848+ dma_pool_free(sdev->pool, v_lli, p_lli);
825849 kfree(txd);
826850 return NULL;
827851 }
@@ -1174,8 +1198,6 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = { · 346 unmodified lines
11741198 };
11751199
11761200 /*
1177− * TODO: Add support for more than 4g physical addressing.
1178− *
11791201 * The A100 binding uses the number of dma channels from the
11801202 * device tree node.
11811203 */
@@ -1194,6 +1216,7 @@ static struct sun6i_dma_config sun50i_a100_dma_cfg = { · 12 unmodified lines
11941216 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
11951217 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
11961218 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1219+ .has_high_addr = true,
11971220 .has_mbus_clk = true,
11981221 };
11991222
@@ -1248,6 +1271,7 @@ static const struct of_device_id sun6i_dma_match[] = { · 48 unmodified lines
12481271 { .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg },
12491272 { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
12501273 { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
1274+ { .compatible = "allwinner,sun20i-d1-dma", .data = &sun50i_a100_dma_cfg },
12511275 { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg },
12521276 { .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg },
12531277 { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg },
drivers/dma/tegra186-gpc-dma.c+1498−0
@@ -0,0 +1,1498 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * DMA driver for NVIDIA Tegra GPC DMA controller.
4+ *
5+ * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved.
6+ */
7+
8+#include <linux/bitfield.h>
9+#include <linux/dmaengine.h>
10+#include <linux/dma-mapping.h>
11+#include <linux/interrupt.h>
12+#include <linux/iommu.h>
13+#include <linux/iopoll.h>
14+#include <linux/minmax.h>
15+#include <linux/module.h>
16+#include <linux/of_device.h>
17+#include <linux/of_dma.h>
18+#include <linux/platform_device.h>
19+#include <linux/reset.h>
20+#include <linux/slab.h>
21+#include <dt-bindings/memory/tegra186-mc.h>
22+#include "virt-dma.h"
23+
24+/* CSR register */
25+#define TEGRA_GPCDMA_CHAN_CSR 0x00
26+#define TEGRA_GPCDMA_CSR_ENB BIT(31)
27+#define TEGRA_GPCDMA_CSR_IE_EOC BIT(30)
28+#define TEGRA_GPCDMA_CSR_ONCE BIT(27)
29+
30+#define TEGRA_GPCDMA_CSR_FC_MODE GENMASK(25, 24)
31+#define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO \
32+ FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0)
33+#define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO \
34+ FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1)
35+#define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO \
36+ FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2)
37+#define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO \
38+ FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3)
39+
40+#define TEGRA_GPCDMA_CSR_DMA GENMASK(23, 21)
41+#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC \
42+ FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0)
43+#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC \
44+ FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1)
45+#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC \
46+ FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2)
47+#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC \
48+ FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3)
49+#define TEGRA_GPCDMA_CSR_DMA_MEM2MEM \
50+ FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4)
51+#define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT \
52+ FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6)
53+
54+#define TEGRA_GPCDMA_CSR_REQ_SEL_MASK GENMASK(20, 16)
55+#define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED \
56+ FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4)
57+#define TEGRA_GPCDMA_CSR_IRQ_MASK BIT(15)
58+#define TEGRA_GPCDMA_CSR_WEIGHT GENMASK(13, 10)
59+
60+/* STATUS register */
61+#define TEGRA_GPCDMA_CHAN_STATUS 0x004
62+#define TEGRA_GPCDMA_STATUS_BUSY BIT(31)
63+#define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30)
64+#define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28)
65+#define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY BIT(27)
66+#define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE BIT(26)
67+#define TEGRA_GPCDMA_STATUS_CHANNEL_RX BIT(25)
68+#define TEGRA_GPCDMA_STATUS_CHANNEL_TX BIT(24)
69+#define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA BIT(23)
70+#define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21)
71+#define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20)
72+
73+#define TEGRA_GPCDMA_CHAN_CSRE 0x008
74+#define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31)
75+
76+/* Source address */
77+#define TEGRA_GPCDMA_CHAN_SRC_PTR 0x00C
78+
79+/* Destination address */
80+#define TEGRA_GPCDMA_CHAN_DST_PTR 0x010
81+
82+/* High address pointer */
83+#define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR 0x014
84+#define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR GENMASK(7, 0)
85+#define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR GENMASK(23, 16)
86+
87+/* MC sequence register */
88+#define TEGRA_GPCDMA_CHAN_MCSEQ 0x18
89+#define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31)
90+#define TEGRA_GPCDMA_MCSEQ_REQ_COUNT GENMASK(30, 25)
91+#define TEGRA_GPCDMA_MCSEQ_BURST GENMASK(24, 23)
92+#define TEGRA_GPCDMA_MCSEQ_BURST_2 \
93+ FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0)
94+#define TEGRA_GPCDMA_MCSEQ_BURST_16 \
95+ FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3)
96+#define TEGRA_GPCDMA_MCSEQ_WRAP1 GENMASK(22, 20)
97+#define TEGRA_GPCDMA_MCSEQ_WRAP0 GENMASK(19, 17)
98+#define TEGRA_GPCDMA_MCSEQ_WRAP_NONE 0
99+
100+#define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK GENMASK(13, 7)
101+#define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK GENMASK(6, 0)
102+
103+/* MMIO sequence register */
104+#define TEGRA_GPCDMA_CHAN_MMIOSEQ 0x01c
105+#define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31)
106+#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH GENMASK(30, 28)
107+#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 \
108+ FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0)
109+#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16 \
110+ FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1)
111+#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32 \
112+ FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2)
113+#define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP BIT(27)
114+#define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT 23
115+#define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN 2U
116+#define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX 32U
117+#define TEGRA_GPCDMA_MMIOSEQ_BURST(bs) \
118+ (GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT)
119+#define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID GENMASK(22, 19)
120+#define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD GENMASK(18, 16)
121+#define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT GENMASK(8, 7)
122+
123+/* Channel WCOUNT */
124+#define TEGRA_GPCDMA_CHAN_WCOUNT 0x20
125+
126+/* Transfer count */
127+#define TEGRA_GPCDMA_CHAN_XFER_COUNT 0x24
128+
129+/* DMA byte count status */
130+#define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS 0x28
131+
132+/* Error Status Register */
133+#define TEGRA_GPCDMA_CHAN_ERR_STATUS 0x30
134+#define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT 8
135+#define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK 0xF
136+#define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ( \
137+ ((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) & \
138+ TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK)
139+#define TEGRA_DMA_BM_FIFO_FULL_ERR 0xF
140+#define TEGRA_DMA_PERIPH_FIFO_FULL_ERR 0xE
141+#define TEGRA_DMA_PERIPH_ID_ERR 0xD
142+#define TEGRA_DMA_STREAM_ID_ERR 0xC
143+#define TEGRA_DMA_MC_SLAVE_ERR 0xB
144+#define TEGRA_DMA_MMIO_SLAVE_ERR 0xA
145+
146+/* Fixed Pattern */
147+#define TEGRA_GPCDMA_CHAN_FIXED_PATTERN 0x34
148+
149+#define TEGRA_GPCDMA_CHAN_TZ 0x38
150+#define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1 BIT(0)
151+#define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1 BIT(1)
152+
153+#define TEGRA_GPCDMA_CHAN_SPARE 0x3c
154+#define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC BIT(16)
155+
156+/*
157+ * If any burst is in flight and DMA paused then this is the time to complete
158+ * on-flight burst and update DMA status register.
159+ */
160+#define TEGRA_GPCDMA_BURST_COMPLETE_TIME 20
161+#define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT 100
162+
163+/* Channel base address offset from GPCDMA base address */
164+#define TEGRA_GPCDMA_CHANNEL_BASE_ADD_OFFSET 0x20000
165+
166+struct tegra_dma;
167+struct tegra_dma_channel;
168+
169+/*
170+ * tegra_dma_chip_data Tegra chip specific DMA data
171+ * @nr_channels: Number of channels available in the controller.
172+ * @channel_reg_size: Channel register size.
173+ * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
174+ * @hw_support_pause: DMA HW engine support pause of the channel.
175+ */
176+struct tegra_dma_chip_data {
177+ bool hw_support_pause;
178+ unsigned int nr_channels;
179+ unsigned int channel_reg_size;
180+ unsigned int max_dma_count;
181+ int (*terminate)(struct tegra_dma_channel *tdc);
182+};
183+
184+/* DMA channel registers */
185+struct tegra_dma_channel_regs {
186+ u32 csr;
187+ u32 src_ptr;
188+ u32 dst_ptr;
189+ u32 high_addr_ptr;
190+ u32 mc_seq;
191+ u32 mmio_seq;
192+ u32 wcount;
193+ u32 fixed_pattern;
194+};
195+
196+/*
197+ * tegra_dma_sg_req: DMA request details to configure hardware. This
198+ * contains the details for one transfer to configure DMA hw.
199+ * The client's request for data transfer can be broken into multiple
200+ * sub-transfer as per requester details and hw support. This sub transfer
201+ * get added as an array in Tegra DMA desc which manages the transfer details.
202+ */
203+struct tegra_dma_sg_req {
204+ unsigned int len;
205+ struct tegra_dma_channel_regs ch_regs;
206+};
207+
208+/*
209+ * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to
210+ * manage client request and keep track of transfer status, callbacks
211+ * and request counts etc.
212+ */
213+struct tegra_dma_desc {
214+ bool cyclic;
215+ unsigned int bytes_req;
216+ unsigned int bytes_xfer;
217+ unsigned int sg_idx;
218+ unsigned int sg_count;
219+ struct virt_dma_desc vd;
220+ struct tegra_dma_channel *tdc;
221+ struct tegra_dma_sg_req sg_req[];
222+};
223+
224+/*
225+ * tegra_dma_channel: Channel specific information
226+ */
227+struct tegra_dma_channel {
228+ bool config_init;
229+ char name[30];
230+ enum dma_transfer_direction sid_dir;
231+ int id;
232+ int irq;
233+ int slave_id;
234+ struct tegra_dma *tdma;
235+ struct virt_dma_chan vc;
236+ struct tegra_dma_desc *dma_desc;
237+ struct dma_slave_config dma_sconfig;
238+ unsigned int stream_id;
239+ unsigned long chan_base_offset;
240+};
241+
242+/*
243+ * tegra_dma: Tegra DMA specific information
244+ */
245+struct tegra_dma {
246+ const struct tegra_dma_chip_data *chip_data;
247+ unsigned long sid_m2d_reserved;
248+ unsigned long sid_d2m_reserved;
249+ void __iomem *base_addr;
250+ struct device *dev;
251+ struct dma_device dma_dev;
252+ struct reset_control *rst;
253+ struct tegra_dma_channel channels[];
254+};
255+
256+static inline void tdc_write(struct tegra_dma_channel *tdc,
257+ u32 reg, u32 val)
258+{
259+ writel_relaxed(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);
260+}
261+
262+static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
263+{
264+ return readl_relaxed(tdc->tdma->base_addr + tdc->chan_base_offset + reg);
265+}
266+
267+static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
268+{
269+ return container_of(dc, struct tegra_dma_channel, vc.chan);
270+}
271+
272+static inline struct tegra_dma_desc *vd_to_tegra_dma_desc(struct virt_dma_desc *vd)
273+{
274+ return container_of(vd, struct tegra_dma_desc, vd);
275+}
276+
277+static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
278+{
279+ return tdc->vc.chan.device->dev;
280+}
281+
282+static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc)
283+{
284+ dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n",
285+ tdc->id, tdc->name);
286+ dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n",
287+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR),
288+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS),
289+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE),
290+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR),
291+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR)
292+ );
293+ dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n",
294+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ),
295+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ),
296+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT),
297+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT),
298+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS)
299+ );
300+ dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n",
301+ tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS));
302+}
303+
304+static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc,
305+ enum dma_transfer_direction direction)
306+{
307+ struct tegra_dma *tdma = tdc->tdma;
308+ int sid = tdc->slave_id;
309+
310+ if (!is_slave_direction(direction))
311+ return 0;
312+
313+ switch (direction) {
314+ case DMA_MEM_TO_DEV:
315+ if (test_and_set_bit(sid, &tdma->sid_m2d_reserved)) {
316+ dev_err(tdma->dev, "slave id already in use\n");
317+ return -EINVAL;
318+ }
319+ break;
320+ case DMA_DEV_TO_MEM:
321+ if (test_and_set_bit(sid, &tdma->sid_d2m_reserved)) {
322+ dev_err(tdma->dev, "slave id already in use\n");
323+ return -EINVAL;
324+ }
325+ break;
326+ default:
327+ break;
328+ }
329+
330+ tdc->sid_dir = direction;
331+
332+ return 0;
333+}
334+
335+static void tegra_dma_sid_free(struct tegra_dma_channel *tdc)
336+{
337+ struct tegra_dma *tdma = tdc->tdma;
338+ int sid = tdc->slave_id;
339+
340+ switch (tdc->sid_dir) {
341+ case DMA_MEM_TO_DEV:
342+ clear_bit(sid, &tdma->sid_m2d_reserved);
343+ break;
344+ case DMA_DEV_TO_MEM:
345+ clear_bit(sid, &tdma->sid_d2m_reserved);
346+ break;
347+ default:
348+ break;
349+ }
350+
351+ tdc->sid_dir = DMA_TRANS_NONE;
352+}
353+
354+static void tegra_dma_desc_free(struct virt_dma_desc *vd)
355+{
356+ kfree(container_of(vd, struct tegra_dma_desc, vd));
357+}
358+
359+static int tegra_dma_slave_config(struct dma_chan *dc,
360+ struct dma_slave_config *sconfig)
361+{
362+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
363+
364+ memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
365+ tdc->config_init = true;
366+
367+ return 0;
368+}
369+
370+static int tegra_dma_pause(struct tegra_dma_channel *tdc)
371+{
372+ int ret;
373+ u32 val;
374+
375+ val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
376+ val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
377+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
378+
379+ /* Wait until busy bit is de-asserted */
380+ ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
381+ tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
382+ val,
383+ !(val & TEGRA_GPCDMA_STATUS_BUSY),
384+ TEGRA_GPCDMA_BURST_COMPLETE_TIME,
385+ TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
386+
387+ if (ret) {
388+ dev_err(tdc2dev(tdc), "DMA pause timed out\n");
389+ tegra_dma_dump_chan_regs(tdc);
390+ }
391+
392+ return ret;
393+}
394+
395+static int tegra_dma_device_pause(struct dma_chan *dc)
396+{
397+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
398+ unsigned long flags;
399+ int ret;
400+
401+ if (!tdc->tdma->chip_data->hw_support_pause)
402+ return -ENOSYS;
403+
404+ spin_lock_irqsave(&tdc->vc.lock, flags);
405+ ret = tegra_dma_pause(tdc);
406+ spin_unlock_irqrestore(&tdc->vc.lock, flags);
407+
408+ return ret;
409+}
410+
411+static void tegra_dma_resume(struct tegra_dma_channel *tdc)
412+{
413+ u32 val;
414+
415+ val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
416+ val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
417+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
418+}
419+
420+static int tegra_dma_device_resume(struct dma_chan *dc)
421+{
422+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
423+ unsigned long flags;
424+
425+ if (!tdc->tdma->chip_data->hw_support_pause)
426+ return -ENOSYS;
427+
428+ spin_lock_irqsave(&tdc->vc.lock, flags);
429+ tegra_dma_resume(tdc);
430+ spin_unlock_irqrestore(&tdc->vc.lock, flags);
431+
432+ return 0;
433+}
434+
435+static void tegra_dma_disable(struct tegra_dma_channel *tdc)
436+{
437+ u32 csr, status;
438+
439+ csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
440+
441+ /* Disable interrupts */
442+ csr &= ~TEGRA_GPCDMA_CSR_IE_EOC;
443+
444+ /* Disable DMA */
445+ csr &= ~TEGRA_GPCDMA_CSR_ENB;
446+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
447+
448+ /* Clear interrupt status if it is there */
449+ status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
450+ if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) {
451+ dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
452+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status);
453+ }
454+}
455+
456+static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc)
457+{
458+ struct tegra_dma_desc *dma_desc = tdc->dma_desc;
459+ struct tegra_dma_channel_regs *ch_regs;
460+ int ret;
461+ u32 val;
462+
463+ dma_desc->sg_idx++;
464+
465+ /* Reset the sg index for cyclic transfers */
466+ if (dma_desc->sg_idx == dma_desc->sg_count)
467+ dma_desc->sg_idx = 0;
468+
469+ /* Configure next transfer immediately after DMA is busy */
470+ ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
471+ tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
472+ val,
473+ (val & TEGRA_GPCDMA_STATUS_BUSY), 0,
474+ TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
475+ if (ret)
476+ return;
477+
478+ ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
479+
480+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
481+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
482+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
483+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
484+
485+ /* Start DMA */
486+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
487+ ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
488+}
489+
490+static void tegra_dma_start(struct tegra_dma_channel *tdc)
491+{
492+ struct tegra_dma_desc *dma_desc = tdc->dma_desc;
493+ struct tegra_dma_channel_regs *ch_regs;
494+ struct virt_dma_desc *vdesc;
495+
496+ if (!dma_desc) {
497+ vdesc = vchan_next_desc(&tdc->vc);
498+ if (!vdesc)
499+ return;
500+
501+ dma_desc = vd_to_tegra_dma_desc(vdesc);
502+ list_del(&vdesc->node);
503+ dma_desc->tdc = tdc;
504+ tdc->dma_desc = dma_desc;
505+
506+ tegra_dma_resume(tdc);
507+ }
508+
509+ ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
510+
511+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
512+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0);
513+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
514+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
515+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
516+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern);
517+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq);
518+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq);
519+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr);
520+
521+ /* Start DMA */
522+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
523+ ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
524+}
525+
526+static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc)
527+{
528+ vchan_cookie_complete(&tdc->dma_desc->vd);
529+
530+ tegra_dma_sid_free(tdc);
531+ tdc->dma_desc = NULL;
532+}
533+
534+static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc,
535+ unsigned int err_status)
536+{
537+ switch (TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) {
538+ case TEGRA_DMA_BM_FIFO_FULL_ERR:
539+ dev_err(tdc->tdma->dev,
540+ "GPCDMA CH%d bm fifo full\n", tdc->id);
541+ break;
542+
543+ case TEGRA_DMA_PERIPH_FIFO_FULL_ERR:
544+ dev_err(tdc->tdma->dev,
545+ "GPCDMA CH%d peripheral fifo full\n", tdc->id);
546+ break;
547+
548+ case TEGRA_DMA_PERIPH_ID_ERR:
549+ dev_err(tdc->tdma->dev,
550+ "GPCDMA CH%d illegal peripheral id\n", tdc->id);
551+ break;
552+
553+ case TEGRA_DMA_STREAM_ID_ERR:
554+ dev_err(tdc->tdma->dev,
555+ "GPCDMA CH%d illegal stream id\n", tdc->id);
556+ break;
557+
558+ case TEGRA_DMA_MC_SLAVE_ERR:
559+ dev_err(tdc->tdma->dev,
560+ "GPCDMA CH%d mc slave error\n", tdc->id);
561+ break;
562+
563+ case TEGRA_DMA_MMIO_SLAVE_ERR:
564+ dev_err(tdc->tdma->dev,
565+ "GPCDMA CH%d mmio slave error\n", tdc->id);
566+ break;
567+
568+ default:
569+ dev_err(tdc->tdma->dev,
570+ "GPCDMA CH%d security violation %x\n", tdc->id,
571+ err_status);
572+ }
573+}
574+
575+static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
576+{
577+ struct tegra_dma_channel *tdc = dev_id;
578+ struct tegra_dma_desc *dma_desc = tdc->dma_desc;
579+ struct tegra_dma_sg_req *sg_req;
580+ u32 status;
581+
582+ /* Check channel error status register */
583+ status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS);
584+ if (status) {
585+ tegra_dma_chan_decode_error(tdc, status);
586+ tegra_dma_dump_chan_regs(tdc);
587+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF);
588+ }
589+
590+ spin_lock(&tdc->vc.lock);
591+ status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
592+ if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC))
593+ goto irq_done;
594+
595+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS,
596+ TEGRA_GPCDMA_STATUS_ISE_EOC);
597+
598+ if (!dma_desc)
599+ goto irq_done;
600+
601+ sg_req = dma_desc->sg_req;
602+ dma_desc->bytes_xfer += sg_req[dma_desc->sg_idx].len;
603+
604+ if (dma_desc->cyclic) {
605+ vchan_cyclic_callback(&dma_desc->vd);
606+ tegra_dma_configure_next_sg(tdc);
607+ } else {
608+ dma_desc->sg_idx++;
609+ if (dma_desc->sg_idx == dma_desc->sg_count)
610+ tegra_dma_xfer_complete(tdc);
611+ else
612+ tegra_dma_start(tdc);
613+ }
614+
615+irq_done:
616+ spin_unlock(&tdc->vc.lock);
617+ return IRQ_HANDLED;
618+}
619+
620+static void tegra_dma_issue_pending(struct dma_chan *dc)
621+{
622+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
623+ unsigned long flags;
624+
625+ if (tdc->dma_desc)
626+ return;
627+
628+ spin_lock_irqsave(&tdc->vc.lock, flags);
629+ if (vchan_issue_pending(&tdc->vc))
630+ tegra_dma_start(tdc);
631+
632+ /*
633+ * For cyclic DMA transfers, program the second
634+ * transfer parameters as soon as the first DMA
635+ * transfer is started inorder for the DMA
636+ * controller to trigger the second transfer
637+ * with the correct parameters.
638+ */
639+ if (tdc->dma_desc && tdc->dma_desc->cyclic)
640+ tegra_dma_configure_next_sg(tdc);
641+
642+ spin_unlock_irqrestore(&tdc->vc.lock, flags);
643+}
644+
645+static int tegra_dma_stop_client(struct tegra_dma_channel *tdc)
646+{
647+ int ret;
648+ u32 status, csr;
649+
650+ /*
651+ * Change the client associated with the DMA channel
652+ * to stop DMA engine from starting any more bursts for
653+ * the given client and wait for in flight bursts to complete
654+ */
655+ csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
656+ csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK);
657+ csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED;
658+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
659+
660+ /* Wait for in flight data transfer to finish */
661+ udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME);
662+
663+ /* If TX/RX path is still active wait till it becomes
664+ * inactive
665+ */
666+
667+ ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
668+ tdc->chan_base_offset +
669+ TEGRA_GPCDMA_CHAN_STATUS,
670+ status,
671+ !(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX |
672+ TEGRA_GPCDMA_STATUS_CHANNEL_RX)),
673+ 5,
674+ TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
675+ if (ret) {
676+ dev_err(tdc2dev(tdc), "Timeout waiting for DMA burst completion!\n");
677+ tegra_dma_dump_chan_regs(tdc);
678+ }
679+
680+ return ret;
681+}
682+
683+static int tegra_dma_terminate_all(struct dma_chan *dc)
684+{
685+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
686+ unsigned long flags;
687+ LIST_HEAD(head);
688+ int err;
689+
690+ spin_lock_irqsave(&tdc->vc.lock, flags);
691+
692+ if (tdc->dma_desc) {
693+ err = tdc->tdma->chip_data->terminate(tdc);
694+ if (err) {
695+ spin_unlock_irqrestore(&tdc->vc.lock, flags);
696+ return err;
697+ }
698+
699+ tegra_dma_disable(tdc);
700+ tdc->dma_desc = NULL;
701+ }
702+
703+ tegra_dma_sid_free(tdc);
704+ vchan_get_all_descriptors(&tdc->vc, &head);
705+ spin_unlock_irqrestore(&tdc->vc.lock, flags);
706+
707+ vchan_dma_desc_free_list(&tdc->vc, &head);
708+
709+ return 0;
710+}
711+
712+static int tegra_dma_get_residual(struct tegra_dma_channel *tdc)
713+{
714+ struct tegra_dma_desc *dma_desc = tdc->dma_desc;
715+ struct tegra_dma_sg_req *sg_req = dma_desc->sg_req;
716+ unsigned int bytes_xfer, residual;
717+ u32 wcount = 0, status;
718+
719+ wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT);
720+
721+ /*
722+ * Set wcount = 0 if EOC bit is set. The transfer would have
723+ * already completed and the CHAN_XFER_COUNT could have updated
724+ * for the next transfer, specifically in case of cyclic transfers.
725+ */
726+ status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
727+ if (status & TEGRA_GPCDMA_STATUS_ISE_EOC)
728+ wcount = 0;
729+
730+ bytes_xfer = dma_desc->bytes_xfer +
731+ sg_req[dma_desc->sg_idx].len - (wcount * 4);
732+
733+ residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req);
734+
735+ return residual;
736+}
737+
738+static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
739+ dma_cookie_t cookie,
740+ struct dma_tx_state *txstate)
741+{
742+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
743+ struct tegra_dma_desc *dma_desc;
744+ struct virt_dma_desc *vd;
745+ unsigned int residual;
746+ unsigned long flags;
747+ enum dma_status ret;
748+
749+ ret = dma_cookie_status(dc, cookie, txstate);
750+ if (ret == DMA_COMPLETE)
751+ return ret;
752+
753+ spin_lock_irqsave(&tdc->vc.lock, flags);
754+ vd = vchan_find_desc(&tdc->vc, cookie);
755+ if (vd) {
756+ dma_desc = vd_to_tegra_dma_desc(vd);
757+ residual = dma_desc->bytes_req;
758+ dma_set_residue(txstate, residual);
759+ } else if (tdc->dma_desc && tdc->dma_desc->vd.tx.cookie == cookie) {
760+ residual = tegra_dma_get_residual(tdc);
761+ dma_set_residue(txstate, residual);
762+ } else {
763+ dev_err(tdc2dev(tdc), "cookie %d is not found\n", cookie);
764+ }
765+ spin_unlock_irqrestore(&tdc->vc.lock, flags);
766+
767+ return ret;
768+}
769+
770+static inline int get_bus_width(struct tegra_dma_channel *tdc,
771+ enum dma_slave_buswidth slave_bw)
772+{
773+ switch (slave_bw) {
774+ case DMA_SLAVE_BUSWIDTH_1_BYTE:
775+ return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8;
776+ case DMA_SLAVE_BUSWIDTH_2_BYTES:
777+ return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16;
778+ case DMA_SLAVE_BUSWIDTH_4_BYTES:
779+ return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32;
780+ default:
781+ dev_err(tdc2dev(tdc), "given slave bus width is not supported\n");
782+ return -EINVAL;
783+ }
784+}
785+
786+static unsigned int get_burst_size(struct tegra_dma_channel *tdc,
787+ u32 burst_size, enum dma_slave_buswidth slave_bw,
788+ int len)
789+{
790+ unsigned int burst_mmio_width, burst_byte;
791+
792+ /*
793+ * burst_size from client is in terms of the bus_width.
794+ * convert that into words.
795+ * If burst_size is not specified from client, then use
796+ * len to calculate the optimum burst size
797+ */
798+ burst_byte = burst_size ? burst_size * slave_bw : len;
799+ burst_mmio_width = burst_byte / 4;
800+
801+ if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN)
802+ return 0;
803+
804+ burst_mmio_width = min(burst_mmio_width, TEGRA_GPCDMA_MMIOSEQ_BURST_MAX);
805+
806+ return TEGRA_GPCDMA_MMIOSEQ_BURST(burst_mmio_width);
807+}
808+
809+static int get_transfer_param(struct tegra_dma_channel *tdc,
810+ enum dma_transfer_direction direction,
811+ u32 *apb_addr,
812+ u32 *mmio_seq,
813+ u32 *csr,
814+ unsigned int *burst_size,
815+ enum dma_slave_buswidth *slave_bw)
816+{
817+ switch (direction) {
818+ case DMA_MEM_TO_DEV:
819+ *apb_addr = tdc->dma_sconfig.dst_addr;
820+ *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);
821+ *burst_size = tdc->dma_sconfig.dst_maxburst;
822+ *slave_bw = tdc->dma_sconfig.dst_addr_width;
823+ *csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC;
824+ return 0;
825+ case DMA_DEV_TO_MEM:
826+ *apb_addr = tdc->dma_sconfig.src_addr;
827+ *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);
828+ *burst_size = tdc->dma_sconfig.src_maxburst;
829+ *slave_bw = tdc->dma_sconfig.src_addr_width;
830+ *csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC;
831+ return 0;
832+ default:
833+ dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
834+ }
835+
836+ return -EINVAL;
837+}
838+
839+static struct dma_async_tx_descriptor *
840+tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value,
841+ size_t len, unsigned long flags)
842+{
843+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
844+ unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
845+ struct tegra_dma_sg_req *sg_req;
846+ struct tegra_dma_desc *dma_desc;
847+ u32 csr, mc_seq;
848+
849+ if ((len & 3) || (dest & 3) || len > max_dma_count) {
850+ dev_err(tdc2dev(tdc),
851+ "DMA length/memory address is not supported\n");
852+ return NULL;
853+ }
854+
855+ /* Set DMA mode to fixed pattern */
856+ csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT;
857+ /* Enable once or continuous mode */
858+ csr |= TEGRA_GPCDMA_CSR_ONCE;
859+ /* Enable IRQ mask */
860+ csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
861+ /* Enable the DMA interrupt */
862+ if (flags & DMA_PREP_INTERRUPT)
863+ csr |= TEGRA_GPCDMA_CSR_IE_EOC;
864+ /* Configure default priority weight for the channel */
865+ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
866+
867+ mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
868+ /* retain stream-id and clean rest */
869+ mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
870+
871+ /* Set the address wrapping */
872+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
873+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
874+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
875+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
876+
877+ /* Program outstanding MC requests */
878+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
879+ /* Set burst size */
880+ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
881+
882+ dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
883+ if (!dma_desc)
884+ return NULL;
885+
886+ dma_desc->bytes_req = len;
887+ dma_desc->sg_count = 1;
888+ sg_req = dma_desc->sg_req;
889+
890+ sg_req[0].ch_regs.src_ptr = 0;
891+ sg_req[0].ch_regs.dst_ptr = dest;
892+ sg_req[0].ch_regs.high_addr_ptr =
893+ FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
894+ sg_req[0].ch_regs.fixed_pattern = value;
895+ /* Word count reg takes value as (N +1) words */
896+ sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
897+ sg_req[0].ch_regs.csr = csr;
898+ sg_req[0].ch_regs.mmio_seq = 0;
899+ sg_req[0].ch_regs.mc_seq = mc_seq;
900+ sg_req[0].len = len;
901+
902+ dma_desc->cyclic = false;
903+ return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
904+}
905+
906+static struct dma_async_tx_descriptor *
907+tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest,
908+ dma_addr_t src, size_t len, unsigned long flags)
909+{
910+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
911+ struct tegra_dma_sg_req *sg_req;
912+ struct tegra_dma_desc *dma_desc;
913+ unsigned int max_dma_count;
914+ u32 csr, mc_seq;
915+
916+ max_dma_count = tdc->tdma->chip_data->max_dma_count;
917+ if ((len & 3) || (src & 3) || (dest & 3) || len > max_dma_count) {
918+ dev_err(tdc2dev(tdc),
919+ "DMA length/memory address is not supported\n");
920+ return NULL;
921+ }
922+
923+ /* Set DMA mode to memory to memory transfer */
924+ csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM;
925+ /* Enable once or continuous mode */
926+ csr |= TEGRA_GPCDMA_CSR_ONCE;
927+ /* Enable IRQ mask */
928+ csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
929+ /* Enable the DMA interrupt */
930+ if (flags & DMA_PREP_INTERRUPT)
931+ csr |= TEGRA_GPCDMA_CSR_IE_EOC;
932+ /* Configure default priority weight for the channel */
933+ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
934+
935+ mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
936+ /* retain stream-id and clean rest */
937+ mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) |
938+ (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
939+
940+ /* Set the address wrapping */
941+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
942+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
943+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
944+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
945+
946+ /* Program outstanding MC requests */
947+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
948+ /* Set burst size */
949+ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
950+
951+ dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
952+ if (!dma_desc)
953+ return NULL;
954+
955+ dma_desc->bytes_req = len;
956+ dma_desc->sg_count = 1;
957+ sg_req = dma_desc->sg_req;
958+
959+ sg_req[0].ch_regs.src_ptr = src;
960+ sg_req[0].ch_regs.dst_ptr = dest;
961+ sg_req[0].ch_regs.high_addr_ptr =
962+ FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32));
963+ sg_req[0].ch_regs.high_addr_ptr |=
964+ FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
965+ /* Word count reg takes value as (N +1) words */
966+ sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
967+ sg_req[0].ch_regs.csr = csr;
968+ sg_req[0].ch_regs.mmio_seq = 0;
969+ sg_req[0].ch_regs.mc_seq = mc_seq;
970+ sg_req[0].len = len;
971+
972+ dma_desc->cyclic = false;
973+ return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
974+}
975+
976+static struct dma_async_tx_descriptor *
977+tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,
978+ unsigned int sg_len, enum dma_transfer_direction direction,
979+ unsigned long flags, void *context)
980+{
981+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
982+ unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
983+ enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
984+ u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0;
985+ struct tegra_dma_sg_req *sg_req;
986+ struct tegra_dma_desc *dma_desc;
987+ struct scatterlist *sg;
988+ u32 burst_size;
989+ unsigned int i;
990+ int ret;
991+
992+ if (!tdc->config_init) {
993+ dev_err(tdc2dev(tdc), "DMA channel is not configured\n");
994+ return NULL;
995+ }
996+ if (sg_len < 1) {
997+ dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);
998+ return NULL;
999+ }
1000+
1001+ ret = tegra_dma_sid_reserve(tdc, direction);
1002+ if (ret)
1003+ return NULL;
1004+
1005+ ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1006+ &burst_size, &slave_bw);
1007+ if (ret < 0)
1008+ return NULL;
1009+
1010+ /* Enable once or continuous mode */
1011+ csr |= TEGRA_GPCDMA_CSR_ONCE;
1012+ /* Program the slave id in requestor select */
1013+ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1014+ /* Enable IRQ mask */
1015+ csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1016+ /* Configure default priority weight for the channel*/
1017+ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1018+
1019+ /* Enable the DMA interrupt */
1020+ if (flags & DMA_PREP_INTERRUPT)
1021+ csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1022+
1023+ mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1024+ /* retain stream-id and clean rest */
1025+ mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1026+
1027+ /* Set the address wrapping on both MC and MMIO side */
1028+
1029+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1030+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1031+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1032+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1033+ mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1034+
1035+ /* Program 2 MC outstanding requests by default. */
1036+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1037+
1038+ /* Setting MC burst size depending on MMIO burst size */
1039+ if (burst_size == 64)
1040+ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1041+ else
1042+ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1043+
1044+ dma_desc = kzalloc(struct_size(dma_desc, sg_req, sg_len), GFP_NOWAIT);
1045+ if (!dma_desc)
1046+ return NULL;
1047+
1048+ dma_desc->sg_count = sg_len;
1049+ sg_req = dma_desc->sg_req;
1050+
1051+ /* Make transfer requests */
1052+ for_each_sg(sgl, sg, sg_len, i) {
1053+ u32 len;
1054+ dma_addr_t mem;
1055+
1056+ mem = sg_dma_address(sg);
1057+ len = sg_dma_len(sg);
1058+
1059+ if ((len & 3) || (mem & 3) || len > max_dma_count) {
1060+ dev_err(tdc2dev(tdc),
1061+ "DMA length/memory address is not supported\n");
1062+ kfree(dma_desc);
1063+ return NULL;
1064+ }
1065+
1066+ mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1067+ dma_desc->bytes_req += len;
1068+
1069+ if (direction == DMA_MEM_TO_DEV) {
1070+ sg_req[i].ch_regs.src_ptr = mem;
1071+ sg_req[i].ch_regs.dst_ptr = apb_ptr;
1072+ sg_req[i].ch_regs.high_addr_ptr =
1073+ FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1074+ } else if (direction == DMA_DEV_TO_MEM) {
1075+ sg_req[i].ch_regs.src_ptr = apb_ptr;
1076+ sg_req[i].ch_regs.dst_ptr = mem;
1077+ sg_req[i].ch_regs.high_addr_ptr =
1078+ FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1079+ }
1080+
1081+ /*
1082+ * Word count register takes input in words. Writing a value
1083+ * of N into word count register means a req of (N+1) words.
1084+ */
1085+ sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1086+ sg_req[i].ch_regs.csr = csr;
1087+ sg_req[i].ch_regs.mmio_seq = mmio_seq;
1088+ sg_req[i].ch_regs.mc_seq = mc_seq;
1089+ sg_req[i].len = len;
1090+ }
1091+
1092+ dma_desc->cyclic = false;
1093+ return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1094+}
1095+
1096+static struct dma_async_tx_descriptor *
1097+tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
1098+ size_t period_len, enum dma_transfer_direction direction,
1099+ unsigned long flags)
1100+{
1101+ enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
1102+ u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size;
1103+ unsigned int max_dma_count, len, period_count, i;
1104+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1105+ struct tegra_dma_desc *dma_desc;
1106+ struct tegra_dma_sg_req *sg_req;
1107+ dma_addr_t mem = buf_addr;
1108+ int ret;
1109+
1110+ if (!buf_len || !period_len) {
1111+ dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
1112+ return NULL;
1113+ }
1114+
1115+ if (!tdc->config_init) {
1116+ dev_err(tdc2dev(tdc), "DMA slave is not configured\n");
1117+ return NULL;
1118+ }
1119+
1120+ ret = tegra_dma_sid_reserve(tdc, direction);
1121+ if (ret)
1122+ return NULL;
1123+
1124+ /*
1125+ * We only support cycle transfer when buf_len is multiple of
1126+ * period_len.
1127+ */
1128+ if (buf_len % period_len) {
1129+ dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");
1130+ return NULL;
1131+ }
1132+
1133+ len = period_len;
1134+ max_dma_count = tdc->tdma->chip_data->max_dma_count;
1135+ if ((len & 3) || (buf_addr & 3) || len > max_dma_count) {
1136+ dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
1137+ return NULL;
1138+ }
1139+
1140+ ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1141+ &burst_size, &slave_bw);
1142+ if (ret < 0)
1143+ return NULL;
1144+
1145+ /* Enable once or continuous mode */
1146+ csr &= ~TEGRA_GPCDMA_CSR_ONCE;
1147+ /* Program the slave id in requestor select */
1148+ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1149+ /* Enable IRQ mask */
1150+ csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1151+ /* Configure default priority weight for the channel*/
1152+ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1153+
1154+ /* Enable the DMA interrupt */
1155+ if (flags & DMA_PREP_INTERRUPT)
1156+ csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1157+
1158+ mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1159+
1160+ mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1161+ /* retain stream-id and clean rest */
1162+ mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1163+
1164+ /* Set the address wrapping on both MC and MMIO side */
1165+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1166+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1167+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1168+ TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1169+
1170+ /* Program 2 MC outstanding requests by default. */
1171+ mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1172+ /* Setting MC burst size depending on MMIO burst size */
1173+ if (burst_size == 64)
1174+ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1175+ else
1176+ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1177+
1178+ period_count = buf_len / period_len;
1179+ dma_desc = kzalloc(struct_size(dma_desc, sg_req, period_count),
1180+ GFP_NOWAIT);
1181+ if (!dma_desc)
1182+ return NULL;
1183+
1184+ dma_desc->bytes_req = buf_len;
1185+ dma_desc->sg_count = period_count;
1186+ sg_req = dma_desc->sg_req;
1187+
1188+ /* Split transfer equal to period size */
1189+ for (i = 0; i < period_count; i++) {
1190+ mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1191+ if (direction == DMA_MEM_TO_DEV) {
1192+ sg_req[i].ch_regs.src_ptr = mem;
1193+ sg_req[i].ch_regs.dst_ptr = apb_ptr;
1194+ sg_req[i].ch_regs.high_addr_ptr =
1195+ FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1196+ } else if (direction == DMA_DEV_TO_MEM) {
1197+ sg_req[i].ch_regs.src_ptr = apb_ptr;
1198+ sg_req[i].ch_regs.dst_ptr = mem;
1199+ sg_req[i].ch_regs.high_addr_ptr =
1200+ FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1201+ }
1202+ /*
1203+ * Word count register takes input in words. Writing a value
1204+ * of N into word count register means a req of (N+1) words.
1205+ */
1206+ sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1207+ sg_req[i].ch_regs.csr = csr;
1208+ sg_req[i].ch_regs.mmio_seq = mmio_seq;
1209+ sg_req[i].ch_regs.mc_seq = mc_seq;
1210+ sg_req[i].len = len;
1211+
1212+ mem += len;
1213+ }
1214+
1215+ dma_desc->cyclic = true;
1216+
1217+ return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1218+}
1219+
1220+static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
1221+{
1222+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1223+ int ret;
1224+
1225+ ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc);
1226+ if (ret) {
1227+ dev_err(tdc2dev(tdc), "request_irq failed for %s\n", tdc->name);
1228+ return ret;
1229+ }
1230+
1231+ dma_cookie_init(&tdc->vc.chan);
1232+ tdc->config_init = false;
1233+ return 0;
1234+}
1235+
1236+static void tegra_dma_chan_synchronize(struct dma_chan *dc)
1237+{
1238+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1239+
1240+ synchronize_irq(tdc->irq);
1241+ vchan_synchronize(&tdc->vc);
1242+}
1243+
1244+static void tegra_dma_free_chan_resources(struct dma_chan *dc)
1245+{
1246+ struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1247+
1248+ dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
1249+
1250+ tegra_dma_terminate_all(dc);
1251+ synchronize_irq(tdc->irq);
1252+
1253+ tasklet_kill(&tdc->vc.task);
1254+ tdc->config_init = false;
1255+ tdc->slave_id = -1;
1256+ tdc->sid_dir = DMA_TRANS_NONE;
1257+ free_irq(tdc->irq, tdc);
1258+
1259+ vchan_free_chan_resources(&tdc->vc);
1260+}
1261+
1262+static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
1263+ struct of_dma *ofdma)
1264+{
1265+ struct tegra_dma *tdma = ofdma->of_dma_data;
1266+ struct tegra_dma_channel *tdc;
1267+ struct dma_chan *chan;
1268+
1269+ chan = dma_get_any_slave_channel(&tdma->dma_dev);
1270+ if (!chan)
1271+ return NULL;
1272+
1273+ tdc = to_tegra_dma_chan(chan);
1274+ tdc->slave_id = dma_spec->args[0];
1275+
1276+ return chan;
1277+}
1278+
1279+static const struct tegra_dma_chip_data tegra186_dma_chip_data = {
1280+ .nr_channels = 31,
1281+ .channel_reg_size = SZ_64K,
1282+ .max_dma_count = SZ_1G,
1283+ .hw_support_pause = false,
1284+ .terminate = tegra_dma_stop_client,
1285+};
1286+
1287+static const struct tegra_dma_chip_data tegra194_dma_chip_data = {
1288+ .nr_channels = 31,
1289+ .channel_reg_size = SZ_64K,
1290+ .max_dma_count = SZ_1G,
1291+ .hw_support_pause = true,
1292+ .terminate = tegra_dma_pause,
1293+};
1294+
1295+static const struct of_device_id tegra_dma_of_match[] = {
1296+ {
1297+ .compatible = "nvidia,tegra186-gpcdma",
1298+ .data = &tegra186_dma_chip_data,
1299+ }, {
1300+ .compatible = "nvidia,tegra194-gpcdma",
1301+ .data = &tegra194_dma_chip_data,
1302+ }, {
1303+ },
1304+};
1305+MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
1306+
1307+static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id)
1308+{
1309+ unsigned int reg_val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1310+
1311+ reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK);
1312+ reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
1313+
1314+ reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id);
1315+ reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id);
1316+
1317+ tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val);
1318+ return 0;
1319+}
1320+
1321+static int tegra_dma_probe(struct platform_device *pdev)
1322+{
1323+ const struct tegra_dma_chip_data *cdata = NULL;
1324+ struct iommu_fwspec *iommu_spec;
1325+ unsigned int stream_id, i;
1326+ struct tegra_dma *tdma;
1327+ int ret;
1328+
1329+ cdata = of_device_get_match_data(&pdev->dev);
1330+
1331+ tdma = devm_kzalloc(&pdev->dev,
1332+ struct_size(tdma, channels, cdata->nr_channels),
1333+ GFP_KERNEL);
1334+ if (!tdma)
1335+ return -ENOMEM;
1336+
1337+ tdma->dev = &pdev->dev;
1338+ tdma->chip_data = cdata;
1339+ platform_set_drvdata(pdev, tdma);
1340+
1341+ tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);
1342+ if (IS_ERR(tdma->base_addr))
1343+ return PTR_ERR(tdma->base_addr);
1344+
1345+ tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma");
1346+ if (IS_ERR(tdma->rst)) {
1347+ return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst),
1348+ "Missing controller reset\n");
1349+ }
1350+ reset_control_reset(tdma->rst);
1351+
1352+ tdma->dma_dev.dev = &pdev->dev;
1353+
1354+ iommu_spec = dev_iommu_fwspec_get(&pdev->dev);
1355+ if (!iommu_spec) {
1356+ dev_err(&pdev->dev, "Missing iommu stream-id\n");
1357+ return -EINVAL;
1358+ }
1359+ stream_id = iommu_spec->ids[0] & 0xffff;
1360+
1361+ INIT_LIST_HEAD(&tdma->dma_dev.channels);
1362+ for (i = 0; i < cdata->nr_channels; i++) {
1363+ struct tegra_dma_channel *tdc = &tdma->channels[i];
1364+
1365+ tdc->irq = platform_get_irq(pdev, i);
1366+ if (tdc->irq < 0)
1367+ return tdc->irq;
1368+
1369+ tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADD_OFFSET +
1370+ i * cdata->channel_reg_size;
1371+ snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i);
1372+ tdc->tdma = tdma;
1373+ tdc->id = i;
1374+ tdc->slave_id = -1;
1375+
1376+ vchan_init(&tdc->vc, &tdma->dma_dev);
1377+ tdc->vc.desc_free = tegra_dma_desc_free;
1378+
1379+ /* program stream-id for this channel */
1380+ tegra_dma_program_sid(tdc, stream_id);
1381+ tdc->stream_id = stream_id;
1382+ }
1383+
1384+ dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
1385+ dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
1386+ dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask);
1387+ dma_cap_set(DMA_MEMSET, tdma->dma_dev.cap_mask);
1388+ dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
1389+
1390+ /*
1391+ * Only word aligned transfers are supported. Set the copy
1392+ * alignment shift.
1393+ */
1394+ tdma->dma_dev.copy_align = 2;
1395+ tdma->dma_dev.fill_align = 2;
1396+ tdma->dma_dev.device_alloc_chan_resources =
1397+ tegra_dma_alloc_chan_resources;
1398+ tdma->dma_dev.device_free_chan_resources =
1399+ tegra_dma_free_chan_resources;
1400+ tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;
1401+ tdma->dma_dev.device_prep_dma_memcpy = tegra_dma_prep_dma_memcpy;
1402+ tdma->dma_dev.device_prep_dma_memset = tegra_dma_prep_dma_memset;
1403+ tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;
1404+ tdma->dma_dev.device_config = tegra_dma_slave_config;
1405+ tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;
1406+ tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
1407+ tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
1408+ tdma->dma_dev.device_pause = tegra_dma_device_pause;
1409+ tdma->dma_dev.device_resume = tegra_dma_device_resume;
1410+ tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize;
1411+ tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1412+
1413+ ret = dma_async_device_register(&tdma->dma_dev);
1414+ if (ret < 0) {
1415+ dev_err_probe(&pdev->dev, ret,
1416+ "GPC DMA driver registration failed\n");
1417+ return ret;
1418+ }
1419+
1420+ ret = of_dma_controller_register(pdev->dev.of_node,
1421+ tegra_dma_of_xlate, tdma);
1422+ if (ret < 0) {
1423+ dev_err_probe(&pdev->dev, ret,
1424+ "GPC DMA OF registration failed\n");
1425+
1426+ dma_async_device_unregister(&tdma->dma_dev);
1427+ return ret;
1428+ }
1429+
1430+ dev_info(&pdev->dev, "GPC DMA driver register %d channels\n",
1431+ cdata->nr_channels);
1432+
1433+ return 0;
1434+}
1435+
1436+static int tegra_dma_remove(struct platform_device *pdev)
1437+{
1438+ struct tegra_dma *tdma = platform_get_drvdata(pdev);
1439+
1440+ of_dma_controller_free(pdev->dev.of_node);
1441+ dma_async_device_unregister(&tdma->dma_dev);
1442+
1443+ return 0;
1444+}
1445+
1446+static int __maybe_unused tegra_dma_pm_suspend(struct device *dev)
1447+{
1448+ struct tegra_dma *tdma = dev_get_drvdata(dev);
1449+ unsigned int i;
1450+
1451+ for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1452+ struct tegra_dma_channel *tdc = &tdma->channels[i];
1453+
1454+ if (tdc->dma_desc) {
1455+ dev_err(tdma->dev, "channel %u busy\n", i);
1456+ return -EBUSY;
1457+ }
1458+ }
1459+
1460+ return 0;
1461+}
1462+
1463+static int __maybe_unused tegra_dma_pm_resume(struct device *dev)
1464+{
1465+ struct tegra_dma *tdma = dev_get_drvdata(dev);
1466+ unsigned int i;
1467+
1468+ reset_control_reset(tdma->rst);
1469+
1470+ for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1471+ struct tegra_dma_channel *tdc = &tdma->channels[i];
1472+
1473+ tegra_dma_program_sid(tdc, tdc->stream_id);
1474+ }
1475+
1476+ return 0;
1477+}
1478+
1479+static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
1480+ SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume)
1481+};
1482+
1483+static struct platform_driver tegra_dma_driver = {
1484+ .driver = {
1485+ .name = "tegra-gpcdma",
1486+ .pm = &tegra_dma_dev_pm_ops,
1487+ .of_match_table = tegra_dma_of_match,
1488+ },
1489+ .probe = tegra_dma_probe,
1490+ .remove = tegra_dma_remove,
1491+};
1492+
1493+module_platform_driver(tegra_dma_driver);
1494+
1495+MODULE_DESCRIPTION("NVIDIA Tegra GPC DMA Controller driver");
1496+MODULE_AUTHOR("Pavan Kunapuli <pkunapuli@nvidia.com>");
1497+MODULE_AUTHOR("Rajesh Gumasta <rgumasta@nvidia.com>");
1498+MODULE_LICENSE("GPL");
drivers/dma/ti/cppi41.c+5−1
@@ -1105,8 +1105,12 @@ static int cppi41_dma_probe(struct platform_device *pdev) · 1104 unmodified lines
11051105 cdd->qmgr_num_pend = glue_info->qmgr_num_pend;
11061106 cdd->first_completion_queue = glue_info->first_completion_queue;
11071107
1108+ /* Parse new and deprecated dma-channels properties */
11081109 ret = of_property_read_u32(dev->of_node,
1109− "#dma-channels", &cdd->n_chans);
1110+ "dma-channels", &cdd->n_chans);
1111+ if (ret)
1112+ ret = of_property_read_u32(dev->of_node,
1113+ "#dma-channels", &cdd->n_chans);
11101114 if (ret)
11111115 goto err_get_n_chans;
11121116
drivers/dma/ti/k3-psil-am62.c+4−4
@@ -70,10 +70,10 @@ · 69 unmodified lines
7070 /* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */
7171 static struct psil_ep am62_src_ep_map[] = {
7272 /* SAUL */
73− PSIL_SAUL(0x7500, 20, 35, 8, 35, 0),
74− PSIL_SAUL(0x7501, 21, 35, 8, 36, 0),
75− PSIL_SAUL(0x7502, 22, 43, 8, 43, 0),
76− PSIL_SAUL(0x7503, 23, 43, 8, 44, 0),
73+ PSIL_SAUL(0x7504, 20, 35, 8, 35, 0),
74+ PSIL_SAUL(0x7505, 21, 35, 8, 36, 0),
75+ PSIL_SAUL(0x7506, 22, 43, 8, 43, 0),
76+ PSIL_SAUL(0x7507, 23, 43, 8, 44, 0),
7777 /* PDMA_MAIN0 - SPI0-3 */
7878 PSIL_PDMA_XY_PKT(0x4302),
7979 PSIL_PDMA_XY_PKT(0x4303),
drivers/dma/xilinx/zynqmp_dma.c+13−4
@@ -229,7 +229,7 @@ struct zynqmp_dma_chan { · 228 unmodified lines
229229 bool is_dmacoherent;
230230 struct tasklet_struct tasklet;
231231 bool idle;
232− u32 desc_size;
232+ size_t desc_size;
233233 bool err;
234234 u32 bus_width;
235235 u32 src_burst_len;
@@ -486,7 +486,8 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) · 250 unmodified lines
486486 }
487487
488488 chan->desc_pool_v = dma_alloc_coherent(chan->dev,
489− (2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS),
489+ (2 * ZYNQMP_DMA_DESC_SIZE(chan) *
490+ ZYNQMP_DMA_NUM_DESCS),
490491 &chan->desc_pool_p, GFP_KERNEL);
491492 if (!chan->desc_pool_v)
492493 return -ENOMEM;
@@ -1077,7 +1078,11 @@ static int zynqmp_dma_probe(struct platform_device *pdev) · 584 unmodified lines
10771078 pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT);
10781079 pm_runtime_use_autosuspend(zdev->dev);
10791080 pm_runtime_enable(zdev->dev);
1080− pm_runtime_get_sync(zdev->dev);
1081+ ret = pm_runtime_resume_and_get(zdev->dev);
1082+ if (ret < 0) {
1083+ dev_err(&pdev->dev, "device wakeup failed.\n");
1084+ pm_runtime_disable(zdev->dev);
1085+ }
10811086 if (!pm_runtime_enabled(zdev->dev)) {
10821087 ret = zynqmp_dma_runtime_resume(zdev->dev);
10831088 if (ret)
@@ -1093,7 +1098,11 @@ static int zynqmp_dma_probe(struct platform_device *pdev) · 9 unmodified lines
10931098 p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);
10941099 p->src_addr_widths = BIT(zdev->chan->bus_width / 8);
10951100
1096− dma_async_device_register(&zdev->common);
1101+ ret = dma_async_device_register(&zdev->common);
1102+ if (ret) {
1103+ dev_err(zdev->dev, "failed to register the dma device\n");
1104+ goto free_chan_resources;
1105+ }
10971106
10981107 ret = of_dma_controller_register(pdev->dev.of_node,
10991108 of_zynqmp_dma_xlate, zdev);
include/linux/dmaengine.h+8−1
@@ -870,7 +870,6 @@ struct dma_device { · 869 unmodified lines
870870 struct device *dev;
871871 struct module *owner;
872872 struct ida chan_ida;
873− struct mutex chan_mutex; /* to protect chan_ida */
874873
875874 u32 src_addr_widths;
876875 u32 dst_addr_widths;
@@ -1031,6 +1030,14 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_interleaved_dma( · 154 unmodified lines
10311030 return chan->device->device_prep_interleaved_dma(chan, xt, flags);
10321031 }
10331032
1033+/**
1034+ * dmaengine_prep_dma_memset() - Prepare a DMA memset descriptor.
1035+ * @chan: The channel to be used for this descriptor
1036+ * @dest: Address of buffer to be set
1037+ * @value: Treated as a single byte value that fills the destination buffer
1038+ * @len: The total size of dest
1039+ * @flags: DMA engine flags
1040+ */
10341041 static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memset(
10351042 struct dma_chan *chan, dma_addr_t dest, int value, size_t len,
10361043 unsigned long flags)
include/linux/soc/renesas/r9a06g032-sysctrl.h+11−0
@@ -0,0 +1,11 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef __LINUX_SOC_RENESAS_R9A06G032_SYSCTRL_H__
3+#define __LINUX_SOC_RENESAS_R9A06G032_SYSCTRL_H__
4+
5+#ifdef CONFIG_CLK_R9A06G032
6+int r9a06g032_sysctrl_set_dmamux(u32 mask, u32 val);
7+#else
8+static inline int r9a06g032_sysctrl_set_dmamux(u32 mask, u32 val) { return -ENODEV; }
9+#endif
10+
11+#endif /* __LINUX_SOC_RENESAS_R9A06G032_SYSCTRL_H__ */
include/uapi/linux/idxd.h+28−3
@@ -53,6 +53,11 @@ enum idxd_scmd_stat { · 52 unmodified lines
5353
5454 /* IAX */
5555 #define IDXD_OP_FLAG_RD_SRC2_AECS 0x010000
56+#define IDXD_OP_FLAG_RD_SRC2_2ND 0x020000
57+#define IDXD_OP_FLAG_WR_SRC2_AECS_COMP 0x040000
58+#define IDXD_OP_FLAG_WR_SRC2_AECS_OVFL 0x080000
59+#define IDXD_OP_FLAG_SRC2_STS 0x100000
60+#define IDXD_OP_FLAG_CRC_RFC3720 0x200000
5661
5762 /* Opcode */
5863 enum dsa_opcode {
@@ -81,6 +86,18 @@ enum iax_opcode { · 22 unmodified lines
8186 IAX_OPCODE_MEMMOVE,
8287 IAX_OPCODE_DECOMPRESS = 0x42,
8388 IAX_OPCODE_COMPRESS,
89+ IAX_OPCODE_CRC64,
90+ IAX_OPCODE_ZERO_DECOMP_32 = 0x48,
91+ IAX_OPCODE_ZERO_DECOMP_16,
92+ IAX_OPCODE_DECOMP_32 = 0x4c,
93+ IAX_OPCODE_DECOMP_16,
94+ IAX_OPCODE_SCAN = 0x50,
95+ IAX_OPCODE_SET_MEMBER,
96+ IAX_OPCODE_EXTRACT,
97+ IAX_OPCODE_SELECT,
98+ IAX_OPCODE_RLE_BURST,
99+ IAX_OPCDE_FIND_UNIQUE,
100+ IAX_OPCODE_EXPAND,
84101 };
85102
86103 /* Completion record status */
@@ -120,6 +137,7 @@ enum iax_completion_status { · 33 unmodified lines
120137 IAX_COMP_NONE = 0,
121138 IAX_COMP_SUCCESS,
122139 IAX_COMP_PAGE_FAULT_IR = 0x04,
140+ IAX_COMP_ANALYTICS_ERROR = 0x0a,
123141 IAX_COMP_OUTBUF_OVERFLOW,
124142 IAX_COMP_BAD_OPCODE = 0x10,
125143 IAX_COMP_INVALID_FLAGS,
@@ -140,7 +158,10 @@ enum iax_completion_status { · 14 unmodified lines
140158 IAX_COMP_WATCHDOG,
141159 IAX_COMP_INVALID_COMP_FLAG = 0x30,
142160 IAX_COMP_INVALID_FILTER_FLAG,
143− IAX_COMP_INVALID_NUM_ELEMS = 0x33,
161+ IAX_COMP_INVALID_INPUT_SIZE,
162+ IAX_COMP_INVALID_NUM_ELEMS,
163+ IAX_COMP_INVALID_SRC1_WIDTH,
164+ IAX_COMP_INVALID_INVERT_OUT,
144165 };
145166
146167 #define DSA_COMP_STATUS_MASK 0x7f
@@ -319,8 +340,12 @@ struct iax_completion_record { · 172 unmodified lines
319340 uint32_t output_size;
320341 uint8_t output_bits;
321342 uint8_t rsvd3;
322− uint16_t rsvd4;
323− uint64_t rsvd5[4];
343+ uint16_t xor_csum;
344+ uint32_t crc;
345+ uint32_t min;
346+ uint32_t max;
347+ uint32_t sum;
348+ uint64_t rsvd4[2];
324349 } __attribute__((packed));
325350
326351 struct iax_raw_completion_record {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t