History

Merge tag 'cxl-for-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl

Pull cxl updates from Dan Williams: "Compute Express Link (CXL) updates for 6.0: - Introduce a 'struct cxl_region' object with support for provisioning and assembling persistent memory regions. - Introduce alloc_free_mem_region() to accompany the existing request_free_mem_region() as a method to allocate physical memory capacity out of an existing resource. - Export insert_resource_expand_to_fit() for the CXL subsystem to late-publish CXL platform windows in iomem_resource. - Add a polled mode PCI DOE (Data Object Exchange) driver service and use it in cxl_pci to retrieve the CDAT (Coherent Device Attribute Table)" * tag 'cxl-for-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl: (74 commits) cxl/hdm: Fix skip allocations vs multiple pmem allocations cxl/region: Disallow region granularity != window granularity cxl/region: Fix x1 interleave to greater than x1 interleave routing cxl/region: Move HPA setup to cxl_region_attach() cxl/region: Fix decoder interleave programming Documentation: cxl: remove dangling kernel-doc reference cxl/region: describe targets and nr_targets members of cxl_region_params cxl/regions: add padding for cxl_rr_ep_add nested lists cxl/region: Fix IS_ERR() vs NULL check cxl/region: Fix region reference target accounting cxl/region: Fix region commit uninitialized variable warning cxl/region: Fix port setup uninitialized variable warnings cxl/region: Stop initializing interleave granularity cxl/hdm: Fix DPA reservation vs cxl_endpoint_decoder lifetime cxl/acpi: Minimize granularity for x1 interleaves cxl/region: Delete 'region' attribute from root decoders cxl/acpi: Autoload driver for 'cxl_acpi' test devices cxl/region: decrement ->nr_targets on error in cxl_region_attach() cxl/region: prevent underflow in ways_to_cxl() cxl/region: uninitialized variable in alloc_hpa() ...

Linus Torvalds committed 4 years ago

Commit c2356982 parents 5e2e738 1cd8a2539 files · +5498 −576Browse files

Unified · Split

39 files
  1. .clang-format +1 −0
  2. Documentation/ABI/testing/sysfs-bus-cxl +265 −40
  3. Documentation/driver-api/cxl/memory-devices.rst +8 −0
  4. arch/powerpc/mm/mem.c +1 −0
  5. drivers/cxl/Kconfig +9 −0
  6. drivers/cxl/acpi.c +217 −26
  7. drivers/cxl/core/Makefile +1 −0
  8. drivers/cxl/core/core.h +50 −1
  9. drivers/cxl/core/hdm.c +635 −56
  10. drivers/cxl/core/mbox.c +47 −48
  11. drivers/cxl/core/memdev.c +2 −2
  12. drivers/cxl/core/pci.c +174 −7
  13. drivers/cxl/core/pmem.c +2 −2
  14. drivers/cxl/core/port.c +536 −202
  15. drivers/cxl/core/region.c +1896 −0
  16. drivers/cxl/cxl.h +283 −29
  17. drivers/cxl/cxlmem.h +38 −4
  18. drivers/cxl/cxlpci.h +1 −0
  19. drivers/cxl/mem.c +23 −26
  20. drivers/cxl/pci.c +45 −1
  21. drivers/cxl/pmem.c +254 −5
  22. drivers/cxl/port.c +53 −0
  23. drivers/nvdimm/region_devs.c +21 −7
  24. drivers/pci/Kconfig +3 −0
  25. drivers/pci/Makefile +1 −0
  26. drivers/pci/doe.c +536 −0
  27. drivers/pci/probe.c +1 −1
  28. include/linux/ioport.h +3 −0
  29. include/linux/libnvdimm.h +5 −0
  30. include/linux/pci-doe.h +77 −0
  31. include/linux/pci_ids.h +1 −0
  32. include/linux/sysfs.h +16 −0
  33. include/uapi/linux/pci_regs.h +28 −1
  34. kernel/resource.c +150 −35
  35. mm/Kconfig +5 −0
  36. tools/testing/cxl/Kbuild +1 −0
  37. tools/testing/cxl/test/cxl.c +76 −55
  38. tools/testing/cxl/test/mem.c +28 −25
  39. tools/testing/cxl/test/mock.c +5 −3
.clang-format+1−0
@@ -516,6 +516,7 @@ ForEachMacros: · 515 unmodified lines
516 - 'of_property_for_each_string'516 - 'of_property_for_each_string'
517 - 'of_property_for_each_u32'517 - 'of_property_for_each_u32'
518 - 'pci_bus_for_each_resource'518 - 'pci_bus_for_each_resource'
519 - 'pci_doe_for_each_off'
519 - 'pcl_for_each_chunk'520 - 'pcl_for_each_chunk'
520 - 'pcl_for_each_segment'521 - 'pcl_for_each_segment'
521 - 'pcm_for_each_format'522 - 'pcm_for_each_format'
Documentation/ABI/testing/sysfs-bus-cxl+265−40
@@ -7,6 +7,7 @@ Description: · 6 unmodified lines
7 all descendant memdevs for unbind. Writing '1' to this attribute7 all descendant memdevs for unbind. Writing '1' to this attribute
8 flushes that work.8 flushes that work.
9 9
10
10What: /sys/bus/cxl/devices/memX/firmware_version11What: /sys/bus/cxl/devices/memX/firmware_version
11Date: December, 202012Date: December, 2020
12KernelVersion: v5.1213KernelVersion: v5.12
@@ -16,6 +17,7 @@ Description: · 3 unmodified lines
16 Memory Device Output Payload in the CXL-2.017 Memory Device Output Payload in the CXL-2.0
17 specification.18 specification.
18 19
20
19What: /sys/bus/cxl/devices/memX/ram/size21What: /sys/bus/cxl/devices/memX/ram/size
20Date: December, 202022Date: December, 2020
21KernelVersion: v5.1223KernelVersion: v5.12
@@ -25,6 +27,7 @@ Description: · 3 unmodified lines
25 identically named field in the Identify Memory Device Output27 identically named field in the Identify Memory Device Output
26 Payload in the CXL-2.0 specification.28 Payload in the CXL-2.0 specification.
27 29
30
28What: /sys/bus/cxl/devices/memX/pmem/size31What: /sys/bus/cxl/devices/memX/pmem/size
29Date: December, 202032Date: December, 2020
30KernelVersion: v5.1233KernelVersion: v5.12
@@ -34,6 +37,7 @@ Description: · 3 unmodified lines
34 identically named field in the Identify Memory Device Output37 identically named field in the Identify Memory Device Output
35 Payload in the CXL-2.0 specification.38 Payload in the CXL-2.0 specification.
36 39
40
37What: /sys/bus/cxl/devices/memX/serial41What: /sys/bus/cxl/devices/memX/serial
38Date: January, 202242Date: January, 2022
39KernelVersion: v5.1843KernelVersion: v5.18
@@ -43,6 +47,7 @@ Description: · 3 unmodified lines
43 capability. Mandatory for CXL devices, see CXL 2.0 8.1.12.247 capability. Mandatory for CXL devices, see CXL 2.0 8.1.12.2
44 Memory Device PCIe Capabilities and Extended Capabilities.48 Memory Device PCIe Capabilities and Extended Capabilities.
45 49
50
46What: /sys/bus/cxl/devices/memX/numa_node51What: /sys/bus/cxl/devices/memX/numa_node
47Date: January, 202252Date: January, 2022
48KernelVersion: v5.1853KernelVersion: v5.18
@@ -52,114 +57,334 @@ Description: · 3 unmodified lines
52 host PCI device for this memory device, emit the CPU node57 host PCI device for this memory device, emit the CPU node
53 affinity for this device.58 affinity for this device.
54 59
60
55What: /sys/bus/cxl/devices/*/devtype61What: /sys/bus/cxl/devices/*/devtype
56Date: June, 202162Date: June, 2021
57KernelVersion: v5.1463KernelVersion: v5.14
58Contact: linux-cxl@vger.kernel.org64Contact: linux-cxl@vger.kernel.org
59Description:65Description:
60 CXL device objects export the devtype attribute which mirrors66 (RO) CXL device objects export the devtype attribute which
61 the same value communicated in the DEVTYPE environment variable67 mirrors the same value communicated in the DEVTYPE environment
62 for uevents for devices on the "cxl" bus.68 variable for uevents for devices on the "cxl" bus.
69
63 70
64What: /sys/bus/cxl/devices/*/modalias71What: /sys/bus/cxl/devices/*/modalias
65Date: December, 202172Date: December, 2021
66KernelVersion: v5.1873KernelVersion: v5.18
67Contact: linux-cxl@vger.kernel.org74Contact: linux-cxl@vger.kernel.org
68Description:75Description:
69 CXL device objects export the modalias attribute which mirrors76 (RO) CXL device objects export the modalias attribute which
70 the same value communicated in the MODALIAS environment variable77 mirrors the same value communicated in the MODALIAS environment
71 for uevents for devices on the "cxl" bus.78 variable for uevents for devices on the "cxl" bus.
79
72 80
73What: /sys/bus/cxl/devices/portX/uport81What: /sys/bus/cxl/devices/portX/uport
74Date: June, 202182Date: June, 2021
75KernelVersion: v5.1483KernelVersion: v5.14
76Contact: linux-cxl@vger.kernel.org84Contact: linux-cxl@vger.kernel.org
77Description:85Description:
78 CXL port objects are enumerated from either a platform firmware86 (RO) CXL port objects are enumerated from either a platform
79 device (ACPI0017 and ACPI0016) or PCIe switch upstream port with87 firmware device (ACPI0017 and ACPI0016) or PCIe switch upstream
80 CXL component registers. The 'uport' symlink connects the CXL88 port with CXL component registers. The 'uport' symlink connects
81 portX object to the device that published the CXL port89 the CXL portX object to the device that published the CXL port
82 capability.90 capability.
83 91
92
84What: /sys/bus/cxl/devices/portX/dportY93What: /sys/bus/cxl/devices/portX/dportY
85Date: June, 202194Date: June, 2021
86KernelVersion: v5.1495KernelVersion: v5.14
87Contact: linux-cxl@vger.kernel.org96Contact: linux-cxl@vger.kernel.org
88Description:97Description:
89 CXL port objects are enumerated from either a platform firmware98 (RO) CXL port objects are enumerated from either a platform
90 device (ACPI0017 and ACPI0016) or PCIe switch upstream port with99 firmware device (ACPI0017 and ACPI0016) or PCIe switch upstream
91 CXL component registers. The 'dportY' symlink identifies one or100 port with CXL component registers. The 'dportY' symlink
92 more downstream ports that the upstream port may target in its101 identifies one or more downstream ports that the upstream port
93 decode of CXL memory resources. The 'Y' integer reflects the102 may target in its decode of CXL memory resources. The 'Y'
94 hardware port unique-id used in the hardware decoder target103 integer reflects the hardware port unique-id used in the
95 list.104 hardware decoder target list.
105
96 106
97What: /sys/bus/cxl/devices/decoderX.Y107What: /sys/bus/cxl/devices/decoderX.Y
98Date: June, 2021108Date: June, 2021
99KernelVersion: v5.14109KernelVersion: v5.14
100Contact: linux-cxl@vger.kernel.org110Contact: linux-cxl@vger.kernel.org
101Description:111Description:
102 CXL decoder objects are enumerated from either a platform112 (RO) CXL decoder objects are enumerated from either a platform
103 firmware description, or a CXL HDM decoder register set in a113 firmware description, or a CXL HDM decoder register set in a
104 PCIe device (see CXL 2.0 section 8.2.5.12 CXL HDM Decoder114 PCIe device (see CXL 2.0 section 8.2.5.12 CXL HDM Decoder
105 Capability Structure). The 'X' in decoderX.Y represents the115 Capability Structure). The 'X' in decoderX.Y represents the
106 cxl_port container of this decoder, and 'Y' represents the116 cxl_port container of this decoder, and 'Y' represents the
107 instance id of a given decoder resource.117 instance id of a given decoder resource.
108 118
119
109What: /sys/bus/cxl/devices/decoderX.Y/{start,size}120What: /sys/bus/cxl/devices/decoderX.Y/{start,size}
110Date: June, 2021121Date: June, 2021
111KernelVersion: v5.14122KernelVersion: v5.14
112Contact: linux-cxl@vger.kernel.org123Contact: linux-cxl@vger.kernel.org
113Description:124Description:
114 The 'start' and 'size' attributes together convey the physical125 (RO) The 'start' and 'size' attributes together convey the
115 address base and number of bytes mapped in the decoder's decode126 physical address base and number of bytes mapped in the
116 window. For decoders of devtype "cxl_decoder_root" the address127 decoder's decode window. For decoders of devtype
117 range is fixed. For decoders of devtype "cxl_decoder_switch" the128 "cxl_decoder_root" the address range is fixed. For decoders of
118 address is bounded by the decode range of the cxl_port ancestor129 devtype "cxl_decoder_switch" the address is bounded by the
119 of the decoder's cxl_port, and dynamically updates based on the130 decode range of the cxl_port ancestor of the decoder's cxl_port,
120 active memory regions in that address space.131 and dynamically updates based on the active memory regions in
132 that address space.
133
121 134
122What: /sys/bus/cxl/devices/decoderX.Y/locked135What: /sys/bus/cxl/devices/decoderX.Y/locked
123Date: June, 2021136Date: June, 2021
124KernelVersion: v5.14137KernelVersion: v5.14
125Contact: linux-cxl@vger.kernel.org138Contact: linux-cxl@vger.kernel.org
126Description:139Description:
127 CXL HDM decoders have the capability to lock the configuration140 (RO) CXL HDM decoders have the capability to lock the
128 until the next device reset. For decoders of devtype141 configuration until the next device reset. For decoders of
129 "cxl_decoder_root" there is no standard facility to unlock them.142 devtype "cxl_decoder_root" there is no standard facility to
130 For decoders of devtype "cxl_decoder_switch" a secondary bus143 unlock them. For decoders of devtype "cxl_decoder_switch" a
131 reset, of the PCIe bridge that provides the bus for this144 secondary bus reset, of the PCIe bridge that provides the bus
132 decoders uport, unlocks / resets the decoder.145 for this decoders uport, unlocks / resets the decoder.
146
133 147
134What: /sys/bus/cxl/devices/decoderX.Y/target_list148What: /sys/bus/cxl/devices/decoderX.Y/target_list
135Date: June, 2021149Date: June, 2021
136KernelVersion: v5.14150KernelVersion: v5.14
137Contact: linux-cxl@vger.kernel.org151Contact: linux-cxl@vger.kernel.org
138Description:152Description:
139 Display a comma separated list of the current decoder target153 (RO) Display a comma separated list of the current decoder
140 configuration. The list is ordered by the current configured154 target configuration. The list is ordered by the current
141 interleave order of the decoder's dport instances. Each entry in155 configured interleave order of the decoder's dport instances.
142 the list is a dport id.156 Each entry in the list is a dport id.
157
143 158
144What: /sys/bus/cxl/devices/decoderX.Y/cap_{pmem,ram,type2,type3}159What: /sys/bus/cxl/devices/decoderX.Y/cap_{pmem,ram,type2,type3}
145Date: June, 2021160Date: June, 2021
146KernelVersion: v5.14161KernelVersion: v5.14
147Contact: linux-cxl@vger.kernel.org162Contact: linux-cxl@vger.kernel.org
148Description:163Description:
149 When a CXL decoder is of devtype "cxl_decoder_root", it164 (RO) When a CXL decoder is of devtype "cxl_decoder_root", it
150 represents a fixed memory window identified by platform165 represents a fixed memory window identified by platform
151 firmware. A fixed window may only support a subset of memory166 firmware. A fixed window may only support a subset of memory
152 types. The 'cap_*' attributes indicate whether persistent167 types. The 'cap_*' attributes indicate whether persistent
153 memory, volatile memory, accelerator memory, and / or expander168 memory, volatile memory, accelerator memory, and / or expander
154 memory may be mapped behind this decoder's memory window.169 memory may be mapped behind this decoder's memory window.
155 170
171
156What: /sys/bus/cxl/devices/decoderX.Y/target_type172What: /sys/bus/cxl/devices/decoderX.Y/target_type
157Date: June, 2021173Date: June, 2021
158KernelVersion: v5.14174KernelVersion: v5.14
159Contact: linux-cxl@vger.kernel.org175Contact: linux-cxl@vger.kernel.org
160Description:176Description:
161 When a CXL decoder is of devtype "cxl_decoder_switch", it can177 (RO) When a CXL decoder is of devtype "cxl_decoder_switch", it
162 optionally decode either accelerator memory (type-2) or expander178 can optionally decode either accelerator memory (type-2) or
163 memory (type-3). The 'target_type' attribute indicates the179 expander memory (type-3). The 'target_type' attribute indicates
164 current setting which may dynamically change based on what180 the current setting which may dynamically change based on what
165 memory regions are activated in this decode hierarchy.181 memory regions are activated in this decode hierarchy.
182
183
184What: /sys/bus/cxl/devices/endpointX/CDAT
185Date: July, 2022
186KernelVersion: v5.20
187Contact: linux-cxl@vger.kernel.org
188Description:
189 (RO) If this sysfs entry is not present no DOE mailbox was
190 found to support CDAT data. If it is present and the length of
191 the data is 0 reading the CDAT data failed. Otherwise the CDAT
192 data is reported.
193
194
195What: /sys/bus/cxl/devices/decoderX.Y/mode
196Date: May, 2022
197KernelVersion: v5.20
198Contact: linux-cxl@vger.kernel.org
199Description:
200 (RW) When a CXL decoder is of devtype "cxl_decoder_endpoint" it
201 translates from a host physical address range, to a device local
202 address range. Device-local address ranges are further split
203 into a 'ram' (volatile memory) range and 'pmem' (persistent
204 memory) range. The 'mode' attribute emits one of 'ram', 'pmem',
205 'mixed', or 'none'. The 'mixed' indication is for error cases
206 when a decoder straddles the volatile/persistent partition
207 boundary, and 'none' indicates the decoder is not actively
208 decoding, or no DPA allocation policy has been set.
209
210 'mode' can be written, when the decoder is in the 'disabled'
211 state, with either 'ram' or 'pmem' to set the boundaries for the
212 next allocation.
213
214
215What: /sys/bus/cxl/devices/decoderX.Y/dpa_resource
216Date: May, 2022
217KernelVersion: v5.20
218Contact: linux-cxl@vger.kernel.org
219Description:
220 (RO) When a CXL decoder is of devtype "cxl_decoder_endpoint",
221 and its 'dpa_size' attribute is non-zero, this attribute
222 indicates the device physical address (DPA) base address of the
223 allocation.
224
225
226What: /sys/bus/cxl/devices/decoderX.Y/dpa_size
227Date: May, 2022
228KernelVersion: v5.20
229Contact: linux-cxl@vger.kernel.org
230Description:
231 (RW) When a CXL decoder is of devtype "cxl_decoder_endpoint" it
232 translates from a host physical address range, to a device local
233 address range. The range, base address plus length in bytes, of
234 DPA allocated to this decoder is conveyed in these 2 attributes.
235 Allocations can be mutated as long as the decoder is in the
236 disabled state. A write to 'dpa_size' releases the previous DPA
237 allocation and then attempts to allocate from the free capacity
238 in the device partition referred to by 'decoderX.Y/mode'.
239 Allocate and free requests can only be performed on the highest
240 instance number disabled decoder with non-zero size. I.e.
241 allocations are enforced to occur in increasing 'decoderX.Y/id'
242 order and frees are enforced to occur in decreasing
243 'decoderX.Y/id' order.
244
245
246What: /sys/bus/cxl/devices/decoderX.Y/interleave_ways
247Date: May, 2022
248KernelVersion: v5.20
249Contact: linux-cxl@vger.kernel.org
250Description:
251 (RO) The number of targets across which this decoder's host
252 physical address (HPA) memory range is interleaved. The device
253 maps every Nth block of HPA (of size ==
254 'interleave_granularity') to consecutive DPA addresses. The
255 decoder's position in the interleave is determined by the
256 device's (endpoint or switch) switch ancestry. For root
257 decoders their interleave is specified by platform firmware and
258 they only specify a downstream target order for host bridges.
259
260
261What: /sys/bus/cxl/devices/decoderX.Y/interleave_granularity
262Date: May, 2022
263KernelVersion: v5.20
264Contact: linux-cxl@vger.kernel.org
265Description:
266 (RO) The number of consecutive bytes of host physical address
267 space this decoder claims at address N before the decode rotates
268 to the next target in the interleave at address N +
269 interleave_granularity (assuming N is aligned to
270 interleave_granularity).
271
272
273What: /sys/bus/cxl/devices/decoderX.Y/create_pmem_region
274Date: May, 2022
275KernelVersion: v5.20
276Contact: linux-cxl@vger.kernel.org
277Description:
278 (RW) Write a string in the form 'regionZ' to start the process
279 of defining a new persistent memory region (interleave-set)
280 within the decode range bounded by root decoder 'decoderX.Y'.
281 The value written must match the current value returned from
282 reading this attribute. An atomic compare exchange operation is
283 done on write to assign the requested id to a region and
284 allocate the region-id for the next creation attempt. EBUSY is
285 returned if the region name written does not match the current
286 cached value.
287
288
289What: /sys/bus/cxl/devices/decoderX.Y/delete_region
290Date: May, 2022
291KernelVersion: v5.20
292Contact: linux-cxl@vger.kernel.org
293Description:
294 (WO) Write a string in the form 'regionZ' to delete that region,
295 provided it is currently idle / not bound to a driver.
296
297
298What: /sys/bus/cxl/devices/regionZ/uuid
299Date: May, 2022
300KernelVersion: v5.20
301Contact: linux-cxl@vger.kernel.org
302Description:
303 (RW) Write a unique identifier for the region. This field must
304 be set for persistent regions and it must not conflict with the
305 UUID of another region.
306
307
308What: /sys/bus/cxl/devices/regionZ/interleave_granularity
309Date: May, 2022
310KernelVersion: v5.20
311Contact: linux-cxl@vger.kernel.org
312Description:
313 (RW) Set the number of consecutive bytes each device in the
314 interleave set will claim. The possible interleave granularity
315 values are determined by the CXL spec and the participating
316 devices.
317
318
319What: /sys/bus/cxl/devices/regionZ/interleave_ways
320Date: May, 2022
321KernelVersion: v5.20
322Contact: linux-cxl@vger.kernel.org
323Description:
324 (RW) Configures the number of devices participating in the
325 region is set by writing this value. Each device will provide
326 1/interleave_ways of storage for the region.
327
328
329What: /sys/bus/cxl/devices/regionZ/size
330Date: May, 2022
331KernelVersion: v5.20
332Contact: linux-cxl@vger.kernel.org
333Description:
334 (RW) System physical address space to be consumed by the region.
335 When written trigger the driver to allocate space out of the
336 parent root decoder's address space. When read the size of the
337 address space is reported and should match the span of the
338 region's resource attribute. Size shall be set after the
339 interleave configuration parameters. Once set it cannot be
340 changed, only freed by writing 0. The kernel makes no guarantees
341 that data is maintained over an address space freeing event, and
342 there is no guarantee that a free followed by an allocate
343 results in the same address being allocated.
344
345
346What: /sys/bus/cxl/devices/regionZ/resource
347Date: May, 2022
348KernelVersion: v5.20
349Contact: linux-cxl@vger.kernel.org
350Description:
351 (RO) A region is a contiguous partition of a CXL root decoder
352 address space. Region capacity is allocated by writing to the
353 size attribute, the resulting physical address space determined
354 by the driver is reflected here. It is therefore not useful to
355 read this before writing a value to the size attribute.
356
357
358What: /sys/bus/cxl/devices/regionZ/target[0..N]
359Date: May, 2022
360KernelVersion: v5.20
361Contact: linux-cxl@vger.kernel.org
362Description:
363 (RW) Write an endpoint decoder object name to 'targetX' where X
364 is the intended position of the endpoint device in the region
365 interleave and N is the 'interleave_ways' setting for the
366 region. ENXIO is returned if the write results in an impossible
367 to map decode scenario, like the endpoint is unreachable at that
368 position relative to the root decoder interleave. EBUSY is
369 returned if the position in the region is already occupied, or
370 if the region is not in a state to accept interleave
371 configuration changes. EINVAL is returned if the object name is
372 not an endpoint decoder. Once all positions have been
373 successfully written a final validation for decode conflicts is
374 performed before activating the region.
375
376
377What: /sys/bus/cxl/devices/regionZ/commit
378Date: May, 2022
379KernelVersion: v5.20
380Contact: linux-cxl@vger.kernel.org
381Description:
382 (RW) Write a boolean 'true' string value to this attribute to
383 trigger the region to transition from the software programmed
384 state to the actively decoding in hardware state. The commit
385 operation in addition to validating that the region is in proper
386 configured state, validates that the decoders are being
387 committed in spec mandated order (last committed decoder id +
388 1), and checks that the hardware accepts the commit request.
389 Reading this value indicates whether the region is committed or
390 not.
Documentation/driver-api/cxl/memory-devices.rst+8−0
@@ -362,6 +362,14 @@ CXL Core · 361 unmodified lines
362.. kernel-doc:: drivers/cxl/core/mbox.c362.. kernel-doc:: drivers/cxl/core/mbox.c
363 :doc: cxl mbox363 :doc: cxl mbox
364 364
365CXL Regions
366-----------
367.. kernel-doc:: drivers/cxl/core/region.c
368 :doc: cxl core region
369
370.. kernel-doc:: drivers/cxl/core/region.c
371 :identifiers:
372
365External Interfaces373External Interfaces
366===================374===================
367 375
arch/powerpc/mm/mem.c+1−0
@@ -55,6 +55,7 @@ int memory_add_physaddr_to_nid(u64 start) · 54 unmodified lines
55{55{
56 return hot_add_scn_to_nid(start);56 return hot_add_scn_to_nid(start);
57}57}
58EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
58#endif59#endif
59 60
60int __weak create_section_mapping(unsigned long start, unsigned long end,61int __weak create_section_mapping(unsigned long start, unsigned long end,
drivers/cxl/Kconfig+9−0
@@ -2,6 +2,7 @@ · 1 unmodified line
2menuconfig CXL_BUS2menuconfig CXL_BUS
3 tristate "CXL (Compute Express Link) Devices Support"3 tristate "CXL (Compute Express Link) Devices Support"
4 depends on PCI4 depends on PCI
5 select PCI_DOE
5 help6 help
6 CXL is a bus that is electrically compatible with PCI Express, but7 CXL is a bus that is electrically compatible with PCI Express, but
7 layers three protocols on that signalling (CXL.io, CXL.cache, and8 layers three protocols on that signalling (CXL.io, CXL.cache, and
@@ -102,4 +103,12 @@ config CXL_SUSPEND · 94 unmodified lines
102 def_bool y103 def_bool y
103 depends on SUSPEND && CXL_MEM104 depends on SUSPEND && CXL_MEM
104 105
106config CXL_REGION
107 bool
108 default CXL_BUS
109 # For MAX_PHYSMEM_BITS
110 depends on SPARSEMEM
111 select MEMREGION
112 select GET_FREE_REGION
113
105endif114endif
drivers/cxl/acpi.c+217−26
@@ -9,10 +9,6 @@ · 8 unmodified lines
9#include "cxlpci.h"9#include "cxlpci.h"
10#include "cxl.h"10#include "cxl.h"
11 11
12/* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */
13#define CFMWS_INTERLEAVE_WAYS(x) (1 << (x)->interleave_ways)
14#define CFMWS_INTERLEAVE_GRANULARITY(x) ((x)->granularity + 8)
15
16static unsigned long cfmws_to_decoder_flags(int restrictions)12static unsigned long cfmws_to_decoder_flags(int restrictions)
17{13{
18 unsigned long flags = CXL_DECODER_F_ENABLE;14 unsigned long flags = CXL_DECODER_F_ENABLE;
@@ -34,7 +30,8 @@ static unsigned long cfmws_to_decoder_flags(int restrictions) · 15 unmodified lines
34static int cxl_acpi_cfmws_verify(struct device *dev,30static int cxl_acpi_cfmws_verify(struct device *dev,
35 struct acpi_cedt_cfmws *cfmws)31 struct acpi_cedt_cfmws *cfmws)
36{32{
37 int expected_len;33 int rc, expected_len;
34 unsigned int ways;
38 35
39 if (cfmws->interleave_arithmetic != ACPI_CEDT_CFMWS_ARITHMETIC_MODULO) {36 if (cfmws->interleave_arithmetic != ACPI_CEDT_CFMWS_ARITHMETIC_MODULO) {
40 dev_err(dev, "CFMWS Unsupported Interleave Arithmetic\n");37 dev_err(dev, "CFMWS Unsupported Interleave Arithmetic\n");
@@ -51,14 +48,14 @@ static int cxl_acpi_cfmws_verify(struct device *dev, · 10 unmodified lines
51 return -EINVAL;48 return -EINVAL;
52 }49 }
53 50
54 if (CFMWS_INTERLEAVE_WAYS(cfmws) > CXL_DECODER_MAX_INTERLEAVE) {51 rc = cxl_to_ways(cfmws->interleave_ways, &ways);
55 dev_err(dev, "CFMWS Interleave Ways (%d) too large\n",52 if (rc) {
56 CFMWS_INTERLEAVE_WAYS(cfmws));53 dev_err(dev, "CFMWS Interleave Ways (%d) invalid\n",
54 cfmws->interleave_ways);
57 return -EINVAL;55 return -EINVAL;
58 }56 }
59 57
60 expected_len = struct_size((cfmws), interleave_targets,58 expected_len = struct_size(cfmws, interleave_targets, ways);
61 CFMWS_INTERLEAVE_WAYS(cfmws));
62 59
63 if (cfmws->header.length < expected_len) {60 if (cfmws->header.length < expected_len) {
64 dev_err(dev, "CFMWS length %d less than expected %d\n",61 dev_err(dev, "CFMWS length %d less than expected %d\n",
@@ -76,6 +73,8 @@ static int cxl_acpi_cfmws_verify(struct device *dev, · 11 unmodified lines
76struct cxl_cfmws_context {73struct cxl_cfmws_context {
77 struct device *dev;74 struct device *dev;
78 struct cxl_port *root_port;75 struct cxl_port *root_port;
76 struct resource *cxl_res;
77 int id;
79};78};
80 79
81static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg,80static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg,
@@ -84,10 +83,14 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, · 2 unmodified lines
84 int target_map[CXL_DECODER_MAX_INTERLEAVE];83 int target_map[CXL_DECODER_MAX_INTERLEAVE];
85 struct cxl_cfmws_context *ctx = arg;84 struct cxl_cfmws_context *ctx = arg;
86 struct cxl_port *root_port = ctx->root_port;85 struct cxl_port *root_port = ctx->root_port;
86 struct resource *cxl_res = ctx->cxl_res;
87 struct cxl_root_decoder *cxlrd;
87 struct device *dev = ctx->dev;88 struct device *dev = ctx->dev;
88 struct acpi_cedt_cfmws *cfmws;89 struct acpi_cedt_cfmws *cfmws;
89 struct cxl_decoder *cxld;90 struct cxl_decoder *cxld;
90 int rc, i;91 unsigned int ways, i, ig;
92 struct resource *res;
93 int rc;
91 94
92 cfmws = (struct acpi_cedt_cfmws *) header;95 cfmws = (struct acpi_cedt_cfmws *) header;
93 96
@@ -99,19 +102,51 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, · 5 unmodified lines
99 return 0;102 return 0;
100 }103 }
101 104
102 for (i = 0; i < CFMWS_INTERLEAVE_WAYS(cfmws); i++)105 rc = cxl_to_ways(cfmws->interleave_ways, &ways);
106 if (rc)
107 return rc;
108 rc = cxl_to_granularity(cfmws->granularity, &ig);
109 if (rc)
110 return rc;
111 for (i = 0; i < ways; i++)
103 target_map[i] = cfmws->interleave_targets[i];112 target_map[i] = cfmws->interleave_targets[i];
104 113
105 cxld = cxl_root_decoder_alloc(root_port, CFMWS_INTERLEAVE_WAYS(cfmws));114 res = kzalloc(sizeof(*res), GFP_KERNEL);
106 if (IS_ERR(cxld))115 if (!res)
116 return -ENOMEM;
117
118 res->name = kasprintf(GFP_KERNEL, "CXL Window %d", ctx->id++);
119 if (!res->name)
120 goto err_name;
121
122 res->start = cfmws->base_hpa;
123 res->end = cfmws->base_hpa + cfmws->window_size - 1;
124 res->flags = IORESOURCE_MEM;
125
126 /* add to the local resource tracking to establish a sort order */
127 rc = insert_resource(cxl_res, res);
128 if (rc)
129 goto err_insert;
130
131 cxlrd = cxl_root_decoder_alloc(root_port, ways);
132 if (IS_ERR(cxlrd))
107 return 0;133 return 0;
108 134
135 cxld = &cxlrd->cxlsd.cxld;
109 cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions);136 cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions);
110 cxld->target_type = CXL_DECODER_EXPANDER;137 cxld->target_type = CXL_DECODER_EXPANDER;
111 cxld->platform_res = (struct resource)DEFINE_RES_MEM(cfmws->base_hpa,138 cxld->hpa_range = (struct range) {
112 cfmws->window_size);139 .start = res->start,
113 cxld->interleave_ways = CFMWS_INTERLEAVE_WAYS(cfmws);140 .end = res->end,
114 cxld->interleave_granularity = CFMWS_INTERLEAVE_GRANULARITY(cfmws);141 };
142 cxld->interleave_ways = ways;
143 /*
144 * Minimize the x1 granularity to advertise support for any
145 * valid region granularity
146 */
147 if (ways == 1)
148 ig = CXL_DECODER_MIN_GRANULARITY;
149 cxld->interleave_granularity = ig;
115 150
116 rc = cxl_decoder_add(cxld, target_map);151 rc = cxl_decoder_add(cxld, target_map);
117 if (rc)152 if (rc)
@@ -119,15 +154,22 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg, · 1 unmodified line
119 else154 else
120 rc = cxl_decoder_autoremove(dev, cxld);155 rc = cxl_decoder_autoremove(dev, cxld);
121 if (rc) {156 if (rc) {
122 dev_err(dev, "Failed to add decoder for %pr\n",157 dev_err(dev, "Failed to add decode range [%#llx - %#llx]\n",
123 &cxld->platform_res);158 cxld->hpa_range.start, cxld->hpa_range.end);
124 return 0;159 return 0;
125 }160 }
126 dev_dbg(dev, "add: %s node: %d range %pr\n", dev_name(&cxld->dev),161 dev_dbg(dev, "add: %s node: %d range [%#llx - %#llx]\n",
127 phys_to_target_node(cxld->platform_res.start),162 dev_name(&cxld->dev),
128 &cxld->platform_res);163 phys_to_target_node(cxld->hpa_range.start),
164 cxld->hpa_range.start, cxld->hpa_range.end);
129 165
130 return 0;166 return 0;
167
168err_insert:
169 kfree(res->name);
170err_name:
171 kfree(res);
172 return -ENOMEM;
131}173}
132 174
133__mock struct acpi_device *to_cxl_host_bridge(struct device *host,175__mock struct acpi_device *to_cxl_host_bridge(struct device *host,
@@ -175,8 +217,7 @@ static int add_host_bridge_uport(struct device *match, void *arg) · 41 unmodified lines
175 if (rc)217 if (rc)
176 return rc;218 return rc;
177 219
178 port = devm_cxl_add_port(host, match, dport->component_reg_phys,220 port = devm_cxl_add_port(host, match, dport->component_reg_phys, dport);
179 root_port);
180 if (IS_ERR(port))221 if (IS_ERR(port))
181 return PTR_ERR(port);222 return PTR_ERR(port);
182 dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev));223 dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev));
@@ -282,9 +323,127 @@ static void cxl_acpi_lock_reset_class(void *dev) · 99 unmodified lines
282 device_lock_reset_class(dev);323 device_lock_reset_class(dev);
283}324}
284 325
326static void del_cxl_resource(struct resource *res)
327{
328 kfree(res->name);
329 kfree(res);
330}
331
332static void cxl_set_public_resource(struct resource *priv, struct resource *pub)
333{
334 priv->desc = (unsigned long) pub;
335}
336
337static struct resource *cxl_get_public_resource(struct resource *priv)
338{
339 return (struct resource *) priv->desc;
340}
341
342static void remove_cxl_resources(void *data)
343{
344 struct resource *res, *next, *cxl = data;
345
346 for (res = cxl->child; res; res = next) {
347 struct resource *victim = cxl_get_public_resource(res);
348
349 next = res->sibling;
350 remove_resource(res);
351
352 if (victim) {
353 remove_resource(victim);
354 kfree(victim);
355 }
356
357 del_cxl_resource(res);
358 }
359}
360
361/**
362 * add_cxl_resources() - reflect CXL fixed memory windows in iomem_resource
363 * @cxl_res: A standalone resource tree where each CXL window is a sibling
364 *
365 * Walk each CXL window in @cxl_res and add it to iomem_resource potentially
366 * expanding its boundaries to ensure that any conflicting resources become
367 * children. If a window is expanded it may then conflict with a another window
368 * entry and require the window to be truncated or trimmed. Consider this
369 * situation:
370 *
371 * |-- "CXL Window 0" --||----- "CXL Window 1" -----|
372 * |--------------- "System RAM" -------------|
373 *
374 * ...where platform firmware has established as System RAM resource across 2
375 * windows, but has left some portion of window 1 for dynamic CXL region
376 * provisioning. In this case "Window 0" will span the entirety of the "System
377 * RAM" span, and "CXL Window 1" is truncated to the remaining tail past the end
378 * of that "System RAM" resource.
379 */
380static int add_cxl_resources(struct resource *cxl_res)
381{
382 struct resource *res, *new, *next;
383
384 for (res = cxl_res->child; res; res = next) {
385 new = kzalloc(sizeof(*new), GFP_KERNEL);
386 if (!new)
387 return -ENOMEM;
388 new->name = res->name;
389 new->start = res->start;
390 new->end = res->end;
391 new->flags = IORESOURCE_MEM;
392 new->desc = IORES_DESC_CXL;
393
394 /*
395 * Record the public resource in the private cxl_res tree for
396 * later removal.
397 */
398 cxl_set_public_resource(res, new);
399
400 insert_resource_expand_to_fit(&iomem_resource, new);
401
402 next = res->sibling;
403 while (next && resource_overlaps(new, next)) {
404 if (resource_contains(new, next)) {
405 struct resource *_next = next->sibling;
406
407 remove_resource(next);
408 del_cxl_resource(next);
409 next = _next;
410 } else
411 next->start = new->end + 1;
412 }
413 }
414 return 0;
415}
416
417static int pair_cxl_resource(struct device *dev, void *data)
418{
419 struct resource *cxl_res = data;
420 struct resource *p;
421
422 if (!is_root_decoder(dev))
423 return 0;
424
425 for (p = cxl_res->child; p; p = p->sibling) {
426 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
427 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
428 struct resource res = {
429 .start = cxld->hpa_range.start,
430 .end = cxld->hpa_range.end,
431 .flags = IORESOURCE_MEM,
432 };
433
434 if (resource_contains(p, &res)) {
435 cxlrd->res = cxl_get_public_resource(p);
436 break;
437 }
438 }
439
440 return 0;
441}
442
285static int cxl_acpi_probe(struct platform_device *pdev)443static int cxl_acpi_probe(struct platform_device *pdev)
286{444{
287 int rc;445 int rc;
446 struct resource *cxl_res;
288 struct cxl_port *root_port;447 struct cxl_port *root_port;
289 struct device *host = &pdev->dev;448 struct device *host = &pdev->dev;
290 struct acpi_device *adev = ACPI_COMPANION(host);449 struct acpi_device *adev = ACPI_COMPANION(host);
@@ -296,6 +455,14 @@ static int cxl_acpi_probe(struct platform_device *pdev) · 5 unmodified lines
296 if (rc)455 if (rc)
297 return rc;456 return rc;
298 457
458 cxl_res = devm_kzalloc(host, sizeof(*cxl_res), GFP_KERNEL);
459 if (!cxl_res)
460 return -ENOMEM;
461 cxl_res->name = "CXL mem";
462 cxl_res->start = 0;
463 cxl_res->end = -1;
464 cxl_res->flags = IORESOURCE_MEM;
465
299 root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);466 root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
300 if (IS_ERR(root_port))467 if (IS_ERR(root_port))
301 return PTR_ERR(root_port);468 return PTR_ERR(root_port);
@@ -306,11 +473,28 @@ static int cxl_acpi_probe(struct platform_device *pdev) · 4 unmodified lines
306 if (rc < 0)473 if (rc < 0)
307 return rc;474 return rc;
308 475
476 rc = devm_add_action_or_reset(host, remove_cxl_resources, cxl_res);
477 if (rc)
478 return rc;
479
309 ctx = (struct cxl_cfmws_context) {480 ctx = (struct cxl_cfmws_context) {
310 .dev = host,481 .dev = host,
311 .root_port = root_port,482 .root_port = root_port,
483 .cxl_res = cxl_res,
312 };484 };
313 acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, cxl_parse_cfmws, &ctx);485 rc = acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, cxl_parse_cfmws, &ctx);
486 if (rc < 0)
487 return -ENXIO;
488
489 rc = add_cxl_resources(cxl_res);
490 if (rc)
491 return rc;
492
493 /*
494 * Populate the root decoders with their related iomem resource,
495 * if present
496 */
497 device_for_each_child(&root_port->dev, cxl_res, pair_cxl_resource);
314 498
315 /*499 /*
316 * Root level scanned with host-bridge as dports, now scan host-bridges500 * Root level scanned with host-bridge as dports, now scan host-bridges
@@ -337,12 +521,19 @@ static const struct acpi_device_id cxl_acpi_ids[] = { · 20 unmodified lines
337};521};
338MODULE_DEVICE_TABLE(acpi, cxl_acpi_ids);522MODULE_DEVICE_TABLE(acpi, cxl_acpi_ids);
339 523
524static const struct platform_device_id cxl_test_ids[] = {
525 { "cxl_acpi" },
526 { },
527};
528MODULE_DEVICE_TABLE(platform, cxl_test_ids);
529
340static struct platform_driver cxl_acpi_driver = {530static struct platform_driver cxl_acpi_driver = {
341 .probe = cxl_acpi_probe,531 .probe = cxl_acpi_probe,
342 .driver = {532 .driver = {
343 .name = KBUILD_MODNAME,533 .name = KBUILD_MODNAME,
344 .acpi_match_table = cxl_acpi_ids,534 .acpi_match_table = cxl_acpi_ids,
345 },535 },
536 .id_table = cxl_test_ids,
346};537};
347 538
348module_platform_driver(cxl_acpi_driver);539module_platform_driver(cxl_acpi_driver);
drivers/cxl/core/Makefile+1−0
@@ -10,3 +10,4 @@ cxl_core-y += memdev.o · 9 unmodified lines
10cxl_core-y += mbox.o10cxl_core-y += mbox.o
11cxl_core-y += pci.o11cxl_core-y += pci.o
12cxl_core-y += hdm.o12cxl_core-y += hdm.o
13cxl_core-$(CONFIG_CXL_REGION) += region.o
drivers/cxl/core/core.h+50−1
@@ -9,6 +9,36 @@ extern const struct device_type cxl_nvdimm_type; · 8 unmodified lines
9 9
10extern struct attribute_group cxl_base_attribute_group;10extern struct attribute_group cxl_base_attribute_group;
11 11
12#ifdef CONFIG_CXL_REGION
13extern struct device_attribute dev_attr_create_pmem_region;
14extern struct device_attribute dev_attr_delete_region;
15extern struct device_attribute dev_attr_region;
16extern const struct device_type cxl_pmem_region_type;
17extern const struct device_type cxl_region_type;
18void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled);
19#define CXL_REGION_ATTR(x) (&dev_attr_##x.attr)
20#define CXL_REGION_TYPE(x) (&cxl_region_type)
21#define SET_CXL_REGION_ATTR(x) (&dev_attr_##x.attr),
22#define CXL_PMEM_REGION_TYPE(x) (&cxl_pmem_region_type)
23int cxl_region_init(void);
24void cxl_region_exit(void);
25#else
26static inline void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
27{
28}
29static inline int cxl_region_init(void)
30{
31 return 0;
32}
33static inline void cxl_region_exit(void)
34{
35}
36#define CXL_REGION_ATTR(x) NULL
37#define CXL_REGION_TYPE(x) NULL
38#define SET_CXL_REGION_ATTR(x)
39#define CXL_PMEM_REGION_TYPE(x) NULL
40#endif
41
12struct cxl_send_command;42struct cxl_send_command;
13struct cxl_mem_query_commands;43struct cxl_mem_query_commands;
14int cxl_query_cmd(struct cxl_memdev *cxlmd,44int cxl_query_cmd(struct cxl_memdev *cxlmd,
@@ -17,9 +47,28 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s); · 2 unmodified lines
17void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,47void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
18 resource_size_t length);48 resource_size_t length);
19 49
50struct dentry *cxl_debugfs_create_dir(const char *dir);
51int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
52 enum cxl_decoder_mode mode);
53int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size);
54int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
55resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled);
56resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled);
57extern struct rw_semaphore cxl_dpa_rwsem;
58
59bool is_switch_decoder(struct device *dev);
60struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
61static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port,
62 struct cxl_memdev *cxlmd)
63{
64 if (!port)
65 return NULL;
66
67 return xa_load(&port->endpoints, (unsigned long)&cxlmd->dev);
68}
69
20int cxl_memdev_init(void);70int cxl_memdev_init(void);
21void cxl_memdev_exit(void);71void cxl_memdev_exit(void);
22void cxl_mbox_init(void);72void cxl_mbox_init(void);
23void cxl_mbox_exit(void);
24 73
25#endif /* __CXL_CORE_H__ */74#endif /* __CXL_CORE_H__ */
drivers/cxl/core/hdm.c+635−56
@@ -1,6 +1,7 @@
1// SPDX-License-Identifier: GPL-2.0-only1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/io-64-nonatomic-hi-lo.h>3#include <linux/io-64-nonatomic-hi-lo.h>
4#include <linux/seq_file.h>
4#include <linux/device.h>5#include <linux/device.h>
5#include <linux/delay.h>6#include <linux/delay.h>
6 7
@@ -16,6 +17,8 @@ · 9 unmodified lines
16 * for enumerating these registers and capabilities.17 * for enumerating these registers and capabilities.
17 */18 */
18 19
20DECLARE_RWSEM(cxl_dpa_rwsem);
21
19static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,22static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
20 int *target_map)23 int *target_map)
21{24{
@@ -46,20 +49,22 @@ static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, · 24 unmodified lines
46 */49 */
47int devm_cxl_add_passthrough_decoder(struct cxl_port *port)50int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
48{51{
49 struct cxl_decoder *cxld;52 struct cxl_switch_decoder *cxlsd;
50 struct cxl_dport *dport;53 struct cxl_dport *dport = NULL;
51 int single_port_map[1];54 int single_port_map[1];
55 unsigned long index;
52 56
53 cxld = cxl_switch_decoder_alloc(port, 1);57 cxlsd = cxl_switch_decoder_alloc(port, 1);
54 if (IS_ERR(cxld))58 if (IS_ERR(cxlsd))
55 return PTR_ERR(cxld);59 return PTR_ERR(cxlsd);
56 60
57 device_lock_assert(&port->dev);61 device_lock_assert(&port->dev);
58 62
59 dport = list_first_entry(&port->dports, typeof(*dport), list);63 xa_for_each(&port->dports, index, dport)
64 break;
60 single_port_map[0] = dport->port_id;65 single_port_map[0] = dport->port_id;
61 66
62 return add_hdm_decoder(port, cxld, single_port_map);67 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
63}68}
64EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);69EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
65 70
@@ -124,47 +129,577 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port) · 58 unmodified lines
124 return ERR_PTR(-ENXIO);129 return ERR_PTR(-ENXIO);
125 }130 }
126 131
132 dev_set_drvdata(dev, cxlhdm);
133
127 return cxlhdm;134 return cxlhdm;
128}135}
129EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);136EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
130 137
131static int to_interleave_granularity(u32 ctrl)138static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
139{
140 unsigned long long start = r->start, end = r->end;
141
142 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
143 r->name);
144}
145
146void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
147{
148 struct resource *p1, *p2;
149
150 down_read(&cxl_dpa_rwsem);
151 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
152 __cxl_dpa_debug(file, p1, 0);
153 for (p2 = p1->child; p2; p2 = p2->sibling)
154 __cxl_dpa_debug(file, p2, 1);
155 }
156 up_read(&cxl_dpa_rwsem);
157}
158EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
159
160/*
161 * Must be called in a context that synchronizes against this decoder's
162 * port ->remove() callback (like an endpoint decoder sysfs attribute)
163 */
164static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
165{
166 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
167 struct cxl_port *port = cxled_to_port(cxled);
168 struct cxl_dev_state *cxlds = cxlmd->cxlds;
169 struct resource *res = cxled->dpa_res;
170 resource_size_t skip_start;
171
172 lockdep_assert_held_write(&cxl_dpa_rwsem);
173
174 /* save @skip_start, before @res is released */
175 skip_start = res->start - cxled->skip;
176 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
177 if (cxled->skip)
178 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
179 cxled->skip = 0;
180 cxled->dpa_res = NULL;
181 put_device(&cxled->cxld.dev);
182 port->hdm_end--;
183}
184
185static void cxl_dpa_release(void *cxled)
186{
187 down_write(&cxl_dpa_rwsem);
188 __cxl_dpa_release(cxled);
189 up_write(&cxl_dpa_rwsem);
190}
191
192/*
193 * Must be called from context that will not race port device
194 * unregistration, like decoder sysfs attribute methods
195 */
196static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
197{
198 struct cxl_port *port = cxled_to_port(cxled);
199
200 lockdep_assert_held_write(&cxl_dpa_rwsem);
201 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
202 __cxl_dpa_release(cxled);
203}
204
205static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
206 resource_size_t base, resource_size_t len,
207 resource_size_t skipped)
208{
209 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
210 struct cxl_port *port = cxled_to_port(cxled);
211 struct cxl_dev_state *cxlds = cxlmd->cxlds;
212 struct device *dev = &port->dev;
213 struct resource *res;
214
215 lockdep_assert_held_write(&cxl_dpa_rwsem);
216
217 if (!len)
218 goto success;
219
220 if (cxled->dpa_res) {
221 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
222 port->id, cxled->cxld.id, cxled->dpa_res);
223 return -EBUSY;
224 }
225
226 if (port->hdm_end + 1 != cxled->cxld.id) {
227 /*
228 * Assumes alloc and commit order is always in hardware instance
229 * order per expectations from 8.2.5.12.20 Committing Decoder
230 * Programming that enforce decoder[m] committed before
231 * decoder[m+1] commit start.
232 */
233 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
234 cxled->cxld.id, port->id, port->hdm_end + 1);
235 return -EBUSY;
236 }
237
238 if (skipped) {
239 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
240 dev_name(&cxled->cxld.dev), 0);
241 if (!res) {
242 dev_dbg(dev,
243 "decoder%d.%d: failed to reserve skipped space\n",
244 port->id, cxled->cxld.id);
245 return -EBUSY;
246 }
247 }
248 res = __request_region(&cxlds->dpa_res, base, len,
249 dev_name(&cxled->cxld.dev), 0);
250 if (!res) {
251 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
252 port->id, cxled->cxld.id);
253 if (skipped)
254 __release_region(&cxlds->dpa_res, base - skipped,
255 skipped);
256 return -EBUSY;
257 }
258 cxled->dpa_res = res;
259 cxled->skip = skipped;
260
261 if (resource_contains(&cxlds->pmem_res, res))
262 cxled->mode = CXL_DECODER_PMEM;
263 else if (resource_contains(&cxlds->ram_res, res))
264 cxled->mode = CXL_DECODER_RAM;
265 else {
266 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
267 cxled->cxld.id, cxled->dpa_res);
268 cxled->mode = CXL_DECODER_MIXED;
269 }
270
271success:
272 port->hdm_end++;
273 get_device(&cxled->cxld.dev);
274 return 0;
275}
276
277static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
278 resource_size_t base, resource_size_t len,
279 resource_size_t skipped)
132{280{
133 int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl);281 struct cxl_port *port = cxled_to_port(cxled);
282 int rc;
283
284 down_write(&cxl_dpa_rwsem);
285 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
286 up_write(&cxl_dpa_rwsem);
287
288 if (rc)
289 return rc;
134 290
135 return 256 << val;291 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
136}292}
137 293
138static int to_interleave_ways(u32 ctrl)294resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
139{295{
140 int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl);296 resource_size_t size = 0;
297
298 down_read(&cxl_dpa_rwsem);
299 if (cxled->dpa_res)
300 size = resource_size(cxled->dpa_res);
301 up_read(&cxl_dpa_rwsem);
141 302
142 switch (val) {303 return size;
143 case 0 ... 4:304}
144 return 1 << val;305
145 case 8 ... 10:306resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
146 return 3 << (val - 8);307{
308 resource_size_t base = -1;
309
310 down_read(&cxl_dpa_rwsem);
311 if (cxled->dpa_res)
312 base = cxled->dpa_res->start;
313 up_read(&cxl_dpa_rwsem);
314
315 return base;
316}
317
318int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
319{
320 struct cxl_port *port = cxled_to_port(cxled);
321 struct device *dev = &cxled->cxld.dev;
322 int rc;
323
324 down_write(&cxl_dpa_rwsem);
325 if (!cxled->dpa_res) {
326 rc = 0;
327 goto out;
328 }
329 if (cxled->cxld.region) {
330 dev_dbg(dev, "decoder assigned to: %s\n",
331 dev_name(&cxled->cxld.region->dev));
332 rc = -EBUSY;
333 goto out;
334 }
335 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
336 dev_dbg(dev, "decoder enabled\n");
337 rc = -EBUSY;
338 goto out;
339 }
340 if (cxled->cxld.id != port->hdm_end) {
341 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
342 port->hdm_end);
343 rc = -EBUSY;
344 goto out;
345 }
346 devm_cxl_dpa_release(cxled);
347 rc = 0;
348out:
349 up_write(&cxl_dpa_rwsem);
350 return rc;
351}
352
353int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
354 enum cxl_decoder_mode mode)
355{
356 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
357 struct cxl_dev_state *cxlds = cxlmd->cxlds;
358 struct device *dev = &cxled->cxld.dev;
359 int rc;
360
361 switch (mode) {
362 case CXL_DECODER_RAM:
363 case CXL_DECODER_PMEM:
364 break;
147 default:365 default:
366 dev_dbg(dev, "unsupported mode: %d\n", mode);
367 return -EINVAL;
368 }
369
370 down_write(&cxl_dpa_rwsem);
371 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
372 rc = -EBUSY;
373 goto out;
374 }
375
376 /*
377 * Only allow modes that are supported by the current partition
378 * configuration
379 */
380 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
381 dev_dbg(dev, "no available pmem capacity\n");
382 rc = -ENXIO;
383 goto out;
384 }
385 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
386 dev_dbg(dev, "no available ram capacity\n");
387 rc = -ENXIO;
388 goto out;
389 }
390
391 cxled->mode = mode;
392 rc = 0;
393out:
394 up_write(&cxl_dpa_rwsem);
395
396 return rc;
397}
398
399int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
400{
401 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
402 resource_size_t free_ram_start, free_pmem_start;
403 struct cxl_port *port = cxled_to_port(cxled);
404 struct cxl_dev_state *cxlds = cxlmd->cxlds;
405 struct device *dev = &cxled->cxld.dev;
406 resource_size_t start, avail, skip;
407 struct resource *p, *last;
408 int rc;
409
410 down_write(&cxl_dpa_rwsem);
411 if (cxled->cxld.region) {
412 dev_dbg(dev, "decoder attached to %s\n",
413 dev_name(&cxled->cxld.region->dev));
414 rc = -EBUSY;
415 goto out;
416 }
417
418 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
419 dev_dbg(dev, "decoder enabled\n");
420 rc = -EBUSY;
421 goto out;
422 }
423
424 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
425 last = p;
426 if (last)
427 free_ram_start = last->end + 1;
428 else
429 free_ram_start = cxlds->ram_res.start;
430
431 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
432 last = p;
433 if (last)
434 free_pmem_start = last->end + 1;
435 else
436 free_pmem_start = cxlds->pmem_res.start;
437
438 if (cxled->mode == CXL_DECODER_RAM) {
439 start = free_ram_start;
440 avail = cxlds->ram_res.end - start + 1;
441 skip = 0;
442 } else if (cxled->mode == CXL_DECODER_PMEM) {
443 resource_size_t skip_start, skip_end;
444
445 start = free_pmem_start;
446 avail = cxlds->pmem_res.end - start + 1;
447 skip_start = free_ram_start;
448
449 /*
450 * If some pmem is already allocated, then that allocation
451 * already handled the skip.
452 */
453 if (cxlds->pmem_res.child &&
454 skip_start == cxlds->pmem_res.child->start)
455 skip_end = skip_start - 1;
456 else
457 skip_end = start - 1;
458 skip = skip_end - skip_start + 1;
459 } else {
460 dev_dbg(dev, "mode not set\n");
461 rc = -EINVAL;
462 goto out;
463 }
464
465 if (size > avail) {
466 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
467 cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
468 &avail);
469 rc = -ENOSPC;
470 goto out;
471 }
472
473 rc = __cxl_dpa_reserve(cxled, start, size, skip);
474out:
475 up_write(&cxl_dpa_rwsem);
476
477 if (rc)
478 return rc;
479
480 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
481}
482
483static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
484{
485 u16 eig;
486 u8 eiw;
487
488 /*
489 * Input validation ensures these warns never fire, but otherwise
490 * suppress unititalized variable usage warnings.
491 */
492 if (WARN_ONCE(ways_to_cxl(cxld->interleave_ways, &eiw),
493 "invalid interleave_ways: %d\n", cxld->interleave_ways))
494 return;
495 if (WARN_ONCE(granularity_to_cxl(cxld->interleave_granularity, &eig),
496 "invalid interleave_granularity: %d\n",
497 cxld->interleave_granularity))
498 return;
499
500 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
501 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
502 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
503}
504
505static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
506{
507 u32p_replace_bits(ctrl, !!(cxld->target_type == 3),
508 CXL_HDM_DECODER0_CTRL_TYPE);
509}
510
511static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
512{
513 struct cxl_dport **t = &cxlsd->target[0];
514 int ways = cxlsd->cxld.interleave_ways;
515
516 if (dev_WARN_ONCE(&cxlsd->cxld.dev,
517 ways > 8 || ways > cxlsd->nr_targets,
518 "ways: %d overflows targets: %d\n", ways,
519 cxlsd->nr_targets))
520 return -ENXIO;
521
522 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
523 if (ways > 1)
524 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
525 if (ways > 2)
526 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
527 if (ways > 3)
528 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
529 if (ways > 4)
530 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
531 if (ways > 5)
532 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
533 if (ways > 6)
534 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
535 if (ways > 7)
536 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
537
538 return 0;
539}
540
541/*
542 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
543 * committed or error within 10ms, but just be generous with 20ms to account for
544 * clock skew and other marginal behavior
545 */
546#define COMMIT_TIMEOUT_MS 20
547static int cxld_await_commit(void __iomem *hdm, int id)
548{
549 u32 ctrl;
550 int i;
551
552 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
553 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
554 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
555 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
556 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
557 return -EIO;
558 }
559 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
560 return 0;
561 fsleep(1000);
562 }
563
564 return -ETIMEDOUT;
565}
566
567static int cxl_decoder_commit(struct cxl_decoder *cxld)
568{
569 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
570 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
571 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
572 int id = cxld->id, rc;
573 u64 base, size;
574 u32 ctrl;
575
576 if (cxld->flags & CXL_DECODER_F_ENABLE)
577 return 0;
578
579 if (port->commit_end + 1 != id) {
580 dev_dbg(&port->dev,
581 "%s: out of order commit, expected decoder%d.%d\n",
582 dev_name(&cxld->dev), port->id, port->commit_end + 1);
583 return -EBUSY;
584 }
585
586 down_read(&cxl_dpa_rwsem);
587 /* common decoder settings */
588 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
589 cxld_set_interleave(cxld, &ctrl);
590 cxld_set_type(cxld, &ctrl);
591 base = cxld->hpa_range.start;
592 size = range_len(&cxld->hpa_range);
593
594 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
595 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
596 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
597 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
598
599 if (is_switch_decoder(&cxld->dev)) {
600 struct cxl_switch_decoder *cxlsd =
601 to_cxl_switch_decoder(&cxld->dev);
602 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
603 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
604 u64 targets;
605
606 rc = cxlsd_set_targets(cxlsd, &targets);
607 if (rc) {
608 dev_dbg(&port->dev, "%s: target configuration error\n",
609 dev_name(&cxld->dev));
610 goto err;
611 }
612
613 writel(upper_32_bits(targets), tl_hi);
614 writel(lower_32_bits(targets), tl_lo);
615 } else {
616 struct cxl_endpoint_decoder *cxled =
617 to_cxl_endpoint_decoder(&cxld->dev);
618 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
619 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
620
621 writel(upper_32_bits(cxled->skip), sk_hi);
622 writel(lower_32_bits(cxled->skip), sk_lo);
623 }
624
625 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
626 up_read(&cxl_dpa_rwsem);
627
628 port->commit_end++;
629 rc = cxld_await_commit(hdm, cxld->id);
630err:
631 if (rc) {
632 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
633 dev_name(&cxld->dev), rc);
634 cxld->reset(cxld);
635 return rc;
636 }
637 cxld->flags |= CXL_DECODER_F_ENABLE;
638
639 return 0;
640}
641
642static int cxl_decoder_reset(struct cxl_decoder *cxld)
643{
644 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
645 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
646 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
647 int id = cxld->id;
648 u32 ctrl;
649
650 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
148 return 0;651 return 0;
652
653 if (port->commit_end != id) {
654 dev_dbg(&port->dev,
655 "%s: out of order reset, expected decoder%d.%d\n",
656 dev_name(&cxld->dev), port->id, port->commit_end);
657 return -EBUSY;
149 }658 }
659
660 down_read(&cxl_dpa_rwsem);
661 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
662 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
663 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
664
665 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
666 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
667 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
668 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
669 up_read(&cxl_dpa_rwsem);
670
671 port->commit_end--;
672 cxld->flags &= ~CXL_DECODER_F_ENABLE;
673
674 return 0;
150}675}
151 676
152static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,677static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
153 int *target_map, void __iomem *hdm, int which)678 int *target_map, void __iomem *hdm, int which,
679 u64 *dpa_base)
154{680{
155 u64 size, base;681 struct cxl_endpoint_decoder *cxled = NULL;
682 u64 size, base, skip, dpa_size;
683 bool committed;
684 u32 remainder;
685 int i, rc;
156 u32 ctrl;686 u32 ctrl;
157 int i;
158 union {687 union {
159 u64 value;688 u64 value;
160 unsigned char target_id[8];689 unsigned char target_id[8];
161 } target_list;690 } target_list;
162 691
692 if (is_endpoint_decoder(&cxld->dev))
693 cxled = to_cxl_endpoint_decoder(&cxld->dev);
694
163 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));695 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
164 base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));696 base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
165 size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));697 size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
698 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
699 cxld->commit = cxl_decoder_commit;
700 cxld->reset = cxl_decoder_reset;
166 701
167 if (!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED))702 if (!committed)
168 size = 0;703 size = 0;
169 if (base == U64_MAX || size == U64_MAX) {704 if (base == U64_MAX || size == U64_MAX) {
170 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",705 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
@@ -172,39 +707,77 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, · 1 unmodified line
172 return -ENXIO;707 return -ENXIO;
173 }708 }
174 709
175 cxld->decoder_range = (struct range) {710 cxld->hpa_range = (struct range) {
176 .start = base,711 .start = base,
177 .end = base + size - 1,712 .end = base + size - 1,
178 };713 };
179 714
180 /* switch decoders are always enabled if committed */715 /* decoders are enabled if committed */
181 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) {716 if (committed) {
182 cxld->flags |= CXL_DECODER_F_ENABLE;717 cxld->flags |= CXL_DECODER_F_ENABLE;
183 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)718 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
184 cxld->flags |= CXL_DECODER_F_LOCK;719 cxld->flags |= CXL_DECODER_F_LOCK;
720 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
721 cxld->target_type = CXL_DECODER_EXPANDER;
722 else
723 cxld->target_type = CXL_DECODER_ACCELERATOR;
724 if (cxld->id != port->commit_end + 1) {
725 dev_warn(&port->dev,
726 "decoder%d.%d: Committed out of order\n",
727 port->id, cxld->id);
728 return -ENXIO;
729 }
730 port->commit_end = cxld->id;
731 } else {
732 /* unless / until type-2 drivers arrive, assume type-3 */
733 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) {
734 ctrl |= CXL_HDM_DECODER0_CTRL_TYPE;
735 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
736 }
737 cxld->target_type = CXL_DECODER_EXPANDER;
185 }738 }
186 cxld->interleave_ways = to_interleave_ways(ctrl);739 rc = cxl_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
187 if (!cxld->interleave_ways) {740 &cxld->interleave_ways);
741 if (rc) {
188 dev_warn(&port->dev,742 dev_warn(&port->dev,
189 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",743 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
190 port->id, cxld->id, ctrl);744 port->id, cxld->id, ctrl);
191 return -ENXIO;745 return rc;
192 }746 }
193 cxld->interleave_granularity = to_interleave_granularity(ctrl);747 rc = cxl_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
748 &cxld->interleave_granularity);
749 if (rc)
750 return rc;
194 751
195 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))752 if (!cxled) {
196 cxld->target_type = CXL_DECODER_EXPANDER;753 target_list.value =
197 else754 ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
198 cxld->target_type = CXL_DECODER_ACCELERATOR;755 for (i = 0; i < cxld->interleave_ways; i++)
756 target_map[i] = target_list.target_id[i];
199 757
200 if (is_endpoint_decoder(&cxld->dev))
201 return 0;758 return 0;
759 }
202 760
203 target_list.value =761 if (!committed)
204 ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));762 return 0;
205 for (i = 0; i < cxld->interleave_ways; i++)
206 target_map[i] = target_list.target_id[i];
207 763
764 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
765 if (remainder) {
766 dev_err(&port->dev,
767 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
768 port->id, cxld->id, size, cxld->interleave_ways);
769 return -ENXIO;
770 }
771 skip = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
772 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
773 if (rc) {
774 dev_err(&port->dev,
775 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
776 port->id, cxld->id, *dpa_base,
777 *dpa_base + dpa_size + skip - 1, rc);
778 return rc;
779 }
780 *dpa_base += dpa_size + skip;
208 return 0;781 return 0;
209}782}
210 783
@@ -216,7 +789,8 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 5 unmodified lines
216{789{
217 void __iomem *hdm = cxlhdm->regs.hdm_decoder;790 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
218 struct cxl_port *port = cxlhdm->port;791 struct cxl_port *port = cxlhdm->port;
219 int i, committed, failed;792 int i, committed;
793 u64 dpa_base = 0;
220 u32 ctrl;794 u32 ctrl;
221 795
222 /*796 /*
@@ -236,27 +810,37 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 13 unmodified lines
236 if (committed != cxlhdm->decoder_count)810 if (committed != cxlhdm->decoder_count)
237 msleep(20);811 msleep(20);
238 812
239 for (i = 0, failed = 0; i < cxlhdm->decoder_count; i++) {813 for (i = 0; i < cxlhdm->decoder_count; i++) {
240 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };814 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
241 int rc, target_count = cxlhdm->target_count;815 int rc, target_count = cxlhdm->target_count;
242 struct cxl_decoder *cxld;816 struct cxl_decoder *cxld;
243 817
244 if (is_cxl_endpoint(port))818 if (is_cxl_endpoint(port)) {
245 cxld = cxl_endpoint_decoder_alloc(port);819 struct cxl_endpoint_decoder *cxled;
246 else820
247 cxld = cxl_switch_decoder_alloc(port, target_count);821 cxled = cxl_endpoint_decoder_alloc(port);
248 if (IS_ERR(cxld)) {822 if (IS_ERR(cxled)) {
249 dev_warn(&port->dev,823 dev_warn(&port->dev,
250 "Failed to allocate the decoder\n");824 "Failed to allocate the decoder\n");
251 return PTR_ERR(cxld);825 return PTR_ERR(cxled);
826 }
827 cxld = &cxled->cxld;
828 } else {
829 struct cxl_switch_decoder *cxlsd;
830
831 cxlsd = cxl_switch_decoder_alloc(port, target_count);
832 if (IS_ERR(cxlsd)) {
833 dev_warn(&port->dev,
834 "Failed to allocate the decoder\n");
835 return PTR_ERR(cxlsd);
836 }
837 cxld = &cxlsd->cxld;
252 }838 }
253 839
254 rc = init_hdm_decoder(port, cxld, target_map,840 rc = init_hdm_decoder(port, cxld, target_map, hdm, i, &dpa_base);
255 cxlhdm->regs.hdm_decoder, i);
256 if (rc) {841 if (rc) {
257 put_device(&cxld->dev);842 put_device(&cxld->dev);
258 failed++;843 return rc;
259 continue;
260 }844 }
261 rc = add_hdm_decoder(port, cxld, target_map);845 rc = add_hdm_decoder(port, cxld, target_map);
262 if (rc) {846 if (rc) {
@@ -266,11 +850,6 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 3 unmodified lines
266 }850 }
267 }851 }
268 852
269 if (failed == cxlhdm->decoder_count) {
270 dev_err(&port->dev, "No valid decoders found\n");
271 return -ENXIO;
272 }
273
274 return 0;853 return 0;
275}854}
276EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);855EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);
drivers/cxl/core/mbox.c+47−48
@@ -718,12 +718,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL); · 717 unmodified lines
718 */718 */
719static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds)719static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds)
720{720{
721 struct cxl_mbox_get_partition_info {721 struct cxl_mbox_get_partition_info pi;
722 __le64 active_volatile_cap;
723 __le64 active_persistent_cap;
724 __le64 next_volatile_cap;
725 __le64 next_persistent_cap;
726 } __packed pi;
727 int rc;722 int rc;
728 723
729 rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_GET_PARTITION_INFO, NULL, 0,724 rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_GET_PARTITION_INFO, NULL, 0,
@@ -773,15 +768,6 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) · 43 unmodified lines
773 cxlds->partition_align_bytes =768 cxlds->partition_align_bytes =
774 le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;769 le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
775 770
776 dev_dbg(cxlds->dev,
777 "Identify Memory Device\n"
778 " total_bytes = %#llx\n"
779 " volatile_only_bytes = %#llx\n"
780 " persistent_only_bytes = %#llx\n"
781 " partition_align_bytes = %#llx\n",
782 cxlds->total_bytes, cxlds->volatile_only_bytes,
783 cxlds->persistent_only_bytes, cxlds->partition_align_bytes);
784
785 cxlds->lsa_size = le32_to_cpu(id.lsa_size);771 cxlds->lsa_size = le32_to_cpu(id.lsa_size);
786 memcpy(cxlds->firmware_version, id.fw_revision, sizeof(id.fw_revision));772 memcpy(cxlds->firmware_version, id.fw_revision, sizeof(id.fw_revision));
787 773
@@ -789,42 +775,63 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) · 1 unmodified line
789}775}
790EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);776EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);
791 777
792int cxl_mem_create_range_info(struct cxl_dev_state *cxlds)778static int add_dpa_res(struct device *dev, struct resource *parent,
779 struct resource *res, resource_size_t start,
780 resource_size_t size, const char *type)
793{781{
794 int rc;782 int rc;
795 783
796 if (cxlds->partition_align_bytes == 0) {784 res->name = type;
797 cxlds->ram_range.start = 0;785 res->start = start;
798 cxlds->ram_range.end = cxlds->volatile_only_bytes - 1;786 res->end = start + size - 1;
799 cxlds->pmem_range.start = cxlds->volatile_only_bytes;787 res->flags = IORESOURCE_MEM;
800 cxlds->pmem_range.end = cxlds->volatile_only_bytes +788 if (resource_size(res) == 0) {
801 cxlds->persistent_only_bytes - 1;789 dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
802 return 0;790 return 0;
803 }791 }
804 792 rc = request_resource(parent, res);
805 rc = cxl_mem_get_partition_info(cxlds);
806 if (rc) {793 if (rc) {
807 dev_err(cxlds->dev, "Failed to query partition information\n");794 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
795 res, rc);
808 return rc;796 return rc;
809 }797 }
810 798
811 dev_dbg(cxlds->dev,799 dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
812 "Get Partition Info\n"800
813 " active_volatile_bytes = %#llx\n"801 return 0;
814 " active_persistent_bytes = %#llx\n"802}
815 " next_volatile_bytes = %#llx\n"
816 " next_persistent_bytes = %#llx\n",
817 cxlds->active_volatile_bytes, cxlds->active_persistent_bytes,
818 cxlds->next_volatile_bytes, cxlds->next_persistent_bytes);
819 803
820 cxlds->ram_range.start = 0;804int cxl_mem_create_range_info(struct cxl_dev_state *cxlds)
821 cxlds->ram_range.end = cxlds->active_volatile_bytes - 1;805{
806 struct device *dev = cxlds->dev;
807 int rc;
822 808
823 cxlds->pmem_range.start = cxlds->active_volatile_bytes;809 cxlds->dpa_res =
824 cxlds->pmem_range.end =810 (struct resource)DEFINE_RES_MEM(0, cxlds->total_bytes);
825 cxlds->active_volatile_bytes + cxlds->active_persistent_bytes - 1;
826 811
827 return 0;812 if (cxlds->partition_align_bytes == 0) {
813 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
814 cxlds->volatile_only_bytes, "ram");
815 if (rc)
816 return rc;
817 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
818 cxlds->volatile_only_bytes,
819 cxlds->persistent_only_bytes, "pmem");
820 }
821
822 rc = cxl_mem_get_partition_info(cxlds);
823 if (rc) {
824 dev_err(dev, "Failed to query partition information\n");
825 return rc;
826 }
827
828 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
829 cxlds->active_volatile_bytes, "ram");
830 if (rc)
831 return rc;
832 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
833 cxlds->active_volatile_bytes,
834 cxlds->active_persistent_bytes, "pmem");
828}835}
829EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);836EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);
830 837
@@ -845,19 +852,11 @@ struct cxl_dev_state *cxl_dev_state_create(struct device *dev) · 14 unmodified lines
845}852}
846EXPORT_SYMBOL_NS_GPL(cxl_dev_state_create, CXL);853EXPORT_SYMBOL_NS_GPL(cxl_dev_state_create, CXL);
847 854
848static struct dentry *cxl_debugfs;
849
850void __init cxl_mbox_init(void)855void __init cxl_mbox_init(void)
851{856{
852 struct dentry *mbox_debugfs;857 struct dentry *mbox_debugfs;
853 858
854 cxl_debugfs = debugfs_create_dir("cxl", NULL);859 mbox_debugfs = cxl_debugfs_create_dir("mbox");
855 mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs);
856 debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,860 debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
857 &cxl_raw_allow_all);861 &cxl_raw_allow_all);
858}862}
859
860void cxl_mbox_exit(void)
861{
862 debugfs_remove_recursive(cxl_debugfs);
863}
drivers/cxl/core/memdev.c+2−2
@@ -68,7 +68,7 @@ static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, · 67 unmodified lines
68{68{
69 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);69 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
70 struct cxl_dev_state *cxlds = cxlmd->cxlds;70 struct cxl_dev_state *cxlds = cxlmd->cxlds;
71 unsigned long long len = range_len(&cxlds->ram_range);71 unsigned long long len = resource_size(&cxlds->ram_res);
72 72
73 return sysfs_emit(buf, "%#llx\n", len);73 return sysfs_emit(buf, "%#llx\n", len);
74}74}
@@ -81,7 +81,7 @@ static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, · 6 unmodified lines
81{81{
82 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);82 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
83 struct cxl_dev_state *cxlds = cxlmd->cxlds;83 struct cxl_dev_state *cxlds = cxlmd->cxlds;
84 unsigned long long len = range_len(&cxlds->pmem_range);84 unsigned long long len = resource_size(&cxlds->pmem_res);
85 85
86 return sysfs_emit(buf, "%#llx\n", len);86 return sysfs_emit(buf, "%#llx\n", len);
87}87}
drivers/cxl/core/pci.c+174−7
@@ -4,6 +4,7 @@ · 3 unmodified lines
4#include <linux/device.h>4#include <linux/device.h>
5#include <linux/delay.h>5#include <linux/delay.h>
6#include <linux/pci.h>6#include <linux/pci.h>
7#include <linux/pci-doe.h>
7#include <cxlpci.h>8#include <cxlpci.h>
8#include <cxlmem.h>9#include <cxlmem.h>
9#include <cxl.h>10#include <cxl.h>
@@ -225,7 +226,6 @@ static int dvsec_range_allowed(struct device *dev, void *arg) · 215 unmodified lines
225{226{
226 struct range *dev_range = arg;227 struct range *dev_range = arg;
227 struct cxl_decoder *cxld;228 struct cxl_decoder *cxld;
228 struct range root_range;
229 229
230 if (!is_root_decoder(dev))230 if (!is_root_decoder(dev))
231 return 0;231 return 0;
@@ -237,12 +237,7 @@ static int dvsec_range_allowed(struct device *dev, void *arg) · 5 unmodified lines
237 if (!(cxld->flags & CXL_DECODER_F_RAM))237 if (!(cxld->flags & CXL_DECODER_F_RAM))
238 return 0;238 return 0;
239 239
240 root_range = (struct range) {240 return range_contains(&cxld->hpa_range, dev_range);
241 .start = cxld->platform_res.start,
242 .end = cxld->platform_res.end,
243 };
244
245 return range_contains(&root_range, dev_range);
246}241}
247 242
248static void disable_hdm(void *_cxlhdm)243static void disable_hdm(void *_cxlhdm)
@@ -458,3 +453,175 @@ hdm_init: · 209 unmodified lines
458 return 0;453 return 0;
459}454}
460EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL);455EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL);
456
457#define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff
458#define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0
459#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00
460#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0
461#define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000
462#define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff
463#define CXL_DOE_PROTOCOL_TABLE_ACCESS 2
464
465static struct pci_doe_mb *find_cdat_doe(struct device *uport)
466{
467 struct cxl_memdev *cxlmd;
468 struct cxl_dev_state *cxlds;
469 unsigned long index;
470 void *entry;
471
472 cxlmd = to_cxl_memdev(uport);
473 cxlds = cxlmd->cxlds;
474
475 xa_for_each(&cxlds->doe_mbs, index, entry) {
476 struct pci_doe_mb *cur = entry;
477
478 if (pci_doe_supports_prot(cur, PCI_DVSEC_VENDOR_ID_CXL,
479 CXL_DOE_PROTOCOL_TABLE_ACCESS))
480 return cur;
481 }
482
483 return NULL;
484}
485
486#define CDAT_DOE_REQ(entry_handle) \
487 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \
488 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \
489 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \
490 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \
491 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle)))
492
493static void cxl_doe_task_complete(struct pci_doe_task *task)
494{
495 complete(task->private);
496}
497
498struct cdat_doe_task {
499 u32 request_pl;
500 u32 response_pl[32];
501 struct completion c;
502 struct pci_doe_task task;
503};
504
505#define DECLARE_CDAT_DOE_TASK(req, cdt) \
506struct cdat_doe_task cdt = { \
507 .c = COMPLETION_INITIALIZER_ONSTACK(cdt.c), \
508 .request_pl = req, \
509 .task = { \
510 .prot.vid = PCI_DVSEC_VENDOR_ID_CXL, \
511 .prot.type = CXL_DOE_PROTOCOL_TABLE_ACCESS, \
512 .request_pl = &cdt.request_pl, \
513 .request_pl_sz = sizeof(cdt.request_pl), \
514 .response_pl = cdt.response_pl, \
515 .response_pl_sz = sizeof(cdt.response_pl), \
516 .complete = cxl_doe_task_complete, \
517 .private = &cdt.c, \
518 } \
519}
520
521static int cxl_cdat_get_length(struct device *dev,
522 struct pci_doe_mb *cdat_doe,
523 size_t *length)
524{
525 DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(0), t);
526 int rc;
527
528 rc = pci_doe_submit_task(cdat_doe, &t.task);
529 if (rc < 0) {
530 dev_err(dev, "DOE submit failed: %d", rc);
531 return rc;
532 }
533 wait_for_completion(&t.c);
534 if (t.task.rv < sizeof(u32))
535 return -EIO;
536
537 *length = t.response_pl[1];
538 dev_dbg(dev, "CDAT length %zu\n", *length);
539
540 return 0;
541}
542
543static int cxl_cdat_read_table(struct device *dev,
544 struct pci_doe_mb *cdat_doe,
545 struct cxl_cdat *cdat)
546{
547 size_t length = cdat->length;
548 u32 *data = cdat->table;
549 int entry_handle = 0;
550
551 do {
552 DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(entry_handle), t);
553 size_t entry_dw;
554 u32 *entry;
555 int rc;
556
557 rc = pci_doe_submit_task(cdat_doe, &t.task);
558 if (rc < 0) {
559 dev_err(dev, "DOE submit failed: %d", rc);
560 return rc;
561 }
562 wait_for_completion(&t.c);
563 /* 1 DW header + 1 DW data min */
564 if (t.task.rv < (2 * sizeof(u32)))
565 return -EIO;
566
567 /* Get the CXL table access header entry handle */
568 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE,
569 t.response_pl[0]);
570 entry = t.response_pl + 1;
571 entry_dw = t.task.rv / sizeof(u32);
572 /* Skip Header */
573 entry_dw -= 1;
574 entry_dw = min(length / sizeof(u32), entry_dw);
575 /* Prevent length < 1 DW from causing a buffer overflow */
576 if (entry_dw) {
577 memcpy(data, entry, entry_dw * sizeof(u32));
578 length -= entry_dw * sizeof(u32);
579 data += entry_dw;
580 }
581 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY);
582
583 return 0;
584}
585
586/**
587 * read_cdat_data - Read the CDAT data on this port
588 * @port: Port to read data from
589 *
590 * This call will sleep waiting for responses from the DOE mailbox.
591 */
592void read_cdat_data(struct cxl_port *port)
593{
594 struct pci_doe_mb *cdat_doe;
595 struct device *dev = &port->dev;
596 struct device *uport = port->uport;
597 size_t cdat_length;
598 int rc;
599
600 cdat_doe = find_cdat_doe(uport);
601 if (!cdat_doe) {
602 dev_dbg(dev, "No CDAT mailbox\n");
603 return;
604 }
605
606 port->cdat_available = true;
607
608 if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) {
609 dev_dbg(dev, "No CDAT length\n");
610 return;
611 }
612
613 port->cdat.table = devm_kzalloc(dev, cdat_length, GFP_KERNEL);
614 if (!port->cdat.table)
615 return;
616
617 port->cdat.length = cdat_length;
618 rc = cxl_cdat_read_table(dev, cdat_doe, &port->cdat);
619 if (rc) {
620 /* Don't leave table data allocated on error */
621 devm_kfree(dev, port->cdat.table);
622 port->cdat.table = NULL;
623 port->cdat.length = 0;
624 dev_err(dev, "CDAT data read error\n");
625 }
626}
627EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL);
drivers/cxl/core/pmem.c+2−2
@@ -62,9 +62,9 @@ static int match_nvdimm_bridge(struct device *dev, void *data) · 61 unmodified lines
62 return is_cxl_nvdimm_bridge(dev);62 return is_cxl_nvdimm_bridge(dev);
63}63}
64 64
65struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd)65struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct device *start)
66{66{
67 struct cxl_port *port = find_cxl_root(&cxl_nvd->dev);67 struct cxl_port *port = find_cxl_root(start);
68 struct device *dev;68 struct device *dev;
69 69
70 if (!port)70 if (!port)
drivers/cxl/core/port.c+536−202
@@ -1,7 +1,9 @@
1// SPDX-License-Identifier: GPL-2.0-only1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3#include <linux/io-64-nonatomic-lo-hi.h>3#include <linux/io-64-nonatomic-lo-hi.h>
4#include <linux/memregion.h>
4#include <linux/workqueue.h>5#include <linux/workqueue.h>
6#include <linux/debugfs.h>
5#include <linux/device.h>7#include <linux/device.h>
6#include <linux/module.h>8#include <linux/module.h>
7#include <linux/pci.h>9#include <linux/pci.h>
@@ -42,6 +44,8 @@ static int cxl_device_id(struct device *dev) · 34 unmodified lines
42 return CXL_DEVICE_NVDIMM_BRIDGE;44 return CXL_DEVICE_NVDIMM_BRIDGE;
43 if (dev->type == &cxl_nvdimm_type)45 if (dev->type == &cxl_nvdimm_type)
44 return CXL_DEVICE_NVDIMM;46 return CXL_DEVICE_NVDIMM;
47 if (dev->type == CXL_PMEM_REGION_TYPE())
48 return CXL_DEVICE_PMEM_REGION;
45 if (is_cxl_port(dev)) {49 if (is_cxl_port(dev)) {
46 if (is_cxl_root(to_cxl_port(dev)))50 if (is_cxl_root(to_cxl_port(dev)))
47 return CXL_DEVICE_ROOT;51 return CXL_DEVICE_ROOT;
@@ -49,6 +53,8 @@ static int cxl_device_id(struct device *dev) · 1 unmodified line
49 }53 }
50 if (is_cxl_memdev(dev))54 if (is_cxl_memdev(dev))
51 return CXL_DEVICE_MEMORY_EXPANDER;55 return CXL_DEVICE_MEMORY_EXPANDER;
56 if (dev->type == CXL_REGION_TYPE())
57 return CXL_DEVICE_REGION;
52 return 0;58 return 0;
53}59}
54 60
@@ -73,14 +79,8 @@ static ssize_t start_show(struct device *dev, struct device_attribute *attr, · 18 unmodified lines
73 char *buf)79 char *buf)
74{80{
75 struct cxl_decoder *cxld = to_cxl_decoder(dev);81 struct cxl_decoder *cxld = to_cxl_decoder(dev);
76 u64 start;
77 82
78 if (is_root_decoder(dev))83 return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
79 start = cxld->platform_res.start;
80 else
81 start = cxld->decoder_range.start;
82
83 return sysfs_emit(buf, "%#llx\n", start);
84}84}
85static DEVICE_ATTR_ADMIN_RO(start);85static DEVICE_ATTR_ADMIN_RO(start);
86 86
@@ -88,14 +88,8 @@ static ssize_t size_show(struct device *dev, struct device_attribute *attr, · 1 unmodified line
88 char *buf)88 char *buf)
89{89{
90 struct cxl_decoder *cxld = to_cxl_decoder(dev);90 struct cxl_decoder *cxld = to_cxl_decoder(dev);
91 u64 size;
92
93 if (is_root_decoder(dev))
94 size = resource_size(&cxld->platform_res);
95 else
96 size = range_len(&cxld->decoder_range);
97 91
98 return sysfs_emit(buf, "%#llx\n", size);92 return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
99}93}
100static DEVICE_ATTR_RO(size);94static DEVICE_ATTR_RO(size);
101 95
@@ -131,20 +125,21 @@ static ssize_t target_type_show(struct device *dev, · 29 unmodified lines
131}125}
132static DEVICE_ATTR_RO(target_type);126static DEVICE_ATTR_RO(target_type);
133 127
134static ssize_t emit_target_list(struct cxl_decoder *cxld, char *buf)128static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
135{129{
130 struct cxl_decoder *cxld = &cxlsd->cxld;
136 ssize_t offset = 0;131 ssize_t offset = 0;
137 int i, rc = 0;132 int i, rc = 0;
138 133
139 for (i = 0; i < cxld->interleave_ways; i++) {134 for (i = 0; i < cxld->interleave_ways; i++) {
140 struct cxl_dport *dport = cxld->target[i];135 struct cxl_dport *dport = cxlsd->target[i];
141 struct cxl_dport *next = NULL;136 struct cxl_dport *next = NULL;
142 137
143 if (!dport)138 if (!dport)
144 break;139 break;
145 140
146 if (i + 1 < cxld->interleave_ways)141 if (i + 1 < cxld->interleave_ways)
147 next = cxld->target[i + 1];142 next = cxlsd->target[i + 1];
148 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,143 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
149 next ? "," : "");144 next ? "," : "");
150 if (rc < 0)145 if (rc < 0)
@@ -158,15 +153,15 @@ static ssize_t emit_target_list(struct cxl_decoder *cxld, char *buf) · 7 unmodified lines
158static ssize_t target_list_show(struct device *dev,153static ssize_t target_list_show(struct device *dev,
159 struct device_attribute *attr, char *buf)154 struct device_attribute *attr, char *buf)
160{155{
161 struct cxl_decoder *cxld = to_cxl_decoder(dev);156 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
162 ssize_t offset;157 ssize_t offset;
163 unsigned int seq;158 unsigned int seq;
164 int rc;159 int rc;
165 160
166 do {161 do {
167 seq = read_seqbegin(&cxld->target_lock);162 seq = read_seqbegin(&cxlsd->target_lock);
168 rc = emit_target_list(cxld, buf);163 rc = emit_target_list(cxlsd, buf);
169 } while (read_seqretry(&cxld->target_lock, seq));164 } while (read_seqretry(&cxlsd->target_lock, seq));
170 165
171 if (rc < 0)166 if (rc < 0)
172 return rc;167 return rc;
@@ -180,10 +175,121 @@ static ssize_t target_list_show(struct device *dev, · 7 unmodified lines
180}175}
181static DEVICE_ATTR_RO(target_list);176static DEVICE_ATTR_RO(target_list);
182 177
178static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
179 char *buf)
180{
181 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
182
183 switch (cxled->mode) {
184 case CXL_DECODER_RAM:
185 return sysfs_emit(buf, "ram\n");
186 case CXL_DECODER_PMEM:
187 return sysfs_emit(buf, "pmem\n");
188 case CXL_DECODER_NONE:
189 return sysfs_emit(buf, "none\n");
190 case CXL_DECODER_MIXED:
191 default:
192 return sysfs_emit(buf, "mixed\n");
193 }
194}
195
196static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
197 const char *buf, size_t len)
198{
199 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
200 enum cxl_decoder_mode mode;
201 ssize_t rc;
202
203 if (sysfs_streq(buf, "pmem"))
204 mode = CXL_DECODER_PMEM;
205 else if (sysfs_streq(buf, "ram"))
206 mode = CXL_DECODER_RAM;
207 else
208 return -EINVAL;
209
210 rc = cxl_dpa_set_mode(cxled, mode);
211 if (rc)
212 return rc;
213
214 return len;
215}
216static DEVICE_ATTR_RW(mode);
217
218static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
219 char *buf)
220{
221 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
222 u64 base = cxl_dpa_resource_start(cxled);
223
224 return sysfs_emit(buf, "%#llx\n", base);
225}
226static DEVICE_ATTR_RO(dpa_resource);
227
228static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
229 char *buf)
230{
231 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
232 resource_size_t size = cxl_dpa_size(cxled);
233
234 return sysfs_emit(buf, "%pa\n", &size);
235}
236
237static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
238 const char *buf, size_t len)
239{
240 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
241 unsigned long long size;
242 ssize_t rc;
243
244 rc = kstrtoull(buf, 0, &size);
245 if (rc)
246 return rc;
247
248 if (!IS_ALIGNED(size, SZ_256M))
249 return -EINVAL;
250
251 rc = cxl_dpa_free(cxled);
252 if (rc)
253 return rc;
254
255 if (size == 0)
256 return len;
257
258 rc = cxl_dpa_alloc(cxled, size);
259 if (rc)
260 return rc;
261
262 return len;
263}
264static DEVICE_ATTR_RW(dpa_size);
265
266static ssize_t interleave_granularity_show(struct device *dev,
267 struct device_attribute *attr,
268 char *buf)
269{
270 struct cxl_decoder *cxld = to_cxl_decoder(dev);
271
272 return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
273}
274
275static DEVICE_ATTR_RO(interleave_granularity);
276
277static ssize_t interleave_ways_show(struct device *dev,
278 struct device_attribute *attr, char *buf)
279{
280 struct cxl_decoder *cxld = to_cxl_decoder(dev);
281
282 return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
283}
284
285static DEVICE_ATTR_RO(interleave_ways);
286
183static struct attribute *cxl_decoder_base_attrs[] = {287static struct attribute *cxl_decoder_base_attrs[] = {
184 &dev_attr_start.attr,288 &dev_attr_start.attr,
185 &dev_attr_size.attr,289 &dev_attr_size.attr,
186 &dev_attr_locked.attr,290 &dev_attr_locked.attr,
291 &dev_attr_interleave_granularity.attr,
292 &dev_attr_interleave_ways.attr,
187 NULL,293 NULL,
188};294};
189 295
@@ -197,11 +303,35 @@ static struct attribute *cxl_decoder_root_attrs[] = { · 7 unmodified lines
197 &dev_attr_cap_type2.attr,303 &dev_attr_cap_type2.attr,
198 &dev_attr_cap_type3.attr,304 &dev_attr_cap_type3.attr,
199 &dev_attr_target_list.attr,305 &dev_attr_target_list.attr,
306 SET_CXL_REGION_ATTR(create_pmem_region)
307 SET_CXL_REGION_ATTR(delete_region)
200 NULL,308 NULL,
201};309};
202 310
311static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
312{
313 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
314
315 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
316}
317
318static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
319{
320 struct device *dev = kobj_to_dev(kobj);
321 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
322
323 if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
324 return 0;
325
326 if (a == CXL_REGION_ATTR(delete_region) && !can_create_pmem(cxlrd))
327 return 0;
328
329 return a->mode;
330}
331
203static struct attribute_group cxl_decoder_root_attribute_group = {332static struct attribute_group cxl_decoder_root_attribute_group = {
204 .attrs = cxl_decoder_root_attrs,333 .attrs = cxl_decoder_root_attrs,
334 .is_visible = cxl_root_decoder_visible,
205};335};
206 336
207static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {337static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
@@ -214,6 +344,7 @@ static const struct attribute_group *cxl_decoder_root_attribute_groups[] = { · 6 unmodified lines
214static struct attribute *cxl_decoder_switch_attrs[] = {344static struct attribute *cxl_decoder_switch_attrs[] = {
215 &dev_attr_target_type.attr,345 &dev_attr_target_type.attr,
216 &dev_attr_target_list.attr,346 &dev_attr_target_list.attr,
347 SET_CXL_REGION_ATTR(region)
217 NULL,348 NULL,
218};349};
219 350
@@ -230,6 +361,10 @@ static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = { · 10 unmodified lines
230 361
231static struct attribute *cxl_decoder_endpoint_attrs[] = {362static struct attribute *cxl_decoder_endpoint_attrs[] = {
232 &dev_attr_target_type.attr,363 &dev_attr_target_type.attr,
364 &dev_attr_mode.attr,
365 &dev_attr_dpa_size.attr,
366 &dev_attr_dpa_resource.attr,
367 SET_CXL_REGION_ATTR(region)
233 NULL,368 NULL,
234};369};
235 370
@@ -244,31 +379,64 @@ static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = { · 8 unmodified lines
244 NULL,379 NULL,
245};380};
246 381
247static void cxl_decoder_release(struct device *dev)382static void __cxl_decoder_release(struct cxl_decoder *cxld)
248{383{
249 struct cxl_decoder *cxld = to_cxl_decoder(dev);384 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
250 struct cxl_port *port = to_cxl_port(dev->parent);
251 385
252 ida_free(&port->decoder_ida, cxld->id);386 ida_free(&port->decoder_ida, cxld->id);
253 kfree(cxld);
254 put_device(&port->dev);387 put_device(&port->dev);
255}388}
256 389
390static void cxl_endpoint_decoder_release(struct device *dev)
391{
392 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
393
394 __cxl_decoder_release(&cxled->cxld);
395 kfree(cxled);
396}
397
398static void cxl_switch_decoder_release(struct device *dev)
399{
400 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
401
402 __cxl_decoder_release(&cxlsd->cxld);
403 kfree(cxlsd);
404}
405
406struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
407{
408 if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
409 "not a cxl_root_decoder device\n"))
410 return NULL;
411 return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
412}
413EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
414
415static void cxl_root_decoder_release(struct device *dev)
416{
417 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
418
419 if (atomic_read(&cxlrd->region_id) >= 0)
420 memregion_free(atomic_read(&cxlrd->region_id));
421 __cxl_decoder_release(&cxlrd->cxlsd.cxld);
422 kfree(cxlrd);
423}
424
257static const struct device_type cxl_decoder_endpoint_type = {425static const struct device_type cxl_decoder_endpoint_type = {
258 .name = "cxl_decoder_endpoint",426 .name = "cxl_decoder_endpoint",
259 .release = cxl_decoder_release,427 .release = cxl_endpoint_decoder_release,
260 .groups = cxl_decoder_endpoint_attribute_groups,428 .groups = cxl_decoder_endpoint_attribute_groups,
261};429};
262 430
263static const struct device_type cxl_decoder_switch_type = {431static const struct device_type cxl_decoder_switch_type = {
264 .name = "cxl_decoder_switch",432 .name = "cxl_decoder_switch",
265 .release = cxl_decoder_release,433 .release = cxl_switch_decoder_release,
266 .groups = cxl_decoder_switch_attribute_groups,434 .groups = cxl_decoder_switch_attribute_groups,
267};435};
268 436
269static const struct device_type cxl_decoder_root_type = {437static const struct device_type cxl_decoder_root_type = {
270 .name = "cxl_decoder_root",438 .name = "cxl_decoder_root",
271 .release = cxl_decoder_release,439 .release = cxl_root_decoder_release,
272 .groups = cxl_decoder_root_attribute_groups,440 .groups = cxl_decoder_root_attribute_groups,
273};441};
274 442
@@ -283,39 +451,63 @@ bool is_root_decoder(struct device *dev) · 8 unmodified lines
283}451}
284EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);452EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
285 453
286bool is_cxl_decoder(struct device *dev)454bool is_switch_decoder(struct device *dev)
287{455{
288 return dev->type && dev->type->release == cxl_decoder_release;456 return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
289}457}
290EXPORT_SYMBOL_NS_GPL(is_cxl_decoder, CXL);
291 458
292struct cxl_decoder *to_cxl_decoder(struct device *dev)459struct cxl_decoder *to_cxl_decoder(struct device *dev)
293{460{
294 if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release,461 if (dev_WARN_ONCE(dev,
462 !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
295 "not a cxl_decoder device\n"))463 "not a cxl_decoder device\n"))
296 return NULL;464 return NULL;
297 return container_of(dev, struct cxl_decoder, dev);465 return container_of(dev, struct cxl_decoder, dev);
298}466}
299EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);467EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
300 468
469struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
470{
471 if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
472 "not a cxl_endpoint_decoder device\n"))
473 return NULL;
474 return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
475}
476EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
477
478struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
479{
480 if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
481 "not a cxl_switch_decoder device\n"))
482 return NULL;
483 return container_of(dev, struct cxl_switch_decoder, cxld.dev);
484}
485
301static void cxl_ep_release(struct cxl_ep *ep)486static void cxl_ep_release(struct cxl_ep *ep)
302{487{
303 if (!ep)
304 return;
305 list_del(&ep->list);
306 put_device(ep->ep);488 put_device(ep->ep);
307 kfree(ep);489 kfree(ep);
308}490}
309 491
492static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
493{
494 if (!ep)
495 return;
496 xa_erase(&port->endpoints, (unsigned long) ep->ep);
497 cxl_ep_release(ep);
498}
499
310static void cxl_port_release(struct device *dev)500static void cxl_port_release(struct device *dev)
311{501{
312 struct cxl_port *port = to_cxl_port(dev);502 struct cxl_port *port = to_cxl_port(dev);
313 struct cxl_ep *ep, *_e;503 unsigned long index;
504 struct cxl_ep *ep;
314 505
315 device_lock(dev);506 xa_for_each(&port->endpoints, index, ep)
316 list_for_each_entry_safe(ep, _e, &port->endpoints, list)507 cxl_ep_remove(port, ep);
317 cxl_ep_release(ep);508 xa_destroy(&port->endpoints);
318 device_unlock(dev);509 xa_destroy(&port->dports);
510 xa_destroy(&port->regions);
319 ida_free(&cxl_port_ida, port->id);511 ida_free(&cxl_port_ida, port->id);
320 kfree(port);512 kfree(port);
321}513}
@@ -370,7 +562,7 @@ static void unregister_port(void *_port) · 48 unmodified lines
370 lock_dev = &parent->dev;562 lock_dev = &parent->dev;
371 563
372 device_lock_assert(lock_dev);564 device_lock_assert(lock_dev);
373 port->uport = NULL;565 port->dead = true;
374 device_unregister(&port->dev);566 device_unregister(&port->dev);
375}567}
376 568
@@ -395,7 +587,7 @@ static struct lock_class_key cxl_port_key; · 18 unmodified lines
395 587
396static struct cxl_port *cxl_port_alloc(struct device *uport,588static struct cxl_port *cxl_port_alloc(struct device *uport,
397 resource_size_t component_reg_phys,589 resource_size_t component_reg_phys,
398 struct cxl_port *parent_port)590 struct cxl_dport *parent_dport)
399{591{
400 struct cxl_port *port;592 struct cxl_port *port;
401 struct device *dev;593 struct device *dev;
@@ -409,6 +601,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport, · 7 unmodified lines
409 if (rc < 0)601 if (rc < 0)
410 goto err;602 goto err;
411 port->id = rc;603 port->id = rc;
604 port->uport = uport;
412 605
413 /*606 /*
414 * The top-level cxl_port "cxl_root" does not have a cxl_port as607 * The top-level cxl_port "cxl_root" does not have a cxl_port as
@@ -417,17 +610,37 @@ static struct cxl_port *cxl_port_alloc(struct device *uport, · 2 unmodified lines
417 * description.610 * description.
418 */611 */
419 dev = &port->dev;612 dev = &port->dev;
420 if (parent_port) {613 if (parent_dport) {
614 struct cxl_port *parent_port = parent_dport->port;
615 struct cxl_port *iter;
616
421 dev->parent = &parent_port->dev;617 dev->parent = &parent_port->dev;
422 port->depth = parent_port->depth + 1;618 port->depth = parent_port->depth + 1;
619 port->parent_dport = parent_dport;
620
621 /*
622 * walk to the host bridge, or the first ancestor that knows
623 * the host bridge
624 */
625 iter = port;
626 while (!iter->host_bridge &&
627 !is_cxl_root(to_cxl_port(iter->dev.parent)))
628 iter = to_cxl_port(iter->dev.parent);
629 if (iter->host_bridge)
630 port->host_bridge = iter->host_bridge;
631 else
632 port->host_bridge = iter->uport;
633 dev_dbg(uport, "host-bridge: %s\n", dev_name(port->host_bridge));
423 } else634 } else
424 dev->parent = uport;635 dev->parent = uport;
425 636
426 port->uport = uport;
427 port->component_reg_phys = component_reg_phys;637 port->component_reg_phys = component_reg_phys;
428 ida_init(&port->decoder_ida);638 ida_init(&port->decoder_ida);
429 INIT_LIST_HEAD(&port->dports);639 port->hdm_end = -1;
430 INIT_LIST_HEAD(&port->endpoints);640 port->commit_end = -1;
641 xa_init(&port->dports);
642 xa_init(&port->endpoints);
643 xa_init(&port->regions);
431 644
432 device_initialize(dev);645 device_initialize(dev);
433 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);646 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
@@ -447,24 +660,24 @@ err: · 13 unmodified lines
447 * @host: host device for devm operations660 * @host: host device for devm operations
448 * @uport: "physical" device implementing this upstream port661 * @uport: "physical" device implementing this upstream port
449 * @component_reg_phys: (optional) for configurable cxl_port instances662 * @component_reg_phys: (optional) for configurable cxl_port instances
450 * @parent_port: next hop up in the CXL memory decode hierarchy663 * @parent_dport: next hop up in the CXL memory decode hierarchy
451 */664 */
452struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,665struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
453 resource_size_t component_reg_phys,666 resource_size_t component_reg_phys,
454 struct cxl_port *parent_port)667 struct cxl_dport *parent_dport)
455{668{
456 struct cxl_port *port;669 struct cxl_port *port;
457 struct device *dev;670 struct device *dev;
458 int rc;671 int rc;
459 672
460 port = cxl_port_alloc(uport, component_reg_phys, parent_port);673 port = cxl_port_alloc(uport, component_reg_phys, parent_dport);
461 if (IS_ERR(port))674 if (IS_ERR(port))
462 return port;675 return port;
463 676
464 dev = &port->dev;677 dev = &port->dev;
465 if (is_cxl_memdev(uport))678 if (is_cxl_memdev(uport))
466 rc = dev_set_name(dev, "endpoint%d", port->id);679 rc = dev_set_name(dev, "endpoint%d", port->id);
467 else if (parent_port)680 else if (parent_dport)
468 rc = dev_set_name(dev, "port%d", port->id);681 rc = dev_set_name(dev, "port%d", port->id);
469 else682 else
470 rc = dev_set_name(dev, "root%d", port->id);683 rc = dev_set_name(dev, "root%d", port->id);
@@ -556,17 +769,13 @@ static int match_root_child(struct device *dev, const void *match) · 85 unmodified lines
556 return 0;769 return 0;
557 770
558 port = to_cxl_port(dev);771 port = to_cxl_port(dev);
559 device_lock(dev);772 iter = match;
560 list_for_each_entry(dport, &port->dports, list) {773 while (iter) {
561 iter = match;774 dport = cxl_find_dport_by_dev(port, iter);
562 while (iter) {775 if (dport)
563 if (iter == dport->dport)776 break;
564 goto out;777 iter = iter->parent;
565 iter = iter->parent;
566 }
567 }778 }
568out:
569 device_unlock(dev);
570 779
571 return !!iter;780 return !!iter;
572}781}
@@ -590,9 +799,10 @@ EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL); · 17 unmodified lines
590static struct cxl_dport *find_dport(struct cxl_port *port, int id)799static struct cxl_dport *find_dport(struct cxl_port *port, int id)
591{800{
592 struct cxl_dport *dport;801 struct cxl_dport *dport;
802 unsigned long index;
593 803
594 device_lock_assert(&port->dev);804 device_lock_assert(&port->dev);
595 list_for_each_entry (dport, &port->dports, list)805 xa_for_each(&port->dports, index, dport)
596 if (dport->port_id == id)806 if (dport->port_id == id)
597 return dport;807 return dport;
598 return NULL;808 return NULL;
@@ -604,15 +814,15 @@ static int add_dport(struct cxl_port *port, struct cxl_dport *new) · 5 unmodified lines
604 814
605 device_lock_assert(&port->dev);815 device_lock_assert(&port->dev);
606 dup = find_dport(port, new->port_id);816 dup = find_dport(port, new->port_id);
607 if (dup)817 if (dup) {
608 dev_err(&port->dev,818 dev_err(&port->dev,
609 "unable to add dport%d-%s non-unique port id (%s)\n",819 "unable to add dport%d-%s non-unique port id (%s)\n",
610 new->port_id, dev_name(new->dport),820 new->port_id, dev_name(new->dport),
611 dev_name(dup->dport));821 dev_name(dup->dport));
612 else822 return -EBUSY;
613 list_add_tail(&new->list, &port->dports);823 }
614 824 return xa_insert(&port->dports, (unsigned long)new->dport, new,
615 return dup ? -EEXIST : 0;825 GFP_KERNEL);
616}826}
617 827
618/*828/*
@@ -639,10 +849,8 @@ static void cxl_dport_remove(void *data) · 20 unmodified lines
639 struct cxl_dport *dport = data;849 struct cxl_dport *dport = data;
640 struct cxl_port *port = dport->port;850 struct cxl_port *port = dport->port;
641 851
852 xa_erase(&port->dports, (unsigned long) dport->dport);
642 put_device(dport->dport);853 put_device(dport->dport);
643 cond_cxl_root_lock(port);
644 list_del(&dport->list);
645 cond_cxl_root_unlock(port);
646}854}
647 855
648static void cxl_dport_unlink(void *data)856static void cxl_dport_unlink(void *data)
@@ -694,7 +902,6 @@ struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, · 45 unmodified lines
694 if (!dport)902 if (!dport)
695 return ERR_PTR(-ENOMEM);903 return ERR_PTR(-ENOMEM);
696 904
697 INIT_LIST_HEAD(&dport->list);
698 dport->dport = dport_dev;905 dport->dport = dport_dev;
699 dport->port_id = port_id;906 dport->port_id = port_id;
700 dport->component_reg_phys = component_reg_phys;907 dport->component_reg_phys = component_reg_phys;
@@ -723,44 +930,33 @@ struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, · 22 unmodified lines
723}930}
724EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);931EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
725 932
726static struct cxl_ep *find_ep(struct cxl_port *port, struct device *ep_dev)933static int add_ep(struct cxl_ep *new)
727{
728 struct cxl_ep *ep;
729
730 device_lock_assert(&port->dev);
731 list_for_each_entry(ep, &port->endpoints, list)
732 if (ep->ep == ep_dev)
733 return ep;
734 return NULL;
735}
736
737static int add_ep(struct cxl_port *port, struct cxl_ep *new)
738{934{
739 struct cxl_ep *dup;935 struct cxl_port *port = new->dport->port;
936 int rc;
740 937
741 device_lock(&port->dev);938 device_lock(&port->dev);
742 if (port->dead) {939 if (port->dead) {
743 device_unlock(&port->dev);940 device_unlock(&port->dev);
744 return -ENXIO;941 return -ENXIO;
745 }942 }
746 dup = find_ep(port, new->ep);943 rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
747 if (!dup)944 GFP_KERNEL);
748 list_add_tail(&new->list, &port->endpoints);
749 device_unlock(&port->dev);945 device_unlock(&port->dev);
750 946
751 return dup ? -EEXIST : 0;947 return rc;
752}948}
753 949
754/**950/**
755 * cxl_add_ep - register an endpoint's interest in a port951 * cxl_add_ep - register an endpoint's interest in a port
756 * @port: a port in the endpoint's topology ancestry952 * @dport: the dport that routes to @ep_dev
757 * @ep_dev: device representing the endpoint953 * @ep_dev: device representing the endpoint
758 *954 *
759 * Intermediate CXL ports are scanned based on the arrival of endpoints.955 * Intermediate CXL ports are scanned based on the arrival of endpoints.
760 * When those endpoints depart the port can be destroyed once all956 * When those endpoints depart the port can be destroyed once all
761 * endpoints that care about that port have been removed.957 * endpoints that care about that port have been removed.
762 */958 */
763static int cxl_add_ep(struct cxl_port *port, struct device *ep_dev)959static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
764{960{
765 struct cxl_ep *ep;961 struct cxl_ep *ep;
766 int rc;962 int rc;
@@ -769,10 +965,10 @@ static int cxl_add_ep(struct cxl_port *port, struct device *ep_dev) · 2 unmodified lines
769 if (!ep)965 if (!ep)
770 return -ENOMEM;966 return -ENOMEM;
771 967
772 INIT_LIST_HEAD(&ep->list);
773 ep->ep = get_device(ep_dev);968 ep->ep = get_device(ep_dev);
969 ep->dport = dport;
774 970
775 rc = add_ep(port, ep);971 rc = add_ep(ep);
776 if (rc)972 if (rc)
777 cxl_ep_release(ep);973 cxl_ep_release(ep);
778 return rc;974 return rc;
@@ -781,11 +977,13 @@ static int cxl_add_ep(struct cxl_port *port, struct device *ep_dev) · 2 unmodified lines
781struct cxl_find_port_ctx {977struct cxl_find_port_ctx {
782 const struct device *dport_dev;978 const struct device *dport_dev;
783 const struct cxl_port *parent_port;979 const struct cxl_port *parent_port;
980 struct cxl_dport **dport;
784};981};
785 982
786static int match_port_by_dport(struct device *dev, const void *data)983static int match_port_by_dport(struct device *dev, const void *data)
787{984{
788 const struct cxl_find_port_ctx *ctx = data;985 const struct cxl_find_port_ctx *ctx = data;
986 struct cxl_dport *dport;
789 struct cxl_port *port;987 struct cxl_port *port;
790 988
791 if (!is_cxl_port(dev))989 if (!is_cxl_port(dev))
@@ -794,7 +992,10 @@ static int match_port_by_dport(struct device *dev, const void *data) · 2 unmodified lines
794 return 0;992 return 0;
795 993
796 port = to_cxl_port(dev);994 port = to_cxl_port(dev);
797 return cxl_find_dport_by_dev(port, ctx->dport_dev) != NULL;995 dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
996 if (ctx->dport)
997 *ctx->dport = dport;
998 return dport != NULL;
798}999}
799 1000
800static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)1001static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
@@ -810,24 +1011,32 @@ static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx) · 9 unmodified lines
810 return NULL;1011 return NULL;
811}1012}
812 1013
813static struct cxl_port *find_cxl_port(struct device *dport_dev)1014static struct cxl_port *find_cxl_port(struct device *dport_dev,
1015 struct cxl_dport **dport)
814{1016{
815 struct cxl_find_port_ctx ctx = {1017 struct cxl_find_port_ctx ctx = {
816 .dport_dev = dport_dev,1018 .dport_dev = dport_dev,
1019 .dport = dport,
817 };1020 };
1021 struct cxl_port *port;
818 1022
819 return __find_cxl_port(&ctx);1023 port = __find_cxl_port(&ctx);
1024 return port;
820}1025}
821 1026
822static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,1027static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
823 struct device *dport_dev)1028 struct device *dport_dev,
1029 struct cxl_dport **dport)
824{1030{
825 struct cxl_find_port_ctx ctx = {1031 struct cxl_find_port_ctx ctx = {
826 .dport_dev = dport_dev,1032 .dport_dev = dport_dev,
827 .parent_port = parent_port,1033 .parent_port = parent_port,
1034 .dport = dport,
828 };1035 };
1036 struct cxl_port *port;
829 1037
830 return __find_cxl_port(&ctx);1038 port = __find_cxl_port(&ctx);
1039 return port;
831}1040}
832 1041
833/*1042/*
@@ -851,13 +1060,13 @@ static void delete_endpoint(void *data) · 17 unmodified lines
851 struct cxl_port *parent_port;1060 struct cxl_port *parent_port;
852 struct device *parent;1061 struct device *parent;
853 1062
854 parent_port = cxl_mem_find_port(cxlmd);1063 parent_port = cxl_mem_find_port(cxlmd, NULL);
855 if (!parent_port)1064 if (!parent_port)
856 goto out;1065 goto out;
857 parent = &parent_port->dev;1066 parent = &parent_port->dev;
858 1067
859 device_lock(parent);1068 device_lock(parent);
860 if (parent->driver && endpoint->uport) {1069 if (parent->driver && !endpoint->dead) {
861 devm_release_action(parent, cxl_unlink_uport, endpoint);1070 devm_release_action(parent, cxl_unlink_uport, endpoint);
862 devm_release_action(parent, unregister_port, endpoint);1071 devm_release_action(parent, unregister_port, endpoint);
863 }1072 }
@@ -883,21 +1092,70 @@ EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL); · 19 unmodified lines
883 * for a port to be unregistered is when all memdevs beneath that port have gone1092 * for a port to be unregistered is when all memdevs beneath that port have gone
884 * through ->remove(). This "bottom-up" removal selectively removes individual1093 * through ->remove(). This "bottom-up" removal selectively removes individual
885 * child ports manually. This depends on devm_cxl_add_port() to not change is1094 * child ports manually. This depends on devm_cxl_add_port() to not change is
886 * devm action registration order.1095 * devm action registration order, and for dports to have already been
1096 * destroyed by reap_dports().
887 */1097 */
888static void delete_switch_port(struct cxl_port *port, struct list_head *dports)1098static void delete_switch_port(struct cxl_port *port)
1099{
1100 devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1101 devm_release_action(port->dev.parent, unregister_port, port);
1102}
1103
1104static void reap_dports(struct cxl_port *port)
889{1105{
890 struct cxl_dport *dport, *_d;1106 struct cxl_dport *dport;
1107 unsigned long index;
1108
1109 device_lock_assert(&port->dev);
891 1110
892 list_for_each_entry_safe(dport, _d, dports, list) {1111 xa_for_each(&port->dports, index, dport) {
893 devm_release_action(&port->dev, cxl_dport_unlink, dport);1112 devm_release_action(&port->dev, cxl_dport_unlink, dport);
894 devm_release_action(&port->dev, cxl_dport_remove, dport);1113 devm_release_action(&port->dev, cxl_dport_remove, dport);
895 devm_kfree(&port->dev, dport);1114 devm_kfree(&port->dev, dport);
896 }1115 }
897 devm_release_action(port->dev.parent, cxl_unlink_uport, port);
898 devm_release_action(port->dev.parent, unregister_port, port);
899}1116}
900 1117
1118int devm_cxl_add_endpoint(struct cxl_memdev *cxlmd,
1119 struct cxl_dport *parent_dport)
1120{
1121 struct cxl_port *parent_port = parent_dport->port;
1122 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1123 struct cxl_port *endpoint, *iter, *down;
1124 int rc;
1125
1126 /*
1127 * Now that the path to the root is established record all the
1128 * intervening ports in the chain.
1129 */
1130 for (iter = parent_port, down = NULL; !is_cxl_root(iter);
1131 down = iter, iter = to_cxl_port(iter->dev.parent)) {
1132 struct cxl_ep *ep;
1133
1134 ep = cxl_ep_load(iter, cxlmd);
1135 ep->next = down;
1136 }
1137
1138 endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev,
1139 cxlds->component_reg_phys, parent_dport);
1140 if (IS_ERR(endpoint))
1141 return PTR_ERR(endpoint);
1142
1143 dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev));
1144
1145 rc = cxl_endpoint_autoremove(cxlmd, endpoint);
1146 if (rc)
1147 return rc;
1148
1149 if (!endpoint->dev.driver) {
1150 dev_err(&cxlmd->dev, "%s failed probe\n",
1151 dev_name(&endpoint->dev));
1152 return -ENXIO;
1153 }
1154
1155 return 0;
1156}
1157EXPORT_SYMBOL_NS_GPL(devm_cxl_add_endpoint, CXL);
1158
901static void cxl_detach_ep(void *data)1159static void cxl_detach_ep(void *data)
902{1160{
903 struct cxl_memdev *cxlmd = data;1161 struct cxl_memdev *cxlmd = data;
@@ -906,13 +1164,13 @@ static void cxl_detach_ep(void *data) · 2 unmodified lines
906 for (iter = &cxlmd->dev; iter; iter = grandparent(iter)) {1164 for (iter = &cxlmd->dev; iter; iter = grandparent(iter)) {
907 struct device *dport_dev = grandparent(iter);1165 struct device *dport_dev = grandparent(iter);
908 struct cxl_port *port, *parent_port;1166 struct cxl_port *port, *parent_port;
909 LIST_HEAD(reap_dports);
910 struct cxl_ep *ep;1167 struct cxl_ep *ep;
1168 bool died = false;
911 1169
912 if (!dport_dev)1170 if (!dport_dev)
913 break;1171 break;
914 1172
915 port = find_cxl_port(dport_dev);1173 port = find_cxl_port(dport_dev, NULL);
916 if (!port)1174 if (!port)
917 continue;1175 continue;
918 1176
@@ -936,26 +1194,27 @@ static void cxl_detach_ep(void *data) · 17 unmodified lines
936 }1194 }
937 1195
938 device_lock(&port->dev);1196 device_lock(&port->dev);
939 ep = find_ep(port, &cxlmd->dev);1197 ep = cxl_ep_load(port, cxlmd);
940 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",1198 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
941 ep ? dev_name(ep->ep) : "", dev_name(&port->dev));1199 ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
942 cxl_ep_release(ep);1200 cxl_ep_remove(port, ep);
943 if (ep && !port->dead && list_empty(&port->endpoints) &&1201 if (ep && !port->dead && xa_empty(&port->endpoints) &&
944 !is_cxl_root(parent_port)) {1202 !is_cxl_root(parent_port)) {
945 /*1203 /*
946 * This was the last ep attached to a dynamically1204 * This was the last ep attached to a dynamically
947 * enumerated port. Block new cxl_add_ep() and garbage1205 * enumerated port. Block new cxl_add_ep() and garbage
948 * collect the port.1206 * collect the port.
949 */1207 */
1208 died = true;
950 port->dead = true;1209 port->dead = true;
951 list_splice_init(&port->dports, &reap_dports);1210 reap_dports(port);
952 }1211 }
953 device_unlock(&port->dev);1212 device_unlock(&port->dev);
954 1213
955 if (!list_empty(&reap_dports)) {1214 if (died) {
956 dev_dbg(&cxlmd->dev, "delete %s\n",1215 dev_dbg(&cxlmd->dev, "delete %s\n",
957 dev_name(&port->dev));1216 dev_name(&port->dev));
958 delete_switch_port(port, &reap_dports);1217 delete_switch_port(port);
959 }1218 }
960 put_device(&port->dev);1219 put_device(&port->dev);
961 device_unlock(&parent_port->dev);1220 device_unlock(&parent_port->dev);
@@ -986,6 +1245,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, · 24 unmodified lines
986{1245{
987 struct device *dparent = grandparent(dport_dev);1246 struct device *dparent = grandparent(dport_dev);
988 struct cxl_port *port, *parent_port = NULL;1247 struct cxl_port *port, *parent_port = NULL;
1248 struct cxl_dport *dport, *parent_dport;
989 resource_size_t component_reg_phys;1249 resource_size_t component_reg_phys;
990 int rc;1250 int rc;
991 1251
@@ -1000,7 +1260,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, · 8 unmodified lines
1000 return -ENXIO;1260 return -ENXIO;
1001 }1261 }
1002 1262
1003 parent_port = find_cxl_port(dparent);1263 parent_port = find_cxl_port(dparent, &parent_dport);
1004 if (!parent_port) {1264 if (!parent_port) {
1005 /* iterate to create this parent_port */1265 /* iterate to create this parent_port */
1006 return -EAGAIN;1266 return -EAGAIN;
@@ -1015,13 +1275,14 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, · 8 unmodified lines
1015 goto out;1275 goto out;
1016 }1276 }
1017 1277
1018 port = find_cxl_port_at(parent_port, dport_dev);1278 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1019 if (!port) {1279 if (!port) {
1020 component_reg_phys = find_component_registers(uport_dev);1280 component_reg_phys = find_component_registers(uport_dev);
1021 port = devm_cxl_add_port(&parent_port->dev, uport_dev,1281 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1022 component_reg_phys, parent_port);1282 component_reg_phys, parent_dport);
1283 /* retry find to pick up the new dport information */
1023 if (!IS_ERR(port))1284 if (!IS_ERR(port))
1024 get_device(&port->dev);1285 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1025 }1286 }
1026out:1287out:
1027 device_unlock(&parent_port->dev);1288 device_unlock(&parent_port->dev);
@@ -1031,8 +1292,8 @@ out: · 3 unmodified lines
1031 else {1292 else {
1032 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",1293 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1033 dev_name(&port->dev), dev_name(port->uport));1294 dev_name(&port->dev), dev_name(port->uport));
1034 rc = cxl_add_ep(port, &cxlmd->dev);1295 rc = cxl_add_ep(dport, &cxlmd->dev);
1035 if (rc == -EEXIST) {1296 if (rc == -EBUSY) {
1036 /*1297 /*
1037 * "can't" happen, but this error code means1298 * "can't" happen, but this error code means
1038 * something to the caller, so translate it.1299 * something to the caller, so translate it.
@@ -1065,6 +1326,7 @@ retry: · 26 unmodified lines
1065 for (iter = dev; iter; iter = grandparent(iter)) {1326 for (iter = dev; iter; iter = grandparent(iter)) {
1066 struct device *dport_dev = grandparent(iter);1327 struct device *dport_dev = grandparent(iter);
1067 struct device *uport_dev;1328 struct device *uport_dev;
1329 struct cxl_dport *dport;
1068 struct cxl_port *port;1330 struct cxl_port *port;
1069 1331
1070 if (!dport_dev)1332 if (!dport_dev)
@@ -1080,12 +1342,12 @@ retry: · 9 unmodified lines
1080 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",1342 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1081 dev_name(iter), dev_name(dport_dev),1343 dev_name(iter), dev_name(dport_dev),
1082 dev_name(uport_dev));1344 dev_name(uport_dev));
1083 port = find_cxl_port(dport_dev);1345 port = find_cxl_port(dport_dev, &dport);
1084 if (port) {1346 if (port) {
1085 dev_dbg(&cxlmd->dev,1347 dev_dbg(&cxlmd->dev,
1086 "found already registered port %s:%s\n",1348 "found already registered port %s:%s\n",
1087 dev_name(&port->dev), dev_name(port->uport));1349 dev_name(&port->dev), dev_name(port->uport));
1088 rc = cxl_add_ep(port, &cxlmd->dev);1350 rc = cxl_add_ep(dport, &cxlmd->dev);
1089 1351
1090 /*1352 /*
1091 * If the endpoint already exists in the port's list,1353 * If the endpoint already exists in the port's list,
@@ -1094,7 +1356,7 @@ retry: · 2 unmodified lines
1094 * the parent_port lock as the current port may be being1356 * the parent_port lock as the current port may be being
1095 * reaped.1357 * reaped.
1096 */1358 */
1097 if (rc && rc != -EEXIST) {1359 if (rc && rc != -EBUSY) {
1098 put_device(&port->dev);1360 put_device(&port->dev);
1099 return rc;1361 return rc;
1100 }1362 }
@@ -1124,30 +1386,14 @@ retry: · 23 unmodified lines
1124}1386}
1125EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);1387EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1126 1388
1127struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd)1389struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1390 struct cxl_dport **dport)
1128{1391{
1129 return find_cxl_port(grandparent(&cxlmd->dev));1392 return find_cxl_port(grandparent(&cxlmd->dev), dport);
1130}1393}
1131EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);1394EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1132 1395
1133struct cxl_dport *cxl_find_dport_by_dev(struct cxl_port *port,1396static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1134 const struct device *dev)
1135{
1136 struct cxl_dport *dport;
1137
1138 device_lock(&port->dev);
1139 list_for_each_entry(dport, &port->dports, list)
1140 if (dport->dport == dev) {
1141 device_unlock(&port->dev);
1142 return dport;
1143 }
1144
1145 device_unlock(&port->dev);
1146 return NULL;
1147}
1148EXPORT_SYMBOL_NS_GPL(cxl_find_dport_by_dev, CXL);
1149
1150static int decoder_populate_targets(struct cxl_decoder *cxld,
1151 struct cxl_port *port, int *target_map)1397 struct cxl_port *port, int *target_map)
1152{1398{
1153 int i, rc = 0;1399 int i, rc = 0;
@@ -1157,88 +1403,92 @@ static int decoder_populate_targets(struct cxl_decoder *cxld, · 3 unmodified lines
1157 1403
1158 device_lock_assert(&port->dev);1404 device_lock_assert(&port->dev);
1159 1405
1160 if (list_empty(&port->dports))1406 if (xa_empty(&port->dports))
1161 return -EINVAL;1407 return -EINVAL;
1162 1408
1163 write_seqlock(&cxld->target_lock);1409 write_seqlock(&cxlsd->target_lock);
1164 for (i = 0; i < cxld->nr_targets; i++) {1410 for (i = 0; i < cxlsd->nr_targets; i++) {
1165 struct cxl_dport *dport = find_dport(port, target_map[i]);1411 struct cxl_dport *dport = find_dport(port, target_map[i]);
1166 1412
1167 if (!dport) {1413 if (!dport) {
1168 rc = -ENXIO;1414 rc = -ENXIO;
1169 break;1415 break;
1170 }1416 }
1171 cxld->target[i] = dport;1417 cxlsd->target[i] = dport;
1172 }1418 }
1173 write_sequnlock(&cxld->target_lock);1419 write_sequnlock(&cxlsd->target_lock);
1174 1420
1175 return rc;1421 return rc;
1176}1422}
1177 1423
1424static struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos)
1425{
1426 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1427 struct cxl_decoder *cxld = &cxlsd->cxld;
1428 int iw;
1429
1430 iw = cxld->interleave_ways;
1431 if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
1432 "misconfigured root decoder\n"))
1433 return NULL;
1434
1435 return cxlrd->cxlsd.target[pos % iw];
1436}
1437
1178static struct lock_class_key cxl_decoder_key;1438static struct lock_class_key cxl_decoder_key;
1179 1439
1180/**1440/**
1181 * cxl_decoder_alloc - Allocate a new CXL decoder1441 * cxl_decoder_init - Common decoder setup / initialization
1182 * @port: owning port of this decoder1442 * @port: owning port of this decoder
1183 * @nr_targets: downstream targets accessible by this decoder. All upstream1443 * @cxld: common decoder properties to initialize
1184 * ports and root ports must have at least 1 target. Endpoint
1185 * devices will have 0 targets. Callers wishing to register an
1186 * endpoint device should specify 0.
1187 *
1188 * A port should contain one or more decoders. Each of those decoders enable
1189 * some address space for CXL.mem utilization. A decoder is expected to be
1190 * configured by the caller before registering.
1191 *1444 *
1192 * Return: A new cxl decoder to be registered by cxl_decoder_add(). The decoder1445 * A port may contain one or more decoders. Each of those decoders
1193 * is initialized to be a "passthrough" decoder.1446 * enable some address space for CXL.mem utilization. A decoder is
1447 * expected to be configured by the caller before registering via
1448 * cxl_decoder_add()
1194 */1449 */
1195static struct cxl_decoder *cxl_decoder_alloc(struct cxl_port *port,1450static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1196 unsigned int nr_targets)
1197{1451{
1198 struct cxl_decoder *cxld;
1199 struct device *dev;1452 struct device *dev;
1200 int rc = 0;1453 int rc;
1201
1202 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1203 return ERR_PTR(-EINVAL);
1204
1205 cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);
1206 if (!cxld)
1207 return ERR_PTR(-ENOMEM);
1208 1454
1209 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);1455 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1210 if (rc < 0)1456 if (rc < 0)
1211 goto err;1457 return rc;
1212 1458
1213 /* need parent to stick around to release the id */1459 /* need parent to stick around to release the id */
1214 get_device(&port->dev);1460 get_device(&port->dev);
1215 cxld->id = rc;1461 cxld->id = rc;
1216 1462
1217 cxld->nr_targets = nr_targets;
1218 seqlock_init(&cxld->target_lock);
1219 dev = &cxld->dev;1463 dev = &cxld->dev;
1220 device_initialize(dev);1464 device_initialize(dev);
1221 lockdep_set_class(&dev->mutex, &cxl_decoder_key);1465 lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1222 device_set_pm_not_required(dev);1466 device_set_pm_not_required(dev);
1223 dev->parent = &port->dev;1467 dev->parent = &port->dev;
1224 dev->bus = &cxl_bus_type;1468 dev->bus = &cxl_bus_type;
1225 if (is_cxl_root(port))
1226 cxld->dev.type = &cxl_decoder_root_type;
1227 else if (is_cxl_endpoint(port))
1228 cxld->dev.type = &cxl_decoder_endpoint_type;
1229 else
1230 cxld->dev.type = &cxl_decoder_switch_type;
1231 1469
1232 /* Pre initialize an "empty" decoder */1470 /* Pre initialize an "empty" decoder */
1233 cxld->interleave_ways = 1;1471 cxld->interleave_ways = 1;
1234 cxld->interleave_granularity = PAGE_SIZE;1472 cxld->interleave_granularity = PAGE_SIZE;
1235 cxld->target_type = CXL_DECODER_EXPANDER;1473 cxld->target_type = CXL_DECODER_EXPANDER;
1236 cxld->platform_res = (struct resource)DEFINE_RES_MEM(0, 0);1474 cxld->hpa_range = (struct range) {
1475 .start = 0,
1476 .end = -1,
1477 };
1237 1478
1238 return cxld;1479 return 0;
1239err:1480}
1240 kfree(cxld);1481
1241 return ERR_PTR(rc);1482static int cxl_switch_decoder_init(struct cxl_port *port,
1483 struct cxl_switch_decoder *cxlsd,
1484 int nr_targets)
1485{
1486 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1487 return -EINVAL;
1488
1489 cxlsd->nr_targets = nr_targets;
1490 seqlock_init(&cxlsd->target_lock);
1491 return cxl_decoder_init(port, &cxlsd->cxld);
1242}1492}
1243 1493
1244/**1494/**
@@ -1251,13 +1501,46 @@ err: · 6 unmodified lines
1251 * firmware description of CXL resources into a CXL standard decode1501 * firmware description of CXL resources into a CXL standard decode
1252 * topology.1502 * topology.
1253 */1503 */
1254struct cxl_decoder *cxl_root_decoder_alloc(struct cxl_port *port,1504struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1255 unsigned int nr_targets)1505 unsigned int nr_targets)
1256{1506{
1507 struct cxl_root_decoder *cxlrd;
1508 struct cxl_switch_decoder *cxlsd;
1509 struct cxl_decoder *cxld;
1510 int rc;
1511
1257 if (!is_cxl_root(port))1512 if (!is_cxl_root(port))
1258 return ERR_PTR(-EINVAL);1513 return ERR_PTR(-EINVAL);
1259 1514
1260 return cxl_decoder_alloc(port, nr_targets);1515 cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1516 GFP_KERNEL);
1517 if (!cxlrd)
1518 return ERR_PTR(-ENOMEM);
1519
1520 cxlsd = &cxlrd->cxlsd;
1521 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1522 if (rc) {
1523 kfree(cxlrd);
1524 return ERR_PTR(rc);
1525 }
1526
1527 cxlrd->calc_hb = cxl_hb_modulo;
1528
1529 cxld = &cxlsd->cxld;
1530 cxld->dev.type = &cxl_decoder_root_type;
1531 /*
1532 * cxl_root_decoder_release() special cases negative ids to
1533 * detect memregion_alloc() failures.
1534 */
1535 atomic_set(&cxlrd->region_id, -1);
1536 rc = memregion_alloc(GFP_KERNEL);
1537 if (rc < 0) {
1538 put_device(&cxld->dev);
1539 return ERR_PTR(rc);
1540 }
1541
1542 atomic_set(&cxlrd->region_id, rc);
1543 return cxlrd;
1261}1544}
1262EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);1545EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1263 1546
@@ -1272,13 +1555,29 @@ EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL); · 8 unmodified lines
1272 * that sit between Switch Upstream Ports / Switch Downstream Ports and1555 * that sit between Switch Upstream Ports / Switch Downstream Ports and
1273 * Host Bridges / Root Ports.1556 * Host Bridges / Root Ports.
1274 */1557 */
1275struct cxl_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,1558struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1276 unsigned int nr_targets)1559 unsigned int nr_targets)
1277{1560{
1561 struct cxl_switch_decoder *cxlsd;
1562 struct cxl_decoder *cxld;
1563 int rc;
1564
1278 if (is_cxl_root(port) || is_cxl_endpoint(port))1565 if (is_cxl_root(port) || is_cxl_endpoint(port))
1279 return ERR_PTR(-EINVAL);1566 return ERR_PTR(-EINVAL);
1280 1567
1281 return cxl_decoder_alloc(port, nr_targets);1568 cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1569 if (!cxlsd)
1570 return ERR_PTR(-ENOMEM);
1571
1572 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1573 if (rc) {
1574 kfree(cxlsd);
1575 return ERR_PTR(rc);
1576 }
1577
1578 cxld = &cxlsd->cxld;
1579 cxld->dev.type = &cxl_decoder_switch_type;
1580 return cxlsd;
1282}1581}
1283EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);1582EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1284 1583
@@ -1288,18 +1587,35 @@ EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL); · 3 unmodified lines
1288 *1587 *
1289 * Return: A new cxl decoder to be registered by cxl_decoder_add()1588 * Return: A new cxl decoder to be registered by cxl_decoder_add()
1290 */1589 */
1291struct cxl_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)1590struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1292{1591{
1592 struct cxl_endpoint_decoder *cxled;
1593 struct cxl_decoder *cxld;
1594 int rc;
1595
1293 if (!is_cxl_endpoint(port))1596 if (!is_cxl_endpoint(port))
1294 return ERR_PTR(-EINVAL);1597 return ERR_PTR(-EINVAL);
1295 1598
1296 return cxl_decoder_alloc(port, 0);1599 cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1600 if (!cxled)
1601 return ERR_PTR(-ENOMEM);
1602
1603 cxled->pos = -1;
1604 cxld = &cxled->cxld;
1605 rc = cxl_decoder_init(port, cxld);
1606 if (rc) {
1607 kfree(cxled);
1608 return ERR_PTR(rc);
1609 }
1610
1611 cxld->dev.type = &cxl_decoder_endpoint_type;
1612 return cxled;
1297}1613}
1298EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);1614EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1299 1615
1300/**1616/**
1301 * cxl_decoder_add_locked - Add a decoder with targets1617 * cxl_decoder_add_locked - Add a decoder with targets
1302 * @cxld: The cxl decoder allocated by cxl_decoder_alloc()1618 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1303 * @target_map: A list of downstream ports that this decoder can direct memory1619 * @target_map: A list of downstream ports that this decoder can direct memory
1304 * traffic to. These numbers should correspond with the port number1620 * traffic to. These numbers should correspond with the port number
1305 * in the PCIe Link Capabilities structure.1621 * in the PCIe Link Capabilities structure.
@@ -1335,7 +1651,9 @@ int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map) · 29 unmodified lines
1335 1651
1336 port = to_cxl_port(cxld->dev.parent);1652 port = to_cxl_port(cxld->dev.parent);
1337 if (!is_endpoint_decoder(dev)) {1653 if (!is_endpoint_decoder(dev)) {
1338 rc = decoder_populate_targets(cxld, port, target_map);1654 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1655
1656 rc = decoder_populate_targets(cxlsd, port, target_map);
1339 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {1657 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1340 dev_err(&port->dev,1658 dev_err(&port->dev,
1341 "Failed to populate active decoder targets\n");1659 "Failed to populate active decoder targets\n");
@@ -1347,20 +1665,13 @@ int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map) · 5 unmodified lines
1347 if (rc)1665 if (rc)
1348 return rc;1666 return rc;
1349 1667
1350 /*
1351 * Platform decoder resources should show up with a reasonable name. All
1352 * other resources are just sub ranges within the main decoder resource.
1353 */
1354 if (is_root_decoder(dev))
1355 cxld->platform_res.name = dev_name(dev);
1356
1357 return device_add(dev);1668 return device_add(dev);
1358}1669}
1359EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);1670EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1360 1671
1361/**1672/**
1362 * cxl_decoder_add - Add a decoder with targets1673 * cxl_decoder_add - Add a decoder with targets
1363 * @cxld: The cxl decoder allocated by cxl_decoder_alloc()1674 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1364 * @target_map: A list of downstream ports that this decoder can direct memory1675 * @target_map: A list of downstream ports that this decoder can direct memory
1365 * traffic to. These numbers should correspond with the port number1676 * traffic to. These numbers should correspond with the port number
1366 * in the PCIe Link Capabilities structure.1677 * in the PCIe Link Capabilities structure.
@@ -1394,6 +1705,13 @@ EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL); · 27 unmodified lines
1394 1705
1395static void cxld_unregister(void *dev)1706static void cxld_unregister(void *dev)
1396{1707{
1708 struct cxl_endpoint_decoder *cxled;
1709
1710 if (is_endpoint_decoder(dev)) {
1711 cxled = to_cxl_endpoint_decoder(dev);
1712 cxl_decoder_kill_region(cxled);
1713 }
1714
1397 device_unregister(dev);1715 device_unregister(dev);
1398}1716}
1399 1717
@@ -1521,10 +1839,20 @@ struct bus_type cxl_bus_type = { · 121 unmodified lines
1521};1839};
1522EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);1840EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
1523 1841
1842static struct dentry *cxl_debugfs;
1843
1844struct dentry *cxl_debugfs_create_dir(const char *dir)
1845{
1846 return debugfs_create_dir(dir, cxl_debugfs);
1847}
1848EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
1849
1524static __init int cxl_core_init(void)1850static __init int cxl_core_init(void)
1525{1851{
1526 int rc;1852 int rc;
1527 1853
1854 cxl_debugfs = debugfs_create_dir("cxl", NULL);
1855
1528 cxl_mbox_init();1856 cxl_mbox_init();
1529 1857
1530 rc = cxl_memdev_init();1858 rc = cxl_memdev_init();
@@ -1541,22 +1869,28 @@ static __init int cxl_core_init(void) · 10 unmodified lines
1541 if (rc)1869 if (rc)
1542 goto err_bus;1870 goto err_bus;
1543 1871
1872 rc = cxl_region_init();
1873 if (rc)
1874 goto err_region;
1875
1544 return 0;1876 return 0;
1545 1877
1878err_region:
1879 bus_unregister(&cxl_bus_type);
1546err_bus:1880err_bus:
1547 destroy_workqueue(cxl_bus_wq);1881 destroy_workqueue(cxl_bus_wq);
1548err_wq:1882err_wq:
1549 cxl_memdev_exit();1883 cxl_memdev_exit();
1550 cxl_mbox_exit();
1551 return rc;1884 return rc;
1552}1885}
1553 1886
1554static void cxl_core_exit(void)1887static void cxl_core_exit(void)
1555{1888{
1889 cxl_region_exit();
1556 bus_unregister(&cxl_bus_type);1890 bus_unregister(&cxl_bus_type);
1557 destroy_workqueue(cxl_bus_wq);1891 destroy_workqueue(cxl_bus_wq);
1558 cxl_memdev_exit();1892 cxl_memdev_exit();
1559 cxl_mbox_exit();1893 debugfs_remove_recursive(cxl_debugfs);
1560}1894}
1561 1895
1562module_init(cxl_core_init);1896module_init(cxl_core_init);
drivers/cxl/core/region.c+1896−0
@@ -0,0 +1,1896 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/memregion.h>
4#include <linux/genalloc.h>
5#include <linux/device.h>
6#include <linux/module.h>
7#include <linux/slab.h>
8#include <linux/uuid.h>
9#include <linux/idr.h>
10#include <cxlmem.h>
11#include <cxl.h>
12#include "core.h"
13
14/**
15 * DOC: cxl core region
16 *
17 * CXL Regions represent mapped memory capacity in system physical address
18 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
19 * Memory ranges, Regions represent the active mapped capacity by the HDM
20 * Decoder Capability structures throughout the Host Bridges, Switches, and
21 * Endpoints in the topology.
22 *
23 * Region configuration has ordering constraints. UUID may be set at any time
24 * but is only visible for persistent regions.
25 * 1. Interleave granularity
26 * 2. Interleave size
27 * 3. Decoder targets
28 */
29
30/*
31 * All changes to the interleave configuration occur with this lock held
32 * for write.
33 */
34static DECLARE_RWSEM(cxl_region_rwsem);
35
36static struct cxl_region *to_cxl_region(struct device *dev);
37
38static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
39 char *buf)
40{
41 struct cxl_region *cxlr = to_cxl_region(dev);
42 struct cxl_region_params *p = &cxlr->params;
43 ssize_t rc;
44
45 rc = down_read_interruptible(&cxl_region_rwsem);
46 if (rc)
47 return rc;
48 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
49 up_read(&cxl_region_rwsem);
50
51 return rc;
52}
53
54static int is_dup(struct device *match, void *data)
55{
56 struct cxl_region_params *p;
57 struct cxl_region *cxlr;
58 uuid_t *uuid = data;
59
60 if (!is_cxl_region(match))
61 return 0;
62
63 lockdep_assert_held(&cxl_region_rwsem);
64 cxlr = to_cxl_region(match);
65 p = &cxlr->params;
66
67 if (uuid_equal(&p->uuid, uuid)) {
68 dev_dbg(match, "already has uuid: %pUb\n", uuid);
69 return -EBUSY;
70 }
71
72 return 0;
73}
74
75static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
76 const char *buf, size_t len)
77{
78 struct cxl_region *cxlr = to_cxl_region(dev);
79 struct cxl_region_params *p = &cxlr->params;
80 uuid_t temp;
81 ssize_t rc;
82
83 if (len != UUID_STRING_LEN + 1)
84 return -EINVAL;
85
86 rc = uuid_parse(buf, &temp);
87 if (rc)
88 return rc;
89
90 if (uuid_is_null(&temp))
91 return -EINVAL;
92
93 rc = down_write_killable(&cxl_region_rwsem);
94 if (rc)
95 return rc;
96
97 if (uuid_equal(&p->uuid, &temp))
98 goto out;
99
100 rc = -EBUSY;
101 if (p->state >= CXL_CONFIG_ACTIVE)
102 goto out;
103
104 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
105 if (rc < 0)
106 goto out;
107
108 uuid_copy(&p->uuid, &temp);
109out:
110 up_write(&cxl_region_rwsem);
111
112 if (rc)
113 return rc;
114 return len;
115}
116static DEVICE_ATTR_RW(uuid);
117
118static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
119 struct cxl_region *cxlr)
120{
121 return xa_load(&port->regions, (unsigned long)cxlr);
122}
123
124static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
125{
126 struct cxl_region_params *p = &cxlr->params;
127 int i;
128
129 for (i = count - 1; i >= 0; i--) {
130 struct cxl_endpoint_decoder *cxled = p->targets[i];
131 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
132 struct cxl_port *iter = cxled_to_port(cxled);
133 struct cxl_ep *ep;
134 int rc;
135
136 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
137 iter = to_cxl_port(iter->dev.parent);
138
139 for (ep = cxl_ep_load(iter, cxlmd); iter;
140 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
141 struct cxl_region_ref *cxl_rr;
142 struct cxl_decoder *cxld;
143
144 cxl_rr = cxl_rr_load(iter, cxlr);
145 cxld = cxl_rr->decoder;
146 rc = cxld->reset(cxld);
147 if (rc)
148 return rc;
149 }
150
151 rc = cxled->cxld.reset(&cxled->cxld);
152 if (rc)
153 return rc;
154 }
155
156 return 0;
157}
158
159static int cxl_region_decode_commit(struct cxl_region *cxlr)
160{
161 struct cxl_region_params *p = &cxlr->params;
162 int i, rc = 0;
163
164 for (i = 0; i < p->nr_targets; i++) {
165 struct cxl_endpoint_decoder *cxled = p->targets[i];
166 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
167 struct cxl_region_ref *cxl_rr;
168 struct cxl_decoder *cxld;
169 struct cxl_port *iter;
170 struct cxl_ep *ep;
171
172 /* commit bottom up */
173 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
174 iter = to_cxl_port(iter->dev.parent)) {
175 cxl_rr = cxl_rr_load(iter, cxlr);
176 cxld = cxl_rr->decoder;
177 rc = cxld->commit(cxld);
178 if (rc)
179 break;
180 }
181
182 if (rc) {
183 /* programming @iter failed, teardown */
184 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
185 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
186 cxl_rr = cxl_rr_load(iter, cxlr);
187 cxld = cxl_rr->decoder;
188 cxld->reset(cxld);
189 }
190
191 cxled->cxld.reset(&cxled->cxld);
192 goto err;
193 }
194 }
195
196 return 0;
197
198err:
199 /* undo the targets that were successfully committed */
200 cxl_region_decode_reset(cxlr, i);
201 return rc;
202}
203
204static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
205 const char *buf, size_t len)
206{
207 struct cxl_region *cxlr = to_cxl_region(dev);
208 struct cxl_region_params *p = &cxlr->params;
209 bool commit;
210 ssize_t rc;
211
212 rc = kstrtobool(buf, &commit);
213 if (rc)
214 return rc;
215
216 rc = down_write_killable(&cxl_region_rwsem);
217 if (rc)
218 return rc;
219
220 /* Already in the requested state? */
221 if (commit && p->state >= CXL_CONFIG_COMMIT)
222 goto out;
223 if (!commit && p->state < CXL_CONFIG_COMMIT)
224 goto out;
225
226 /* Not ready to commit? */
227 if (commit && p->state < CXL_CONFIG_ACTIVE) {
228 rc = -ENXIO;
229 goto out;
230 }
231
232 if (commit)
233 rc = cxl_region_decode_commit(cxlr);
234 else {
235 p->state = CXL_CONFIG_RESET_PENDING;
236 up_write(&cxl_region_rwsem);
237 device_release_driver(&cxlr->dev);
238 down_write(&cxl_region_rwsem);
239
240 /*
241 * The lock was dropped, so need to revalidate that the reset is
242 * still pending.
243 */
244 if (p->state == CXL_CONFIG_RESET_PENDING)
245 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
246 }
247
248 if (rc)
249 goto out;
250
251 if (commit)
252 p->state = CXL_CONFIG_COMMIT;
253 else if (p->state == CXL_CONFIG_RESET_PENDING)
254 p->state = CXL_CONFIG_ACTIVE;
255
256out:
257 up_write(&cxl_region_rwsem);
258
259 if (rc)
260 return rc;
261 return len;
262}
263
264static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
265 char *buf)
266{
267 struct cxl_region *cxlr = to_cxl_region(dev);
268 struct cxl_region_params *p = &cxlr->params;
269 ssize_t rc;
270
271 rc = down_read_interruptible(&cxl_region_rwsem);
272 if (rc)
273 return rc;
274 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
275 up_read(&cxl_region_rwsem);
276
277 return rc;
278}
279static DEVICE_ATTR_RW(commit);
280
281static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
282 int n)
283{
284 struct device *dev = kobj_to_dev(kobj);
285 struct cxl_region *cxlr = to_cxl_region(dev);
286
287 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
288 return 0;
289 return a->mode;
290}
291
292static ssize_t interleave_ways_show(struct device *dev,
293 struct device_attribute *attr, char *buf)
294{
295 struct cxl_region *cxlr = to_cxl_region(dev);
296 struct cxl_region_params *p = &cxlr->params;
297 ssize_t rc;
298
299 rc = down_read_interruptible(&cxl_region_rwsem);
300 if (rc)
301 return rc;
302 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
303 up_read(&cxl_region_rwsem);
304
305 return rc;
306}
307
308static const struct attribute_group *get_cxl_region_target_group(void);
309
310static ssize_t interleave_ways_store(struct device *dev,
311 struct device_attribute *attr,
312 const char *buf, size_t len)
313{
314 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
315 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
316 struct cxl_region *cxlr = to_cxl_region(dev);
317 struct cxl_region_params *p = &cxlr->params;
318 unsigned int val, save;
319 int rc;
320 u8 iw;
321
322 rc = kstrtouint(buf, 0, &val);
323 if (rc)
324 return rc;
325
326 rc = ways_to_cxl(val, &iw);
327 if (rc)
328 return rc;
329
330 /*
331 * Even for x3, x9, and x12 interleaves the region interleave must be a
332 * power of 2 multiple of the host bridge interleave.
333 */
334 if (!is_power_of_2(val / cxld->interleave_ways) ||
335 (val % cxld->interleave_ways)) {
336 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
337 return -EINVAL;
338 }
339
340 rc = down_write_killable(&cxl_region_rwsem);
341 if (rc)
342 return rc;
343 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
344 rc = -EBUSY;
345 goto out;
346 }
347
348 save = p->interleave_ways;
349 p->interleave_ways = val;
350 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
351 if (rc)
352 p->interleave_ways = save;
353out:
354 up_write(&cxl_region_rwsem);
355 if (rc)
356 return rc;
357 return len;
358}
359static DEVICE_ATTR_RW(interleave_ways);
360
361static ssize_t interleave_granularity_show(struct device *dev,
362 struct device_attribute *attr,
363 char *buf)
364{
365 struct cxl_region *cxlr = to_cxl_region(dev);
366 struct cxl_region_params *p = &cxlr->params;
367 ssize_t rc;
368
369 rc = down_read_interruptible(&cxl_region_rwsem);
370 if (rc)
371 return rc;
372 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
373 up_read(&cxl_region_rwsem);
374
375 return rc;
376}
377
378static ssize_t interleave_granularity_store(struct device *dev,
379 struct device_attribute *attr,
380 const char *buf, size_t len)
381{
382 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
383 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
384 struct cxl_region *cxlr = to_cxl_region(dev);
385 struct cxl_region_params *p = &cxlr->params;
386 int rc, val;
387 u16 ig;
388
389 rc = kstrtoint(buf, 0, &val);
390 if (rc)
391 return rc;
392
393 rc = granularity_to_cxl(val, &ig);
394 if (rc)
395 return rc;
396
397 /*
398 * When the host-bridge is interleaved, disallow region granularity !=
399 * root granularity. Regions with a granularity less than the root
400 * interleave result in needing multiple endpoints to support a single
401 * slot in the interleave (possible to suport in the future). Regions
402 * with a granularity greater than the root interleave result in invalid
403 * DPA translations (invalid to support).
404 */
405 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
406 return -EINVAL;
407
408 rc = down_write_killable(&cxl_region_rwsem);
409 if (rc)
410 return rc;
411 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
412 rc = -EBUSY;
413 goto out;
414 }
415
416 p->interleave_granularity = val;
417out:
418 up_write(&cxl_region_rwsem);
419 if (rc)
420 return rc;
421 return len;
422}
423static DEVICE_ATTR_RW(interleave_granularity);
424
425static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
426 char *buf)
427{
428 struct cxl_region *cxlr = to_cxl_region(dev);
429 struct cxl_region_params *p = &cxlr->params;
430 u64 resource = -1ULL;
431 ssize_t rc;
432
433 rc = down_read_interruptible(&cxl_region_rwsem);
434 if (rc)
435 return rc;
436 if (p->res)
437 resource = p->res->start;
438 rc = sysfs_emit(buf, "%#llx\n", resource);
439 up_read(&cxl_region_rwsem);
440
441 return rc;
442}
443static DEVICE_ATTR_RO(resource);
444
445static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
446{
447 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
448 struct cxl_region_params *p = &cxlr->params;
449 struct resource *res;
450 u32 remainder = 0;
451
452 lockdep_assert_held_write(&cxl_region_rwsem);
453
454 /* Nothing to do... */
455 if (p->res && resource_size(p->res) == size)
456 return 0;
457
458 /* To change size the old size must be freed first */
459 if (p->res)
460 return -EBUSY;
461
462 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
463 return -EBUSY;
464
465 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
466 if (!p->interleave_ways || !p->interleave_granularity ||
467 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
468 return -ENXIO;
469
470 div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
471 if (remainder)
472 return -EINVAL;
473
474 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
475 dev_name(&cxlr->dev));
476 if (IS_ERR(res)) {
477 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
478 PTR_ERR(res));
479 return PTR_ERR(res);
480 }
481
482 p->res = res;
483 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
484
485 return 0;
486}
487
488static void cxl_region_iomem_release(struct cxl_region *cxlr)
489{
490 struct cxl_region_params *p = &cxlr->params;
491
492 if (device_is_registered(&cxlr->dev))
493 lockdep_assert_held_write(&cxl_region_rwsem);
494 if (p->res) {
495 remove_resource(p->res);
496 kfree(p->res);
497 p->res = NULL;
498 }
499}
500
501static int free_hpa(struct cxl_region *cxlr)
502{
503 struct cxl_region_params *p = &cxlr->params;
504
505 lockdep_assert_held_write(&cxl_region_rwsem);
506
507 if (!p->res)
508 return 0;
509
510 if (p->state >= CXL_CONFIG_ACTIVE)
511 return -EBUSY;
512
513 cxl_region_iomem_release(cxlr);
514 p->state = CXL_CONFIG_IDLE;
515 return 0;
516}
517
518static ssize_t size_store(struct device *dev, struct device_attribute *attr,
519 const char *buf, size_t len)
520{
521 struct cxl_region *cxlr = to_cxl_region(dev);
522 u64 val;
523 int rc;
524
525 rc = kstrtou64(buf, 0, &val);
526 if (rc)
527 return rc;
528
529 rc = down_write_killable(&cxl_region_rwsem);
530 if (rc)
531 return rc;
532
533 if (val)
534 rc = alloc_hpa(cxlr, val);
535 else
536 rc = free_hpa(cxlr);
537 up_write(&cxl_region_rwsem);
538
539 if (rc)
540 return rc;
541
542 return len;
543}
544
545static ssize_t size_show(struct device *dev, struct device_attribute *attr,
546 char *buf)
547{
548 struct cxl_region *cxlr = to_cxl_region(dev);
549 struct cxl_region_params *p = &cxlr->params;
550 u64 size = 0;
551 ssize_t rc;
552
553 rc = down_read_interruptible(&cxl_region_rwsem);
554 if (rc)
555 return rc;
556 if (p->res)
557 size = resource_size(p->res);
558 rc = sysfs_emit(buf, "%#llx\n", size);
559 up_read(&cxl_region_rwsem);
560
561 return rc;
562}
563static DEVICE_ATTR_RW(size);
564
565static struct attribute *cxl_region_attrs[] = {
566 &dev_attr_uuid.attr,
567 &dev_attr_commit.attr,
568 &dev_attr_interleave_ways.attr,
569 &dev_attr_interleave_granularity.attr,
570 &dev_attr_resource.attr,
571 &dev_attr_size.attr,
572 NULL,
573};
574
575static const struct attribute_group cxl_region_group = {
576 .attrs = cxl_region_attrs,
577 .is_visible = cxl_region_visible,
578};
579
580static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
581{
582 struct cxl_region_params *p = &cxlr->params;
583 struct cxl_endpoint_decoder *cxled;
584 int rc;
585
586 rc = down_read_interruptible(&cxl_region_rwsem);
587 if (rc)
588 return rc;
589
590 if (pos >= p->interleave_ways) {
591 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
592 p->interleave_ways);
593 rc = -ENXIO;
594 goto out;
595 }
596
597 cxled = p->targets[pos];
598 if (!cxled)
599 rc = sysfs_emit(buf, "\n");
600 else
601 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
602out:
603 up_read(&cxl_region_rwsem);
604
605 return rc;
606}
607
608static int match_free_decoder(struct device *dev, void *data)
609{
610 struct cxl_decoder *cxld;
611 int *id = data;
612
613 if (!is_switch_decoder(dev))
614 return 0;
615
616 cxld = to_cxl_decoder(dev);
617
618 /* enforce ordered allocation */
619 if (cxld->id != *id)
620 return 0;
621
622 if (!cxld->region)
623 return 1;
624
625 (*id)++;
626
627 return 0;
628}
629
630static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
631 struct cxl_region *cxlr)
632{
633 struct device *dev;
634 int id = 0;
635
636 dev = device_find_child(&port->dev, &id, match_free_decoder);
637 if (!dev)
638 return NULL;
639 /*
640 * This decoder is pinned registered as long as the endpoint decoder is
641 * registered, and endpoint decoder unregistration holds the
642 * cxl_region_rwsem over unregister events, so no need to hold on to
643 * this extra reference.
644 */
645 put_device(dev);
646 return to_cxl_decoder(dev);
647}
648
649static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
650 struct cxl_region *cxlr)
651{
652 struct cxl_region_params *p = &cxlr->params;
653 struct cxl_region_ref *cxl_rr, *iter;
654 unsigned long index;
655 int rc;
656
657 xa_for_each(&port->regions, index, iter) {
658 struct cxl_region_params *ip = &iter->region->params;
659
660 if (ip->res->start > p->res->start) {
661 dev_dbg(&cxlr->dev,
662 "%s: HPA order violation %s:%pr vs %pr\n",
663 dev_name(&port->dev),
664 dev_name(&iter->region->dev), ip->res, p->res);
665 return ERR_PTR(-EBUSY);
666 }
667 }
668
669 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
670 if (!cxl_rr)
671 return ERR_PTR(-ENOMEM);
672 cxl_rr->port = port;
673 cxl_rr->region = cxlr;
674 cxl_rr->nr_targets = 1;
675 xa_init(&cxl_rr->endpoints);
676
677 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
678 if (rc) {
679 dev_dbg(&cxlr->dev,
680 "%s: failed to track region reference: %d\n",
681 dev_name(&port->dev), rc);
682 kfree(cxl_rr);
683 return ERR_PTR(rc);
684 }
685
686 return cxl_rr;
687}
688
689static void free_region_ref(struct cxl_region_ref *cxl_rr)
690{
691 struct cxl_port *port = cxl_rr->port;
692 struct cxl_region *cxlr = cxl_rr->region;
693 struct cxl_decoder *cxld = cxl_rr->decoder;
694
695 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
696 if (cxld->region == cxlr) {
697 cxld->region = NULL;
698 put_device(&cxlr->dev);
699 }
700
701 xa_erase(&port->regions, (unsigned long)cxlr);
702 xa_destroy(&cxl_rr->endpoints);
703 kfree(cxl_rr);
704}
705
706static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
707 struct cxl_endpoint_decoder *cxled)
708{
709 int rc;
710 struct cxl_port *port = cxl_rr->port;
711 struct cxl_region *cxlr = cxl_rr->region;
712 struct cxl_decoder *cxld = cxl_rr->decoder;
713 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
714
715 if (ep) {
716 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
717 GFP_KERNEL);
718 if (rc)
719 return rc;
720 }
721 cxl_rr->nr_eps++;
722
723 if (!cxld->region) {
724 cxld->region = cxlr;
725 get_device(&cxlr->dev);
726 }
727
728 return 0;
729}
730
731/**
732 * cxl_port_attach_region() - track a region's interest in a port by endpoint
733 * @port: port to add a new region reference 'struct cxl_region_ref'
734 * @cxlr: region to attach to @port
735 * @cxled: endpoint decoder used to create or further pin a region reference
736 * @pos: interleave position of @cxled in @cxlr
737 *
738 * The attach event is an opportunity to validate CXL decode setup
739 * constraints and record metadata needed for programming HDM decoders,
740 * in particular decoder target lists.
741 *
742 * The steps are:
743 *
744 * - validate that there are no other regions with a higher HPA already
745 * associated with @port
746 * - establish a region reference if one is not already present
747 *
748 * - additionally allocate a decoder instance that will host @cxlr on
749 * @port
750 *
751 * - pin the region reference by the endpoint
752 * - account for how many entries in @port's target list are needed to
753 * cover all of the added endpoints.
754 */
755static int cxl_port_attach_region(struct cxl_port *port,
756 struct cxl_region *cxlr,
757 struct cxl_endpoint_decoder *cxled, int pos)
758{
759 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
760 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
761 struct cxl_region_ref *cxl_rr;
762 bool nr_targets_inc = false;
763 struct cxl_decoder *cxld;
764 unsigned long index;
765 int rc = -EBUSY;
766
767 lockdep_assert_held_write(&cxl_region_rwsem);
768
769 cxl_rr = cxl_rr_load(port, cxlr);
770 if (cxl_rr) {
771 struct cxl_ep *ep_iter;
772 int found = 0;
773
774 /*
775 * Walk the existing endpoints that have been attached to
776 * @cxlr at @port and see if they share the same 'next' port
777 * in the downstream direction. I.e. endpoints that share common
778 * upstream switch.
779 */
780 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
781 if (ep_iter == ep)
782 continue;
783 if (ep_iter->next == ep->next) {
784 found++;
785 break;
786 }
787 }
788
789 /*
790 * New target port, or @port is an endpoint port that always
791 * accounts its own local decode as a target.
792 */
793 if (!found || !ep->next) {
794 cxl_rr->nr_targets++;
795 nr_targets_inc = true;
796 }
797
798 /*
799 * The decoder for @cxlr was allocated when the region was first
800 * attached to @port.
801 */
802 cxld = cxl_rr->decoder;
803 } else {
804 cxl_rr = alloc_region_ref(port, cxlr);
805 if (IS_ERR(cxl_rr)) {
806 dev_dbg(&cxlr->dev,
807 "%s: failed to allocate region reference\n",
808 dev_name(&port->dev));
809 return PTR_ERR(cxl_rr);
810 }
811 nr_targets_inc = true;
812
813 if (port == cxled_to_port(cxled))
814 cxld = &cxled->cxld;
815 else
816 cxld = cxl_region_find_decoder(port, cxlr);
817 if (!cxld) {
818 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
819 dev_name(&port->dev));
820 goto out_erase;
821 }
822
823 if (cxld->region) {
824 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
825 dev_name(&port->dev), dev_name(&cxld->dev),
826 dev_name(&cxld->region->dev));
827 rc = -EBUSY;
828 goto out_erase;
829 }
830
831 cxl_rr->decoder = cxld;
832 }
833
834 rc = cxl_rr_ep_add(cxl_rr, cxled);
835 if (rc) {
836 dev_dbg(&cxlr->dev,
837 "%s: failed to track endpoint %s:%s reference\n",
838 dev_name(&port->dev), dev_name(&cxlmd->dev),
839 dev_name(&cxld->dev));
840 goto out_erase;
841 }
842
843 dev_dbg(&cxlr->dev,
844 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
845 dev_name(port->uport), dev_name(&port->dev),
846 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
847 dev_name(&cxled->cxld.dev), pos,
848 ep ? ep->next ? dev_name(ep->next->uport) :
849 dev_name(&cxlmd->dev) :
850 "none",
851 cxl_rr->nr_eps, cxl_rr->nr_targets);
852
853 return 0;
854out_erase:
855 if (nr_targets_inc)
856 cxl_rr->nr_targets--;
857 if (cxl_rr->nr_eps == 0)
858 free_region_ref(cxl_rr);
859 return rc;
860}
861
862static void cxl_port_detach_region(struct cxl_port *port,
863 struct cxl_region *cxlr,
864 struct cxl_endpoint_decoder *cxled)
865{
866 struct cxl_region_ref *cxl_rr;
867 struct cxl_ep *ep = NULL;
868
869 lockdep_assert_held_write(&cxl_region_rwsem);
870
871 cxl_rr = cxl_rr_load(port, cxlr);
872 if (!cxl_rr)
873 return;
874
875 /*
876 * Endpoint ports do not carry cxl_ep references, and they
877 * never target more than one endpoint by definition
878 */
879 if (cxl_rr->decoder == &cxled->cxld)
880 cxl_rr->nr_eps--;
881 else
882 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
883 if (ep) {
884 struct cxl_ep *ep_iter;
885 unsigned long index;
886 int found = 0;
887
888 cxl_rr->nr_eps--;
889 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
890 if (ep_iter->next == ep->next) {
891 found++;
892 break;
893 }
894 }
895 if (!found)
896 cxl_rr->nr_targets--;
897 }
898
899 if (cxl_rr->nr_eps == 0)
900 free_region_ref(cxl_rr);
901}
902
903static int check_last_peer(struct cxl_endpoint_decoder *cxled,
904 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
905 int distance)
906{
907 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
908 struct cxl_region *cxlr = cxl_rr->region;
909 struct cxl_region_params *p = &cxlr->params;
910 struct cxl_endpoint_decoder *cxled_peer;
911 struct cxl_port *port = cxl_rr->port;
912 struct cxl_memdev *cxlmd_peer;
913 struct cxl_ep *ep_peer;
914 int pos = cxled->pos;
915
916 /*
917 * If this position wants to share a dport with the last endpoint mapped
918 * then that endpoint, at index 'position - distance', must also be
919 * mapped by this dport.
920 */
921 if (pos < distance) {
922 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
923 dev_name(port->uport), dev_name(&port->dev),
924 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
925 return -ENXIO;
926 }
927 cxled_peer = p->targets[pos - distance];
928 cxlmd_peer = cxled_to_memdev(cxled_peer);
929 ep_peer = cxl_ep_load(port, cxlmd_peer);
930 if (ep->dport != ep_peer->dport) {
931 dev_dbg(&cxlr->dev,
932 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
933 dev_name(port->uport), dev_name(&port->dev),
934 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
935 dev_name(&cxlmd_peer->dev),
936 dev_name(&cxled_peer->cxld.dev));
937 return -ENXIO;
938 }
939
940 return 0;
941}
942
943static int cxl_port_setup_targets(struct cxl_port *port,
944 struct cxl_region *cxlr,
945 struct cxl_endpoint_decoder *cxled)
946{
947 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
948 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
949 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
950 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
951 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
952 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
953 struct cxl_region_params *p = &cxlr->params;
954 struct cxl_decoder *cxld = cxl_rr->decoder;
955 struct cxl_switch_decoder *cxlsd;
956 u16 eig, peig;
957 u8 eiw, peiw;
958
959 /*
960 * While root level decoders support x3, x6, x12, switch level
961 * decoders only support powers of 2 up to x16.
962 */
963 if (!is_power_of_2(cxl_rr->nr_targets)) {
964 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
965 dev_name(port->uport), dev_name(&port->dev),
966 cxl_rr->nr_targets);
967 return -EINVAL;
968 }
969
970 cxlsd = to_cxl_switch_decoder(&cxld->dev);
971 if (cxl_rr->nr_targets_set) {
972 int i, distance;
973
974 distance = p->nr_targets / cxl_rr->nr_targets;
975 for (i = 0; i < cxl_rr->nr_targets_set; i++)
976 if (ep->dport == cxlsd->target[i]) {
977 rc = check_last_peer(cxled, ep, cxl_rr,
978 distance);
979 if (rc)
980 return rc;
981 goto out_target_set;
982 }
983 goto add_target;
984 }
985
986 if (is_cxl_root(parent_port)) {
987 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
988 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
989 /*
990 * For purposes of address bit routing, use power-of-2 math for
991 * switch ports.
992 */
993 if (!is_power_of_2(parent_iw))
994 parent_iw /= 3;
995 } else {
996 struct cxl_region_ref *parent_rr;
997 struct cxl_decoder *parent_cxld;
998
999 parent_rr = cxl_rr_load(parent_port, cxlr);
1000 parent_cxld = parent_rr->decoder;
1001 parent_ig = parent_cxld->interleave_granularity;
1002 parent_iw = parent_cxld->interleave_ways;
1003 }
1004
1005 rc = granularity_to_cxl(parent_ig, &peig);
1006 if (rc) {
1007 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1008 dev_name(parent_port->uport),
1009 dev_name(&parent_port->dev), parent_ig);
1010 return rc;
1011 }
1012
1013 rc = ways_to_cxl(parent_iw, &peiw);
1014 if (rc) {
1015 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1016 dev_name(parent_port->uport),
1017 dev_name(&parent_port->dev), parent_iw);
1018 return rc;
1019 }
1020
1021 iw = cxl_rr->nr_targets;
1022 rc = ways_to_cxl(iw, &eiw);
1023 if (rc) {
1024 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1025 dev_name(port->uport), dev_name(&port->dev), iw);
1026 return rc;
1027 }
1028
1029 /*
1030 * If @parent_port is masking address bits, pick the next unused address
1031 * bit to route @port's targets.
1032 */
1033 if (parent_iw > 1 && cxl_rr->nr_targets > 1) {
1034 u32 address_bit = max(peig + peiw, eiw + peig);
1035
1036 eig = address_bit - eiw + 1;
1037 } else {
1038 eiw = peiw;
1039 eig = peig;
1040 }
1041
1042 rc = cxl_to_granularity(eig, &ig);
1043 if (rc) {
1044 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1045 dev_name(port->uport), dev_name(&port->dev),
1046 256 << eig);
1047 return rc;
1048 }
1049
1050 cxld->interleave_ways = iw;
1051 cxld->interleave_granularity = ig;
1052 cxld->hpa_range = (struct range) {
1053 .start = p->res->start,
1054 .end = p->res->end,
1055 };
1056 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport),
1057 dev_name(&port->dev), iw, ig);
1058add_target:
1059 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1060 dev_dbg(&cxlr->dev,
1061 "%s:%s: targets full trying to add %s:%s at %d\n",
1062 dev_name(port->uport), dev_name(&port->dev),
1063 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1064 return -ENXIO;
1065 }
1066 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1067 inc = 1;
1068out_target_set:
1069 cxl_rr->nr_targets_set += inc;
1070 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1071 dev_name(port->uport), dev_name(&port->dev),
1072 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport),
1073 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1074
1075 return 0;
1076}
1077
1078static void cxl_port_reset_targets(struct cxl_port *port,
1079 struct cxl_region *cxlr)
1080{
1081 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1082 struct cxl_decoder *cxld;
1083
1084 /*
1085 * After the last endpoint has been detached the entire cxl_rr may now
1086 * be gone.
1087 */
1088 if (!cxl_rr)
1089 return;
1090 cxl_rr->nr_targets_set = 0;
1091
1092 cxld = cxl_rr->decoder;
1093 cxld->hpa_range = (struct range) {
1094 .start = 0,
1095 .end = -1,
1096 };
1097}
1098
1099static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1100{
1101 struct cxl_region_params *p = &cxlr->params;
1102 struct cxl_endpoint_decoder *cxled;
1103 struct cxl_memdev *cxlmd;
1104 struct cxl_port *iter;
1105 struct cxl_ep *ep;
1106 int i;
1107
1108 for (i = 0; i < p->nr_targets; i++) {
1109 cxled = p->targets[i];
1110 cxlmd = cxled_to_memdev(cxled);
1111
1112 iter = cxled_to_port(cxled);
1113 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1114 iter = to_cxl_port(iter->dev.parent);
1115
1116 for (ep = cxl_ep_load(iter, cxlmd); iter;
1117 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1118 cxl_port_reset_targets(iter, cxlr);
1119 }
1120}
1121
1122static int cxl_region_setup_targets(struct cxl_region *cxlr)
1123{
1124 struct cxl_region_params *p = &cxlr->params;
1125 struct cxl_endpoint_decoder *cxled;
1126 struct cxl_memdev *cxlmd;
1127 struct cxl_port *iter;
1128 struct cxl_ep *ep;
1129 int i, rc;
1130
1131 for (i = 0; i < p->nr_targets; i++) {
1132 cxled = p->targets[i];
1133 cxlmd = cxled_to_memdev(cxled);
1134
1135 iter = cxled_to_port(cxled);
1136 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1137 iter = to_cxl_port(iter->dev.parent);
1138
1139 /*
1140 * Descend the topology tree programming targets while
1141 * looking for conflicts.
1142 */
1143 for (ep = cxl_ep_load(iter, cxlmd); iter;
1144 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1145 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1146 if (rc) {
1147 cxl_region_teardown_targets(cxlr);
1148 return rc;
1149 }
1150 }
1151 }
1152
1153 return 0;
1154}
1155
1156static int cxl_region_attach(struct cxl_region *cxlr,
1157 struct cxl_endpoint_decoder *cxled, int pos)
1158{
1159 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1160 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1161 struct cxl_port *ep_port, *root_port, *iter;
1162 struct cxl_region_params *p = &cxlr->params;
1163 struct cxl_dport *dport;
1164 int i, rc = -ENXIO;
1165
1166 if (cxled->mode == CXL_DECODER_DEAD) {
1167 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1168 return -ENODEV;
1169 }
1170
1171 /* all full of members, or interleave config not established? */
1172 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1173 dev_dbg(&cxlr->dev, "region already active\n");
1174 return -EBUSY;
1175 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1176 dev_dbg(&cxlr->dev, "interleave config missing\n");
1177 return -ENXIO;
1178 }
1179
1180 if (pos < 0 || pos >= p->interleave_ways) {
1181 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1182 p->interleave_ways);
1183 return -ENXIO;
1184 }
1185
1186 if (p->targets[pos] == cxled)
1187 return 0;
1188
1189 if (p->targets[pos]) {
1190 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1191 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1192
1193 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1194 pos, dev_name(&cxlmd_target->dev),
1195 dev_name(&cxled_target->cxld.dev));
1196 return -EBUSY;
1197 }
1198
1199 for (i = 0; i < p->interleave_ways; i++) {
1200 struct cxl_endpoint_decoder *cxled_target;
1201 struct cxl_memdev *cxlmd_target;
1202
1203 cxled_target = p->targets[pos];
1204 if (!cxled_target)
1205 continue;
1206
1207 cxlmd_target = cxled_to_memdev(cxled_target);
1208 if (cxlmd_target == cxlmd) {
1209 dev_dbg(&cxlr->dev,
1210 "%s already specified at position %d via: %s\n",
1211 dev_name(&cxlmd->dev), pos,
1212 dev_name(&cxled_target->cxld.dev));
1213 return -EBUSY;
1214 }
1215 }
1216
1217 ep_port = cxled_to_port(cxled);
1218 root_port = cxlrd_to_port(cxlrd);
1219 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1220 if (!dport) {
1221 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1222 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1223 dev_name(cxlr->dev.parent));
1224 return -ENXIO;
1225 }
1226
1227 if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1228 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1229 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1230 dev_name(&cxlrd->cxlsd.cxld.dev));
1231 return -ENXIO;
1232 }
1233
1234 if (cxled->cxld.target_type != cxlr->type) {
1235 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1236 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1237 cxled->cxld.target_type, cxlr->type);
1238 return -ENXIO;
1239 }
1240
1241 if (!cxled->dpa_res) {
1242 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1243 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1244 return -ENXIO;
1245 }
1246
1247 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1248 resource_size(p->res)) {
1249 dev_dbg(&cxlr->dev,
1250 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1251 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1252 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1253 (u64)resource_size(p->res));
1254 return -EINVAL;
1255 }
1256
1257 for (iter = ep_port; !is_cxl_root(iter);
1258 iter = to_cxl_port(iter->dev.parent)) {
1259 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1260 if (rc)
1261 goto err;
1262 }
1263
1264 p->targets[pos] = cxled;
1265 cxled->pos = pos;
1266 p->nr_targets++;
1267
1268 if (p->nr_targets == p->interleave_ways) {
1269 rc = cxl_region_setup_targets(cxlr);
1270 if (rc)
1271 goto err_decrement;
1272 p->state = CXL_CONFIG_ACTIVE;
1273 }
1274
1275 cxled->cxld.interleave_ways = p->interleave_ways;
1276 cxled->cxld.interleave_granularity = p->interleave_granularity;
1277 cxled->cxld.hpa_range = (struct range) {
1278 .start = p->res->start,
1279 .end = p->res->end,
1280 };
1281
1282 return 0;
1283
1284err_decrement:
1285 p->nr_targets--;
1286err:
1287 for (iter = ep_port; !is_cxl_root(iter);
1288 iter = to_cxl_port(iter->dev.parent))
1289 cxl_port_detach_region(iter, cxlr, cxled);
1290 return rc;
1291}
1292
1293static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1294{
1295 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1296 struct cxl_region *cxlr = cxled->cxld.region;
1297 struct cxl_region_params *p;
1298 int rc = 0;
1299
1300 lockdep_assert_held_write(&cxl_region_rwsem);
1301
1302 if (!cxlr)
1303 return 0;
1304
1305 p = &cxlr->params;
1306 get_device(&cxlr->dev);
1307
1308 if (p->state > CXL_CONFIG_ACTIVE) {
1309 /*
1310 * TODO: tear down all impacted regions if a device is
1311 * removed out of order
1312 */
1313 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1314 if (rc)
1315 goto out;
1316 p->state = CXL_CONFIG_ACTIVE;
1317 }
1318
1319 for (iter = ep_port; !is_cxl_root(iter);
1320 iter = to_cxl_port(iter->dev.parent))
1321 cxl_port_detach_region(iter, cxlr, cxled);
1322
1323 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1324 p->targets[cxled->pos] != cxled) {
1325 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1326
1327 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1328 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1329 cxled->pos);
1330 goto out;
1331 }
1332
1333 if (p->state == CXL_CONFIG_ACTIVE) {
1334 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1335 cxl_region_teardown_targets(cxlr);
1336 }
1337 p->targets[cxled->pos] = NULL;
1338 p->nr_targets--;
1339 cxled->cxld.hpa_range = (struct range) {
1340 .start = 0,
1341 .end = -1,
1342 };
1343
1344 /* notify the region driver that one of its targets has departed */
1345 up_write(&cxl_region_rwsem);
1346 device_release_driver(&cxlr->dev);
1347 down_write(&cxl_region_rwsem);
1348out:
1349 put_device(&cxlr->dev);
1350 return rc;
1351}
1352
1353void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1354{
1355 down_write(&cxl_region_rwsem);
1356 cxled->mode = CXL_DECODER_DEAD;
1357 cxl_region_detach(cxled);
1358 up_write(&cxl_region_rwsem);
1359}
1360
1361static int attach_target(struct cxl_region *cxlr, const char *decoder, int pos)
1362{
1363 struct device *dev;
1364 int rc;
1365
1366 dev = bus_find_device_by_name(&cxl_bus_type, NULL, decoder);
1367 if (!dev)
1368 return -ENODEV;
1369
1370 if (!is_endpoint_decoder(dev)) {
1371 put_device(dev);
1372 return -EINVAL;
1373 }
1374
1375 rc = down_write_killable(&cxl_region_rwsem);
1376 if (rc)
1377 goto out;
1378 down_read(&cxl_dpa_rwsem);
1379 rc = cxl_region_attach(cxlr, to_cxl_endpoint_decoder(dev), pos);
1380 up_read(&cxl_dpa_rwsem);
1381 up_write(&cxl_region_rwsem);
1382out:
1383 put_device(dev);
1384 return rc;
1385}
1386
1387static int detach_target(struct cxl_region *cxlr, int pos)
1388{
1389 struct cxl_region_params *p = &cxlr->params;
1390 int rc;
1391
1392 rc = down_write_killable(&cxl_region_rwsem);
1393 if (rc)
1394 return rc;
1395
1396 if (pos >= p->interleave_ways) {
1397 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1398 p->interleave_ways);
1399 rc = -ENXIO;
1400 goto out;
1401 }
1402
1403 if (!p->targets[pos]) {
1404 rc = 0;
1405 goto out;
1406 }
1407
1408 rc = cxl_region_detach(p->targets[pos]);
1409out:
1410 up_write(&cxl_region_rwsem);
1411 return rc;
1412}
1413
1414static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1415 size_t len)
1416{
1417 int rc;
1418
1419 if (sysfs_streq(buf, "\n"))
1420 rc = detach_target(cxlr, pos);
1421 else
1422 rc = attach_target(cxlr, buf, pos);
1423
1424 if (rc < 0)
1425 return rc;
1426 return len;
1427}
1428
1429#define TARGET_ATTR_RW(n) \
1430static ssize_t target##n##_show( \
1431 struct device *dev, struct device_attribute *attr, char *buf) \
1432{ \
1433 return show_targetN(to_cxl_region(dev), buf, (n)); \
1434} \
1435static ssize_t target##n##_store(struct device *dev, \
1436 struct device_attribute *attr, \
1437 const char *buf, size_t len) \
1438{ \
1439 return store_targetN(to_cxl_region(dev), buf, (n), len); \
1440} \
1441static DEVICE_ATTR_RW(target##n)
1442
1443TARGET_ATTR_RW(0);
1444TARGET_ATTR_RW(1);
1445TARGET_ATTR_RW(2);
1446TARGET_ATTR_RW(3);
1447TARGET_ATTR_RW(4);
1448TARGET_ATTR_RW(5);
1449TARGET_ATTR_RW(6);
1450TARGET_ATTR_RW(7);
1451TARGET_ATTR_RW(8);
1452TARGET_ATTR_RW(9);
1453TARGET_ATTR_RW(10);
1454TARGET_ATTR_RW(11);
1455TARGET_ATTR_RW(12);
1456TARGET_ATTR_RW(13);
1457TARGET_ATTR_RW(14);
1458TARGET_ATTR_RW(15);
1459
1460static struct attribute *target_attrs[] = {
1461 &dev_attr_target0.attr,
1462 &dev_attr_target1.attr,
1463 &dev_attr_target2.attr,
1464 &dev_attr_target3.attr,
1465 &dev_attr_target4.attr,
1466 &dev_attr_target5.attr,
1467 &dev_attr_target6.attr,
1468 &dev_attr_target7.attr,
1469 &dev_attr_target8.attr,
1470 &dev_attr_target9.attr,
1471 &dev_attr_target10.attr,
1472 &dev_attr_target11.attr,
1473 &dev_attr_target12.attr,
1474 &dev_attr_target13.attr,
1475 &dev_attr_target14.attr,
1476 &dev_attr_target15.attr,
1477 NULL,
1478};
1479
1480static umode_t cxl_region_target_visible(struct kobject *kobj,
1481 struct attribute *a, int n)
1482{
1483 struct device *dev = kobj_to_dev(kobj);
1484 struct cxl_region *cxlr = to_cxl_region(dev);
1485 struct cxl_region_params *p = &cxlr->params;
1486
1487 if (n < p->interleave_ways)
1488 return a->mode;
1489 return 0;
1490}
1491
1492static const struct attribute_group cxl_region_target_group = {
1493 .attrs = target_attrs,
1494 .is_visible = cxl_region_target_visible,
1495};
1496
1497static const struct attribute_group *get_cxl_region_target_group(void)
1498{
1499 return &cxl_region_target_group;
1500}
1501
1502static const struct attribute_group *region_groups[] = {
1503 &cxl_base_attribute_group,
1504 &cxl_region_group,
1505 &cxl_region_target_group,
1506 NULL,
1507};
1508
1509static void cxl_region_release(struct device *dev)
1510{
1511 struct cxl_region *cxlr = to_cxl_region(dev);
1512
1513 memregion_free(cxlr->id);
1514 kfree(cxlr);
1515}
1516
1517const struct device_type cxl_region_type = {
1518 .name = "cxl_region",
1519 .release = cxl_region_release,
1520 .groups = region_groups
1521};
1522
1523bool is_cxl_region(struct device *dev)
1524{
1525 return dev->type == &cxl_region_type;
1526}
1527EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
1528
1529static struct cxl_region *to_cxl_region(struct device *dev)
1530{
1531 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
1532 "not a cxl_region device\n"))
1533 return NULL;
1534
1535 return container_of(dev, struct cxl_region, dev);
1536}
1537
1538static void unregister_region(void *dev)
1539{
1540 struct cxl_region *cxlr = to_cxl_region(dev);
1541
1542 device_del(dev);
1543 cxl_region_iomem_release(cxlr);
1544 put_device(dev);
1545}
1546
1547static struct lock_class_key cxl_region_key;
1548
1549static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
1550{
1551 struct cxl_region *cxlr;
1552 struct device *dev;
1553
1554 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
1555 if (!cxlr) {
1556 memregion_free(id);
1557 return ERR_PTR(-ENOMEM);
1558 }
1559
1560 dev = &cxlr->dev;
1561 device_initialize(dev);
1562 lockdep_set_class(&dev->mutex, &cxl_region_key);
1563 dev->parent = &cxlrd->cxlsd.cxld.dev;
1564 device_set_pm_not_required(dev);
1565 dev->bus = &cxl_bus_type;
1566 dev->type = &cxl_region_type;
1567 cxlr->id = id;
1568
1569 return cxlr;
1570}
1571
1572/**
1573 * devm_cxl_add_region - Adds a region to a decoder
1574 * @cxlrd: root decoder
1575 * @id: memregion id to create, or memregion_free() on failure
1576 * @mode: mode for the endpoint decoders of this region
1577 * @type: select whether this is an expander or accelerator (type-2 or type-3)
1578 *
1579 * This is the second step of region initialization. Regions exist within an
1580 * address space which is mapped by a @cxlrd.
1581 *
1582 * Return: 0 if the region was added to the @cxlrd, else returns negative error
1583 * code. The region will be named "regionZ" where Z is the unique region number.
1584 */
1585static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
1586 int id,
1587 enum cxl_decoder_mode mode,
1588 enum cxl_decoder_type type)
1589{
1590 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
1591 struct cxl_region *cxlr;
1592 struct device *dev;
1593 int rc;
1594
1595 cxlr = cxl_region_alloc(cxlrd, id);
1596 if (IS_ERR(cxlr))
1597 return cxlr;
1598 cxlr->mode = mode;
1599 cxlr->type = type;
1600
1601 dev = &cxlr->dev;
1602 rc = dev_set_name(dev, "region%d", id);
1603 if (rc)
1604 goto err;
1605
1606 rc = device_add(dev);
1607 if (rc)
1608 goto err;
1609
1610 rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr);
1611 if (rc)
1612 return ERR_PTR(rc);
1613
1614 dev_dbg(port->uport, "%s: created %s\n",
1615 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
1616 return cxlr;
1617
1618err:
1619 put_device(dev);
1620 return ERR_PTR(rc);
1621}
1622
1623static ssize_t create_pmem_region_show(struct device *dev,
1624 struct device_attribute *attr, char *buf)
1625{
1626 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
1627
1628 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
1629}
1630
1631static ssize_t create_pmem_region_store(struct device *dev,
1632 struct device_attribute *attr,
1633 const char *buf, size_t len)
1634{
1635 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
1636 struct cxl_region *cxlr;
1637 int id, rc;
1638
1639 rc = sscanf(buf, "region%d\n", &id);
1640 if (rc != 1)
1641 return -EINVAL;
1642
1643 rc = memregion_alloc(GFP_KERNEL);
1644 if (rc < 0)
1645 return rc;
1646
1647 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
1648 memregion_free(rc);
1649 return -EBUSY;
1650 }
1651
1652 cxlr = devm_cxl_add_region(cxlrd, id, CXL_DECODER_PMEM,
1653 CXL_DECODER_EXPANDER);
1654 if (IS_ERR(cxlr))
1655 return PTR_ERR(cxlr);
1656
1657 return len;
1658}
1659DEVICE_ATTR_RW(create_pmem_region);
1660
1661static ssize_t region_show(struct device *dev, struct device_attribute *attr,
1662 char *buf)
1663{
1664 struct cxl_decoder *cxld = to_cxl_decoder(dev);
1665 ssize_t rc;
1666
1667 rc = down_read_interruptible(&cxl_region_rwsem);
1668 if (rc)
1669 return rc;
1670
1671 if (cxld->region)
1672 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
1673 else
1674 rc = sysfs_emit(buf, "\n");
1675 up_read(&cxl_region_rwsem);
1676
1677 return rc;
1678}
1679DEVICE_ATTR_RO(region);
1680
1681static struct cxl_region *
1682cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
1683{
1684 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
1685 struct device *region_dev;
1686
1687 region_dev = device_find_child_by_name(&cxld->dev, name);
1688 if (!region_dev)
1689 return ERR_PTR(-ENODEV);
1690
1691 return to_cxl_region(region_dev);
1692}
1693
1694static ssize_t delete_region_store(struct device *dev,
1695 struct device_attribute *attr,
1696 const char *buf, size_t len)
1697{
1698 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
1699 struct cxl_port *port = to_cxl_port(dev->parent);
1700 struct cxl_region *cxlr;
1701
1702 cxlr = cxl_find_region_by_name(cxlrd, buf);
1703 if (IS_ERR(cxlr))
1704 return PTR_ERR(cxlr);
1705
1706 devm_release_action(port->uport, unregister_region, cxlr);
1707 put_device(&cxlr->dev);
1708
1709 return len;
1710}
1711DEVICE_ATTR_WO(delete_region);
1712
1713static void cxl_pmem_region_release(struct device *dev)
1714{
1715 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
1716 int i;
1717
1718 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
1719 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
1720
1721 put_device(&cxlmd->dev);
1722 }
1723
1724 kfree(cxlr_pmem);
1725}
1726
1727static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
1728 &cxl_base_attribute_group,
1729 NULL,
1730};
1731
1732const struct device_type cxl_pmem_region_type = {
1733 .name = "cxl_pmem_region",
1734 .release = cxl_pmem_region_release,
1735 .groups = cxl_pmem_region_attribute_groups,
1736};
1737
1738bool is_cxl_pmem_region(struct device *dev)
1739{
1740 return dev->type == &cxl_pmem_region_type;
1741}
1742EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
1743
1744struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
1745{
1746 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
1747 "not a cxl_pmem_region device\n"))
1748 return NULL;
1749 return container_of(dev, struct cxl_pmem_region, dev);
1750}
1751EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
1752
1753static struct lock_class_key cxl_pmem_region_key;
1754
1755static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
1756{
1757 struct cxl_region_params *p = &cxlr->params;
1758 struct cxl_pmem_region *cxlr_pmem;
1759 struct device *dev;
1760 int i;
1761
1762 down_read(&cxl_region_rwsem);
1763 if (p->state != CXL_CONFIG_COMMIT) {
1764 cxlr_pmem = ERR_PTR(-ENXIO);
1765 goto out;
1766 }
1767
1768 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
1769 GFP_KERNEL);
1770 if (!cxlr_pmem) {
1771 cxlr_pmem = ERR_PTR(-ENOMEM);
1772 goto out;
1773 }
1774
1775 cxlr_pmem->hpa_range.start = p->res->start;
1776 cxlr_pmem->hpa_range.end = p->res->end;
1777
1778 /* Snapshot the region configuration underneath the cxl_region_rwsem */
1779 cxlr_pmem->nr_mappings = p->nr_targets;
1780 for (i = 0; i < p->nr_targets; i++) {
1781 struct cxl_endpoint_decoder *cxled = p->targets[i];
1782 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1783 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
1784
1785 m->cxlmd = cxlmd;
1786 get_device(&cxlmd->dev);
1787 m->start = cxled->dpa_res->start;
1788 m->size = resource_size(cxled->dpa_res);
1789 m->position = i;
1790 }
1791
1792 dev = &cxlr_pmem->dev;
1793 cxlr_pmem->cxlr = cxlr;
1794 device_initialize(dev);
1795 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
1796 device_set_pm_not_required(dev);
1797 dev->parent = &cxlr->dev;
1798 dev->bus = &cxl_bus_type;
1799 dev->type = &cxl_pmem_region_type;
1800out:
1801 up_read(&cxl_region_rwsem);
1802
1803 return cxlr_pmem;
1804}
1805
1806static void cxlr_pmem_unregister(void *dev)
1807{
1808 device_unregister(dev);
1809}
1810
1811/**
1812 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
1813 * @cxlr: parent CXL region for this pmem region bridge device
1814 *
1815 * Return: 0 on success negative error code on failure.
1816 */
1817static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
1818{
1819 struct cxl_pmem_region *cxlr_pmem;
1820 struct device *dev;
1821 int rc;
1822
1823 cxlr_pmem = cxl_pmem_region_alloc(cxlr);
1824 if (IS_ERR(cxlr_pmem))
1825 return PTR_ERR(cxlr_pmem);
1826
1827 dev = &cxlr_pmem->dev;
1828 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
1829 if (rc)
1830 goto err;
1831
1832 rc = device_add(dev);
1833 if (rc)
1834 goto err;
1835
1836 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
1837 dev_name(dev));
1838
1839 return devm_add_action_or_reset(&cxlr->dev, cxlr_pmem_unregister, dev);
1840
1841err:
1842 put_device(dev);
1843 return rc;
1844}
1845
1846static int cxl_region_probe(struct device *dev)
1847{
1848 struct cxl_region *cxlr = to_cxl_region(dev);
1849 struct cxl_region_params *p = &cxlr->params;
1850 int rc;
1851
1852 rc = down_read_interruptible(&cxl_region_rwsem);
1853 if (rc) {
1854 dev_dbg(&cxlr->dev, "probe interrupted\n");
1855 return rc;
1856 }
1857
1858 if (p->state < CXL_CONFIG_COMMIT) {
1859 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
1860 rc = -ENXIO;
1861 }
1862
1863 /*
1864 * From this point on any path that changes the region's state away from
1865 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
1866 */
1867 up_read(&cxl_region_rwsem);
1868
1869 switch (cxlr->mode) {
1870 case CXL_DECODER_PMEM:
1871 return devm_cxl_add_pmem_region(cxlr);
1872 default:
1873 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
1874 cxlr->mode);
1875 return -ENXIO;
1876 }
1877}
1878
1879static struct cxl_driver cxl_region_driver = {
1880 .name = "cxl_region",
1881 .probe = cxl_region_probe,
1882 .id = CXL_DEVICE_REGION,
1883};
1884
1885int cxl_region_init(void)
1886{
1887 return cxl_driver_register(&cxl_region_driver);
1888}
1889
1890void cxl_region_exit(void)
1891{
1892 cxl_driver_unregister(&cxl_region_driver);
1893}
1894
1895MODULE_IMPORT_NS(CXL);
1896MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
drivers/cxl/cxl.h+283−29
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include <linux/libnvdimm.h>7#include <linux/libnvdimm.h>
8#include <linux/bitfield.h>8#include <linux/bitfield.h>
9#include <linux/bitops.h>9#include <linux/bitops.h>
10#include <linux/log2.h>
10#include <linux/io.h>11#include <linux/io.h>
11 12
12/**13/**
@@ -53,9 +54,12 @@ · 40 unmodified lines
53#define CXL_HDM_DECODER0_CTRL_LOCK BIT(8)54#define CXL_HDM_DECODER0_CTRL_LOCK BIT(8)
54#define CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)55#define CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)
55#define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)56#define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
57#define CXL_HDM_DECODER0_CTRL_COMMIT_ERROR BIT(11)
56#define CXL_HDM_DECODER0_CTRL_TYPE BIT(12)58#define CXL_HDM_DECODER0_CTRL_TYPE BIT(12)
57#define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)59#define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
58#define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)60#define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
61#define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)
62#define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i)
59 63
60static inline int cxl_hdm_decoder_count(u32 cap_hdr)64static inline int cxl_hdm_decoder_count(u32 cap_hdr)
61{65{
@@ -64,6 +68,57 @@ static inline int cxl_hdm_decoder_count(u32 cap_hdr) · 2 unmodified lines
64 return val ? val * 2 : 1;68 return val ? val * 2 : 1;
65}69}
66 70
71/* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */
72static inline int cxl_to_granularity(u16 ig, unsigned int *val)
73{
74 if (ig > 6)
75 return -EINVAL;
76 *val = 256 << ig;
77 return 0;
78}
79
80/* Encode defined in CXL ECN "3, 6, 12 and 16-way memory Interleaving" */
81static inline int cxl_to_ways(u8 eniw, unsigned int *val)
82{
83 switch (eniw) {
84 case 0 ... 4:
85 *val = 1 << eniw;
86 break;
87 case 8 ... 10:
88 *val = 3 << (eniw - 8);
89 break;
90 default:
91 return -EINVAL;
92 }
93
94 return 0;
95}
96
97static inline int granularity_to_cxl(int g, u16 *ig)
98{
99 if (g > SZ_16K || g < 256 || !is_power_of_2(g))
100 return -EINVAL;
101 *ig = ilog2(g) - 8;
102 return 0;
103}
104
105static inline int ways_to_cxl(unsigned int ways, u8 *iw)
106{
107 if (ways > 16)
108 return -EINVAL;
109 if (is_power_of_2(ways)) {
110 *iw = ilog2(ways);
111 return 0;
112 }
113 if (ways % 3)
114 return -EINVAL;
115 ways /= 3;
116 if (!is_power_of_2(ways))
117 return -EINVAL;
118 *iw = ilog2(ways) + 8;
119 return 0;
120}
121
67/* CXL 2.0 8.2.8.1 Device Capabilities Array Register */122/* CXL 2.0 8.2.8.1 Device Capabilities Array Register */
68#define CXLDEV_CAP_ARRAY_OFFSET 0x0123#define CXLDEV_CAP_ARRAY_OFFSET 0x0
69#define CXLDEV_CAP_ARRAY_CAP_ID 0124#define CXLDEV_CAP_ARRAY_CAP_ID 0
@@ -193,37 +248,153 @@ enum cxl_decoder_type { · 123 unmodified lines
193 */248 */
194#define CXL_DECODER_MAX_INTERLEAVE 16249#define CXL_DECODER_MAX_INTERLEAVE 16
195 250
251#define CXL_DECODER_MIN_GRANULARITY 256
252
196/**253/**
197 * struct cxl_decoder - CXL address range decode configuration254 * struct cxl_decoder - Common CXL HDM Decoder Attributes
198 * @dev: this decoder's device255 * @dev: this decoder's device
199 * @id: kernel device name id256 * @id: kernel device name id
200 * @platform_res: address space resources considered by root decoder257 * @hpa_range: Host physical address range mapped by this decoder
201 * @decoder_range: address space resources considered by midlevel decoder
202 * @interleave_ways: number of cxl_dports in this decode258 * @interleave_ways: number of cxl_dports in this decode
203 * @interleave_granularity: data stride per dport259 * @interleave_granularity: data stride per dport
204 * @target_type: accelerator vs expander (type2 vs type3) selector260 * @target_type: accelerator vs expander (type2 vs type3) selector
261 * @region: currently assigned region for this decoder
205 * @flags: memory type capabilities and locking262 * @flags: memory type capabilities and locking
206 * @target_lock: coordinate coherent reads of the target list263 * @commit: device/decoder-type specific callback to commit settings to hw
207 * @nr_targets: number of elements in @target264 * @reset: device/decoder-type specific callback to reset hw settings
208 * @target: active ordered target list in current decoder configuration265*/
209 */
210struct cxl_decoder {266struct cxl_decoder {
211 struct device dev;267 struct device dev;
212 int id;268 int id;
213 union {269 struct range hpa_range;
214 struct resource platform_res;
215 struct range decoder_range;
216 };
217 int interleave_ways;270 int interleave_ways;
218 int interleave_granularity;271 int interleave_granularity;
219 enum cxl_decoder_type target_type;272 enum cxl_decoder_type target_type;
273 struct cxl_region *region;
220 unsigned long flags;274 unsigned long flags;
275 int (*commit)(struct cxl_decoder *cxld);
276 int (*reset)(struct cxl_decoder *cxld);
277};
278
279/*
280 * CXL_DECODER_DEAD prevents endpoints from being reattached to regions
281 * while cxld_unregister() is running
282 */
283enum cxl_decoder_mode {
284 CXL_DECODER_NONE,
285 CXL_DECODER_RAM,
286 CXL_DECODER_PMEM,
287 CXL_DECODER_MIXED,
288 CXL_DECODER_DEAD,
289};
290
291/**
292 * struct cxl_endpoint_decoder - Endpoint / SPA to DPA decoder
293 * @cxld: base cxl_decoder_object
294 * @dpa_res: actively claimed DPA span of this decoder
295 * @skip: offset into @dpa_res where @cxld.hpa_range maps
296 * @mode: which memory type / access-mode-partition this decoder targets
297 * @pos: interleave position in @cxld.region
298 */
299struct cxl_endpoint_decoder {
300 struct cxl_decoder cxld;
301 struct resource *dpa_res;
302 resource_size_t skip;
303 enum cxl_decoder_mode mode;
304 int pos;
305};
306
307/**
308 * struct cxl_switch_decoder - Switch specific CXL HDM Decoder
309 * @cxld: base cxl_decoder object
310 * @target_lock: coordinate coherent reads of the target list
311 * @nr_targets: number of elements in @target
312 * @target: active ordered target list in current decoder configuration
313 *
314 * The 'switch' decoder type represents the decoder instances of cxl_port's that
315 * route from the root of a CXL memory decode topology to the endpoints. They
316 * come in two flavors, root-level decoders, statically defined by platform
317 * firmware, and mid-level decoders, where interleave-granularity,
318 * interleave-width, and the target list are mutable.
319 */
320struct cxl_switch_decoder {
321 struct cxl_decoder cxld;
221 seqlock_t target_lock;322 seqlock_t target_lock;
222 int nr_targets;323 int nr_targets;
223 struct cxl_dport *target[];324 struct cxl_dport *target[];
224};325};
225 326
226 327
328/**
329 * struct cxl_root_decoder - Static platform CXL address decoder
330 * @res: host / parent resource for region allocations
331 * @region_id: region id for next region provisioning event
332 * @calc_hb: which host bridge covers the n'th position by granularity
333 * @cxlsd: base cxl switch decoder
334 */
335struct cxl_root_decoder {
336 struct resource *res;
337 atomic_t region_id;
338 struct cxl_dport *(*calc_hb)(struct cxl_root_decoder *cxlrd, int pos);
339 struct cxl_switch_decoder cxlsd;
340};
341
342/*
343 * enum cxl_config_state - State machine for region configuration
344 * @CXL_CONFIG_IDLE: Any sysfs attribute can be written freely
345 * @CXL_CONFIG_INTERLEAVE_ACTIVE: region size has been set, no more
346 * changes to interleave_ways or interleave_granularity
347 * @CXL_CONFIG_ACTIVE: All targets have been added the region is now
348 * active
349 * @CXL_CONFIG_RESET_PENDING: see commit_store()
350 * @CXL_CONFIG_COMMIT: Soft-config has been committed to hardware
351 */
352enum cxl_config_state {
353 CXL_CONFIG_IDLE,
354 CXL_CONFIG_INTERLEAVE_ACTIVE,
355 CXL_CONFIG_ACTIVE,
356 CXL_CONFIG_RESET_PENDING,
357 CXL_CONFIG_COMMIT,
358};
359
360/**
361 * struct cxl_region_params - region settings
362 * @state: allow the driver to lockdown further parameter changes
363 * @uuid: unique id for persistent regions
364 * @interleave_ways: number of endpoints in the region
365 * @interleave_granularity: capacity each endpoint contributes to a stripe
366 * @res: allocated iomem capacity for this region
367 * @targets: active ordered targets in current decoder configuration
368 * @nr_targets: number of targets
369 *
370 * State transitions are protected by the cxl_region_rwsem
371 */
372struct cxl_region_params {
373 enum cxl_config_state state;
374 uuid_t uuid;
375 int interleave_ways;
376 int interleave_granularity;
377 struct resource *res;
378 struct cxl_endpoint_decoder *targets[CXL_DECODER_MAX_INTERLEAVE];
379 int nr_targets;
380};
381
382/**
383 * struct cxl_region - CXL region
384 * @dev: This region's device
385 * @id: This region's id. Id is globally unique across all regions
386 * @mode: Endpoint decoder allocation / access mode
387 * @type: Endpoint decoder target type
388 * @params: active + config params for the region
389 */
390struct cxl_region {
391 struct device dev;
392 int id;
393 enum cxl_decoder_mode mode;
394 enum cxl_decoder_type type;
395 struct cxl_region_params params;
396};
397
227/**398/**
228 * enum cxl_nvdimm_brige_state - state machine for managing bus rescans399 * enum cxl_nvdimm_brige_state - state machine for managing bus rescans
229 * @CXL_NVB_NEW: Set at bridge create and after cxl_pmem_wq is destroyed400 * @CXL_NVB_NEW: Set at bridge create and after cxl_pmem_wq is destroyed
@@ -251,7 +422,26 @@ struct cxl_nvdimm_bridge { · 21 unmodified lines
251struct cxl_nvdimm {422struct cxl_nvdimm {
252 struct device dev;423 struct device dev;
253 struct cxl_memdev *cxlmd;424 struct cxl_memdev *cxlmd;
254 struct nvdimm *nvdimm;425 struct cxl_nvdimm_bridge *bridge;
426 struct cxl_pmem_region *region;
427};
428
429struct cxl_pmem_region_mapping {
430 struct cxl_memdev *cxlmd;
431 struct cxl_nvdimm *cxl_nvd;
432 u64 start;
433 u64 size;
434 int position;
435};
436
437struct cxl_pmem_region {
438 struct device dev;
439 struct cxl_region *cxlr;
440 struct nd_region *nd_region;
441 struct cxl_nvdimm_bridge *bridge;
442 struct range hpa_range;
443 int nr_mappings;
444 struct cxl_pmem_region_mapping mapping[];
255};445};
256 446
257/**447/**
@@ -260,50 +450,94 @@ struct cxl_nvdimm { · 2 unmodified lines
260 * decode hierarchy.450 * decode hierarchy.
261 * @dev: this port's device451 * @dev: this port's device
262 * @uport: PCI or platform device implementing the upstream port capability452 * @uport: PCI or platform device implementing the upstream port capability
453 * @host_bridge: Shortcut to the platform attach point for this port
263 * @id: id for port device-name454 * @id: id for port device-name
264 * @dports: cxl_dport instances referenced by decoders455 * @dports: cxl_dport instances referenced by decoders
265 * @endpoints: cxl_ep instances, endpoints that are a descendant of this port456 * @endpoints: cxl_ep instances, endpoints that are a descendant of this port
457 * @regions: cxl_region_ref instances, regions mapped by this port
458 * @parent_dport: dport that points to this port in the parent
266 * @decoder_ida: allocator for decoder ids459 * @decoder_ida: allocator for decoder ids
460 * @hdm_end: track last allocated HDM decoder instance for allocation ordering
461 * @commit_end: cursor to track highest committed decoder for commit ordering
267 * @component_reg_phys: component register capability base address (optional)462 * @component_reg_phys: component register capability base address (optional)
268 * @dead: last ep has been removed, force port re-creation463 * @dead: last ep has been removed, force port re-creation
269 * @depth: How deep this port is relative to the root. depth 0 is the root.464 * @depth: How deep this port is relative to the root. depth 0 is the root.
465 * @cdat: Cached CDAT data
466 * @cdat_available: Should a CDAT attribute be available in sysfs
270 */467 */
271struct cxl_port {468struct cxl_port {
272 struct device dev;469 struct device dev;
273 struct device *uport;470 struct device *uport;
471 struct device *host_bridge;
274 int id;472 int id;
275 struct list_head dports;473 struct xarray dports;
276 struct list_head endpoints;474 struct xarray endpoints;
475 struct xarray regions;
476 struct cxl_dport *parent_dport;
277 struct ida decoder_ida;477 struct ida decoder_ida;
478 int hdm_end;
479 int commit_end;
278 resource_size_t component_reg_phys;480 resource_size_t component_reg_phys;
279 bool dead;481 bool dead;
280 unsigned int depth;482 unsigned int depth;
483 struct cxl_cdat {
484 void *table;
485 size_t length;
486 } cdat;
487 bool cdat_available;
281};488};
282 489
490static inline struct cxl_dport *
491cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev)
492{
493 return xa_load(&port->dports, (unsigned long)dport_dev);
494}
495
283/**496/**
284 * struct cxl_dport - CXL downstream port497 * struct cxl_dport - CXL downstream port
285 * @dport: PCI bridge or firmware device representing the downstream link498 * @dport: PCI bridge or firmware device representing the downstream link
286 * @port_id: unique hardware identifier for dport in decoder target list499 * @port_id: unique hardware identifier for dport in decoder target list
287 * @component_reg_phys: downstream port component registers500 * @component_reg_phys: downstream port component registers
288 * @port: reference to cxl_port that contains this downstream port501 * @port: reference to cxl_port that contains this downstream port
289 * @list: node for a cxl_port's list of cxl_dport instances
290 */502 */
291struct cxl_dport {503struct cxl_dport {
292 struct device *dport;504 struct device *dport;
293 int port_id;505 int port_id;
294 resource_size_t component_reg_phys;506 resource_size_t component_reg_phys;
295 struct cxl_port *port;507 struct cxl_port *port;
296 struct list_head list;
297};508};
298 509
299/**510/**
300 * struct cxl_ep - track an endpoint's interest in a port511 * struct cxl_ep - track an endpoint's interest in a port
301 * @ep: device that hosts a generic CXL endpoint (expander or accelerator)512 * @ep: device that hosts a generic CXL endpoint (expander or accelerator)
302 * @list: node on port->endpoints list513 * @dport: which dport routes to this endpoint on @port
514 * @next: cxl switch port across the link attached to @dport NULL if
515 * attached to an endpoint
303 */516 */
304struct cxl_ep {517struct cxl_ep {
305 struct device *ep;518 struct device *ep;
306 struct list_head list;519 struct cxl_dport *dport;
520 struct cxl_port *next;
521};
522
523/**
524 * struct cxl_region_ref - track a region's interest in a port
525 * @port: point in topology to install this reference
526 * @decoder: decoder assigned for @region in @port
527 * @region: region for this reference
528 * @endpoints: cxl_ep references for region members beneath @port
529 * @nr_targets_set: track how many targets have been programmed during setup
530 * @nr_eps: number of endpoints beneath @port
531 * @nr_targets: number of distinct targets needed to reach @nr_eps
532 */
533struct cxl_region_ref {
534 struct cxl_port *port;
535 struct cxl_decoder *decoder;
536 struct cxl_region *region;
537 struct xarray endpoints;
538 int nr_targets_set;
539 int nr_eps;
540 int nr_targets;
307};541};
308 542
309/*543/*
@@ -325,29 +559,31 @@ int devm_cxl_register_pci_bus(struct device *host, struct device *uport, · 15 unmodified lines
325struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port);559struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port);
326struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,560struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
327 resource_size_t component_reg_phys,561 resource_size_t component_reg_phys,
328 struct cxl_port *parent_port);562 struct cxl_dport *parent_dport);
563int devm_cxl_add_endpoint(struct cxl_memdev *cxlmd,
564 struct cxl_dport *parent_dport);
329struct cxl_port *find_cxl_root(struct device *dev);565struct cxl_port *find_cxl_root(struct device *dev);
330int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);566int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
331int cxl_bus_rescan(void);567int cxl_bus_rescan(void);
332struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd);568struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
569 struct cxl_dport **dport);
333bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);570bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);
334 571
335struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,572struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
336 struct device *dport, int port_id,573 struct device *dport, int port_id,
337 resource_size_t component_reg_phys);574 resource_size_t component_reg_phys);
338struct cxl_dport *cxl_find_dport_by_dev(struct cxl_port *port,
339 const struct device *dev);
340 575
341struct cxl_decoder *to_cxl_decoder(struct device *dev);576struct cxl_decoder *to_cxl_decoder(struct device *dev);
577struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
578struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev);
342bool is_root_decoder(struct device *dev);579bool is_root_decoder(struct device *dev);
343bool is_endpoint_decoder(struct device *dev);580bool is_endpoint_decoder(struct device *dev);
344bool is_cxl_decoder(struct device *dev);581struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
345struct cxl_decoder *cxl_root_decoder_alloc(struct cxl_port *port,582 unsigned int nr_targets);
346 unsigned int nr_targets);583struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
347struct cxl_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,584 unsigned int nr_targets);
348 unsigned int nr_targets);
349int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map);585int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map);
350struct cxl_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port);586struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port);
351int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map);587int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map);
352int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);588int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);
353int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);589int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
@@ -357,6 +593,8 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port); · 3 unmodified lines
357int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm);593int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm);
358int devm_cxl_add_passthrough_decoder(struct cxl_port *port);594int devm_cxl_add_passthrough_decoder(struct cxl_port *port);
359 595
596bool is_cxl_region(struct device *dev);
597
360extern struct bus_type cxl_bus_type;598extern struct bus_type cxl_bus_type;
361 599
362struct cxl_driver {600struct cxl_driver {
@@ -385,6 +623,8 @@ void cxl_driver_unregister(struct cxl_driver *cxl_drv); · 22 unmodified lines
385#define CXL_DEVICE_PORT 3623#define CXL_DEVICE_PORT 3
386#define CXL_DEVICE_ROOT 4624#define CXL_DEVICE_ROOT 4
387#define CXL_DEVICE_MEMORY_EXPANDER 5625#define CXL_DEVICE_MEMORY_EXPANDER 5
626#define CXL_DEVICE_REGION 6
627#define CXL_DEVICE_PMEM_REGION 7
388 628
389#define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")629#define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")
390#define CXL_MODALIAS_FMT "cxl:t%d"630#define CXL_MODALIAS_FMT "cxl:t%d"
@@ -396,7 +636,21 @@ struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev); · 5 unmodified lines
396bool is_cxl_nvdimm(struct device *dev);636bool is_cxl_nvdimm(struct device *dev);
397bool is_cxl_nvdimm_bridge(struct device *dev);637bool is_cxl_nvdimm_bridge(struct device *dev);
398int devm_cxl_add_nvdimm(struct device *host, struct cxl_memdev *cxlmd);638int devm_cxl_add_nvdimm(struct device *host, struct cxl_memdev *cxlmd);
399struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd);639struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct device *dev);
640
641#ifdef CONFIG_CXL_REGION
642bool is_cxl_pmem_region(struct device *dev);
643struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev);
644#else
645static inline bool is_cxl_pmem_region(struct device *dev)
646{
647 return false;
648}
649static inline struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
650{
651 return NULL;
652}
653#endif
400 654
401/*655/*
402 * Unit test builds overrides this to __weak, find the 'strong' version656 * Unit test builds overrides this to __weak, find the 'strong' version
drivers/cxl/cxlmem.h+38−4
@@ -50,6 +50,24 @@ static inline struct cxl_memdev *to_cxl_memdev(struct device *dev) · 49 unmodified lines
50 return container_of(dev, struct cxl_memdev, dev);50 return container_of(dev, struct cxl_memdev, dev);
51}51}
52 52
53static inline struct cxl_port *cxled_to_port(struct cxl_endpoint_decoder *cxled)
54{
55 return to_cxl_port(cxled->cxld.dev.parent);
56}
57
58static inline struct cxl_port *cxlrd_to_port(struct cxl_root_decoder *cxlrd)
59{
60 return to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
61}
62
63static inline struct cxl_memdev *
64cxled_to_memdev(struct cxl_endpoint_decoder *cxled)
65{
66 struct cxl_port *port = to_cxl_port(cxled->cxld.dev.parent);
67
68 return to_cxl_memdev(port->uport);
69}
70
53bool is_cxl_memdev(struct device *dev);71bool is_cxl_memdev(struct device *dev);
54static inline bool is_cxl_endpoint(struct cxl_port *port)72static inline bool is_cxl_endpoint(struct cxl_port *port)
55{73{
@@ -178,8 +196,9 @@ struct cxl_endpoint_dvsec_info { · 122 unmodified lines
178 * @firmware_version: Firmware version for the memory device.196 * @firmware_version: Firmware version for the memory device.
179 * @enabled_cmds: Hardware commands found enabled in CEL.197 * @enabled_cmds: Hardware commands found enabled in CEL.
180 * @exclusive_cmds: Commands that are kernel-internal only198 * @exclusive_cmds: Commands that are kernel-internal only
181 * @pmem_range: Active Persistent memory capacity configuration199 * @dpa_res: Overall DPA resource tree for the device
182 * @ram_range: Active Volatile memory capacity configuration200 * @pmem_res: Active Persistent memory capacity configuration
201 * @ram_res: Active Volatile memory capacity configuration
183 * @total_bytes: sum of all possible capacities202 * @total_bytes: sum of all possible capacities
184 * @volatile_only_bytes: hard volatile capacity203 * @volatile_only_bytes: hard volatile capacity
185 * @persistent_only_bytes: hard persistent capacity204 * @persistent_only_bytes: hard persistent capacity
@@ -191,6 +210,7 @@ struct cxl_endpoint_dvsec_info { · 5 unmodified lines
191 * @component_reg_phys: register base of component registers210 * @component_reg_phys: register base of component registers
192 * @info: Cached DVSEC information about the device.211 * @info: Cached DVSEC information about the device.
193 * @serial: PCIe Device Serial Number212 * @serial: PCIe Device Serial Number
213 * @doe_mbs: PCI DOE mailbox array
194 * @mbox_send: @dev specific transport for transmitting mailbox commands214 * @mbox_send: @dev specific transport for transmitting mailbox commands
195 *215 *
196 * See section 8.2.9.5.2 Capacity Configuration and Label Storage for216 * See section 8.2.9.5.2 Capacity Configuration and Label Storage for
@@ -209,8 +229,9 @@ struct cxl_dev_state { · 12 unmodified lines
209 DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);229 DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
210 DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);230 DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
211 231
212 struct range pmem_range;232 struct resource dpa_res;
213 struct range ram_range;233 struct resource pmem_res;
234 struct resource ram_res;
214 u64 total_bytes;235 u64 total_bytes;
215 u64 volatile_only_bytes;236 u64 volatile_only_bytes;
216 u64 persistent_only_bytes;237 u64 persistent_only_bytes;
@@ -224,6 +245,8 @@ struct cxl_dev_state { · 7 unmodified lines
224 resource_size_t component_reg_phys;245 resource_size_t component_reg_phys;
225 u64 serial;246 u64 serial;
226 247
248 struct xarray doe_mbs;
249
227 int (*mbox_send)(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd);250 int (*mbox_send)(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd);
228};251};
229 252
@@ -299,6 +322,13 @@ struct cxl_mbox_identify { · 69 unmodified lines
299 u8 qos_telemetry_caps;322 u8 qos_telemetry_caps;
300} __packed;323} __packed;
301 324
325struct cxl_mbox_get_partition_info {
326 __le64 active_volatile_cap;
327 __le64 active_persistent_cap;
328 __le64 next_volatile_cap;
329 __le64 next_persistent_cap;
330} __packed;
331
302struct cxl_mbox_get_lsa {332struct cxl_mbox_get_lsa {
303 __le32 offset;333 __le32 offset;
304 __le32 length;334 __le32 length;
@@ -370,4 +400,8 @@ struct cxl_hdm { · 65 unmodified lines
370 unsigned int interleave_mask;400 unsigned int interleave_mask;
371 struct cxl_port *port;401 struct cxl_port *port;
372};402};
403
404struct seq_file;
405struct dentry *cxl_debugfs_create_dir(const char *dir);
406void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds);
373#endif /* __CXL_MEM_H__ */407#endif /* __CXL_MEM_H__ */
drivers/cxl/cxlpci.h+1−0
@@ -74,4 +74,5 @@ static inline resource_size_t cxl_regmap_to_base(struct pci_dev *pdev, · 73 unmodified lines
74int devm_cxl_port_enumerate_dports(struct cxl_port *port);74int devm_cxl_port_enumerate_dports(struct cxl_port *port);
75struct cxl_dev_state;75struct cxl_dev_state;
76int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm);76int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm);
77void read_cdat_data(struct cxl_port *port);
77#endif /* __CXL_PCI_H__ */78#endif /* __CXL_PCI_H__ */
drivers/cxl/mem.c+23−26
@@ -1,5 +1,6 @@
1// SPDX-License-Identifier: GPL-2.0-only1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/debugfs.h>
3#include <linux/device.h>4#include <linux/device.h>
4#include <linux/module.h>5#include <linux/module.h>
5#include <linux/pci.h>6#include <linux/pci.h>
@@ -24,42 +25,32 @@ · 18 unmodified lines
24 * in higher level operations.25 * in higher level operations.
25 */26 */
26 27
27static int create_endpoint(struct cxl_memdev *cxlmd,28static void enable_suspend(void *data)
28 struct cxl_port *parent_port)
29{29{
30 struct cxl_dev_state *cxlds = cxlmd->cxlds;30 cxl_mem_active_dec();
31 struct cxl_port *endpoint;31}
32 int rc;
33 32
34 endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev,33static void remove_debugfs(void *dentry)
35 cxlds->component_reg_phys, parent_port);34{
36 if (IS_ERR(endpoint))35 debugfs_remove_recursive(dentry);
37 return PTR_ERR(endpoint);36}
38 37
39 dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev));38static int cxl_mem_dpa_show(struct seq_file *file, void *data)
39{
40 struct device *dev = file->private;
41 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
40 42
41 rc = cxl_endpoint_autoremove(cxlmd, endpoint);43 cxl_dpa_debug(file, cxlmd->cxlds);
42 if (rc)
43 return rc;
44
45 if (!endpoint->dev.driver) {
46 dev_err(&cxlmd->dev, "%s failed probe\n",
47 dev_name(&endpoint->dev));
48 return -ENXIO;
49 }
50 44
51 return 0;45 return 0;
52}46}
53 47
54static void enable_suspend(void *data)
55{
56 cxl_mem_active_dec();
57}
58
59static int cxl_mem_probe(struct device *dev)48static int cxl_mem_probe(struct device *dev)
60{49{
61 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);50 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
62 struct cxl_port *parent_port;51 struct cxl_port *parent_port;
52 struct cxl_dport *dport;
53 struct dentry *dentry;
63 int rc;54 int rc;
64 55
65 /*56 /*
@@ -73,11 +64,17 @@ static int cxl_mem_probe(struct device *dev) · 7 unmodified lines
73 if (work_pending(&cxlmd->detach_work))64 if (work_pending(&cxlmd->detach_work))
74 return -EBUSY;65 return -EBUSY;
75 66
67 dentry = cxl_debugfs_create_dir(dev_name(dev));
68 debugfs_create_devm_seqfile(dev, "dpamem", dentry, cxl_mem_dpa_show);
69 rc = devm_add_action_or_reset(dev, remove_debugfs, dentry);
70 if (rc)
71 return rc;
72
76 rc = devm_cxl_enumerate_ports(cxlmd);73 rc = devm_cxl_enumerate_ports(cxlmd);
77 if (rc)74 if (rc)
78 return rc;75 return rc;
79 76
80 parent_port = cxl_mem_find_port(cxlmd);77 parent_port = cxl_mem_find_port(cxlmd, &dport);
81 if (!parent_port) {78 if (!parent_port) {
82 dev_err(dev, "CXL port topology not found\n");79 dev_err(dev, "CXL port topology not found\n");
83 return -ENXIO;80 return -ENXIO;
@@ -91,7 +88,7 @@ static int cxl_mem_probe(struct device *dev) · 7 unmodified lines
91 goto unlock;88 goto unlock;
92 }89 }
93 90
94 rc = create_endpoint(cxlmd, parent_port);91 rc = devm_cxl_add_endpoint(cxlmd, dport);
95unlock:92unlock:
96 device_unlock(&parent_port->dev);93 device_unlock(&parent_port->dev);
97 put_device(&parent_port->dev);94 put_device(&parent_port->dev);
drivers/cxl/pci.c+45−1
@@ -8,6 +8,7 @@ · 7 unmodified lines
8#include <linux/mutex.h>8#include <linux/mutex.h>
9#include <linux/list.h>9#include <linux/list.h>
10#include <linux/pci.h>10#include <linux/pci.h>
11#include <linux/pci-doe.h>
11#include <linux/io.h>12#include <linux/io.h>
12#include "cxlmem.h"13#include "cxlmem.h"
13#include "cxlpci.h"14#include "cxlpci.h"
@@ -386,6 +387,47 @@ static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, · 372 unmodified lines
386 return rc;387 return rc;
387}388}
388 389
390static void cxl_pci_destroy_doe(void *mbs)
391{
392 xa_destroy(mbs);
393}
394
395static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds)
396{
397 struct device *dev = cxlds->dev;
398 struct pci_dev *pdev = to_pci_dev(dev);
399 u16 off = 0;
400
401 xa_init(&cxlds->doe_mbs);
402 if (devm_add_action(&pdev->dev, cxl_pci_destroy_doe, &cxlds->doe_mbs)) {
403 dev_err(dev, "Failed to create XArray for DOE's\n");
404 return;
405 }
406
407 /*
408 * Mailbox creation is best effort. Higher layers must determine if
409 * the lack of a mailbox for their protocol is a device failure or not.
410 */
411 pci_doe_for_each_off(pdev, off) {
412 struct pci_doe_mb *doe_mb;
413
414 doe_mb = pcim_doe_create_mb(pdev, off);
415 if (IS_ERR(doe_mb)) {
416 dev_err(dev, "Failed to create MB object for MB @ %x\n",
417 off);
418 continue;
419 }
420
421 if (xa_insert(&cxlds->doe_mbs, off, doe_mb, GFP_KERNEL)) {
422 dev_err(dev, "xa_insert failed to insert MB @ %x\n",
423 off);
424 continue;
425 }
426
427 dev_dbg(dev, "Created DOE mailbox @%x\n", off);
428 }
429}
430
389static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)431static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
390{432{
391 struct cxl_register_map map;433 struct cxl_register_map map;
@@ -434,6 +476,8 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) · 42 unmodified lines
434 476
435 cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);477 cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);
436 478
479 devm_cxl_pci_create_doe(cxlds);
480
437 rc = cxl_pci_setup_mailbox(cxlds);481 rc = cxl_pci_setup_mailbox(cxlds);
438 if (rc)482 if (rc)
439 return rc;483 return rc;
@@ -454,7 +498,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) · 14 unmodified lines
454 if (IS_ERR(cxlmd))498 if (IS_ERR(cxlmd))
455 return PTR_ERR(cxlmd);499 return PTR_ERR(cxlmd);
456 500
457 if (range_len(&cxlds->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))501 if (resource_size(&cxlds->pmem_res) && IS_ENABLED(CONFIG_CXL_PMEM))
458 rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd);502 rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd);
459 503
460 return rc;504 return rc;
drivers/cxl/pmem.c+254−5
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include <linux/ndctl.h>7#include <linux/ndctl.h>
8#include <linux/async.h>8#include <linux/async.h>
9#include <linux/slab.h>9#include <linux/slab.h>
10#include <linux/nd.h>
10#include "cxlmem.h"11#include "cxlmem.h"
11#include "cxl.h"12#include "cxl.h"
12 13
@@ -26,7 +27,23 @@ static void clear_exclusive(void *cxlds) · 13 unmodified lines
26 27
27static void unregister_nvdimm(void *nvdimm)28static void unregister_nvdimm(void *nvdimm)
28{29{
30 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
31 struct cxl_nvdimm_bridge *cxl_nvb = cxl_nvd->bridge;
32 struct cxl_pmem_region *cxlr_pmem;
33
34 device_lock(&cxl_nvb->dev);
35 cxlr_pmem = cxl_nvd->region;
36 dev_set_drvdata(&cxl_nvd->dev, NULL);
37 cxl_nvd->region = NULL;
38 device_unlock(&cxl_nvb->dev);
39
40 if (cxlr_pmem) {
41 device_release_driver(&cxlr_pmem->dev);
42 put_device(&cxlr_pmem->dev);
43 }
44
29 nvdimm_delete(nvdimm);45 nvdimm_delete(nvdimm);
46 cxl_nvd->bridge = NULL;
30}47}
31 48
32static int cxl_nvdimm_probe(struct device *dev)49static int cxl_nvdimm_probe(struct device *dev)
@@ -39,7 +56,7 @@ static int cxl_nvdimm_probe(struct device *dev) · 6 unmodified lines
39 struct nvdimm *nvdimm;56 struct nvdimm *nvdimm;
40 int rc;57 int rc;
41 58
42 cxl_nvb = cxl_find_nvdimm_bridge(cxl_nvd);59 cxl_nvb = cxl_find_nvdimm_bridge(dev);
43 if (!cxl_nvb)60 if (!cxl_nvb)
44 return -ENXIO;61 return -ENXIO;
45 62
@@ -66,6 +83,7 @@ static int cxl_nvdimm_probe(struct device *dev) · 20 unmodified lines
66 }83 }
67 84
68 dev_set_drvdata(dev, nvdimm);85 dev_set_drvdata(dev, nvdimm);
86 cxl_nvd->bridge = cxl_nvb;
69 rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);87 rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
70out:88out:
71 device_unlock(&cxl_nvb->dev);89 device_unlock(&cxl_nvb->dev);
@@ -204,15 +222,38 @@ static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb) · 132 unmodified lines
204 return cxl_nvb->nvdimm_bus != NULL;222 return cxl_nvb->nvdimm_bus != NULL;
205}223}
206 224
207static int cxl_nvdimm_release_driver(struct device *dev, void *data)225static int cxl_nvdimm_release_driver(struct device *dev, void *cxl_nvb)
208{226{
227 struct cxl_nvdimm *cxl_nvd;
228
209 if (!is_cxl_nvdimm(dev))229 if (!is_cxl_nvdimm(dev))
210 return 0;230 return 0;
231
232 cxl_nvd = to_cxl_nvdimm(dev);
233 if (cxl_nvd->bridge != cxl_nvb)
234 return 0;
235
211 device_release_driver(dev);236 device_release_driver(dev);
212 return 0;237 return 0;
213}238}
214 239
215static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus)240static int cxl_pmem_region_release_driver(struct device *dev, void *cxl_nvb)
241{
242 struct cxl_pmem_region *cxlr_pmem;
243
244 if (!is_cxl_pmem_region(dev))
245 return 0;
246
247 cxlr_pmem = to_cxl_pmem_region(dev);
248 if (cxlr_pmem->bridge != cxl_nvb)
249 return 0;
250
251 device_release_driver(dev);
252 return 0;
253}
254
255static void offline_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb,
256 struct nvdimm_bus *nvdimm_bus)
216{257{
217 if (!nvdimm_bus)258 if (!nvdimm_bus)
218 return;259 return;
@@ -222,7 +263,10 @@ static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus) · 3 unmodified lines
222 * nvdimm_bus_unregister() rips the nvdimm objects out from263 * nvdimm_bus_unregister() rips the nvdimm objects out from
223 * underneath them.264 * underneath them.
224 */265 */
225 bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_release_driver);266 bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb,
267 cxl_pmem_region_release_driver);
268 bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb,
269 cxl_nvdimm_release_driver);
226 nvdimm_bus_unregister(nvdimm_bus);270 nvdimm_bus_unregister(nvdimm_bus);
227}271}
228 272
@@ -260,7 +304,7 @@ static void cxl_nvb_update_state(struct work_struct *work) · 31 unmodified lines
260 304
261 dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc);305 dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc);
262 }306 }
263 offline_nvdimm_bus(victim_bus);307 offline_nvdimm_bus(cxl_nvb, victim_bus);
264 308
265 put_device(&cxl_nvb->dev);309 put_device(&cxl_nvb->dev);
266}310}
@@ -315,6 +359,203 @@ static struct cxl_driver cxl_nvdimm_bridge_driver = { · 48 unmodified lines
315 .id = CXL_DEVICE_NVDIMM_BRIDGE,359 .id = CXL_DEVICE_NVDIMM_BRIDGE,
316};360};
317 361
362static int match_cxl_nvdimm(struct device *dev, void *data)
363{
364 return is_cxl_nvdimm(dev);
365}
366
367static void unregister_nvdimm_region(void *nd_region)
368{
369 struct cxl_nvdimm_bridge *cxl_nvb;
370 struct cxl_pmem_region *cxlr_pmem;
371 int i;
372
373 cxlr_pmem = nd_region_provider_data(nd_region);
374 cxl_nvb = cxlr_pmem->bridge;
375 device_lock(&cxl_nvb->dev);
376 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
377 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
378 struct cxl_nvdimm *cxl_nvd = m->cxl_nvd;
379
380 if (cxl_nvd->region) {
381 put_device(&cxlr_pmem->dev);
382 cxl_nvd->region = NULL;
383 }
384 }
385 device_unlock(&cxl_nvb->dev);
386
387 nvdimm_region_delete(nd_region);
388}
389
390static void cxlr_pmem_remove_resource(void *res)
391{
392 remove_resource(res);
393}
394
395struct cxl_pmem_region_info {
396 u64 offset;
397 u64 serial;
398};
399
400static int cxl_pmem_region_probe(struct device *dev)
401{
402 struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
403 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
404 struct cxl_region *cxlr = cxlr_pmem->cxlr;
405 struct cxl_pmem_region_info *info = NULL;
406 struct cxl_nvdimm_bridge *cxl_nvb;
407 struct nd_interleave_set *nd_set;
408 struct nd_region_desc ndr_desc;
409 struct cxl_nvdimm *cxl_nvd;
410 struct nvdimm *nvdimm;
411 struct resource *res;
412 int rc, i = 0;
413
414 cxl_nvb = cxl_find_nvdimm_bridge(&cxlr_pmem->mapping[0].cxlmd->dev);
415 if (!cxl_nvb) {
416 dev_dbg(dev, "bridge not found\n");
417 return -ENXIO;
418 }
419 cxlr_pmem->bridge = cxl_nvb;
420
421 device_lock(&cxl_nvb->dev);
422 if (!cxl_nvb->nvdimm_bus) {
423 dev_dbg(dev, "nvdimm bus not found\n");
424 rc = -ENXIO;
425 goto err;
426 }
427
428 memset(&mappings, 0, sizeof(mappings));
429 memset(&ndr_desc, 0, sizeof(ndr_desc));
430
431 res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
432 if (!res) {
433 rc = -ENOMEM;
434 goto err;
435 }
436
437 res->name = "Persistent Memory";
438 res->start = cxlr_pmem->hpa_range.start;
439 res->end = cxlr_pmem->hpa_range.end;
440 res->flags = IORESOURCE_MEM;
441 res->desc = IORES_DESC_PERSISTENT_MEMORY;
442
443 rc = insert_resource(&iomem_resource, res);
444 if (rc)
445 goto err;
446
447 rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
448 if (rc)
449 goto err;
450
451 ndr_desc.res = res;
452 ndr_desc.provider_data = cxlr_pmem;
453
454 ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
455 ndr_desc.target_node = phys_to_target_node(res->start);
456 if (ndr_desc.target_node == NUMA_NO_NODE) {
457 ndr_desc.target_node = ndr_desc.numa_node;
458 dev_dbg(&cxlr->dev, "changing target node from %d to %d",
459 NUMA_NO_NODE, ndr_desc.target_node);
460 }
461
462 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
463 if (!nd_set) {
464 rc = -ENOMEM;
465 goto err;
466 }
467
468 ndr_desc.memregion = cxlr->id;
469 set_bit(ND_REGION_CXL, &ndr_desc.flags);
470 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
471
472 info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
473 if (!info) {
474 rc = -ENOMEM;
475 goto err;
476 }
477
478 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
479 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
480 struct cxl_memdev *cxlmd = m->cxlmd;
481 struct cxl_dev_state *cxlds = cxlmd->cxlds;
482 struct device *d;
483
484 d = device_find_child(&cxlmd->dev, NULL, match_cxl_nvdimm);
485 if (!d) {
486 dev_dbg(dev, "[%d]: %s: no cxl_nvdimm found\n", i,
487 dev_name(&cxlmd->dev));
488 rc = -ENODEV;
489 goto err;
490 }
491
492 /* safe to drop ref now with bridge lock held */
493 put_device(d);
494
495 cxl_nvd = to_cxl_nvdimm(d);
496 nvdimm = dev_get_drvdata(&cxl_nvd->dev);
497 if (!nvdimm) {
498 dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
499 dev_name(&cxlmd->dev));
500 rc = -ENODEV;
501 goto err;
502 }
503 cxl_nvd->region = cxlr_pmem;
504 get_device(&cxlr_pmem->dev);
505 m->cxl_nvd = cxl_nvd;
506 mappings[i] = (struct nd_mapping_desc) {
507 .nvdimm = nvdimm,
508 .start = m->start,
509 .size = m->size,
510 .position = i,
511 };
512 info[i].offset = m->start;
513 info[i].serial = cxlds->serial;
514 }
515 ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
516 ndr_desc.mapping = mappings;
517
518 /*
519 * TODO enable CXL labels which skip the need for 'interleave-set cookie'
520 */
521 nd_set->cookie1 =
522 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
523 nd_set->cookie2 = nd_set->cookie1;
524 ndr_desc.nd_set = nd_set;
525
526 cxlr_pmem->nd_region =
527 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
528 if (!cxlr_pmem->nd_region) {
529 rc = -ENOMEM;
530 goto err;
531 }
532
533 rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
534 cxlr_pmem->nd_region);
535out:
536 kfree(info);
537 device_unlock(&cxl_nvb->dev);
538 put_device(&cxl_nvb->dev);
539
540 return rc;
541
542err:
543 dev_dbg(dev, "failed to create nvdimm region\n");
544 for (i--; i >= 0; i--) {
545 nvdimm = mappings[i].nvdimm;
546 cxl_nvd = nvdimm_provider_data(nvdimm);
547 put_device(&cxl_nvd->region->dev);
548 cxl_nvd->region = NULL;
549 }
550 goto out;
551}
552
553static struct cxl_driver cxl_pmem_region_driver = {
554 .name = "cxl_pmem_region",
555 .probe = cxl_pmem_region_probe,
556 .id = CXL_DEVICE_PMEM_REGION,
557};
558
318/*559/*
319 * Return all bridges to the CXL_NVB_NEW state to invalidate any560 * Return all bridges to the CXL_NVB_NEW state to invalidate any
320 * ->state_work referring to the now destroyed cxl_pmem_wq.561 * ->state_work referring to the now destroyed cxl_pmem_wq.
@@ -359,8 +600,14 @@ static __init int cxl_pmem_init(void) · 38 unmodified lines
359 if (rc)600 if (rc)
360 goto err_nvdimm;601 goto err_nvdimm;
361 602
603 rc = cxl_driver_register(&cxl_pmem_region_driver);
604 if (rc)
605 goto err_region;
606
362 return 0;607 return 0;
363 608
609err_region:
610 cxl_driver_unregister(&cxl_nvdimm_driver);
364err_nvdimm:611err_nvdimm:
365 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);612 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
366err_bridge:613err_bridge:
@@ -370,6 +617,7 @@ err_bridge: · 3 unmodified lines
370 617
371static __exit void cxl_pmem_exit(void)618static __exit void cxl_pmem_exit(void)
372{619{
620 cxl_driver_unregister(&cxl_pmem_region_driver);
373 cxl_driver_unregister(&cxl_nvdimm_driver);621 cxl_driver_unregister(&cxl_nvdimm_driver);
374 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);622 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
375 destroy_cxl_pmem_wq();623 destroy_cxl_pmem_wq();
@@ -381,3 +629,4 @@ module_exit(cxl_pmem_exit); · 5 unmodified lines
381MODULE_IMPORT_NS(CXL);629MODULE_IMPORT_NS(CXL);
382MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);630MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
383MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);631MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
632MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);
drivers/cxl/port.c+53−0
@@ -53,6 +53,9 @@ static int cxl_port_probe(struct device *dev) · 52 unmodified lines
53 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport);53 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport);
54 struct cxl_dev_state *cxlds = cxlmd->cxlds;54 struct cxl_dev_state *cxlds = cxlmd->cxlds;
55 55
56 /* Cache the data early to ensure is_visible() works */
57 read_cdat_data(port);
58
56 get_device(&cxlmd->dev);59 get_device(&cxlmd->dev);
57 rc = devm_add_action_or_reset(dev, schedule_detach, cxlmd);60 rc = devm_add_action_or_reset(dev, schedule_detach, cxlmd);
58 if (rc)61 if (rc)
@@ -78,10 +81,60 @@ static int cxl_port_probe(struct device *dev) · 19 unmodified lines
78 return 0;81 return 0;
79}82}
80 83
84static ssize_t CDAT_read(struct file *filp, struct kobject *kobj,
85 struct bin_attribute *bin_attr, char *buf,
86 loff_t offset, size_t count)
87{
88 struct device *dev = kobj_to_dev(kobj);
89 struct cxl_port *port = to_cxl_port(dev);
90
91 if (!port->cdat_available)
92 return -ENXIO;
93
94 if (!port->cdat.table)
95 return 0;
96
97 return memory_read_from_buffer(buf, count, &offset,
98 port->cdat.table,
99 port->cdat.length);
100}
101
102static BIN_ATTR_ADMIN_RO(CDAT, 0);
103
104static umode_t cxl_port_bin_attr_is_visible(struct kobject *kobj,
105 struct bin_attribute *attr, int i)
106{
107 struct device *dev = kobj_to_dev(kobj);
108 struct cxl_port *port = to_cxl_port(dev);
109
110 if ((attr == &bin_attr_CDAT) && port->cdat_available)
111 return attr->attr.mode;
112
113 return 0;
114}
115
116static struct bin_attribute *cxl_cdat_bin_attributes[] = {
117 &bin_attr_CDAT,
118 NULL,
119};
120
121static struct attribute_group cxl_cdat_attribute_group = {
122 .bin_attrs = cxl_cdat_bin_attributes,
123 .is_bin_visible = cxl_port_bin_attr_is_visible,
124};
125
126static const struct attribute_group *cxl_port_attribute_groups[] = {
127 &cxl_cdat_attribute_group,
128 NULL,
129};
130
81static struct cxl_driver cxl_port_driver = {131static struct cxl_driver cxl_port_driver = {
82 .name = "cxl_port",132 .name = "cxl_port",
83 .probe = cxl_port_probe,133 .probe = cxl_port_probe,
84 .id = CXL_DEVICE_PORT,134 .id = CXL_DEVICE_PORT,
135 .drv = {
136 .dev_groups = cxl_port_attribute_groups,
137 },
85};138};
86 139
87module_cxl_driver(cxl_port_driver);140module_cxl_driver(cxl_port_driver);
drivers/nvdimm/region_devs.c+21−7
@@ -133,7 +133,8 @@ static void nd_region_release(struct device *dev) · 132 unmodified lines
133 put_device(&nvdimm->dev);133 put_device(&nvdimm->dev);
134 }134 }
135 free_percpu(nd_region->lane);135 free_percpu(nd_region->lane);
136 memregion_free(nd_region->id);136 if (!test_bit(ND_REGION_CXL, &nd_region->flags))
137 memregion_free(nd_region->id);
137 kfree(nd_region);138 kfree(nd_region);
138}139}
139 140
@@ -982,9 +983,14 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, · 842 unmodified lines
982 983
983 if (!nd_region)984 if (!nd_region)
984 return NULL;985 return NULL;
985 nd_region->id = memregion_alloc(GFP_KERNEL);986 /* CXL pre-assigns memregion ids before creating nvdimm regions */
986 if (nd_region->id < 0)987 if (test_bit(ND_REGION_CXL, &ndr_desc->flags)) {
987 goto err_id;988 nd_region->id = ndr_desc->memregion;
989 } else {
990 nd_region->id = memregion_alloc(GFP_KERNEL);
991 if (nd_region->id < 0)
992 goto err_id;
993 }
988 994
989 nd_region->lane = alloc_percpu(struct nd_percpu_lane);995 nd_region->lane = alloc_percpu(struct nd_percpu_lane);
990 if (!nd_region->lane)996 if (!nd_region->lane)
@@ -1043,9 +1049,10 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, · 52 unmodified lines
1043 1049
1044 return nd_region;1050 return nd_region;
1045 1051
1046 err_percpu:1052err_percpu:
1047 memregion_free(nd_region->id);1053 if (!test_bit(ND_REGION_CXL, &ndr_desc->flags))
1048 err_id:1054 memregion_free(nd_region->id);
1055err_id:
1049 kfree(nd_region);1056 kfree(nd_region);
1050 return NULL;1057 return NULL;
1051}1058}
@@ -1068,6 +1075,13 @@ struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus, · 16 unmodified lines
1068}1075}
1069EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);1076EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);
1070 1077
1078void nvdimm_region_delete(struct nd_region *nd_region)
1079{
1080 if (nd_region)
1081 nd_device_unregister(&nd_region->dev, ND_SYNC);
1082}
1083EXPORT_SYMBOL_GPL(nvdimm_region_delete);
1084
1071int nvdimm_flush(struct nd_region *nd_region, struct bio *bio)1085int nvdimm_flush(struct nd_region *nd_region, struct bio *bio)
1072{1086{
1073 int rc = 0;1087 int rc = 0;
drivers/pci/Kconfig+3−0
@@ -121,6 +121,9 @@ config XEN_PCIDEV_FRONTEND · 120 unmodified lines
121config PCI_ATS121config PCI_ATS
122 bool122 bool
123 123
124config PCI_DOE
125 bool
126
124config PCI_ECAM127config PCI_ECAM
125 bool128 bool
126 129
drivers/pci/Makefile+1−0
@@ -31,6 +31,7 @@ obj-$(CONFIG_PCI_ECAM) += ecam.o · 30 unmodified lines
31obj-$(CONFIG_PCI_P2PDMA) += p2pdma.o31obj-$(CONFIG_PCI_P2PDMA) += p2pdma.o
32obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o32obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o
33obj-$(CONFIG_VGA_ARB) += vgaarb.o33obj-$(CONFIG_VGA_ARB) += vgaarb.o
34obj-$(CONFIG_PCI_DOE) += doe.o
34 35
35# Endpoint library must be initialized before its users36# Endpoint library must be initialized before its users
36obj-$(CONFIG_PCI_ENDPOINT) += endpoint/37obj-$(CONFIG_PCI_ENDPOINT) += endpoint/
drivers/pci/doe.c+536−0
@@ -0,0 +1,536 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Data Object Exchange
4 * PCIe r6.0, sec 6.30 DOE
5 *
6 * Copyright (C) 2021 Huawei
7 * Jonathan Cameron <Jonathan.Cameron@huawei.com>
8 *
9 * Copyright (C) 2022 Intel Corporation
10 * Ira Weiny <ira.weiny@intel.com>
11 */
12
13#define dev_fmt(fmt) "DOE: " fmt
14
15#include <linux/bitfield.h>
16#include <linux/delay.h>
17#include <linux/jiffies.h>
18#include <linux/mutex.h>
19#include <linux/pci.h>
20#include <linux/pci-doe.h>
21#include <linux/workqueue.h>
22
23#define PCI_DOE_PROTOCOL_DISCOVERY 0
24
25/* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
26#define PCI_DOE_TIMEOUT HZ
27#define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128)
28
29#define PCI_DOE_FLAG_CANCEL 0
30#define PCI_DOE_FLAG_DEAD 1
31
32/**
33 * struct pci_doe_mb - State for a single DOE mailbox
34 *
35 * This state is used to manage a single DOE mailbox capability. All fields
36 * should be considered opaque to the consumers and the structure passed into
37 * the helpers below after being created by devm_pci_doe_create()
38 *
39 * @pdev: PCI device this mailbox belongs to
40 * @cap_offset: Capability offset
41 * @prots: Array of protocols supported (encoded as long values)
42 * @wq: Wait queue for work item
43 * @work_queue: Queue of pci_doe_work items
44 * @flags: Bit array of PCI_DOE_FLAG_* flags
45 */
46struct pci_doe_mb {
47 struct pci_dev *pdev;
48 u16 cap_offset;
49 struct xarray prots;
50
51 wait_queue_head_t wq;
52 struct workqueue_struct *work_queue;
53 unsigned long flags;
54};
55
56static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
57{
58 if (wait_event_timeout(doe_mb->wq,
59 test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
60 timeout))
61 return -EIO;
62 return 0;
63}
64
65static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
66{
67 struct pci_dev *pdev = doe_mb->pdev;
68 int offset = doe_mb->cap_offset;
69
70 pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
71}
72
73static int pci_doe_abort(struct pci_doe_mb *doe_mb)
74{
75 struct pci_dev *pdev = doe_mb->pdev;
76 int offset = doe_mb->cap_offset;
77 unsigned long timeout_jiffies;
78
79 pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
80
81 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
82 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
83
84 do {
85 int rc;
86 u32 val;
87
88 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
89 if (rc)
90 return rc;
91 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
92
93 /* Abort success! */
94 if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
95 !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
96 return 0;
97
98 } while (!time_after(jiffies, timeout_jiffies));
99
100 /* Abort has timed out and the MB is dead */
101 pci_err(pdev, "[%x] ABORT timed out\n", offset);
102 return -EIO;
103}
104
105static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
106 struct pci_doe_task *task)
107{
108 struct pci_dev *pdev = doe_mb->pdev;
109 int offset = doe_mb->cap_offset;
110 u32 val;
111 int i;
112
113 /*
114 * Check the DOE busy bit is not set. If it is set, this could indicate
115 * someone other than Linux (e.g. firmware) is using the mailbox. Note
116 * it is expected that firmware and OS will negotiate access rights via
117 * an, as yet to be defined, method.
118 */
119 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
120 if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
121 return -EBUSY;
122
123 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
124 return -EIO;
125
126 /* Write DOE Header */
127 val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) |
128 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type);
129 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
130 /* Length is 2 DW of header + length of payload in DW */
131 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
132 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
133 2 + task->request_pl_sz /
134 sizeof(u32)));
135 for (i = 0; i < task->request_pl_sz / sizeof(u32); i++)
136 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
137 task->request_pl[i]);
138
139 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
140
141 return 0;
142}
143
144static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
145{
146 struct pci_dev *pdev = doe_mb->pdev;
147 int offset = doe_mb->cap_offset;
148 u32 val;
149
150 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
151 if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
152 return true;
153 return false;
154}
155
156static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
157{
158 struct pci_dev *pdev = doe_mb->pdev;
159 int offset = doe_mb->cap_offset;
160 size_t length, payload_length;
161 u32 val;
162 int i;
163
164 /* Read the first dword to get the protocol */
165 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
166 if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) ||
167 (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) {
168 dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n",
169 doe_mb->cap_offset, task->prot.vid, task->prot.type,
170 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
171 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
172 return -EIO;
173 }
174
175 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
176 /* Read the second dword to get the length */
177 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
178 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
179
180 length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
181 if (length > SZ_1M || length < 2)
182 return -EIO;
183
184 /* First 2 dwords have already been read */
185 length -= 2;
186 payload_length = min(length, task->response_pl_sz / sizeof(u32));
187 /* Read the rest of the response payload */
188 for (i = 0; i < payload_length; i++) {
189 pci_read_config_dword(pdev, offset + PCI_DOE_READ,
190 &task->response_pl[i]);
191 /* Prior to the last ack, ensure Data Object Ready */
192 if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb))
193 return -EIO;
194 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
195 }
196
197 /* Flush excess length */
198 for (; i < length; i++) {
199 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
200 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
201 }
202
203 /* Final error check to pick up on any since Data Object Ready */
204 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
205 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
206 return -EIO;
207
208 return min(length, task->response_pl_sz / sizeof(u32)) * sizeof(u32);
209}
210
211static void signal_task_complete(struct pci_doe_task *task, int rv)
212{
213 task->rv = rv;
214 task->complete(task);
215}
216
217static void signal_task_abort(struct pci_doe_task *task, int rv)
218{
219 struct pci_doe_mb *doe_mb = task->doe_mb;
220 struct pci_dev *pdev = doe_mb->pdev;
221
222 if (pci_doe_abort(doe_mb)) {
223 /*
224 * If the device can't process an abort; set the mailbox dead
225 * - no more submissions
226 */
227 pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
228 doe_mb->cap_offset);
229 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
230 }
231 signal_task_complete(task, rv);
232}
233
234static void doe_statemachine_work(struct work_struct *work)
235{
236 struct pci_doe_task *task = container_of(work, struct pci_doe_task,
237 work);
238 struct pci_doe_mb *doe_mb = task->doe_mb;
239 struct pci_dev *pdev = doe_mb->pdev;
240 int offset = doe_mb->cap_offset;
241 unsigned long timeout_jiffies;
242 u32 val;
243 int rc;
244
245 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
246 signal_task_complete(task, -EIO);
247 return;
248 }
249
250 /* Send request */
251 rc = pci_doe_send_req(doe_mb, task);
252 if (rc) {
253 /*
254 * The specification does not provide any guidance on how to
255 * resolve conflicting requests from other entities.
256 * Furthermore, it is likely that busy will not be detected
257 * most of the time. Flag any detection of status busy with an
258 * error.
259 */
260 if (rc == -EBUSY)
261 dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
262 offset);
263 signal_task_abort(task, rc);
264 return;
265 }
266
267 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
268 /* Poll for response */
269retry_resp:
270 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
271 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
272 signal_task_abort(task, -EIO);
273 return;
274 }
275
276 if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
277 if (time_after(jiffies, timeout_jiffies)) {
278 signal_task_abort(task, -EIO);
279 return;
280 }
281 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
282 if (rc) {
283 signal_task_abort(task, rc);
284 return;
285 }
286 goto retry_resp;
287 }
288
289 rc = pci_doe_recv_resp(doe_mb, task);
290 if (rc < 0) {
291 signal_task_abort(task, rc);
292 return;
293 }
294
295 signal_task_complete(task, rc);
296}
297
298static void pci_doe_task_complete(struct pci_doe_task *task)
299{
300 complete(task->private);
301}
302
303static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid,
304 u8 *protocol)
305{
306 u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
307 *index);
308 u32 response_pl;
309 DECLARE_COMPLETION_ONSTACK(c);
310 struct pci_doe_task task = {
311 .prot.vid = PCI_VENDOR_ID_PCI_SIG,
312 .prot.type = PCI_DOE_PROTOCOL_DISCOVERY,
313 .request_pl = &request_pl,
314 .request_pl_sz = sizeof(request_pl),
315 .response_pl = &response_pl,
316 .response_pl_sz = sizeof(response_pl),
317 .complete = pci_doe_task_complete,
318 .private = &c,
319 };
320 int rc;
321
322 rc = pci_doe_submit_task(doe_mb, &task);
323 if (rc < 0)
324 return rc;
325
326 wait_for_completion(&c);
327
328 if (task.rv != sizeof(response_pl))
329 return -EIO;
330
331 *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
332 *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL,
333 response_pl);
334 *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
335 response_pl);
336
337 return 0;
338}
339
340static void *pci_doe_xa_prot_entry(u16 vid, u8 prot)
341{
342 return xa_mk_value((vid << 8) | prot);
343}
344
345static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb)
346{
347 u8 index = 0;
348 u8 xa_idx = 0;
349
350 do {
351 int rc;
352 u16 vid;
353 u8 prot;
354
355 rc = pci_doe_discovery(doe_mb, &index, &vid, &prot);
356 if (rc)
357 return rc;
358
359 pci_dbg(doe_mb->pdev,
360 "[%x] Found protocol %d vid: %x prot: %x\n",
361 doe_mb->cap_offset, xa_idx, vid, prot);
362
363 rc = xa_insert(&doe_mb->prots, xa_idx++,
364 pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL);
365 if (rc)
366 return rc;
367 } while (index);
368
369 return 0;
370}
371
372static void pci_doe_xa_destroy(void *mb)
373{
374 struct pci_doe_mb *doe_mb = mb;
375
376 xa_destroy(&doe_mb->prots);
377}
378
379static void pci_doe_destroy_workqueue(void *mb)
380{
381 struct pci_doe_mb *doe_mb = mb;
382
383 destroy_workqueue(doe_mb->work_queue);
384}
385
386static void pci_doe_flush_mb(void *mb)
387{
388 struct pci_doe_mb *doe_mb = mb;
389
390 /* Stop all pending work items from starting */
391 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
392
393 /* Cancel an in progress work item, if necessary */
394 set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
395 wake_up(&doe_mb->wq);
396
397 /* Flush all work items */
398 flush_workqueue(doe_mb->work_queue);
399}
400
401/**
402 * pcim_doe_create_mb() - Create a DOE mailbox object
403 *
404 * @pdev: PCI device to create the DOE mailbox for
405 * @cap_offset: Offset of the DOE mailbox
406 *
407 * Create a single mailbox object to manage the mailbox protocol at the
408 * cap_offset specified.
409 *
410 * RETURNS: created mailbox object on success
411 * ERR_PTR(-errno) on failure
412 */
413struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
414{
415 struct pci_doe_mb *doe_mb;
416 struct device *dev = &pdev->dev;
417 int rc;
418
419 doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL);
420 if (!doe_mb)
421 return ERR_PTR(-ENOMEM);
422
423 doe_mb->pdev = pdev;
424 doe_mb->cap_offset = cap_offset;
425 init_waitqueue_head(&doe_mb->wq);
426
427 xa_init(&doe_mb->prots);
428 rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb);
429 if (rc)
430 return ERR_PTR(rc);
431
432 doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
433 dev_driver_string(&pdev->dev),
434 pci_name(pdev),
435 doe_mb->cap_offset);
436 if (!doe_mb->work_queue) {
437 pci_err(pdev, "[%x] failed to allocate work queue\n",
438 doe_mb->cap_offset);
439 return ERR_PTR(-ENOMEM);
440 }
441 rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb);
442 if (rc)
443 return ERR_PTR(rc);
444
445 /* Reset the mailbox by issuing an abort */
446 rc = pci_doe_abort(doe_mb);
447 if (rc) {
448 pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
449 doe_mb->cap_offset, rc);
450 return ERR_PTR(rc);
451 }
452
453 /*
454 * The state machine and the mailbox should be in sync now;
455 * Set up mailbox flush prior to using the mailbox to query protocols.
456 */
457 rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
458 if (rc)
459 return ERR_PTR(rc);
460
461 rc = pci_doe_cache_protocols(doe_mb);
462 if (rc) {
463 pci_err(pdev, "[%x] failed to cache protocols : %d\n",
464 doe_mb->cap_offset, rc);
465 return ERR_PTR(rc);
466 }
467
468 return doe_mb;
469}
470EXPORT_SYMBOL_GPL(pcim_doe_create_mb);
471
472/**
473 * pci_doe_supports_prot() - Return if the DOE instance supports the given
474 * protocol
475 * @doe_mb: DOE mailbox capability to query
476 * @vid: Protocol Vendor ID
477 * @type: Protocol type
478 *
479 * RETURNS: True if the DOE mailbox supports the protocol specified
480 */
481bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
482{
483 unsigned long index;
484 void *entry;
485
486 /* The discovery protocol must always be supported */
487 if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY)
488 return true;
489
490 xa_for_each(&doe_mb->prots, index, entry)
491 if (entry == pci_doe_xa_prot_entry(vid, type))
492 return true;
493
494 return false;
495}
496EXPORT_SYMBOL_GPL(pci_doe_supports_prot);
497
498/**
499 * pci_doe_submit_task() - Submit a task to be processed by the state machine
500 *
501 * @doe_mb: DOE mailbox capability to submit to
502 * @task: task to be queued
503 *
504 * Submit a DOE task (request/response) to the DOE mailbox to be processed.
505 * Returns upon queueing the task object. If the queue is full this function
506 * will sleep until there is room in the queue.
507 *
508 * task->complete will be called when the state machine is done processing this
509 * task.
510 *
511 * Excess data will be discarded.
512 *
513 * RETURNS: 0 when task has been successfully queued, -ERRNO on error
514 */
515int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
516{
517 if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type))
518 return -EINVAL;
519
520 /*
521 * DOE requests must be a whole number of DW and the response needs to
522 * be big enough for at least 1 DW
523 */
524 if (task->request_pl_sz % sizeof(u32) ||
525 task->response_pl_sz < sizeof(u32))
526 return -EINVAL;
527
528 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
529 return -EIO;
530
531 task->doe_mb = doe_mb;
532 INIT_WORK(&task->work, doe_statemachine_work);
533 queue_work(doe_mb->work_queue, &task->work);
534 return 0;
535}
536EXPORT_SYMBOL_GPL(pci_doe_submit_task);
drivers/pci/probe.c+1−1
@@ -2315,7 +2315,7 @@ EXPORT_SYMBOL(pci_alloc_dev); · 2314 unmodified lines
2315 2315
2316static bool pci_bus_crs_vendor_id(u32 l)2316static bool pci_bus_crs_vendor_id(u32 l)
2317{2317{
2318 return (l & 0xffff) == 0x0001;2318 return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG;
2319}2319}
2320 2320
2321static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,2321static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
include/linux/ioport.h+3−0
@@ -141,6 +141,7 @@ enum { · 140 unmodified lines
141 IORES_DESC_DEVICE_PRIVATE_MEMORY = 6,141 IORES_DESC_DEVICE_PRIVATE_MEMORY = 6,
142 IORES_DESC_RESERVED = 7,142 IORES_DESC_RESERVED = 7,
143 IORES_DESC_SOFT_RESERVED = 8,143 IORES_DESC_SOFT_RESERVED = 8,
144 IORES_DESC_CXL = 9,
144};145};
145 146
146/*147/*
@@ -329,6 +330,8 @@ struct resource *devm_request_free_mem_region(struct device *dev, · 182 unmodified lines
329 struct resource *base, unsigned long size);330 struct resource *base, unsigned long size);
330struct resource *request_free_mem_region(struct resource *base,331struct resource *request_free_mem_region(struct resource *base,
331 unsigned long size, const char *name);332 unsigned long size, const char *name);
333struct resource *alloc_free_mem_region(struct resource *base,
334 unsigned long size, unsigned long align, const char *name);
332 335
333static inline void irqresource_disabled(struct resource *res, u32 irq)336static inline void irqresource_disabled(struct resource *res, u32 irq)
334{337{
include/linux/libnvdimm.h+5−0
@@ -59,6 +59,9 @@ enum { · 58 unmodified lines
59 /* Platform provides asynchronous flush mechanism */59 /* Platform provides asynchronous flush mechanism */
60 ND_REGION_ASYNC = 3,60 ND_REGION_ASYNC = 3,
61 61
62 /* Region was created by CXL subsystem */
63 ND_REGION_CXL = 4,
64
62 /* mark newly adjusted resources as requiring a label update */65 /* mark newly adjusted resources as requiring a label update */
63 DPA_RESOURCE_ADJUSTED = 1 << 0,66 DPA_RESOURCE_ADJUSTED = 1 << 0,
64};67};
@@ -122,6 +125,7 @@ struct nd_region_desc { · 57 unmodified lines
122 int numa_node;125 int numa_node;
123 int target_node;126 int target_node;
124 unsigned long flags;127 unsigned long flags;
128 int memregion;
125 struct device_node *of_node;129 struct device_node *of_node;
126 int (*flush)(struct nd_region *nd_region, struct bio *bio);130 int (*flush)(struct nd_region *nd_region, struct bio *bio);
127};131};
@@ -259,6 +263,7 @@ static inline struct nvdimm *nvdimm_create(struct nvdimm_bus *nvdimm_bus, · 131 unmodified lines
259 cmd_mask, num_flush, flush_wpq, NULL, NULL, NULL);263 cmd_mask, num_flush, flush_wpq, NULL, NULL, NULL);
260}264}
261void nvdimm_delete(struct nvdimm *nvdimm);265void nvdimm_delete(struct nvdimm *nvdimm);
266void nvdimm_region_delete(struct nd_region *nd_region);
262 267
263const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd);268const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd);
264const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd);269const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd);
include/linux/pci-doe.h+77−0
@@ -0,0 +1,77 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Data Object Exchange
4 * PCIe r6.0, sec 6.30 DOE
5 *
6 * Copyright (C) 2021 Huawei
7 * Jonathan Cameron <Jonathan.Cameron@huawei.com>
8 *
9 * Copyright (C) 2022 Intel Corporation
10 * Ira Weiny <ira.weiny@intel.com>
11 */
12
13#ifndef LINUX_PCI_DOE_H
14#define LINUX_PCI_DOE_H
15
16struct pci_doe_protocol {
17 u16 vid;
18 u8 type;
19};
20
21struct pci_doe_mb;
22
23/**
24 * struct pci_doe_task - represents a single query/response
25 *
26 * @prot: DOE Protocol
27 * @request_pl: The request payload
28 * @request_pl_sz: Size of the request payload (bytes)
29 * @response_pl: The response payload
30 * @response_pl_sz: Size of the response payload (bytes)
31 * @rv: Return value. Length of received response or error (bytes)
32 * @complete: Called when task is complete
33 * @private: Private data for the consumer
34 * @work: Used internally by the mailbox
35 * @doe_mb: Used internally by the mailbox
36 *
37 * The payload sizes and rv are specified in bytes with the following
38 * restrictions concerning the protocol.
39 *
40 * 1) The request_pl_sz must be a multiple of double words (4 bytes)
41 * 2) The response_pl_sz must be >= a single double word (4 bytes)
42 * 3) rv is returned as bytes but it will be a multiple of double words
43 *
44 * NOTE there is no need for the caller to initialize work or doe_mb.
45 */
46struct pci_doe_task {
47 struct pci_doe_protocol prot;
48 u32 *request_pl;
49 size_t request_pl_sz;
50 u32 *response_pl;
51 size_t response_pl_sz;
52 int rv;
53 void (*complete)(struct pci_doe_task *task);
54 void *private;
55
56 /* No need for the user to initialize these fields */
57 struct work_struct work;
58 struct pci_doe_mb *doe_mb;
59};
60
61/**
62 * pci_doe_for_each_off - Iterate each DOE capability
63 * @pdev: struct pci_dev to iterate
64 * @off: u16 of config space offset of each mailbox capability found
65 */
66#define pci_doe_for_each_off(pdev, off) \
67 for (off = pci_find_next_ext_capability(pdev, off, \
68 PCI_EXT_CAP_ID_DOE); \
69 off > 0; \
70 off = pci_find_next_ext_capability(pdev, off, \
71 PCI_EXT_CAP_ID_DOE))
72
73struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset);
74bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type);
75int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task);
76
77#endif
include/linux/pci_ids.h+1−0
@@ -151,6 +151,7 @@ · 150 unmodified lines
151#define PCI_CLASS_OTHERS 0xff151#define PCI_CLASS_OTHERS 0xff
152 152
153/* Vendors and devices. Sort key: vendor first, device next. */153/* Vendors and devices. Sort key: vendor first, device next. */
154#define PCI_VENDOR_ID_PCI_SIG 0x0001
154 155
155#define PCI_VENDOR_ID_LOONGSON 0x0014156#define PCI_VENDOR_ID_LOONGSON 0x0014
156 157
include/linux/sysfs.h+16−0
@@ -235,6 +235,22 @@ struct bin_attribute bin_attr_##_name = __BIN_ATTR_WO(_name, _size) · 234 unmodified lines
235#define BIN_ATTR_RW(_name, _size) \235#define BIN_ATTR_RW(_name, _size) \
236struct bin_attribute bin_attr_##_name = __BIN_ATTR_RW(_name, _size)236struct bin_attribute bin_attr_##_name = __BIN_ATTR_RW(_name, _size)
237 237
238
239#define __BIN_ATTR_ADMIN_RO(_name, _size) { \
240 .attr = { .name = __stringify(_name), .mode = 0400 }, \
241 .read = _name##_read, \
242 .size = _size, \
243}
244
245#define __BIN_ATTR_ADMIN_RW(_name, _size) \
246 __BIN_ATTR(_name, 0600, _name##_read, _name##_write, _size)
247
248#define BIN_ATTR_ADMIN_RO(_name, _size) \
249struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RO(_name, _size)
250
251#define BIN_ATTR_ADMIN_RW(_name, _size) \
252struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RW(_name, _size)
253
238struct sysfs_ops {254struct sysfs_ops {
239 ssize_t (*show)(struct kobject *, struct attribute *, char *);255 ssize_t (*show)(struct kobject *, struct attribute *, char *);
240 ssize_t (*store)(struct kobject *, struct attribute *, const char *, size_t);256 ssize_t (*store)(struct kobject *, struct attribute *, const char *, size_t);
include/uapi/linux/pci_regs.h+28−1
@@ -737,7 +737,8 @@ · 736 unmodified lines
737#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */737#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
738#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */738#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
739#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */739#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
740#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_PL_16GT740#define PCI_EXT_CAP_ID_DOE 0x2E /* Data Object Exchange */
741#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_DOE
741 742
742#define PCI_EXT_CAP_DSN_SIZEOF 12743#define PCI_EXT_CAP_DSN_SIZEOF 12
743#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40744#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
@@ -1103,4 +1104,30 @@ · 359 unmodified lines
1103#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK 0x000000F01104#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK 0x000000F0
1104#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT 41105#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT 4
1105 1106
1107/* Data Object Exchange */
1108#define PCI_DOE_CAP 0x04 /* DOE Capabilities Register */
1109#define PCI_DOE_CAP_INT_SUP 0x00000001 /* Interrupt Support */
1110#define PCI_DOE_CAP_INT_MSG_NUM 0x00000ffe /* Interrupt Message Number */
1111#define PCI_DOE_CTRL 0x08 /* DOE Control Register */
1112#define PCI_DOE_CTRL_ABORT 0x00000001 /* DOE Abort */
1113#define PCI_DOE_CTRL_INT_EN 0x00000002 /* DOE Interrupt Enable */
1114#define PCI_DOE_CTRL_GO 0x80000000 /* DOE Go */
1115#define PCI_DOE_STATUS 0x0c /* DOE Status Register */
1116#define PCI_DOE_STATUS_BUSY 0x00000001 /* DOE Busy */
1117#define PCI_DOE_STATUS_INT_STATUS 0x00000002 /* DOE Interrupt Status */
1118#define PCI_DOE_STATUS_ERROR 0x00000004 /* DOE Error */
1119#define PCI_DOE_STATUS_DATA_OBJECT_READY 0x80000000 /* Data Object Ready */
1120#define PCI_DOE_WRITE 0x10 /* DOE Write Data Mailbox Register */
1121#define PCI_DOE_READ 0x14 /* DOE Read Data Mailbox Register */
1122
1123/* DOE Data Object - note not actually registers */
1124#define PCI_DOE_DATA_OBJECT_HEADER_1_VID 0x0000ffff
1125#define PCI_DOE_DATA_OBJECT_HEADER_1_TYPE 0x00ff0000
1126#define PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH 0x0003ffff
1127
1128#define PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX 0x000000ff
1129#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID 0x0000ffff
1130#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL 0x00ff0000
1131#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX 0xff000000
1132
1106#endif /* LINUX_PCI_REGS_H */1133#endif /* LINUX_PCI_REGS_H */
kernel/resource.c+150−35
@@ -489,8 +489,9 @@ int __weak page_is_ram(unsigned long pfn) · 488 unmodified lines
489}489}
490EXPORT_SYMBOL_GPL(page_is_ram);490EXPORT_SYMBOL_GPL(page_is_ram);
491 491
492static int __region_intersects(resource_size_t start, size_t size,492static int __region_intersects(struct resource *parent, resource_size_t start,
493 unsigned long flags, unsigned long desc)493 size_t size, unsigned long flags,
494 unsigned long desc)
494{495{
495 struct resource res;496 struct resource res;
496 int type = 0; int other = 0;497 int type = 0; int other = 0;
@@ -499,7 +500,7 @@ static int __region_intersects(resource_size_t start, size_t size, · 2 unmodified lines
499 res.start = start;500 res.start = start;
500 res.end = start + size - 1;501 res.end = start + size - 1;
501 502
502 for (p = iomem_resource.child; p ; p = p->sibling) {503 for (p = parent->child; p ; p = p->sibling) {
503 bool is_type = (((p->flags & flags) == flags) &&504 bool is_type = (((p->flags & flags) == flags) &&
504 ((desc == IORES_DESC_NONE) ||505 ((desc == IORES_DESC_NONE) ||
505 (desc == p->desc)));506 (desc == p->desc)));
@@ -543,7 +544,7 @@ int region_intersects(resource_size_t start, size_t size, unsigned long flags, · 37 unmodified lines
543 int ret;544 int ret;
544 545
545 read_lock(&resource_lock);546 read_lock(&resource_lock);
546 ret = __region_intersects(start, size, flags, desc);547 ret = __region_intersects(&iomem_resource, start, size, flags, desc);
547 read_unlock(&resource_lock);548 read_unlock(&resource_lock);
548 549
549 return ret;550 return ret;
@@ -891,6 +892,13 @@ void insert_resource_expand_to_fit(struct resource *root, struct resource *new) · 341 unmodified lines
891 }892 }
892 write_unlock(&resource_lock);893 write_unlock(&resource_lock);
893}894}
895/*
896 * Not for general consumption, only early boot memory map parsing, PCI
897 * resource discovery, and late discovery of CXL resources are expected
898 * to use this interface. The former are built-in and only the latter,
899 * CXL, is a module.
900 */
901EXPORT_SYMBOL_NS_GPL(insert_resource_expand_to_fit, CXL);
894 902
895/**903/**
896 * remove_resource - Remove a resource in the resource tree904 * remove_resource - Remove a resource in the resource tree
@@ -1773,62 +1781,139 @@ void resource_list_free(struct list_head *head) · 876 unmodified lines
1773}1781}
1774EXPORT_SYMBOL(resource_list_free);1782EXPORT_SYMBOL(resource_list_free);
1775 1783
1776#ifdef CONFIG_DEVICE_PRIVATE1784#ifdef CONFIG_GET_FREE_REGION
1777static struct resource *__request_free_mem_region(struct device *dev,1785#define GFR_DESCENDING (1UL << 0)
1778 struct resource *base, unsigned long size, const char *name)1786#define GFR_REQUEST_REGION (1UL << 1)
1787#define GFR_DEFAULT_ALIGN (1UL << PA_SECTION_SHIFT)
1788
1789static resource_size_t gfr_start(struct resource *base, resource_size_t size,
1790 resource_size_t align, unsigned long flags)
1791{
1792 if (flags & GFR_DESCENDING) {
1793 resource_size_t end;
1794
1795 end = min_t(resource_size_t, base->end,
1796 (1ULL << MAX_PHYSMEM_BITS) - 1);
1797 return end - size + 1;
1798 }
1799
1800 return ALIGN(base->start, align);
1801}
1802
1803static bool gfr_continue(struct resource *base, resource_size_t addr,
1804 resource_size_t size, unsigned long flags)
1805{
1806 if (flags & GFR_DESCENDING)
1807 return addr > size && addr >= base->start;
1808 /*
1809 * In the ascend case be careful that the last increment by
1810 * @size did not wrap 0.
1811 */
1812 return addr > addr - size &&
1813 addr <= min_t(resource_size_t, base->end,
1814 (1ULL << MAX_PHYSMEM_BITS) - 1);
1815}
1816
1817static resource_size_t gfr_next(resource_size_t addr, resource_size_t size,
1818 unsigned long flags)
1779{1819{
1780 resource_size_t end, addr;1820 if (flags & GFR_DESCENDING)
1821 return addr - size;
1822 return addr + size;
1823}
1824
1825static void remove_free_mem_region(void *_res)
1826{
1827 struct resource *res = _res;
1828
1829 if (res->parent)
1830 remove_resource(res);
1831 free_resource(res);
1832}
1833
1834static struct resource *
1835get_free_mem_region(struct device *dev, struct resource *base,
1836 resource_size_t size, const unsigned long align,
1837 const char *name, const unsigned long desc,
1838 const unsigned long flags)
1839{
1840 resource_size_t addr;
1781 struct resource *res;1841 struct resource *res;
1782 struct region_devres *dr = NULL;1842 struct region_devres *dr = NULL;
1783 1843
1784 size = ALIGN(size, 1UL << PA_SECTION_SHIFT);1844 size = ALIGN(size, align);
1785 end = min_t(unsigned long, base->end, (1UL << MAX_PHYSMEM_BITS) - 1);
1786 addr = end - size + 1UL;
1787 1845
1788 res = alloc_resource(GFP_KERNEL);1846 res = alloc_resource(GFP_KERNEL);
1789 if (!res)1847 if (!res)
1790 return ERR_PTR(-ENOMEM);1848 return ERR_PTR(-ENOMEM);
1791 1849
1792 if (dev) {1850 if (dev && (flags & GFR_REQUEST_REGION)) {
1793 dr = devres_alloc(devm_region_release,1851 dr = devres_alloc(devm_region_release,
1794 sizeof(struct region_devres), GFP_KERNEL);1852 sizeof(struct region_devres), GFP_KERNEL);
1795 if (!dr) {1853 if (!dr) {
1796 free_resource(res);1854 free_resource(res);
1797 return ERR_PTR(-ENOMEM);1855 return ERR_PTR(-ENOMEM);
1798 }1856 }
1857 } else if (dev) {
1858 if (devm_add_action_or_reset(dev, remove_free_mem_region, res))
1859 return ERR_PTR(-ENOMEM);
1799 }1860 }
1800 1861
1801 write_lock(&resource_lock);1862 write_lock(&resource_lock);
1802 for (; addr > size && addr >= base->start; addr -= size) {1863 for (addr = gfr_start(base, size, align, flags);
1803 if (__region_intersects(addr, size, 0, IORES_DESC_NONE) !=1864 gfr_continue(base, addr, size, flags);
1804 REGION_DISJOINT)1865 addr = gfr_next(addr, size, flags)) {
1866 if (__region_intersects(base, addr, size, 0, IORES_DESC_NONE) !=
1867 REGION_DISJOINT)
1805 continue;1868 continue;
1806 1869
1807 if (__request_region_locked(res, &iomem_resource, addr, size,1870 if (flags & GFR_REQUEST_REGION) {
1808 name, 0))1871 if (__request_region_locked(res, &iomem_resource, addr,
1809 break;1872 size, name, 0))
1873 break;
1810 1874
1811 if (dev) {1875 if (dev) {
1812 dr->parent = &iomem_resource;1876 dr->parent = &iomem_resource;
1813 dr->start = addr;1877 dr->start = addr;
1814 dr->n = size;1878 dr->n = size;
1815 devres_add(dev, dr);1879 devres_add(dev, dr);
1816 }1880 }
1817 1881
1818 res->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;1882 res->desc = desc;
1819 write_unlock(&resource_lock);1883 write_unlock(&resource_lock);
1884
1885
1886 /*
1887 * A driver is claiming this region so revoke any
1888 * mappings.
1889 */
1890 revoke_iomem(res);
1891 } else {
1892 res->start = addr;
1893 res->end = addr + size - 1;
1894 res->name = name;
1895 res->desc = desc;
1896 res->flags = IORESOURCE_MEM;
1897
1898 /*
1899 * Only succeed if the resource hosts an exclusive
1900 * range after the insert
1901 */
1902 if (__insert_resource(base, res) || res->child)
1903 break;
1904
1905 write_unlock(&resource_lock);
1906 }
1820 1907
1821 /*
1822 * A driver is claiming this region so revoke any mappings.
1823 */
1824 revoke_iomem(res);
1825 return res;1908 return res;
1826 }1909 }
1827 write_unlock(&resource_lock);1910 write_unlock(&resource_lock);
1828 1911
1829 free_resource(res);1912 if (flags & GFR_REQUEST_REGION) {
1830 if (dr)1913 free_resource(res);
1831 devres_free(dr);1914 devres_free(dr);
1915 } else if (dev)
1916 devm_release_action(dev, remove_free_mem_region, res);
1832 1917
1833 return ERR_PTR(-ERANGE);1918 return ERR_PTR(-ERANGE);
1834}1919}
@@ -1847,18 +1932,48 @@ static struct resource *__request_free_mem_region(struct device *dev, · 12 unmodified lines
1847struct resource *devm_request_free_mem_region(struct device *dev,1932struct resource *devm_request_free_mem_region(struct device *dev,
1848 struct resource *base, unsigned long size)1933 struct resource *base, unsigned long size)
1849{1934{
1850 return __request_free_mem_region(dev, base, size, dev_name(dev));1935 unsigned long flags = GFR_DESCENDING | GFR_REQUEST_REGION;
1936
1937 return get_free_mem_region(dev, base, size, GFR_DEFAULT_ALIGN,
1938 dev_name(dev),
1939 IORES_DESC_DEVICE_PRIVATE_MEMORY, flags);
1851}1940}
1852EXPORT_SYMBOL_GPL(devm_request_free_mem_region);1941EXPORT_SYMBOL_GPL(devm_request_free_mem_region);
1853 1942
1854struct resource *request_free_mem_region(struct resource *base,1943struct resource *request_free_mem_region(struct resource *base,
1855 unsigned long size, const char *name)1944 unsigned long size, const char *name)
1856{1945{
1857 return __request_free_mem_region(NULL, base, size, name);1946 unsigned long flags = GFR_DESCENDING | GFR_REQUEST_REGION;
1947
1948 return get_free_mem_region(NULL, base, size, GFR_DEFAULT_ALIGN, name,
1949 IORES_DESC_DEVICE_PRIVATE_MEMORY, flags);
1858}1950}
1859EXPORT_SYMBOL_GPL(request_free_mem_region);1951EXPORT_SYMBOL_GPL(request_free_mem_region);
1860 1952
1861#endif /* CONFIG_DEVICE_PRIVATE */1953/**
1954 * alloc_free_mem_region - find a free region relative to @base
1955 * @base: resource that will parent the new resource
1956 * @size: size in bytes of memory to allocate from @base
1957 * @align: alignment requirements for the allocation
1958 * @name: resource name
1959 *
1960 * Buses like CXL, that can dynamically instantiate new memory regions,
1961 * need a method to allocate physical address space for those regions.
1962 * Allocate and insert a new resource to cover a free, unclaimed by a
1963 * descendant of @base, range in the span of @base.
1964 */
1965struct resource *alloc_free_mem_region(struct resource *base,
1966 unsigned long size, unsigned long align,
1967 const char *name)
1968{
1969 /* Default of ascending direction and insert resource */
1970 unsigned long flags = 0;
1971
1972 return get_free_mem_region(NULL, base, size, align, name,
1973 IORES_DESC_NONE, flags);
1974}
1975EXPORT_SYMBOL_NS_GPL(alloc_free_mem_region, CXL);
1976#endif /* CONFIG_GET_FREE_REGION */
1862 1977
1863static int __init strict_iomem(char *str)1978static int __init strict_iomem(char *str)
1864{1979{
mm/Kconfig+5−0
@@ -983,9 +983,14 @@ config HMM_MIRROR · 982 unmodified lines
983 bool983 bool
984 depends on MMU984 depends on MMU
985 985
986config GET_FREE_REGION
987 depends on SPARSEMEM
988 bool
989
986config DEVICE_PRIVATE990config DEVICE_PRIVATE
987 bool "Unaddressable device memory (GPU memory, ...)"991 bool "Unaddressable device memory (GPU memory, ...)"
988 depends on ZONE_DEVICE992 depends on ZONE_DEVICE
993 select GET_FREE_REGION
989 994
990 help995 help
991 Allows creation of struct pages to represent unaddressable device996 Allows creation of struct pages to represent unaddressable device
tools/testing/cxl/Kbuild+1−0
@@ -47,6 +47,7 @@ cxl_core-y += $(CXL_CORE_SRC)/memdev.o · 46 unmodified lines
47cxl_core-y += $(CXL_CORE_SRC)/mbox.o47cxl_core-y += $(CXL_CORE_SRC)/mbox.o
48cxl_core-y += $(CXL_CORE_SRC)/pci.o48cxl_core-y += $(CXL_CORE_SRC)/pci.o
49cxl_core-y += $(CXL_CORE_SRC)/hdm.o49cxl_core-y += $(CXL_CORE_SRC)/hdm.o
50cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o
50cxl_core-y += config_check.o51cxl_core-y += config_check.o
51 52
52obj-m += test/53obj-m += test/
tools/testing/cxl/test/cxl.c+76−55
@@ -14,7 +14,7 @@ · 13 unmodified lines
14#define NR_CXL_HOST_BRIDGES 214#define NR_CXL_HOST_BRIDGES 2
15#define NR_CXL_ROOT_PORTS 215#define NR_CXL_ROOT_PORTS 2
16#define NR_CXL_SWITCH_PORTS 216#define NR_CXL_SWITCH_PORTS 2
17#define NR_CXL_PORT_DECODERS 217#define NR_CXL_PORT_DECODERS 8
18 18
19static struct platform_device *cxl_acpi;19static struct platform_device *cxl_acpi;
20static struct platform_device *cxl_host_bridge[NR_CXL_HOST_BRIDGES];20static struct platform_device *cxl_host_bridge[NR_CXL_HOST_BRIDGES];
@@ -118,7 +118,7 @@ static struct { · 97 unmodified lines
118 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |118 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
119 ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,119 ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
120 .qtg_id = 0,120 .qtg_id = 0,
121 .window_size = SZ_256M,121 .window_size = SZ_256M * 4UL,
122 },122 },
123 .target = { 0 },123 .target = { 0 },
124 },124 },
@@ -133,7 +133,7 @@ static struct { · 8 unmodified lines
133 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |133 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
134 ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,134 ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
135 .qtg_id = 1,135 .qtg_id = 1,
136 .window_size = SZ_256M * 2,136 .window_size = SZ_256M * 8UL,
137 },137 },
138 .target = { 0, 1, },138 .target = { 0, 1, },
139 },139 },
@@ -148,7 +148,7 @@ static struct { · 8 unmodified lines
148 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |148 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
149 ACPI_CEDT_CFMWS_RESTRICT_PMEM,149 ACPI_CEDT_CFMWS_RESTRICT_PMEM,
150 .qtg_id = 2,150 .qtg_id = 2,
151 .window_size = SZ_256M,151 .window_size = SZ_256M * 4UL,
152 },152 },
153 .target = { 0 },153 .target = { 0 },
154 },154 },
@@ -163,7 +163,7 @@ static struct { · 8 unmodified lines
163 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |163 .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
164 ACPI_CEDT_CFMWS_RESTRICT_PMEM,164 ACPI_CEDT_CFMWS_RESTRICT_PMEM,
165 .qtg_id = 3,165 .qtg_id = 3,
166 .window_size = SZ_256M * 2,166 .window_size = SZ_256M * 8UL,
167 },167 },
168 .target = { 0, 1, },168 .target = { 0, 1, },
169 },169 },
@@ -429,6 +429,50 @@ static int map_targets(struct device *dev, void *data) · 259 unmodified lines
429 return 0;429 return 0;
430}430}
431 431
432static int mock_decoder_commit(struct cxl_decoder *cxld)
433{
434 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
435 int id = cxld->id;
436
437 if (cxld->flags & CXL_DECODER_F_ENABLE)
438 return 0;
439
440 dev_dbg(&port->dev, "%s commit\n", dev_name(&cxld->dev));
441 if (port->commit_end + 1 != id) {
442 dev_dbg(&port->dev,
443 "%s: out of order commit, expected decoder%d.%d\n",
444 dev_name(&cxld->dev), port->id, port->commit_end + 1);
445 return -EBUSY;
446 }
447
448 port->commit_end++;
449 cxld->flags |= CXL_DECODER_F_ENABLE;
450
451 return 0;
452}
453
454static int mock_decoder_reset(struct cxl_decoder *cxld)
455{
456 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
457 int id = cxld->id;
458
459 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
460 return 0;
461
462 dev_dbg(&port->dev, "%s reset\n", dev_name(&cxld->dev));
463 if (port->commit_end != id) {
464 dev_dbg(&port->dev,
465 "%s: out of order reset, expected decoder%d.%d\n",
466 dev_name(&cxld->dev), port->id, port->commit_end);
467 return -EBUSY;
468 }
469
470 port->commit_end--;
471 cxld->flags &= ~CXL_DECODER_F_ENABLE;
472
473 return 0;
474}
475
432static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)476static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
433{477{
434 struct cxl_port *port = cxlhdm->port;478 struct cxl_port *port = cxlhdm->port;
@@ -451,25 +495,39 @@ static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 16 unmodified lines
451 struct cxl_decoder *cxld;495 struct cxl_decoder *cxld;
452 int rc;496 int rc;
453 497
454 if (target_count)498 if (target_count) {
455 cxld = cxl_switch_decoder_alloc(port, target_count);499 struct cxl_switch_decoder *cxlsd;
456 else500
457 cxld = cxl_endpoint_decoder_alloc(port);501 cxlsd = cxl_switch_decoder_alloc(port, target_count);
458 if (IS_ERR(cxld)) {502 if (IS_ERR(cxlsd)) {
459 dev_warn(&port->dev,503 dev_warn(&port->dev,
460 "Failed to allocate the decoder\n");504 "Failed to allocate the decoder\n");
461 return PTR_ERR(cxld);505 return PTR_ERR(cxlsd);
506 }
507 cxld = &cxlsd->cxld;
508 } else {
509 struct cxl_endpoint_decoder *cxled;
510
511 cxled = cxl_endpoint_decoder_alloc(port);
512
513 if (IS_ERR(cxled)) {
514 dev_warn(&port->dev,
515 "Failed to allocate the decoder\n");
516 return PTR_ERR(cxled);
517 }
518 cxld = &cxled->cxld;
462 }519 }
463 520
464 cxld->decoder_range = (struct range) {521 cxld->hpa_range = (struct range) {
465 .start = 0,522 .start = 0,
466 .end = -1,523 .end = -1,
467 };524 };
468 525
469 cxld->flags = CXL_DECODER_F_ENABLE;
470 cxld->interleave_ways = min_not_zero(target_count, 1);526 cxld->interleave_ways = min_not_zero(target_count, 1);
471 cxld->interleave_granularity = SZ_4K;527 cxld->interleave_granularity = SZ_4K;
472 cxld->target_type = CXL_DECODER_EXPANDER;528 cxld->target_type = CXL_DECODER_EXPANDER;
529 cxld->commit = mock_decoder_commit;
530 cxld->reset = mock_decoder_reset;
473 531
474 if (target_count) {532 if (target_count) {
475 rc = device_for_each_child(port->uport, &ctx,533 rc = device_for_each_child(port->uport, &ctx,
@@ -569,44 +627,6 @@ static void mock_companion(struct acpi_device *adev, struct device *dev) · 93 unmodified lines
569#define SZ_512G (SZ_64G * 8)627#define SZ_512G (SZ_64G * 8)
570#endif628#endif
571 629
572static struct platform_device *alloc_memdev(int id)
573{
574 struct resource res[] = {
575 [0] = {
576 .flags = IORESOURCE_MEM,
577 },
578 [1] = {
579 .flags = IORESOURCE_MEM,
580 .desc = IORES_DESC_PERSISTENT_MEMORY,
581 },
582 };
583 struct platform_device *pdev;
584 int i, rc;
585
586 for (i = 0; i < ARRAY_SIZE(res); i++) {
587 struct cxl_mock_res *r = alloc_mock_res(SZ_256M);
588
589 if (!r)
590 return NULL;
591 res[i].start = r->range.start;
592 res[i].end = r->range.end;
593 }
594
595 pdev = platform_device_alloc("cxl_mem", id);
596 if (!pdev)
597 return NULL;
598
599 rc = platform_device_add_resources(pdev, res, ARRAY_SIZE(res));
600 if (rc)
601 goto err;
602
603 return pdev;
604
605err:
606 platform_device_put(pdev);
607 return NULL;
608}
609
610static __init int cxl_test_init(void)630static __init int cxl_test_init(void)
611{631{
612 int rc, i;632 int rc, i;
@@ -619,7 +639,8 @@ static __init int cxl_test_init(void) · 6 unmodified lines
619 goto err_gen_pool_create;639 goto err_gen_pool_create;
620 }640 }
621 641
622 rc = gen_pool_add(cxl_mock_pool, SZ_512G, SZ_64G, NUMA_NO_NODE);642 rc = gen_pool_add(cxl_mock_pool, iomem_resource.end + 1 - SZ_64G,
643 SZ_64G, NUMA_NO_NODE);
623 if (rc)644 if (rc)
624 goto err_gen_pool_add;645 goto err_gen_pool_add;
625 646
@@ -708,7 +729,7 @@ static __init int cxl_test_init(void) · 82 unmodified lines
708 struct platform_device *dport = cxl_switch_dport[i];729 struct platform_device *dport = cxl_switch_dport[i];
709 struct platform_device *pdev;730 struct platform_device *pdev;
710 731
711 pdev = alloc_memdev(i);732 pdev = platform_device_alloc("cxl_mem", i);
712 if (!pdev)733 if (!pdev)
713 goto err_mem;734 goto err_mem;
714 pdev->dev.parent = &dport->dev;735 pdev->dev.parent = &dport->dev;
tools/testing/cxl/test/mem.c+28−25
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#include <cxlmem.h>10#include <cxlmem.h>
11 11
12#define LSA_SIZE SZ_128K12#define LSA_SIZE SZ_128K
13#define DEV_SIZE SZ_2G
13#define EFFECT(x) (1U << x)14#define EFFECT(x) (1U << x)
14 15
15static struct cxl_cel_entry mock_cel[] = {16static struct cxl_cel_entry mock_cel[] = {
@@ -25,6 +26,10 @@ static struct cxl_cel_entry mock_cel[] = { · 9 unmodified lines
25 .opcode = cpu_to_le16(CXL_MBOX_OP_GET_LSA),26 .opcode = cpu_to_le16(CXL_MBOX_OP_GET_LSA),
26 .effect = cpu_to_le16(0),27 .effect = cpu_to_le16(0),
27 },28 },
29 {
30 .opcode = cpu_to_le16(CXL_MBOX_OP_GET_PARTITION_INFO),
31 .effect = cpu_to_le16(0),
32 },
28 {33 {
29 .opcode = cpu_to_le16(CXL_MBOX_OP_SET_LSA),34 .opcode = cpu_to_le16(CXL_MBOX_OP_SET_LSA),
30 .effect = cpu_to_le16(EFFECT(1) | EFFECT(2)),35 .effect = cpu_to_le16(EFFECT(1) | EFFECT(2)),
@@ -97,42 +102,37 @@ static int mock_get_log(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd) · 66 unmodified lines
97 102
98static int mock_id(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)103static int mock_id(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
99{104{
100 struct platform_device *pdev = to_platform_device(cxlds->dev);
101 struct cxl_mbox_identify id = {105 struct cxl_mbox_identify id = {
102 .fw_revision = { "mock fw v1 " },106 .fw_revision = { "mock fw v1 " },
103 .lsa_size = cpu_to_le32(LSA_SIZE),107 .lsa_size = cpu_to_le32(LSA_SIZE),
104 /* FIXME: Add partition support */108 .partition_align =
105 .partition_align = cpu_to_le64(0),109 cpu_to_le64(SZ_256M / CXL_CAPACITY_MULTIPLIER),
110 .total_capacity =
111 cpu_to_le64(DEV_SIZE / CXL_CAPACITY_MULTIPLIER),
106 };112 };
107 u64 capacity = 0;
108 int i;
109 113
110 if (cmd->size_out < sizeof(id))114 if (cmd->size_out < sizeof(id))
111 return -EINVAL;115 return -EINVAL;
112 116
113 for (i = 0; i < 2; i++) {117 memcpy(cmd->payload_out, &id, sizeof(id));
114 struct resource *res;
115
116 res = platform_get_resource(pdev, IORESOURCE_MEM, i);
117 if (!res)
118 break;
119
120 capacity += resource_size(res) / CXL_CAPACITY_MULTIPLIER;
121 118
122 if (le64_to_cpu(id.partition_align))119 return 0;
123 continue;120}
124 121
125 if (res->desc == IORES_DESC_PERSISTENT_MEMORY)122static int mock_partition_info(struct cxl_dev_state *cxlds,
126 id.persistent_capacity = cpu_to_le64(123 struct cxl_mbox_cmd *cmd)
127 resource_size(res) / CXL_CAPACITY_MULTIPLIER);124{
128 else125 struct cxl_mbox_get_partition_info pi = {
129 id.volatile_capacity = cpu_to_le64(126 .active_volatile_cap =
130 resource_size(res) / CXL_CAPACITY_MULTIPLIER);127 cpu_to_le64(DEV_SIZE / 2 / CXL_CAPACITY_MULTIPLIER),
131 }128 .active_persistent_cap =
129 cpu_to_le64(DEV_SIZE / 2 / CXL_CAPACITY_MULTIPLIER),
130 };
132 131
133 id.total_capacity = cpu_to_le64(capacity);132 if (cmd->size_out < sizeof(pi))
133 return -EINVAL;
134 134
135 memcpy(cmd->payload_out, &id, sizeof(id));135 memcpy(cmd->payload_out, &pi, sizeof(pi));
136 136
137 return 0;137 return 0;
138}138}
@@ -221,6 +221,9 @@ static int cxl_mock_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd * · 82 unmodified lines
221 case CXL_MBOX_OP_GET_LSA:221 case CXL_MBOX_OP_GET_LSA:
222 rc = mock_get_lsa(cxlds, cmd);222 rc = mock_get_lsa(cxlds, cmd);
223 break;223 break;
224 case CXL_MBOX_OP_GET_PARTITION_INFO:
225 rc = mock_partition_info(cxlds, cmd);
226 break;
224 case CXL_MBOX_OP_SET_LSA:227 case CXL_MBOX_OP_SET_LSA:
225 rc = mock_set_lsa(cxlds, cmd);228 rc = mock_set_lsa(cxlds, cmd);
226 break;229 break;
@@ -282,7 +285,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) · 55 unmodified lines
282 if (IS_ERR(cxlmd))285 if (IS_ERR(cxlmd))
283 return PTR_ERR(cxlmd);286 return PTR_ERR(cxlmd);
284 287
285 if (range_len(&cxlds->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))288 if (resource_size(&cxlds->pmem_res) && IS_ENABLED(CONFIG_CXL_PMEM))
286 rc = devm_cxl_add_nvdimm(dev, cxlmd);289 rc = devm_cxl_add_nvdimm(dev, cxlmd);
287 290
288 return 0;291 return 0;
tools/testing/cxl/test/mock.c+5−3
@@ -208,13 +208,15 @@ int __wrap_cxl_await_media_ready(struct cxl_dev_state *cxlds) · 207 unmodified lines
208}208}
209EXPORT_SYMBOL_NS_GPL(__wrap_cxl_await_media_ready, CXL);209EXPORT_SYMBOL_NS_GPL(__wrap_cxl_await_media_ready, CXL);
210 210
211bool __wrap_cxl_hdm_decode_init(struct cxl_dev_state *cxlds,211int __wrap_cxl_hdm_decode_init(struct cxl_dev_state *cxlds,
212 struct cxl_hdm *cxlhdm)212 struct cxl_hdm *cxlhdm)
213{213{
214 int rc = 0, index;214 int rc = 0, index;
215 struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);215 struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
216 216
217 if (!ops || !ops->is_mock_dev(cxlds->dev))217 if (ops && ops->is_mock_dev(cxlds->dev))
218 rc = 0;
219 else
218 rc = cxl_hdm_decode_init(cxlds, cxlhdm);220 rc = cxl_hdm_decode_init(cxlds, cxlhdm);
219 put_cxl_mock_ops(index);221 put_cxl_mock_ops(index);
220 222