Merge tag 'cxl-for-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl

Pull cxl updates from Dan Williams: "Compute Express Link (CXL) updates for 6.0: - Introduce a 'struct cxl_region' object with support for provisioning and assembling persistent memory regions. - Introduce alloc_free_mem_region() to accompany the existing request_free_mem_region() as a method to allocate physical memory capacity out of an existing resource. - Export insert_resource_expand_to_fit() for the CXL subsystem to late-publish CXL platform windows in iomem_resource. - Add a polled mode PCI DOE (Data Object Exchange) driver service and use it in cxl_pci to retrieve the CDAT (Coherent Device Attribute Table)" * tag 'cxl-for-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl: (74 commits) cxl/hdm: Fix skip allocations vs multiple pmem allocations cxl/region: Disallow region granularity != window granularity cxl/region: Fix x1 interleave to greater than x1 interleave routing cxl/region: Move HPA setup to cxl_region_attach() cxl/region: Fix decoder interleave programming Documentation: cxl: remove dangling kernel-doc reference cxl/region: describe targets and nr_targets members of cxl_region_params cxl/regions: add padding for cxl_rr_ep_add nested lists cxl/region: Fix IS_ERR() vs NULL check cxl/region: Fix region reference target accounting cxl/region: Fix region commit uninitialized variable warning cxl/region: Fix port setup uninitialized variable warnings cxl/region: Stop initializing interleave granularity cxl/hdm: Fix DPA reservation vs cxl_endpoint_decoder lifetime cxl/acpi: Minimize granularity for x1 interleaves cxl/region: Delete 'region' attribute from root decoders cxl/acpi: Autoload driver for 'cxl_acpi' test devices cxl/region: decrement ->nr_targets on error in cxl_region_attach() cxl/region: prevent underflow in ways_to_cxl() cxl/region: uninitialized variable in alloc_hpa() ...

Linus Torvalds committed 4 years ago

Commit c2356982 parents 5e2e738 1cd8a2539 files · +5498 −576Browse files

Unified · Split

39 files

.clang-format +1 −0
Documentation/ABI/testing/sysfs-bus-cxl +265 −40
Documentation/driver-api/cxl/memory-devices.rst +8 −0
arch/powerpc/mm/mem.c +1 −0
drivers/cxl/Kconfig +9 −0
drivers/cxl/acpi.c +217 −26
drivers/cxl/core/Makefile +1 −0
drivers/cxl/core/core.h +50 −1
drivers/cxl/core/hdm.c +635 −56
drivers/cxl/core/mbox.c +47 −48
drivers/cxl/core/memdev.c +2 −2
drivers/cxl/core/pci.c +174 −7
drivers/cxl/core/pmem.c +2 −2
drivers/cxl/core/port.c +536 −202
drivers/cxl/core/region.c +1896 −0
drivers/cxl/cxl.h +283 −29
drivers/cxl/cxlmem.h +38 −4
drivers/cxl/cxlpci.h +1 −0
drivers/cxl/mem.c +23 −26
drivers/cxl/pci.c +45 −1
drivers/cxl/pmem.c +254 −5
drivers/cxl/port.c +53 −0
drivers/nvdimm/region_devs.c +21 −7
drivers/pci/Kconfig +3 −0
drivers/pci/Makefile +1 −0
drivers/pci/doe.c +536 −0
drivers/pci/probe.c +1 −1
include/linux/ioport.h +3 −0
include/linux/libnvdimm.h +5 −0
include/linux/pci-doe.h +77 −0
include/linux/pci_ids.h +1 −0
include/linux/sysfs.h +16 −0
include/uapi/linux/pci_regs.h +28 −1
kernel/resource.c +150 −35
mm/Kconfig +5 −0
tools/testing/cxl/Kbuild +1 −0
tools/testing/cxl/test/cxl.c +76 −55
tools/testing/cxl/test/mem.c +28 −25
tools/testing/cxl/test/mock.c +5 −3

.clang-format+1−0

@@ -516,6 +516,7 @@ ForEachMacros: · 515 unmodified lines
516	516	- 'of_property_for_each_string'
517	517	- 'of_property_for_each_u32'
518	518	- 'pci_bus_for_each_resource'
	519	+ - 'pci_doe_for_each_off'
519	520	- 'pcl_for_each_chunk'
520	521	- 'pcl_for_each_segment'
521	522	- 'pcm_for_each_format'

Documentation/ABI/testing/sysfs-bus-cxl+265−40

@@ -7,6 +7,7 @@ Description: · 6 unmodified lines
7	7	all descendant memdevs for unbind. Writing '1' to this attribute
8	8	flushes that work.
9	9
	10	+
10	11	What: /sys/bus/cxl/devices/memX/firmware_version
11	12	Date: December, 2020
12	13	KernelVersion: v5.12
@@ -16,6 +17,7 @@ Description: · 3 unmodified lines
16	17	Memory Device Output Payload in the CXL-2.0
17	18	specification.
18	19
	20	+
19	21	What: /sys/bus/cxl/devices/memX/ram/size
20	22	Date: December, 2020
21	23	KernelVersion: v5.12
@@ -25,6 +27,7 @@ Description: · 3 unmodified lines
25	27	identically named field in the Identify Memory Device Output
26	28	Payload in the CXL-2.0 specification.
27	29
	30	+
28	31	What: /sys/bus/cxl/devices/memX/pmem/size
29	32	Date: December, 2020
30	33	KernelVersion: v5.12
@@ -34,6 +37,7 @@ Description: · 3 unmodified lines
34	37	identically named field in the Identify Memory Device Output
35	38	Payload in the CXL-2.0 specification.
36	39
	40	+
37	41	What: /sys/bus/cxl/devices/memX/serial
38	42	Date: January, 2022
39	43	KernelVersion: v5.18
@@ -43,6 +47,7 @@ Description: · 3 unmodified lines
43	47	capability. Mandatory for CXL devices, see CXL 2.0 8.1.12.2
44	48	Memory Device PCIe Capabilities and Extended Capabilities.
45	49
	50	+
46	51	What: /sys/bus/cxl/devices/memX/numa_node
47	52	Date: January, 2022
48	53	KernelVersion: v5.18
@@ -52,114 +57,334 @@ Description: · 3 unmodified lines
52	57	host PCI device for this memory device, emit the CPU node
53	58	affinity for this device.
54	59
	60	+
55	61	What: /sys/bus/cxl/devices/*/devtype
56	62	Date: June, 2021
57	63	KernelVersion: v5.14
58	64	Contact: linux-cxl@vger.kernel.org
59	65	Description:
60		− CXL device objects export the devtype attribute which mirrors
61		− the same value communicated in the DEVTYPE environment variable
62		− for uevents for devices on the "cxl" bus.
	66	+ (RO) CXL device objects export the devtype attribute which
	67	+ mirrors the same value communicated in the DEVTYPE environment
	68	+ variable for uevents for devices on the "cxl" bus.
	69	+
63	70
64	71	What: /sys/bus/cxl/devices/*/modalias
65	72	Date: December, 2021
66	73	KernelVersion: v5.18
67	74	Contact: linux-cxl@vger.kernel.org
68	75	Description:
69		− CXL device objects export the modalias attribute which mirrors
70		− the same value communicated in the MODALIAS environment variable
71		− for uevents for devices on the "cxl" bus.
	76	+ (RO) CXL device objects export the modalias attribute which
	77	+ mirrors the same value communicated in the MODALIAS environment
	78	+ variable for uevents for devices on the "cxl" bus.
	79	+
72	80
73	81	What: /sys/bus/cxl/devices/portX/uport
74	82	Date: June, 2021
75	83	KernelVersion: v5.14
76	84	Contact: linux-cxl@vger.kernel.org
77	85	Description:
78		− CXL port objects are enumerated from either a platform firmware
79		− device (ACPI0017 and ACPI0016) or PCIe switch upstream port with
80		− CXL component registers. The 'uport' symlink connects the CXL
81		− portX object to the device that published the CXL port
	86	+ (RO) CXL port objects are enumerated from either a platform
	87	+ firmware device (ACPI0017 and ACPI0016) or PCIe switch upstream
	88	+ port with CXL component registers. The 'uport' symlink connects
	89	+ the CXL portX object to the device that published the CXL port
82	90	capability.
83	91
	92	+
84	93	What: /sys/bus/cxl/devices/portX/dportY
85	94	Date: June, 2021
86	95	KernelVersion: v5.14
87	96	Contact: linux-cxl@vger.kernel.org
88	97	Description:
89		− CXL port objects are enumerated from either a platform firmware
90		− device (ACPI0017 and ACPI0016) or PCIe switch upstream port with
91		− CXL component registers. The 'dportY' symlink identifies one or
92		− more downstream ports that the upstream port may target in its
93		− decode of CXL memory resources. The 'Y' integer reflects the
94		− hardware port unique-id used in the hardware decoder target
95		− list.
	98	+ (RO) CXL port objects are enumerated from either a platform
	99	+ firmware device (ACPI0017 and ACPI0016) or PCIe switch upstream
	100	+ port with CXL component registers. The 'dportY' symlink
	101	+ identifies one or more downstream ports that the upstream port
	102	+ may target in its decode of CXL memory resources. The 'Y'
	103	+ integer reflects the hardware port unique-id used in the
	104	+ hardware decoder target list.
	105	+
96	106
97	107	What: /sys/bus/cxl/devices/decoderX.Y
98	108	Date: June, 2021
99	109	KernelVersion: v5.14
100	110	Contact: linux-cxl@vger.kernel.org
101	111	Description:
102		− CXL decoder objects are enumerated from either a platform
	112	+ (RO) CXL decoder objects are enumerated from either a platform
103	113	firmware description, or a CXL HDM decoder register set in a
104	114	PCIe device (see CXL 2.0 section 8.2.5.12 CXL HDM Decoder
105	115	Capability Structure). The 'X' in decoderX.Y represents the
106	116	cxl_port container of this decoder, and 'Y' represents the
107	117	instance id of a given decoder resource.
108	118
	119	+
109	120	What: /sys/bus/cxl/devices/decoderX.Y/{start,size}
110	121	Date: June, 2021
111	122	KernelVersion: v5.14
112	123	Contact: linux-cxl@vger.kernel.org
113	124	Description:
114		− The 'start' and 'size' attributes together convey the physical
115		− address base and number of bytes mapped in the decoder's decode
116		− window. For decoders of devtype "cxl_decoder_root" the address
117		− range is fixed. For decoders of devtype "cxl_decoder_switch" the
118		− address is bounded by the decode range of the cxl_port ancestor
119		− of the decoder's cxl_port, and dynamically updates based on the
120		− active memory regions in that address space.
	125	+ (RO) The 'start' and 'size' attributes together convey the
	126	+ physical address base and number of bytes mapped in the
	127	+ decoder's decode window. For decoders of devtype
	128	+ "cxl_decoder_root" the address range is fixed. For decoders of
	129	+ devtype "cxl_decoder_switch" the address is bounded by the
	130	+ decode range of the cxl_port ancestor of the decoder's cxl_port,
	131	+ and dynamically updates based on the active memory regions in
	132	+ that address space.
	133	+
121	134
122	135	What: /sys/bus/cxl/devices/decoderX.Y/locked
123	136	Date: June, 2021
124	137	KernelVersion: v5.14
125	138	Contact: linux-cxl@vger.kernel.org
126	139	Description:
127		− CXL HDM decoders have the capability to lock the configuration
128		− until the next device reset. For decoders of devtype
129		− "cxl_decoder_root" there is no standard facility to unlock them.
130		− For decoders of devtype "cxl_decoder_switch" a secondary bus
131		− reset, of the PCIe bridge that provides the bus for this
132		− decoders uport, unlocks / resets the decoder.
	140	+ (RO) CXL HDM decoders have the capability to lock the
	141	+ configuration until the next device reset. For decoders of
	142	+ devtype "cxl_decoder_root" there is no standard facility to
	143	+ unlock them. For decoders of devtype "cxl_decoder_switch" a
	144	+ secondary bus reset, of the PCIe bridge that provides the bus
	145	+ for this decoders uport, unlocks / resets the decoder.
	146	+
133	147
134	148	What: /sys/bus/cxl/devices/decoderX.Y/target_list
135	149	Date: June, 2021
136	150	KernelVersion: v5.14
137	151	Contact: linux-cxl@vger.kernel.org
138	152	Description:
139		− Display a comma separated list of the current decoder target
140		− configuration. The list is ordered by the current configured
141		− interleave order of the decoder's dport instances. Each entry in
142		− the list is a dport id.
	153	+ (RO) Display a comma separated list of the current decoder
	154	+ target configuration. The list is ordered by the current
	155	+ configured interleave order of the decoder's dport instances.
	156	+ Each entry in the list is a dport id.
	157	+
143	158
144	159	What: /sys/bus/cxl/devices/decoderX.Y/cap_{pmem,ram,type2,type3}
145	160	Date: June, 2021
146	161	KernelVersion: v5.14
147	162	Contact: linux-cxl@vger.kernel.org
148	163	Description:
149		− When a CXL decoder is of devtype "cxl_decoder_root", it
	164	+ (RO) When a CXL decoder is of devtype "cxl_decoder_root", it
150	165	represents a fixed memory window identified by platform
151	166	firmware. A fixed window may only support a subset of memory
152	167	types. The 'cap_*' attributes indicate whether persistent
153	168	memory, volatile memory, accelerator memory, and / or expander
154	169	memory may be mapped behind this decoder's memory window.
155	170
	171	+
156	172	What: /sys/bus/cxl/devices/decoderX.Y/target_type
157	173	Date: June, 2021
158	174	KernelVersion: v5.14
159	175	Contact: linux-cxl@vger.kernel.org
160	176	Description:
161		− When a CXL decoder is of devtype "cxl_decoder_switch", it can
162		− optionally decode either accelerator memory (type-2) or expander
163		− memory (type-3). The 'target_type' attribute indicates the
164		− current setting which may dynamically change based on what
	177	+ (RO) When a CXL decoder is of devtype "cxl_decoder_switch", it
	178	+ can optionally decode either accelerator memory (type-2) or
	179	+ expander memory (type-3). The 'target_type' attribute indicates
	180	+ the current setting which may dynamically change based on what
165	181	memory regions are activated in this decode hierarchy.
	182	+
	183	+
	184	+What: /sys/bus/cxl/devices/endpointX/CDAT
	185	+Date: July, 2022
	186	+KernelVersion: v5.20
	187	+Contact: linux-cxl@vger.kernel.org
	188	+Description:
	189	+ (RO) If this sysfs entry is not present no DOE mailbox was
	190	+ found to support CDAT data. If it is present and the length of
	191	+ the data is 0 reading the CDAT data failed. Otherwise the CDAT
	192	+ data is reported.
	193	+
	194	+
	195	+What: /sys/bus/cxl/devices/decoderX.Y/mode
	196	+Date: May, 2022
	197	+KernelVersion: v5.20
	198	+Contact: linux-cxl@vger.kernel.org
	199	+Description:
	200	+ (RW) When a CXL decoder is of devtype "cxl_decoder_endpoint" it
	201	+ translates from a host physical address range, to a device local
	202	+ address range. Device-local address ranges are further split
	203	+ into a 'ram' (volatile memory) range and 'pmem' (persistent
	204	+ memory) range. The 'mode' attribute emits one of 'ram', 'pmem',
	205	+ 'mixed', or 'none'. The 'mixed' indication is for error cases
	206	+ when a decoder straddles the volatile/persistent partition
	207	+ boundary, and 'none' indicates the decoder is not actively
	208	+ decoding, or no DPA allocation policy has been set.
	209	+
	210	+ 'mode' can be written, when the decoder is in the 'disabled'
	211	+ state, with either 'ram' or 'pmem' to set the boundaries for the
	212	+ next allocation.
	213	+
	214	+
	215	+What: /sys/bus/cxl/devices/decoderX.Y/dpa_resource
	216	+Date: May, 2022
	217	+KernelVersion: v5.20
	218	+Contact: linux-cxl@vger.kernel.org
	219	+Description:
	220	+ (RO) When a CXL decoder is of devtype "cxl_decoder_endpoint",
	221	+ and its 'dpa_size' attribute is non-zero, this attribute
	222	+ indicates the device physical address (DPA) base address of the
	223	+ allocation.
	224	+
	225	+
	226	+What: /sys/bus/cxl/devices/decoderX.Y/dpa_size
	227	+Date: May, 2022
	228	+KernelVersion: v5.20
	229	+Contact: linux-cxl@vger.kernel.org
	230	+Description:
	231	+ (RW) When a CXL decoder is of devtype "cxl_decoder_endpoint" it
	232	+ translates from a host physical address range, to a device local
	233	+ address range. The range, base address plus length in bytes, of
	234	+ DPA allocated to this decoder is conveyed in these 2 attributes.
	235	+ Allocations can be mutated as long as the decoder is in the
	236	+ disabled state. A write to 'dpa_size' releases the previous DPA
	237	+ allocation and then attempts to allocate from the free capacity
	238	+ in the device partition referred to by 'decoderX.Y/mode'.
	239	+ Allocate and free requests can only be performed on the highest
	240	+ instance number disabled decoder with non-zero size. I.e.
	241	+ allocations are enforced to occur in increasing 'decoderX.Y/id'
	242	+ order and frees are enforced to occur in decreasing
	243	+ 'decoderX.Y/id' order.
	244	+
	245	+
	246	+What: /sys/bus/cxl/devices/decoderX.Y/interleave_ways
	247	+Date: May, 2022
	248	+KernelVersion: v5.20
	249	+Contact: linux-cxl@vger.kernel.org
	250	+Description:
	251	+ (RO) The number of targets across which this decoder's host
	252	+ physical address (HPA) memory range is interleaved. The device
	253	+ maps every Nth block of HPA (of size ==
	254	+ 'interleave_granularity') to consecutive DPA addresses. The
	255	+ decoder's position in the interleave is determined by the
	256	+ device's (endpoint or switch) switch ancestry. For root
	257	+ decoders their interleave is specified by platform firmware and
	258	+ they only specify a downstream target order for host bridges.
	259	+
	260	+
	261	+What: /sys/bus/cxl/devices/decoderX.Y/interleave_granularity
	262	+Date: May, 2022
	263	+KernelVersion: v5.20
	264	+Contact: linux-cxl@vger.kernel.org
	265	+Description:
	266	+ (RO) The number of consecutive bytes of host physical address
	267	+ space this decoder claims at address N before the decode rotates
	268	+ to the next target in the interleave at address N +
	269	+ interleave_granularity (assuming N is aligned to
	270	+ interleave_granularity).
	271	+
	272	+
	273	+What: /sys/bus/cxl/devices/decoderX.Y/create_pmem_region
	274	+Date: May, 2022
	275	+KernelVersion: v5.20
	276	+Contact: linux-cxl@vger.kernel.org
	277	+Description:
	278	+ (RW) Write a string in the form 'regionZ' to start the process
	279	+ of defining a new persistent memory region (interleave-set)
	280	+ within the decode range bounded by root decoder 'decoderX.Y'.
	281	+ The value written must match the current value returned from
	282	+ reading this attribute. An atomic compare exchange operation is
	283	+ done on write to assign the requested id to a region and
	284	+ allocate the region-id for the next creation attempt. EBUSY is
	285	+ returned if the region name written does not match the current
	286	+ cached value.
	287	+
	288	+
	289	+What: /sys/bus/cxl/devices/decoderX.Y/delete_region
	290	+Date: May, 2022
	291	+KernelVersion: v5.20
	292	+Contact: linux-cxl@vger.kernel.org
	293	+Description:
	294	+ (WO) Write a string in the form 'regionZ' to delete that region,
	295	+ provided it is currently idle / not bound to a driver.
	296	+
	297	+
	298	+What: /sys/bus/cxl/devices/regionZ/uuid
	299	+Date: May, 2022
	300	+KernelVersion: v5.20
	301	+Contact: linux-cxl@vger.kernel.org
	302	+Description:
	303	+ (RW) Write a unique identifier for the region. This field must
	304	+ be set for persistent regions and it must not conflict with the
	305	+ UUID of another region.
	306	+
	307	+
	308	+What: /sys/bus/cxl/devices/regionZ/interleave_granularity
	309	+Date: May, 2022
	310	+KernelVersion: v5.20
	311	+Contact: linux-cxl@vger.kernel.org
	312	+Description:
	313	+ (RW) Set the number of consecutive bytes each device in the
	314	+ interleave set will claim. The possible interleave granularity
	315	+ values are determined by the CXL spec and the participating
	316	+ devices.
	317	+
	318	+
	319	+What: /sys/bus/cxl/devices/regionZ/interleave_ways
	320	+Date: May, 2022
	321	+KernelVersion: v5.20
	322	+Contact: linux-cxl@vger.kernel.org
	323	+Description:
	324	+ (RW) Configures the number of devices participating in the
	325	+ region is set by writing this value. Each device will provide
	326	+ 1/interleave_ways of storage for the region.
	327	+
	328	+
	329	+What: /sys/bus/cxl/devices/regionZ/size
	330	+Date: May, 2022
	331	+KernelVersion: v5.20
	332	+Contact: linux-cxl@vger.kernel.org
	333	+Description:
	334	+ (RW) System physical address space to be consumed by the region.
	335	+ When written trigger the driver to allocate space out of the
	336	+ parent root decoder's address space. When read the size of the
	337	+ address space is reported and should match the span of the
	338	+ region's resource attribute. Size shall be set after the
	339	+ interleave configuration parameters. Once set it cannot be
	340	+ changed, only freed by writing 0. The kernel makes no guarantees
	341	+ that data is maintained over an address space freeing event, and
	342	+ there is no guarantee that a free followed by an allocate
	343	+ results in the same address being allocated.
	344	+
	345	+
	346	+What: /sys/bus/cxl/devices/regionZ/resource
	347	+Date: May, 2022
	348	+KernelVersion: v5.20
	349	+Contact: linux-cxl@vger.kernel.org
	350	+Description:
	351	+ (RO) A region is a contiguous partition of a CXL root decoder
	352	+ address space. Region capacity is allocated by writing to the
	353	+ size attribute, the resulting physical address space determined
	354	+ by the driver is reflected here. It is therefore not useful to
	355	+ read this before writing a value to the size attribute.
	356	+
	357	+
	358	+What: /sys/bus/cxl/devices/regionZ/target[0..N]
	359	+Date: May, 2022
	360	+KernelVersion: v5.20
	361	+Contact: linux-cxl@vger.kernel.org
	362	+Description:
	363	+ (RW) Write an endpoint decoder object name to 'targetX' where X
	364	+ is the intended position of the endpoint device in the region
	365	+ interleave and N is the 'interleave_ways' setting for the
	366	+ region. ENXIO is returned if the write results in an impossible
	367	+ to map decode scenario, like the endpoint is unreachable at that
	368	+ position relative to the root decoder interleave. EBUSY is
	369	+ returned if the position in the region is already occupied, or
	370	+ if the region is not in a state to accept interleave
	371	+ configuration changes. EINVAL is returned if the object name is
	372	+ not an endpoint decoder. Once all positions have been
	373	+ successfully written a final validation for decode conflicts is
	374	+ performed before activating the region.
	375	+
	376	+
	377	+What: /sys/bus/cxl/devices/regionZ/commit
	378	+Date: May, 2022
	379	+KernelVersion: v5.20
	380	+Contact: linux-cxl@vger.kernel.org
	381	+Description:
	382	+ (RW) Write a boolean 'true' string value to this attribute to
	383	+ trigger the region to transition from the software programmed
	384	+ state to the actively decoding in hardware state. The commit
	385	+ operation in addition to validating that the region is in proper
	386	+ configured state, validates that the decoders are being
	387	+ committed in spec mandated order (last committed decoder id +
	388	+ 1), and checks that the hardware accepts the commit request.
	389	+ Reading this value indicates whether the region is committed or
	390	+ not.

Documentation/driver-api/cxl/memory-devices.rst+8−0

@@ -362,6 +362,14 @@ CXL Core · 361 unmodified lines
362	362	.. kernel-doc:: drivers/cxl/core/mbox.c
363	363	:doc: cxl mbox
364	364
	365	+CXL Regions
	366	+-----------
	367	+.. kernel-doc:: drivers/cxl/core/region.c
	368	+ :doc: cxl core region
	369	+
	370	+.. kernel-doc:: drivers/cxl/core/region.c
	371	+ :identifiers:
	372	+
365	373	External Interfaces
366	374	===================
367	375

arch/powerpc/mm/mem.c+1−0

@@ -55,6 +55,7 @@ int memory_add_physaddr_to_nid(u64 start) · 54 unmodified lines
55	55	{
56	56	return hot_add_scn_to_nid(start);
57	57	}
	58	+EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
58	59	#endif
59	60
60	61	int __weak create_section_mapping(unsigned long start, unsigned long end,

drivers/cxl/Kconfig+9−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	menuconfig CXL_BUS
3	3	tristate "CXL (Compute Express Link) Devices Support"
4	4	depends on PCI
	5	+ select PCI_DOE
5	6	help
6	7	CXL is a bus that is electrically compatible with PCI Express, but
7	8	layers three protocols on that signalling (CXL.io, CXL.cache, and
@@ -102,4 +103,12 @@ config CXL_SUSPEND · 94 unmodified lines
102	103	def_bool y
103	104	depends on SUSPEND && CXL_MEM
104	105
	106	+config CXL_REGION
	107	+ bool
	108	+ default CXL_BUS
	109	+ # For MAX_PHYSMEM_BITS
	110	+ depends on SPARSEMEM
	111	+ select MEMREGION
	112	+ select GET_FREE_REGION
	113	+
105	114	endif

drivers/cxl/acpi.c+217−26

@@ -9,10 +9,6 @@ · 8 unmodified lines
9	9	#include "cxlpci.h"
10	10	#include "cxl.h"
11	11
12		−/* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */
13		−#define CFMWS_INTERLEAVE_WAYS(x) (1 << (x)->interleave_ways)
14		−#define CFMWS_INTERLEAVE_GRANULARITY(x) ((x)->granularity + 8)
15		−
16	12	static unsigned long cfmws_to_decoder_flags(int restrictions)
17	13	{
18	14	unsigned long flags = CXL_DECODER_F_ENABLE;
@@ -34,7 +30,8 @@ static unsigned long cfmws_to_decoder_flags(int restrictions) · 15 unmodified lines
34	30	static int cxl_acpi_cfmws_verify(struct device *dev,
35	31	struct acpi_cedt_cfmws *cfmws)
36	32	{
37		− int expected_len;
	33	+ int rc, expected_len;
	34	+ unsigned int ways;
38	35
39	36	if (cfmws->interleave_arithmetic != ACPI_CEDT_CFMWS_ARITHMETIC_MODULO) {
40	37	dev_err(dev, "CFMWS Unsupported Interleave Arithmetic\n");
@@ -51,14 +48,14 @@ static int cxl_acpi_cfmws_verify(struct device *dev, · 10 unmodified lines
51	48	return -EINVAL;
52	49	}
53	50
54		− if (CFMWS_INTERLEAVE_WAYS(cfmws) > CXL_DECODER_MAX_INTERLEAVE) {
55		− dev_err(dev, "CFMWS Interleave Ways (%d) too large\n",
56		− CFMWS_INTERLEAVE_WAYS(cfmws));
	51	+ rc = cxl_to_ways(cfmws->interleave_ways, &ways);
	52	+ if (rc) {
	53	+ dev_err(dev, "CFMWS Interleave Ways (%d) invalid\n",
	54	+ cfmws->interleave_ways);
57	55	return -EINVAL;
58	56	}
59	57
60		− expected_len = struct_size((cfmws), interleave_targets,
61		− CFMWS_INTERLEAVE_WAYS(cfmws));
	58	+ expected_len = struct_size(cfmws, interleave_targets, ways);
62	59
63	60	if (cfmws->header.length < expected_len) {
64	61	dev_err(dev, "CFMWS length %d less than expected %d\n",
@@ -76,6 +73,8 @@ static int cxl_acpi_cfmws_verify(struct device *dev, · 11 unmodified lines
76	73	struct cxl_cfmws_context {
77	74	struct device *dev;
78	75	struct cxl_port *root_port;
	76	+ struct resource *cxl_res;
	77	+ int id;
79	78	};
80	79
81	80	static int cxl_parse_cfmws(union acpi_subtable_headers header, void arg,
@@ -84,10 +83,14 @@ static int cxl_parse_cfmws(union acpi_subtable_headers header, void arg, · 2 unmodified lines
84	83	int target_map[CXL_DECODER_MAX_INTERLEAVE];
85	84	struct cxl_cfmws_context *ctx = arg;
86	85	struct cxl_port *root_port = ctx->root_port;
	86	+ struct resource *cxl_res = ctx->cxl_res;
	87	+ struct cxl_root_decoder *cxlrd;
87	88	struct device *dev = ctx->dev;
88	89	struct acpi_cedt_cfmws *cfmws;
89	90	struct cxl_decoder *cxld;
90		− int rc, i;
	91	+ unsigned int ways, i, ig;
	92	+ struct resource *res;
	93	+ int rc;
91	94
92	95	cfmws = (struct acpi_cedt_cfmws *) header;
93	96
@@ -99,19 +102,51 @@ static int cxl_parse_cfmws(union acpi_subtable_headers header, void arg, · 5 unmodified lines
99	102	return 0;
100	103	}
101	104
102		− for (i = 0; i < CFMWS_INTERLEAVE_WAYS(cfmws); i++)
	105	+ rc = cxl_to_ways(cfmws->interleave_ways, &ways);
	106	+ if (rc)
	107	+ return rc;
	108	+ rc = cxl_to_granularity(cfmws->granularity, &ig);
	109	+ if (rc)
	110	+ return rc;
	111	+ for (i = 0; i < ways; i++)
103	112	target_map[i] = cfmws->interleave_targets[i];
104	113
105		− cxld = cxl_root_decoder_alloc(root_port, CFMWS_INTERLEAVE_WAYS(cfmws));
106		− if (IS_ERR(cxld))
	114	+ res = kzalloc(sizeof(*res), GFP_KERNEL);
	115	+ if (!res)
	116	+ return -ENOMEM;
	117	+
	118	+ res->name = kasprintf(GFP_KERNEL, "CXL Window %d", ctx->id++);
	119	+ if (!res->name)
	120	+ goto err_name;
	121	+
	122	+ res->start = cfmws->base_hpa;
	123	+ res->end = cfmws->base_hpa + cfmws->window_size - 1;
	124	+ res->flags = IORESOURCE_MEM;
	125	+
	126	+ /* add to the local resource tracking to establish a sort order */
	127	+ rc = insert_resource(cxl_res, res);
	128	+ if (rc)
	129	+ goto err_insert;
	130	+
	131	+ cxlrd = cxl_root_decoder_alloc(root_port, ways);
	132	+ if (IS_ERR(cxlrd))
107	133	return 0;
108	134
	135	+ cxld = &cxlrd->cxlsd.cxld;
109	136	cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions);
110	137	cxld->target_type = CXL_DECODER_EXPANDER;
111		− cxld->platform_res = (struct resource)DEFINE_RES_MEM(cfmws->base_hpa,
112		− cfmws->window_size);
113		− cxld->interleave_ways = CFMWS_INTERLEAVE_WAYS(cfmws);
114		− cxld->interleave_granularity = CFMWS_INTERLEAVE_GRANULARITY(cfmws);
	138	+ cxld->hpa_range = (struct range) {
	139	+ .start = res->start,
	140	+ .end = res->end,
	141	+ };
	142	+ cxld->interleave_ways = ways;
	143	+ /*
	144	+ * Minimize the x1 granularity to advertise support for any
	145	+ * valid region granularity
	146	+ */
	147	+ if (ways == 1)
	148	+ ig = CXL_DECODER_MIN_GRANULARITY;
	149	+ cxld->interleave_granularity = ig;
115	150
116	151	rc = cxl_decoder_add(cxld, target_map);
117	152	if (rc)
@@ -119,15 +154,22 @@ static int cxl_parse_cfmws(union acpi_subtable_headers header, void arg, · 1 unmodified line
119	154	else
120	155	rc = cxl_decoder_autoremove(dev, cxld);
121	156	if (rc) {
122		− dev_err(dev, "Failed to add decoder for %pr\n",
123		− &cxld->platform_res);
	157	+ dev_err(dev, "Failed to add decode range [%#llx - %#llx]\n",
	158	+ cxld->hpa_range.start, cxld->hpa_range.end);
124	159	return 0;
125	160	}
126		− dev_dbg(dev, "add: %s node: %d range %pr\n", dev_name(&cxld->dev),
127		− phys_to_target_node(cxld->platform_res.start),
128		− &cxld->platform_res);
	161	+ dev_dbg(dev, "add: %s node: %d range [%#llx - %#llx]\n",
	162	+ dev_name(&cxld->dev),
	163	+ phys_to_target_node(cxld->hpa_range.start),
	164	+ cxld->hpa_range.start, cxld->hpa_range.end);
129	165
130	166	return 0;
	167	+
	168	+err_insert:
	169	+ kfree(res->name);
	170	+err_name:
	171	+ kfree(res);
	172	+ return -ENOMEM;
131	173	}
132	174
133	175	__mock struct acpi_device to_cxl_host_bridge(struct device host,
@@ -175,8 +217,7 @@ static int add_host_bridge_uport(struct device match, void arg) · 41 unmodified lines
175	217	if (rc)
176	218	return rc;
177	219
178		− port = devm_cxl_add_port(host, match, dport->component_reg_phys,
179		− root_port);
	220	+ port = devm_cxl_add_port(host, match, dport->component_reg_phys, dport);
180	221	if (IS_ERR(port))
181	222	return PTR_ERR(port);
182	223	dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev));
@@ -282,9 +323,127 @@ static void cxl_acpi_lock_reset_class(void *dev) · 99 unmodified lines
282	323	device_lock_reset_class(dev);
283	324	}
284	325
	326	+static void del_cxl_resource(struct resource *res)
	327	+{
	328	+ kfree(res->name);
	329	+ kfree(res);
	330	+}
	331	+
	332	+static void cxl_set_public_resource(struct resource priv, struct resource pub)
	333	+{
	334	+ priv->desc = (unsigned long) pub;
	335	+}
	336	+
	337	+static struct resource cxl_get_public_resource(struct resource priv)
	338	+{
	339	+ return (struct resource *) priv->desc;
	340	+}
	341	+
	342	+static void remove_cxl_resources(void *data)
	343	+{
	344	+ struct resource res, next, *cxl = data;
	345	+
	346	+ for (res = cxl->child; res; res = next) {
	347	+ struct resource *victim = cxl_get_public_resource(res);
	348	+
	349	+ next = res->sibling;
	350	+ remove_resource(res);
	351	+
	352	+ if (victim) {
	353	+ remove_resource(victim);
	354	+ kfree(victim);
	355	+ }
	356	+
	357	+ del_cxl_resource(res);
	358	+ }
	359	+}
	360	+
	361	+/**
	362	+ * add_cxl_resources() - reflect CXL fixed memory windows in iomem_resource
	363	+ * @cxl_res: A standalone resource tree where each CXL window is a sibling
	364	+ *
	365	+ * Walk each CXL window in @cxl_res and add it to iomem_resource potentially
	366	+ * expanding its boundaries to ensure that any conflicting resources become
	367	+ * children. If a window is expanded it may then conflict with a another window
	368	+ * entry and require the window to be truncated or trimmed. Consider this
	369	+ * situation:
	370	+ *
	371	+ * \|-- "CXL Window 0" --\|\|----- "CXL Window 1" -----\|
	372	+ * \|--------------- "System RAM" -------------\|
	373	+ *
	374	+ * ...where platform firmware has established as System RAM resource across 2
	375	+ * windows, but has left some portion of window 1 for dynamic CXL region
	376	+ * provisioning. In this case "Window 0" will span the entirety of the "System
	377	+ * RAM" span, and "CXL Window 1" is truncated to the remaining tail past the end
	378	+ * of that "System RAM" resource.
	379	+ */
	380	+static int add_cxl_resources(struct resource *cxl_res)
	381	+{
	382	+ struct resource res, new, *next;
	383	+
	384	+ for (res = cxl_res->child; res; res = next) {
	385	+ new = kzalloc(sizeof(*new), GFP_KERNEL);
	386	+ if (!new)
	387	+ return -ENOMEM;
	388	+ new->name = res->name;
	389	+ new->start = res->start;
	390	+ new->end = res->end;
	391	+ new->flags = IORESOURCE_MEM;
	392	+ new->desc = IORES_DESC_CXL;
	393	+
	394	+ /*
	395	+ * Record the public resource in the private cxl_res tree for
	396	+ * later removal.
	397	+ */
	398	+ cxl_set_public_resource(res, new);
	399	+
	400	+ insert_resource_expand_to_fit(&iomem_resource, new);
	401	+
	402	+ next = res->sibling;
	403	+ while (next && resource_overlaps(new, next)) {
	404	+ if (resource_contains(new, next)) {
	405	+ struct resource *_next = next->sibling;
	406	+
	407	+ remove_resource(next);
	408	+ del_cxl_resource(next);
	409	+ next = _next;
	410	+ } else
	411	+ next->start = new->end + 1;
	412	+ }
	413	+ }
	414	+ return 0;
	415	+}
	416	+
	417	+static int pair_cxl_resource(struct device dev, void data)
	418	+{
	419	+ struct resource *cxl_res = data;
	420	+ struct resource *p;
	421	+
	422	+ if (!is_root_decoder(dev))
	423	+ return 0;
	424	+
	425	+ for (p = cxl_res->child; p; p = p->sibling) {
	426	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
	427	+ struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
	428	+ struct resource res = {
	429	+ .start = cxld->hpa_range.start,
	430	+ .end = cxld->hpa_range.end,
	431	+ .flags = IORESOURCE_MEM,
	432	+ };
	433	+
	434	+ if (resource_contains(p, &res)) {
	435	+ cxlrd->res = cxl_get_public_resource(p);
	436	+ break;
	437	+ }
	438	+ }
	439	+
	440	+ return 0;
	441	+}
	442	+
285	443	static int cxl_acpi_probe(struct platform_device *pdev)
286	444	{
287	445	int rc;
	446	+ struct resource *cxl_res;
288	447	struct cxl_port *root_port;
289	448	struct device *host = &pdev->dev;
290	449	struct acpi_device *adev = ACPI_COMPANION(host);
@@ -296,6 +455,14 @@ static int cxl_acpi_probe(struct platform_device *pdev) · 5 unmodified lines
296	455	if (rc)
297	456	return rc;
298	457
	458	+ cxl_res = devm_kzalloc(host, sizeof(*cxl_res), GFP_KERNEL);
	459	+ if (!cxl_res)
	460	+ return -ENOMEM;
	461	+ cxl_res->name = "CXL mem";
	462	+ cxl_res->start = 0;
	463	+ cxl_res->end = -1;
	464	+ cxl_res->flags = IORESOURCE_MEM;
	465	+
299	466	root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
300	467	if (IS_ERR(root_port))
301	468	return PTR_ERR(root_port);
@@ -306,11 +473,28 @@ static int cxl_acpi_probe(struct platform_device *pdev) · 4 unmodified lines
306	473	if (rc < 0)
307	474	return rc;
308	475
	476	+ rc = devm_add_action_or_reset(host, remove_cxl_resources, cxl_res);
	477	+ if (rc)
	478	+ return rc;
	479	+
309	480	ctx = (struct cxl_cfmws_context) {
310	481	.dev = host,
311	482	.root_port = root_port,
	483	+ .cxl_res = cxl_res,
312	484	};
313		− acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, cxl_parse_cfmws, &ctx);
	485	+ rc = acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, cxl_parse_cfmws, &ctx);
	486	+ if (rc < 0)
	487	+ return -ENXIO;
	488	+
	489	+ rc = add_cxl_resources(cxl_res);
	490	+ if (rc)
	491	+ return rc;
	492	+
	493	+ /*
	494	+ * Populate the root decoders with their related iomem resource,
	495	+ * if present
	496	+ */
	497	+ device_for_each_child(&root_port->dev, cxl_res, pair_cxl_resource);
314	498
315	499	/*
316	500	* Root level scanned with host-bridge as dports, now scan host-bridges
@@ -337,12 +521,19 @@ static const struct acpi_device_id cxl_acpi_ids[] = { · 20 unmodified lines
337	521	};
338	522	MODULE_DEVICE_TABLE(acpi, cxl_acpi_ids);
339	523
	524	+static const struct platform_device_id cxl_test_ids[] = {
	525	+ { "cxl_acpi" },
	526	+ { },
	527	+};
	528	+MODULE_DEVICE_TABLE(platform, cxl_test_ids);
	529	+
340	530	static struct platform_driver cxl_acpi_driver = {
341	531	.probe = cxl_acpi_probe,
342	532	.driver = {
343	533	.name = KBUILD_MODNAME,
344	534	.acpi_match_table = cxl_acpi_ids,
345	535	},
	536	+ .id_table = cxl_test_ids,
346	537	};
347	538
348	539	module_platform_driver(cxl_acpi_driver);

drivers/cxl/core/Makefile+1−0

@@ -10,3 +10,4 @@ cxl_core-y += memdev.o · 9 unmodified lines
10	10	cxl_core-y += mbox.o
11	11	cxl_core-y += pci.o
12	12	cxl_core-y += hdm.o
	13	+cxl_core-$(CONFIG_CXL_REGION) += region.o

drivers/cxl/core/core.h+50−1

@@ -9,6 +9,36 @@ extern const struct device_type cxl_nvdimm_type; · 8 unmodified lines
9	9
10	10	extern struct attribute_group cxl_base_attribute_group;
11	11
	12	+#ifdef CONFIG_CXL_REGION
	13	+extern struct device_attribute dev_attr_create_pmem_region;
	14	+extern struct device_attribute dev_attr_delete_region;
	15	+extern struct device_attribute dev_attr_region;
	16	+extern const struct device_type cxl_pmem_region_type;
	17	+extern const struct device_type cxl_region_type;
	18	+void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled);
	19	+#define CXL_REGION_ATTR(x) (&dev_attr_##x.attr)
	20	+#define CXL_REGION_TYPE(x) (&cxl_region_type)
	21	+#define SET_CXL_REGION_ATTR(x) (&dev_attr_##x.attr),
	22	+#define CXL_PMEM_REGION_TYPE(x) (&cxl_pmem_region_type)
	23	+int cxl_region_init(void);
	24	+void cxl_region_exit(void);
	25	+#else
	26	+static inline void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
	27	+{
	28	+}
	29	+static inline int cxl_region_init(void)
	30	+{
	31	+ return 0;
	32	+}
	33	+static inline void cxl_region_exit(void)
	34	+{
	35	+}
	36	+#define CXL_REGION_ATTR(x) NULL
	37	+#define CXL_REGION_TYPE(x) NULL
	38	+#define SET_CXL_REGION_ATTR(x)
	39	+#define CXL_PMEM_REGION_TYPE(x) NULL
	40	+#endif
	41	+
12	42	struct cxl_send_command;
13	43	struct cxl_mem_query_commands;
14	44	int cxl_query_cmd(struct cxl_memdev *cxlmd,
@@ -17,9 +47,28 @@ int cxl_send_cmd(struct cxl_memdev cxlmd, struct cxl_send_command __user s); · 2 unmodified lines
17	47	void __iomem devm_cxl_iomap_block(struct device dev, resource_size_t addr,
18	48	resource_size_t length);
19	49
	50	+struct dentry cxl_debugfs_create_dir(const char dir);
	51	+int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
	52	+ enum cxl_decoder_mode mode);
	53	+int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size);
	54	+int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
	55	+resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled);
	56	+resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled);
	57	+extern struct rw_semaphore cxl_dpa_rwsem;
	58	+
	59	+bool is_switch_decoder(struct device *dev);
	60	+struct cxl_switch_decoder to_cxl_switch_decoder(struct device dev);
	61	+static inline struct cxl_ep cxl_ep_load(struct cxl_port port,
	62	+ struct cxl_memdev *cxlmd)
	63	+{
	64	+ if (!port)
	65	+ return NULL;
	66	+
	67	+ return xa_load(&port->endpoints, (unsigned long)&cxlmd->dev);
	68	+}
	69	+
20	70	int cxl_memdev_init(void);
21	71	void cxl_memdev_exit(void);
22	72	void cxl_mbox_init(void);
23		−void cxl_mbox_exit(void);
24	73
25	74	#endif /* __CXL_CORE_H__ */

drivers/cxl/core/hdm.c+635−56

@@ -1,6 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0-only
2	2	/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3	3	#include <linux/io-64-nonatomic-hi-lo.h>
	4	+#include <linux/seq_file.h>
4	5	#include <linux/device.h>
5	6	#include <linux/delay.h>
6	7
@@ -16,6 +17,8 @@ · 9 unmodified lines
16	17	* for enumerating these registers and capabilities.
17	18	*/
18	19
	20	+DECLARE_RWSEM(cxl_dpa_rwsem);
	21	+
19	22	static int add_hdm_decoder(struct cxl_port port, struct cxl_decoder cxld,
20	23	int *target_map)
21	24	{
@@ -46,20 +49,22 @@ static int add_hdm_decoder(struct cxl_port port, struct cxl_decoder cxld, · 24 unmodified lines
46	49	*/
47	50	int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
48	51	{
49		− struct cxl_decoder *cxld;
50		− struct cxl_dport *dport;
	52	+ struct cxl_switch_decoder *cxlsd;
	53	+ struct cxl_dport *dport = NULL;
51	54	int single_port_map[1];
	55	+ unsigned long index;
52	56
53		− cxld = cxl_switch_decoder_alloc(port, 1);
54		− if (IS_ERR(cxld))
55		− return PTR_ERR(cxld);
	57	+ cxlsd = cxl_switch_decoder_alloc(port, 1);
	58	+ if (IS_ERR(cxlsd))
	59	+ return PTR_ERR(cxlsd);
56	60
57	61	device_lock_assert(&port->dev);
58	62
59		− dport = list_first_entry(&port->dports, typeof(*dport), list);
	63	+ xa_for_each(&port->dports, index, dport)
	64	+ break;
60	65	single_port_map[0] = dport->port_id;
61	66
62		− return add_hdm_decoder(port, cxld, single_port_map);
	67	+ return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
63	68	}
64	69	EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
65	70
@@ -124,47 +129,577 @@ struct cxl_hdm devm_cxl_setup_hdm(struct cxl_port port) · 58 unmodified lines
124	129	return ERR_PTR(-ENXIO);
125	130	}
126	131
	132	+ dev_set_drvdata(dev, cxlhdm);
	133	+
127	134	return cxlhdm;
128	135	}
129	136	EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
130	137
131		−static int to_interleave_granularity(u32 ctrl)
	138	+static void __cxl_dpa_debug(struct seq_file file, struct resource r, int depth)
	139	+{
	140	+ unsigned long long start = r->start, end = r->end;
	141	+
	142	+ seq_printf(file, "%s%08llx-%08llx : %s\n", depth 2, "", start, end,
	143	+ r->name);
	144	+}
	145	+
	146	+void cxl_dpa_debug(struct seq_file file, struct cxl_dev_state cxlds)
	147	+{
	148	+ struct resource p1, p2;
	149	+
	150	+ down_read(&cxl_dpa_rwsem);
	151	+ for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
	152	+ __cxl_dpa_debug(file, p1, 0);
	153	+ for (p2 = p1->child; p2; p2 = p2->sibling)
	154	+ __cxl_dpa_debug(file, p2, 1);
	155	+ }
	156	+ up_read(&cxl_dpa_rwsem);
	157	+}
	158	+EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
	159	+
	160	+/*
	161	+ * Must be called in a context that synchronizes against this decoder's
	162	+ * port ->remove() callback (like an endpoint decoder sysfs attribute)
	163	+ */
	164	+static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
	165	+{
	166	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	167	+ struct cxl_port *port = cxled_to_port(cxled);
	168	+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
	169	+ struct resource *res = cxled->dpa_res;
	170	+ resource_size_t skip_start;
	171	+
	172	+ lockdep_assert_held_write(&cxl_dpa_rwsem);
	173	+
	174	+ /* save @skip_start, before @res is released */
	175	+ skip_start = res->start - cxled->skip;
	176	+ __release_region(&cxlds->dpa_res, res->start, resource_size(res));
	177	+ if (cxled->skip)
	178	+ __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
	179	+ cxled->skip = 0;
	180	+ cxled->dpa_res = NULL;
	181	+ put_device(&cxled->cxld.dev);
	182	+ port->hdm_end--;
	183	+}
	184	+
	185	+static void cxl_dpa_release(void *cxled)
	186	+{
	187	+ down_write(&cxl_dpa_rwsem);
	188	+ __cxl_dpa_release(cxled);
	189	+ up_write(&cxl_dpa_rwsem);
	190	+}
	191	+
	192	+/*
	193	+ * Must be called from context that will not race port device
	194	+ * unregistration, like decoder sysfs attribute methods
	195	+ */
	196	+static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
	197	+{
	198	+ struct cxl_port *port = cxled_to_port(cxled);
	199	+
	200	+ lockdep_assert_held_write(&cxl_dpa_rwsem);
	201	+ devm_remove_action(&port->dev, cxl_dpa_release, cxled);
	202	+ __cxl_dpa_release(cxled);
	203	+}
	204	+
	205	+static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
	206	+ resource_size_t base, resource_size_t len,
	207	+ resource_size_t skipped)
	208	+{
	209	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	210	+ struct cxl_port *port = cxled_to_port(cxled);
	211	+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
	212	+ struct device *dev = &port->dev;
	213	+ struct resource *res;
	214	+
	215	+ lockdep_assert_held_write(&cxl_dpa_rwsem);
	216	+
	217	+ if (!len)
	218	+ goto success;
	219	+
	220	+ if (cxled->dpa_res) {
	221	+ dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
	222	+ port->id, cxled->cxld.id, cxled->dpa_res);
	223	+ return -EBUSY;
	224	+ }
	225	+
	226	+ if (port->hdm_end + 1 != cxled->cxld.id) {
	227	+ /*
	228	+ * Assumes alloc and commit order is always in hardware instance
	229	+ * order per expectations from 8.2.5.12.20 Committing Decoder
	230	+ * Programming that enforce decoder[m] committed before
	231	+ * decoder[m+1] commit start.
	232	+ */
	233	+ dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
	234	+ cxled->cxld.id, port->id, port->hdm_end + 1);
	235	+ return -EBUSY;
	236	+ }
	237	+
	238	+ if (skipped) {
	239	+ res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
	240	+ dev_name(&cxled->cxld.dev), 0);
	241	+ if (!res) {
	242	+ dev_dbg(dev,
	243	+ "decoder%d.%d: failed to reserve skipped space\n",
	244	+ port->id, cxled->cxld.id);
	245	+ return -EBUSY;
	246	+ }
	247	+ }
	248	+ res = __request_region(&cxlds->dpa_res, base, len,
	249	+ dev_name(&cxled->cxld.dev), 0);
	250	+ if (!res) {
	251	+ dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
	252	+ port->id, cxled->cxld.id);
	253	+ if (skipped)
	254	+ __release_region(&cxlds->dpa_res, base - skipped,
	255	+ skipped);
	256	+ return -EBUSY;
	257	+ }
	258	+ cxled->dpa_res = res;
	259	+ cxled->skip = skipped;
	260	+
	261	+ if (resource_contains(&cxlds->pmem_res, res))
	262	+ cxled->mode = CXL_DECODER_PMEM;
	263	+ else if (resource_contains(&cxlds->ram_res, res))
	264	+ cxled->mode = CXL_DECODER_RAM;
	265	+ else {
	266	+ dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
	267	+ cxled->cxld.id, cxled->dpa_res);
	268	+ cxled->mode = CXL_DECODER_MIXED;
	269	+ }
	270	+
	271	+success:
	272	+ port->hdm_end++;
	273	+ get_device(&cxled->cxld.dev);
	274	+ return 0;
	275	+}
	276	+
	277	+static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
	278	+ resource_size_t base, resource_size_t len,
	279	+ resource_size_t skipped)
132	280	{
133		− int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl);
	281	+ struct cxl_port *port = cxled_to_port(cxled);
	282	+ int rc;
	283	+
	284	+ down_write(&cxl_dpa_rwsem);
	285	+ rc = __cxl_dpa_reserve(cxled, base, len, skipped);
	286	+ up_write(&cxl_dpa_rwsem);
	287	+
	288	+ if (rc)
	289	+ return rc;
134	290
135		− return 256 << val;
	291	+ return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
136	292	}
137	293
138		−static int to_interleave_ways(u32 ctrl)
	294	+resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
139	295	{
140		− int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl);
	296	+ resource_size_t size = 0;
	297	+
	298	+ down_read(&cxl_dpa_rwsem);
	299	+ if (cxled->dpa_res)
	300	+ size = resource_size(cxled->dpa_res);
	301	+ up_read(&cxl_dpa_rwsem);
141	302
142		− switch (val) {
143		− case 0 ... 4:
144		− return 1 << val;
145		− case 8 ... 10:
146		− return 3 << (val - 8);
	303	+ return size;
	304	+}
	305	+
	306	+resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
	307	+{
	308	+ resource_size_t base = -1;
	309	+
	310	+ down_read(&cxl_dpa_rwsem);
	311	+ if (cxled->dpa_res)
	312	+ base = cxled->dpa_res->start;
	313	+ up_read(&cxl_dpa_rwsem);
	314	+
	315	+ return base;
	316	+}
	317	+
	318	+int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
	319	+{
	320	+ struct cxl_port *port = cxled_to_port(cxled);
	321	+ struct device *dev = &cxled->cxld.dev;
	322	+ int rc;
	323	+
	324	+ down_write(&cxl_dpa_rwsem);
	325	+ if (!cxled->dpa_res) {
	326	+ rc = 0;
	327	+ goto out;
	328	+ }
	329	+ if (cxled->cxld.region) {
	330	+ dev_dbg(dev, "decoder assigned to: %s\n",
	331	+ dev_name(&cxled->cxld.region->dev));
	332	+ rc = -EBUSY;
	333	+ goto out;
	334	+ }
	335	+ if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
	336	+ dev_dbg(dev, "decoder enabled\n");
	337	+ rc = -EBUSY;
	338	+ goto out;
	339	+ }
	340	+ if (cxled->cxld.id != port->hdm_end) {
	341	+ dev_dbg(dev, "expected decoder%d.%d\n", port->id,
	342	+ port->hdm_end);
	343	+ rc = -EBUSY;
	344	+ goto out;
	345	+ }
	346	+ devm_cxl_dpa_release(cxled);
	347	+ rc = 0;
	348	+out:
	349	+ up_write(&cxl_dpa_rwsem);
	350	+ return rc;
	351	+}
	352	+
	353	+int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
	354	+ enum cxl_decoder_mode mode)
	355	+{
	356	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	357	+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
	358	+ struct device *dev = &cxled->cxld.dev;
	359	+ int rc;
	360	+
	361	+ switch (mode) {
	362	+ case CXL_DECODER_RAM:
	363	+ case CXL_DECODER_PMEM:
	364	+ break;
147	365	default:
	366	+ dev_dbg(dev, "unsupported mode: %d\n", mode);
	367	+ return -EINVAL;
	368	+ }
	369	+
	370	+ down_write(&cxl_dpa_rwsem);
	371	+ if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
	372	+ rc = -EBUSY;
	373	+ goto out;
	374	+ }
	375	+
	376	+ /*
	377	+ * Only allow modes that are supported by the current partition
	378	+ * configuration
	379	+ */
	380	+ if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
	381	+ dev_dbg(dev, "no available pmem capacity\n");
	382	+ rc = -ENXIO;
	383	+ goto out;
	384	+ }
	385	+ if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
	386	+ dev_dbg(dev, "no available ram capacity\n");
	387	+ rc = -ENXIO;
	388	+ goto out;
	389	+ }
	390	+
	391	+ cxled->mode = mode;
	392	+ rc = 0;
	393	+out:
	394	+ up_write(&cxl_dpa_rwsem);
	395	+
	396	+ return rc;
	397	+}
	398	+
	399	+int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
	400	+{
	401	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	402	+ resource_size_t free_ram_start, free_pmem_start;
	403	+ struct cxl_port *port = cxled_to_port(cxled);
	404	+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
	405	+ struct device *dev = &cxled->cxld.dev;
	406	+ resource_size_t start, avail, skip;
	407	+ struct resource p, last;
	408	+ int rc;
	409	+
	410	+ down_write(&cxl_dpa_rwsem);
	411	+ if (cxled->cxld.region) {
	412	+ dev_dbg(dev, "decoder attached to %s\n",
	413	+ dev_name(&cxled->cxld.region->dev));
	414	+ rc = -EBUSY;
	415	+ goto out;
	416	+ }
	417	+
	418	+ if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
	419	+ dev_dbg(dev, "decoder enabled\n");
	420	+ rc = -EBUSY;
	421	+ goto out;
	422	+ }
	423	+
	424	+ for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
	425	+ last = p;
	426	+ if (last)
	427	+ free_ram_start = last->end + 1;
	428	+ else
	429	+ free_ram_start = cxlds->ram_res.start;
	430	+
	431	+ for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
	432	+ last = p;
	433	+ if (last)
	434	+ free_pmem_start = last->end + 1;
	435	+ else
	436	+ free_pmem_start = cxlds->pmem_res.start;
	437	+
	438	+ if (cxled->mode == CXL_DECODER_RAM) {
	439	+ start = free_ram_start;
	440	+ avail = cxlds->ram_res.end - start + 1;
	441	+ skip = 0;
	442	+ } else if (cxled->mode == CXL_DECODER_PMEM) {
	443	+ resource_size_t skip_start, skip_end;
	444	+
	445	+ start = free_pmem_start;
	446	+ avail = cxlds->pmem_res.end - start + 1;
	447	+ skip_start = free_ram_start;
	448	+
	449	+ /*
	450	+ * If some pmem is already allocated, then that allocation
	451	+ * already handled the skip.
	452	+ */
	453	+ if (cxlds->pmem_res.child &&
	454	+ skip_start == cxlds->pmem_res.child->start)
	455	+ skip_end = skip_start - 1;
	456	+ else
	457	+ skip_end = start - 1;
	458	+ skip = skip_end - skip_start + 1;
	459	+ } else {
	460	+ dev_dbg(dev, "mode not set\n");
	461	+ rc = -EINVAL;
	462	+ goto out;
	463	+ }
	464	+
	465	+ if (size > avail) {
	466	+ dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
	467	+ cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
	468	+ &avail);
	469	+ rc = -ENOSPC;
	470	+ goto out;
	471	+ }
	472	+
	473	+ rc = __cxl_dpa_reserve(cxled, start, size, skip);
	474	+out:
	475	+ up_write(&cxl_dpa_rwsem);
	476	+
	477	+ if (rc)
	478	+ return rc;
	479	+
	480	+ return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
	481	+}
	482	+
	483	+static void cxld_set_interleave(struct cxl_decoder cxld, u32 ctrl)
	484	+{
	485	+ u16 eig;
	486	+ u8 eiw;
	487	+
	488	+ /*
	489	+ * Input validation ensures these warns never fire, but otherwise
	490	+ * suppress unititalized variable usage warnings.
	491	+ */
	492	+ if (WARN_ONCE(ways_to_cxl(cxld->interleave_ways, &eiw),
	493	+ "invalid interleave_ways: %d\n", cxld->interleave_ways))
	494	+ return;
	495	+ if (WARN_ONCE(granularity_to_cxl(cxld->interleave_granularity, &eig),
	496	+ "invalid interleave_granularity: %d\n",
	497	+ cxld->interleave_granularity))
	498	+ return;
	499	+
	500	+ u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
	501	+ u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
	502	+ *ctrl \|= CXL_HDM_DECODER0_CTRL_COMMIT;
	503	+}
	504	+
	505	+static void cxld_set_type(struct cxl_decoder cxld, u32 ctrl)
	506	+{
	507	+ u32p_replace_bits(ctrl, !!(cxld->target_type == 3),
	508	+ CXL_HDM_DECODER0_CTRL_TYPE);
	509	+}
	510	+
	511	+static int cxlsd_set_targets(struct cxl_switch_decoder cxlsd, u64 tgt)
	512	+{
	513	+ struct cxl_dport **t = &cxlsd->target[0];
	514	+ int ways = cxlsd->cxld.interleave_ways;
	515	+
	516	+ if (dev_WARN_ONCE(&cxlsd->cxld.dev,
	517	+ ways > 8 \|\| ways > cxlsd->nr_targets,
	518	+ "ways: %d overflows targets: %d\n", ways,
	519	+ cxlsd->nr_targets))
	520	+ return -ENXIO;
	521	+
	522	+ *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
	523	+ if (ways > 1)
	524	+ *tgt \|= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
	525	+ if (ways > 2)
	526	+ *tgt \|= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
	527	+ if (ways > 3)
	528	+ *tgt \|= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
	529	+ if (ways > 4)
	530	+ *tgt \|= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
	531	+ if (ways > 5)
	532	+ *tgt \|= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
	533	+ if (ways > 6)
	534	+ *tgt \|= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
	535	+ if (ways > 7)
	536	+ *tgt \|= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
	537	+
	538	+ return 0;
	539	+}
	540	+
	541	+/*
	542	+ * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
	543	+ * committed or error within 10ms, but just be generous with 20ms to account for
	544	+ * clock skew and other marginal behavior
	545	+ */
	546	+#define COMMIT_TIMEOUT_MS 20
	547	+static int cxld_await_commit(void __iomem *hdm, int id)
	548	+{
	549	+ u32 ctrl;
	550	+ int i;
	551	+
	552	+ for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
	553	+ ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
	554	+ if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
	555	+ ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
	556	+ writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
	557	+ return -EIO;
	558	+ }
	559	+ if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
	560	+ return 0;
	561	+ fsleep(1000);
	562	+ }
	563	+
	564	+ return -ETIMEDOUT;
	565	+}
	566	+
	567	+static int cxl_decoder_commit(struct cxl_decoder *cxld)
	568	+{
	569	+ struct cxl_port *port = to_cxl_port(cxld->dev.parent);
	570	+ struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
	571	+ void __iomem *hdm = cxlhdm->regs.hdm_decoder;
	572	+ int id = cxld->id, rc;
	573	+ u64 base, size;
	574	+ u32 ctrl;
	575	+
	576	+ if (cxld->flags & CXL_DECODER_F_ENABLE)
	577	+ return 0;
	578	+
	579	+ if (port->commit_end + 1 != id) {
	580	+ dev_dbg(&port->dev,
	581	+ "%s: out of order commit, expected decoder%d.%d\n",
	582	+ dev_name(&cxld->dev), port->id, port->commit_end + 1);
	583	+ return -EBUSY;
	584	+ }
	585	+
	586	+ down_read(&cxl_dpa_rwsem);
	587	+ /* common decoder settings */
	588	+ ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
	589	+ cxld_set_interleave(cxld, &ctrl);
	590	+ cxld_set_type(cxld, &ctrl);
	591	+ base = cxld->hpa_range.start;
	592	+ size = range_len(&cxld->hpa_range);
	593	+
	594	+ writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
	595	+ writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
	596	+ writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
	597	+ writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
	598	+
	599	+ if (is_switch_decoder(&cxld->dev)) {
	600	+ struct cxl_switch_decoder *cxlsd =
	601	+ to_cxl_switch_decoder(&cxld->dev);
	602	+ void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
	603	+ void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
	604	+ u64 targets;
	605	+
	606	+ rc = cxlsd_set_targets(cxlsd, &targets);
	607	+ if (rc) {
	608	+ dev_dbg(&port->dev, "%s: target configuration error\n",
	609	+ dev_name(&cxld->dev));
	610	+ goto err;
	611	+ }
	612	+
	613	+ writel(upper_32_bits(targets), tl_hi);
	614	+ writel(lower_32_bits(targets), tl_lo);
	615	+ } else {
	616	+ struct cxl_endpoint_decoder *cxled =
	617	+ to_cxl_endpoint_decoder(&cxld->dev);
	618	+ void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
	619	+ void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
	620	+
	621	+ writel(upper_32_bits(cxled->skip), sk_hi);
	622	+ writel(lower_32_bits(cxled->skip), sk_lo);
	623	+ }
	624	+
	625	+ writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
	626	+ up_read(&cxl_dpa_rwsem);
	627	+
	628	+ port->commit_end++;
	629	+ rc = cxld_await_commit(hdm, cxld->id);
	630	+err:
	631	+ if (rc) {
	632	+ dev_dbg(&port->dev, "%s: error %d committing decoder\n",
	633	+ dev_name(&cxld->dev), rc);
	634	+ cxld->reset(cxld);
	635	+ return rc;
	636	+ }
	637	+ cxld->flags \|= CXL_DECODER_F_ENABLE;
	638	+
	639	+ return 0;
	640	+}
	641	+
	642	+static int cxl_decoder_reset(struct cxl_decoder *cxld)
	643	+{
	644	+ struct cxl_port *port = to_cxl_port(cxld->dev.parent);
	645	+ struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
	646	+ void __iomem *hdm = cxlhdm->regs.hdm_decoder;
	647	+ int id = cxld->id;
	648	+ u32 ctrl;
	649	+
	650	+ if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
148	651	return 0;
	652	+
	653	+ if (port->commit_end != id) {
	654	+ dev_dbg(&port->dev,
	655	+ "%s: out of order reset, expected decoder%d.%d\n",
	656	+ dev_name(&cxld->dev), port->id, port->commit_end);
	657	+ return -EBUSY;
149	658	}
	659	+
	660	+ down_read(&cxl_dpa_rwsem);
	661	+ ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
	662	+ ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
	663	+ writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
	664	+
	665	+ writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
	666	+ writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
	667	+ writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
	668	+ writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
	669	+ up_read(&cxl_dpa_rwsem);
	670	+
	671	+ port->commit_end--;
	672	+ cxld->flags &= ~CXL_DECODER_F_ENABLE;
	673	+
	674	+ return 0;
150	675	}
151	676
152	677	static int init_hdm_decoder(struct cxl_port port, struct cxl_decoder cxld,
153		− int target_map, void __iomem hdm, int which)
	678	+ int target_map, void __iomem hdm, int which,
	679	+ u64 *dpa_base)
154	680	{
155		− u64 size, base;
	681	+ struct cxl_endpoint_decoder *cxled = NULL;
	682	+ u64 size, base, skip, dpa_size;
	683	+ bool committed;
	684	+ u32 remainder;
	685	+ int i, rc;
156	686	u32 ctrl;
157		− int i;
158	687	union {
159	688	u64 value;
160	689	unsigned char target_id[8];
161	690	} target_list;
162	691
	692	+ if (is_endpoint_decoder(&cxld->dev))
	693	+ cxled = to_cxl_endpoint_decoder(&cxld->dev);
	694	+
163	695	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
164	696	base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
165	697	size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
	698	+ committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
	699	+ cxld->commit = cxl_decoder_commit;
	700	+ cxld->reset = cxl_decoder_reset;
166	701
167		− if (!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED))
	702	+ if (!committed)
168	703	size = 0;
169	704	if (base == U64_MAX \|\| size == U64_MAX) {
170	705	dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
@@ -172,39 +707,77 @@ static int init_hdm_decoder(struct cxl_port port, struct cxl_decoder cxld, · 1 unmodified line
172	707	return -ENXIO;
173	708	}
174	709
175		− cxld->decoder_range = (struct range) {
	710	+ cxld->hpa_range = (struct range) {
176	711	.start = base,
177	712	.end = base + size - 1,
178	713	};
179	714
180		− /* switch decoders are always enabled if committed */
181		− if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) {
	715	+ /* decoders are enabled if committed */
	716	+ if (committed) {
182	717	cxld->flags \|= CXL_DECODER_F_ENABLE;
183	718	if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
184	719	cxld->flags \|= CXL_DECODER_F_LOCK;
	720	+ if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
	721	+ cxld->target_type = CXL_DECODER_EXPANDER;
	722	+ else
	723	+ cxld->target_type = CXL_DECODER_ACCELERATOR;
	724	+ if (cxld->id != port->commit_end + 1) {
	725	+ dev_warn(&port->dev,
	726	+ "decoder%d.%d: Committed out of order\n",
	727	+ port->id, cxld->id);
	728	+ return -ENXIO;
	729	+ }
	730	+ port->commit_end = cxld->id;
	731	+ } else {
	732	+ /* unless / until type-2 drivers arrive, assume type-3 */
	733	+ if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) {
	734	+ ctrl \|= CXL_HDM_DECODER0_CTRL_TYPE;
	735	+ writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
	736	+ }
	737	+ cxld->target_type = CXL_DECODER_EXPANDER;
185	738	}
186		− cxld->interleave_ways = to_interleave_ways(ctrl);
187		− if (!cxld->interleave_ways) {
	739	+ rc = cxl_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
	740	+ &cxld->interleave_ways);
	741	+ if (rc) {
188	742	dev_warn(&port->dev,
189	743	"decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
190	744	port->id, cxld->id, ctrl);
191		− return -ENXIO;
	745	+ return rc;
192	746	}
193		− cxld->interleave_granularity = to_interleave_granularity(ctrl);
	747	+ rc = cxl_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
	748	+ &cxld->interleave_granularity);
	749	+ if (rc)
	750	+ return rc;
194	751
195		− if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
196		− cxld->target_type = CXL_DECODER_EXPANDER;
197		− else
198		− cxld->target_type = CXL_DECODER_ACCELERATOR;
	752	+ if (!cxled) {
	753	+ target_list.value =
	754	+ ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
	755	+ for (i = 0; i < cxld->interleave_ways; i++)
	756	+ target_map[i] = target_list.target_id[i];
199	757
200		− if (is_endpoint_decoder(&cxld->dev))
201	758	return 0;
	759	+ }
202	760
203		− target_list.value =
204		− ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
205		− for (i = 0; i < cxld->interleave_ways; i++)
206		− target_map[i] = target_list.target_id[i];
	761	+ if (!committed)
	762	+ return 0;
207	763
	764	+ dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
	765	+ if (remainder) {
	766	+ dev_err(&port->dev,
	767	+ "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
	768	+ port->id, cxld->id, size, cxld->interleave_ways);
	769	+ return -ENXIO;
	770	+ }
	771	+ skip = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
	772	+ rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
	773	+ if (rc) {
	774	+ dev_err(&port->dev,
	775	+ "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
	776	+ port->id, cxld->id, *dpa_base,
	777	+ *dpa_base + dpa_size + skip - 1, rc);
	778	+ return rc;
	779	+ }
	780	+ *dpa_base += dpa_size + skip;
208	781	return 0;
209	782	}
210	783
@@ -216,7 +789,8 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 5 unmodified lines
216	789	{
217	790	void __iomem *hdm = cxlhdm->regs.hdm_decoder;
218	791	struct cxl_port *port = cxlhdm->port;
219		− int i, committed, failed;
	792	+ int i, committed;
	793	+ u64 dpa_base = 0;
220	794	u32 ctrl;
221	795
222	796	/*
@@ -236,27 +810,37 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 13 unmodified lines
236	810	if (committed != cxlhdm->decoder_count)
237	811	msleep(20);
238	812
239		− for (i = 0, failed = 0; i < cxlhdm->decoder_count; i++) {
	813	+ for (i = 0; i < cxlhdm->decoder_count; i++) {
240	814	int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
241	815	int rc, target_count = cxlhdm->target_count;
242	816	struct cxl_decoder *cxld;
243	817
244		− if (is_cxl_endpoint(port))
245		− cxld = cxl_endpoint_decoder_alloc(port);
246		− else
247		− cxld = cxl_switch_decoder_alloc(port, target_count);
248		− if (IS_ERR(cxld)) {
249		− dev_warn(&port->dev,
250		− "Failed to allocate the decoder\n");
251		− return PTR_ERR(cxld);
	818	+ if (is_cxl_endpoint(port)) {
	819	+ struct cxl_endpoint_decoder *cxled;
	820	+
	821	+ cxled = cxl_endpoint_decoder_alloc(port);
	822	+ if (IS_ERR(cxled)) {
	823	+ dev_warn(&port->dev,
	824	+ "Failed to allocate the decoder\n");
	825	+ return PTR_ERR(cxled);
	826	+ }
	827	+ cxld = &cxled->cxld;
	828	+ } else {
	829	+ struct cxl_switch_decoder *cxlsd;
	830	+
	831	+ cxlsd = cxl_switch_decoder_alloc(port, target_count);
	832	+ if (IS_ERR(cxlsd)) {
	833	+ dev_warn(&port->dev,
	834	+ "Failed to allocate the decoder\n");
	835	+ return PTR_ERR(cxlsd);
	836	+ }
	837	+ cxld = &cxlsd->cxld;
252	838	}
253	839
254		− rc = init_hdm_decoder(port, cxld, target_map,
255		− cxlhdm->regs.hdm_decoder, i);
	840	+ rc = init_hdm_decoder(port, cxld, target_map, hdm, i, &dpa_base);
256	841	if (rc) {
257	842	put_device(&cxld->dev);
258		− failed++;
259		− continue;
	843	+ return rc;
260	844	}
261	845	rc = add_hdm_decoder(port, cxld, target_map);
262	846	if (rc) {
@@ -266,11 +850,6 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 3 unmodified lines
266	850	}
267	851	}
268	852
269		− if (failed == cxlhdm->decoder_count) {
270		− dev_err(&port->dev, "No valid decoders found\n");
271		− return -ENXIO;
272		− }
273		−
274	853	return 0;
275	854	}
276	855	EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);

drivers/cxl/core/mbox.c+47−48

@@ -718,12 +718,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL); · 717 unmodified lines
718	718	*/
719	719	static int cxl_mem_get_partition_info(struct cxl_dev_state *cxlds)
720	720	{
721		− struct cxl_mbox_get_partition_info {
722		− __le64 active_volatile_cap;
723		− __le64 active_persistent_cap;
724		− __le64 next_volatile_cap;
725		− __le64 next_persistent_cap;
726		− } __packed pi;
	721	+ struct cxl_mbox_get_partition_info pi;
727	722	int rc;
728	723
729	724	rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_GET_PARTITION_INFO, NULL, 0,
@@ -773,15 +768,6 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) · 43 unmodified lines
773	768	cxlds->partition_align_bytes =
774	769	le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
775	770
776		− dev_dbg(cxlds->dev,
777		− "Identify Memory Device\n"
778		− " total_bytes = %#llx\n"
779		− " volatile_only_bytes = %#llx\n"
780		− " persistent_only_bytes = %#llx\n"
781		− " partition_align_bytes = %#llx\n",
782		− cxlds->total_bytes, cxlds->volatile_only_bytes,
783		− cxlds->persistent_only_bytes, cxlds->partition_align_bytes);
784		−
785	771	cxlds->lsa_size = le32_to_cpu(id.lsa_size);
786	772	memcpy(cxlds->firmware_version, id.fw_revision, sizeof(id.fw_revision));
787	773
@@ -789,42 +775,63 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) · 1 unmodified line
789	775	}
790	776	EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);
791	777
792		−int cxl_mem_create_range_info(struct cxl_dev_state *cxlds)
	778	+static int add_dpa_res(struct device dev, struct resource parent,
	779	+ struct resource *res, resource_size_t start,
	780	+ resource_size_t size, const char *type)
793	781	{
794	782	int rc;
795	783
796		− if (cxlds->partition_align_bytes == 0) {
797		− cxlds->ram_range.start = 0;
798		− cxlds->ram_range.end = cxlds->volatile_only_bytes - 1;
799		− cxlds->pmem_range.start = cxlds->volatile_only_bytes;
800		− cxlds->pmem_range.end = cxlds->volatile_only_bytes +
801		− cxlds->persistent_only_bytes - 1;
	784	+ res->name = type;
	785	+ res->start = start;
	786	+ res->end = start + size - 1;
	787	+ res->flags = IORESOURCE_MEM;
	788	+ if (resource_size(res) == 0) {
	789	+ dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
802	790	return 0;
803	791	}
804		−
805		− rc = cxl_mem_get_partition_info(cxlds);
	792	+ rc = request_resource(parent, res);
806	793	if (rc) {
807		− dev_err(cxlds->dev, "Failed to query partition information\n");
	794	+ dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
	795	+ res, rc);
808	796	return rc;
809	797	}
810	798
811		− dev_dbg(cxlds->dev,
812		− "Get Partition Info\n"
813		− " active_volatile_bytes = %#llx\n"
814		− " active_persistent_bytes = %#llx\n"
815		− " next_volatile_bytes = %#llx\n"
816		− " next_persistent_bytes = %#llx\n",
817		− cxlds->active_volatile_bytes, cxlds->active_persistent_bytes,
818		− cxlds->next_volatile_bytes, cxlds->next_persistent_bytes);
	799	+ dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
	800	+
	801	+ return 0;
	802	+}
819	803
820		− cxlds->ram_range.start = 0;
821		− cxlds->ram_range.end = cxlds->active_volatile_bytes - 1;
	804	+int cxl_mem_create_range_info(struct cxl_dev_state *cxlds)
	805	+{
	806	+ struct device *dev = cxlds->dev;
	807	+ int rc;
822	808
823		− cxlds->pmem_range.start = cxlds->active_volatile_bytes;
824		− cxlds->pmem_range.end =
825		− cxlds->active_volatile_bytes + cxlds->active_persistent_bytes - 1;
	809	+ cxlds->dpa_res =
	810	+ (struct resource)DEFINE_RES_MEM(0, cxlds->total_bytes);
826	811
827		− return 0;
	812	+ if (cxlds->partition_align_bytes == 0) {
	813	+ rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
	814	+ cxlds->volatile_only_bytes, "ram");
	815	+ if (rc)
	816	+ return rc;
	817	+ return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
	818	+ cxlds->volatile_only_bytes,
	819	+ cxlds->persistent_only_bytes, "pmem");
	820	+ }
	821	+
	822	+ rc = cxl_mem_get_partition_info(cxlds);
	823	+ if (rc) {
	824	+ dev_err(dev, "Failed to query partition information\n");
	825	+ return rc;
	826	+ }
	827	+
	828	+ rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
	829	+ cxlds->active_volatile_bytes, "ram");
	830	+ if (rc)
	831	+ return rc;
	832	+ return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
	833	+ cxlds->active_volatile_bytes,
	834	+ cxlds->active_persistent_bytes, "pmem");
828	835	}
829	836	EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);
830	837
@@ -845,19 +852,11 @@ struct cxl_dev_state cxl_dev_state_create(struct device dev) · 14 unmodified lines
845	852	}
846	853	EXPORT_SYMBOL_NS_GPL(cxl_dev_state_create, CXL);
847	854
848		−static struct dentry *cxl_debugfs;
849		−
850	855	void __init cxl_mbox_init(void)
851	856	{
852	857	struct dentry *mbox_debugfs;
853	858
854		− cxl_debugfs = debugfs_create_dir("cxl", NULL);
855		− mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs);
	859	+ mbox_debugfs = cxl_debugfs_create_dir("mbox");
856	860	debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
857	861	&cxl_raw_allow_all);
858	862	}
859		−
860		−void cxl_mbox_exit(void)
861		−{
862		− debugfs_remove_recursive(cxl_debugfs);
863		−}

drivers/cxl/core/memdev.c+2−2

@@ -68,7 +68,7 @@ static ssize_t ram_size_show(struct device dev, struct device_attribute attr, · 67 unmodified lines
68	68	{
69	69	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
70	70	struct cxl_dev_state *cxlds = cxlmd->cxlds;
71		− unsigned long long len = range_len(&cxlds->ram_range);
	71	+ unsigned long long len = resource_size(&cxlds->ram_res);
72	72
73	73	return sysfs_emit(buf, "%#llx\n", len);
74	74	}
@@ -81,7 +81,7 @@ static ssize_t pmem_size_show(struct device dev, struct device_attribute attr, · 6 unmodified lines
81	81	{
82	82	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
83	83	struct cxl_dev_state *cxlds = cxlmd->cxlds;
84		− unsigned long long len = range_len(&cxlds->pmem_range);
	84	+ unsigned long long len = resource_size(&cxlds->pmem_res);
85	85
86	86	return sysfs_emit(buf, "%#llx\n", len);
87	87	}

drivers/cxl/core/pci.c+174−7

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	#include <linux/device.h>
5	5	#include <linux/delay.h>
6	6	#include <linux/pci.h>
	7	+#include <linux/pci-doe.h>
7	8	#include <cxlpci.h>
8	9	#include <cxlmem.h>
9	10	#include <cxl.h>
@@ -225,7 +226,6 @@ static int dvsec_range_allowed(struct device dev, void arg) · 215 unmodified lines
225	226	{
226	227	struct range *dev_range = arg;
227	228	struct cxl_decoder *cxld;
228		− struct range root_range;
229	229
230	230	if (!is_root_decoder(dev))
231	231	return 0;
@@ -237,12 +237,7 @@ static int dvsec_range_allowed(struct device dev, void arg) · 5 unmodified lines
237	237	if (!(cxld->flags & CXL_DECODER_F_RAM))
238	238	return 0;
239	239
240		− root_range = (struct range) {
241		− .start = cxld->platform_res.start,
242		− .end = cxld->platform_res.end,
243		− };
244		−
245		− return range_contains(&root_range, dev_range);
	240	+ return range_contains(&cxld->hpa_range, dev_range);
246	241	}
247	242
248	243	static void disable_hdm(void *_cxlhdm)
@@ -458,3 +453,175 @@ hdm_init: · 209 unmodified lines
458	453	return 0;
459	454	}
460	455	EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL);
	456	+
	457	+#define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff
	458	+#define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0
	459	+#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00
	460	+#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0
	461	+#define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000
	462	+#define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff
	463	+#define CXL_DOE_PROTOCOL_TABLE_ACCESS 2
	464	+
	465	+static struct pci_doe_mb find_cdat_doe(struct device uport)
	466	+{
	467	+ struct cxl_memdev *cxlmd;
	468	+ struct cxl_dev_state *cxlds;
	469	+ unsigned long index;
	470	+ void *entry;
	471	+
	472	+ cxlmd = to_cxl_memdev(uport);
	473	+ cxlds = cxlmd->cxlds;
	474	+
	475	+ xa_for_each(&cxlds->doe_mbs, index, entry) {
	476	+ struct pci_doe_mb *cur = entry;
	477	+
	478	+ if (pci_doe_supports_prot(cur, PCI_DVSEC_VENDOR_ID_CXL,
	479	+ CXL_DOE_PROTOCOL_TABLE_ACCESS))
	480	+ return cur;
	481	+ }
	482	+
	483	+ return NULL;
	484	+}
	485	+
	486	+#define CDAT_DOE_REQ(entry_handle) \
	487	+ (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \
	488	+ CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) \| \
	489	+ FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \
	490	+ CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) \| \
	491	+ FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle)))
	492	+
	493	+static void cxl_doe_task_complete(struct pci_doe_task *task)
	494	+{
	495	+ complete(task->private);
	496	+}
	497	+
	498	+struct cdat_doe_task {
	499	+ u32 request_pl;
	500	+ u32 response_pl[32];
	501	+ struct completion c;
	502	+ struct pci_doe_task task;
	503	+};
	504	+
	505	+#define DECLARE_CDAT_DOE_TASK(req, cdt) \
	506	+struct cdat_doe_task cdt = { \
	507	+ .c = COMPLETION_INITIALIZER_ONSTACK(cdt.c), \
	508	+ .request_pl = req, \
	509	+ .task = { \
	510	+ .prot.vid = PCI_DVSEC_VENDOR_ID_CXL, \
	511	+ .prot.type = CXL_DOE_PROTOCOL_TABLE_ACCESS, \
	512	+ .request_pl = &cdt.request_pl, \
	513	+ .request_pl_sz = sizeof(cdt.request_pl), \
	514	+ .response_pl = cdt.response_pl, \
	515	+ .response_pl_sz = sizeof(cdt.response_pl), \
	516	+ .complete = cxl_doe_task_complete, \
	517	+ .private = &cdt.c, \
	518	+ } \
	519	+}
	520	+
	521	+static int cxl_cdat_get_length(struct device *dev,
	522	+ struct pci_doe_mb *cdat_doe,
	523	+ size_t *length)
	524	+{
	525	+ DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(0), t);
	526	+ int rc;
	527	+
	528	+ rc = pci_doe_submit_task(cdat_doe, &t.task);
	529	+ if (rc < 0) {
	530	+ dev_err(dev, "DOE submit failed: %d", rc);
	531	+ return rc;
	532	+ }
	533	+ wait_for_completion(&t.c);
	534	+ if (t.task.rv < sizeof(u32))
	535	+ return -EIO;
	536	+
	537	+ *length = t.response_pl[1];
	538	+ dev_dbg(dev, "CDAT length %zu\n", *length);
	539	+
	540	+ return 0;
	541	+}
	542	+
	543	+static int cxl_cdat_read_table(struct device *dev,
	544	+ struct pci_doe_mb *cdat_doe,
	545	+ struct cxl_cdat *cdat)
	546	+{
	547	+ size_t length = cdat->length;
	548	+ u32 *data = cdat->table;
	549	+ int entry_handle = 0;
	550	+
	551	+ do {
	552	+ DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(entry_handle), t);
	553	+ size_t entry_dw;
	554	+ u32 *entry;
	555	+ int rc;
	556	+
	557	+ rc = pci_doe_submit_task(cdat_doe, &t.task);
	558	+ if (rc < 0) {
	559	+ dev_err(dev, "DOE submit failed: %d", rc);
	560	+ return rc;
	561	+ }
	562	+ wait_for_completion(&t.c);
	563	+ /* 1 DW header + 1 DW data min */
	564	+ if (t.task.rv < (2 * sizeof(u32)))
	565	+ return -EIO;
	566	+
	567	+ /* Get the CXL table access header entry handle */
	568	+ entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE,
	569	+ t.response_pl[0]);
	570	+ entry = t.response_pl + 1;
	571	+ entry_dw = t.task.rv / sizeof(u32);
	572	+ /* Skip Header */
	573	+ entry_dw -= 1;
	574	+ entry_dw = min(length / sizeof(u32), entry_dw);
	575	+ /* Prevent length < 1 DW from causing a buffer overflow */
	576	+ if (entry_dw) {
	577	+ memcpy(data, entry, entry_dw * sizeof(u32));
	578	+ length -= entry_dw * sizeof(u32);
	579	+ data += entry_dw;
	580	+ }
	581	+ } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY);
	582	+
	583	+ return 0;
	584	+}
	585	+
	586	+/**
	587	+ * read_cdat_data - Read the CDAT data on this port
	588	+ * @port: Port to read data from
	589	+ *
	590	+ * This call will sleep waiting for responses from the DOE mailbox.
	591	+ */
	592	+void read_cdat_data(struct cxl_port *port)
	593	+{
	594	+ struct pci_doe_mb *cdat_doe;
	595	+ struct device *dev = &port->dev;
	596	+ struct device *uport = port->uport;
	597	+ size_t cdat_length;
	598	+ int rc;
	599	+
	600	+ cdat_doe = find_cdat_doe(uport);
	601	+ if (!cdat_doe) {
	602	+ dev_dbg(dev, "No CDAT mailbox\n");
	603	+ return;
	604	+ }
	605	+
	606	+ port->cdat_available = true;
	607	+
	608	+ if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) {
	609	+ dev_dbg(dev, "No CDAT length\n");
	610	+ return;
	611	+ }
	612	+
	613	+ port->cdat.table = devm_kzalloc(dev, cdat_length, GFP_KERNEL);
	614	+ if (!port->cdat.table)
	615	+ return;
	616	+
	617	+ port->cdat.length = cdat_length;
	618	+ rc = cxl_cdat_read_table(dev, cdat_doe, &port->cdat);
	619	+ if (rc) {
	620	+ /* Don't leave table data allocated on error */
	621	+ devm_kfree(dev, port->cdat.table);
	622	+ port->cdat.table = NULL;
	623	+ port->cdat.length = 0;
	624	+ dev_err(dev, "CDAT data read error\n");
	625	+ }
	626	+}
	627	+EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL);

drivers/cxl/core/pmem.c+2−2

@@ -62,9 +62,9 @@ static int match_nvdimm_bridge(struct device dev, void data) · 61 unmodified lines
62	62	return is_cxl_nvdimm_bridge(dev);
63	63	}
64	64
65		−struct cxl_nvdimm_bridge cxl_find_nvdimm_bridge(struct cxl_nvdimm cxl_nvd)
	65	+struct cxl_nvdimm_bridge cxl_find_nvdimm_bridge(struct device start)
66	66	{
67		− struct cxl_port *port = find_cxl_root(&cxl_nvd->dev);
	67	+ struct cxl_port *port = find_cxl_root(start);
68	68	struct device *dev;
69	69
70	70	if (!port)

drivers/cxl/core/port.c+536−202

@@ -1,7 +1,9 @@
1	1	// SPDX-License-Identifier: GPL-2.0-only
2	2	/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3	3	#include <linux/io-64-nonatomic-lo-hi.h>
	4	+#include <linux/memregion.h>
4	5	#include <linux/workqueue.h>
	6	+#include <linux/debugfs.h>
5	7	#include <linux/device.h>
6	8	#include <linux/module.h>
7	9	#include <linux/pci.h>
@@ -42,6 +44,8 @@ static int cxl_device_id(struct device *dev) · 34 unmodified lines
42	44	return CXL_DEVICE_NVDIMM_BRIDGE;
43	45	if (dev->type == &cxl_nvdimm_type)
44	46	return CXL_DEVICE_NVDIMM;
	47	+ if (dev->type == CXL_PMEM_REGION_TYPE())
	48	+ return CXL_DEVICE_PMEM_REGION;
45	49	if (is_cxl_port(dev)) {
46	50	if (is_cxl_root(to_cxl_port(dev)))
47	51	return CXL_DEVICE_ROOT;
@@ -49,6 +53,8 @@ static int cxl_device_id(struct device *dev) · 1 unmodified line
49	53	}
50	54	if (is_cxl_memdev(dev))
51	55	return CXL_DEVICE_MEMORY_EXPANDER;
	56	+ if (dev->type == CXL_REGION_TYPE())
	57	+ return CXL_DEVICE_REGION;
52	58	return 0;
53	59	}
54	60
@@ -73,14 +79,8 @@ static ssize_t start_show(struct device dev, struct device_attribute attr, · 18 unmodified lines
73	79	char *buf)
74	80	{
75	81	struct cxl_decoder *cxld = to_cxl_decoder(dev);
76		− u64 start;
77	82
78		− if (is_root_decoder(dev))
79		− start = cxld->platform_res.start;
80		− else
81		− start = cxld->decoder_range.start;
82		−
83		− return sysfs_emit(buf, "%#llx\n", start);
	83	+ return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
84	84	}
85	85	static DEVICE_ATTR_ADMIN_RO(start);
86	86
@@ -88,14 +88,8 @@ static ssize_t size_show(struct device dev, struct device_attribute attr, · 1 unmodified line
88	88	char *buf)
89	89	{
90	90	struct cxl_decoder *cxld = to_cxl_decoder(dev);
91		− u64 size;
92		−
93		− if (is_root_decoder(dev))
94		− size = resource_size(&cxld->platform_res);
95		− else
96		− size = range_len(&cxld->decoder_range);
97	91
98		− return sysfs_emit(buf, "%#llx\n", size);
	92	+ return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
99	93	}
100	94	static DEVICE_ATTR_RO(size);
101	95
@@ -131,20 +125,21 @@ static ssize_t target_type_show(struct device *dev, · 29 unmodified lines
131	125	}
132	126	static DEVICE_ATTR_RO(target_type);
133	127
134		−static ssize_t emit_target_list(struct cxl_decoder cxld, char buf)
	128	+static ssize_t emit_target_list(struct cxl_switch_decoder cxlsd, char buf)
135	129	{
	130	+ struct cxl_decoder *cxld = &cxlsd->cxld;
136	131	ssize_t offset = 0;
137	132	int i, rc = 0;
138	133
139	134	for (i = 0; i < cxld->interleave_ways; i++) {
140		− struct cxl_dport *dport = cxld->target[i];
	135	+ struct cxl_dport *dport = cxlsd->target[i];
141	136	struct cxl_dport *next = NULL;
142	137
143	138	if (!dport)
144	139	break;
145	140
146	141	if (i + 1 < cxld->interleave_ways)
147		− next = cxld->target[i + 1];
	142	+ next = cxlsd->target[i + 1];
148	143	rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
149	144	next ? "," : "");
150	145	if (rc < 0)
@@ -158,15 +153,15 @@ static ssize_t emit_target_list(struct cxl_decoder cxld, char buf) · 7 unmodified lines
158	153	static ssize_t target_list_show(struct device *dev,
159	154	struct device_attribute attr, char buf)
160	155	{
161		− struct cxl_decoder *cxld = to_cxl_decoder(dev);
	156	+ struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
162	157	ssize_t offset;
163	158	unsigned int seq;
164	159	int rc;
165	160
166	161	do {
167		− seq = read_seqbegin(&cxld->target_lock);
168		− rc = emit_target_list(cxld, buf);
169		− } while (read_seqretry(&cxld->target_lock, seq));
	162	+ seq = read_seqbegin(&cxlsd->target_lock);
	163	+ rc = emit_target_list(cxlsd, buf);
	164	+ } while (read_seqretry(&cxlsd->target_lock, seq));
170	165
171	166	if (rc < 0)
172	167	return rc;
@@ -180,10 +175,121 @@ static ssize_t target_list_show(struct device *dev, · 7 unmodified lines
180	175	}
181	176	static DEVICE_ATTR_RO(target_list);
182	177
	178	+static ssize_t mode_show(struct device dev, struct device_attribute attr,
	179	+ char *buf)
	180	+{
	181	+ struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
	182	+
	183	+ switch (cxled->mode) {
	184	+ case CXL_DECODER_RAM:
	185	+ return sysfs_emit(buf, "ram\n");
	186	+ case CXL_DECODER_PMEM:
	187	+ return sysfs_emit(buf, "pmem\n");
	188	+ case CXL_DECODER_NONE:
	189	+ return sysfs_emit(buf, "none\n");
	190	+ case CXL_DECODER_MIXED:
	191	+ default:
	192	+ return sysfs_emit(buf, "mixed\n");
	193	+ }
	194	+}
	195	+
	196	+static ssize_t mode_store(struct device dev, struct device_attribute attr,
	197	+ const char *buf, size_t len)
	198	+{
	199	+ struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
	200	+ enum cxl_decoder_mode mode;
	201	+ ssize_t rc;
	202	+
	203	+ if (sysfs_streq(buf, "pmem"))
	204	+ mode = CXL_DECODER_PMEM;
	205	+ else if (sysfs_streq(buf, "ram"))
	206	+ mode = CXL_DECODER_RAM;
	207	+ else
	208	+ return -EINVAL;
	209	+
	210	+ rc = cxl_dpa_set_mode(cxled, mode);
	211	+ if (rc)
	212	+ return rc;
	213	+
	214	+ return len;
	215	+}
	216	+static DEVICE_ATTR_RW(mode);
	217	+
	218	+static ssize_t dpa_resource_show(struct device dev, struct device_attribute attr,
	219	+ char *buf)
	220	+{
	221	+ struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
	222	+ u64 base = cxl_dpa_resource_start(cxled);
	223	+
	224	+ return sysfs_emit(buf, "%#llx\n", base);
	225	+}
	226	+static DEVICE_ATTR_RO(dpa_resource);
	227	+
	228	+static ssize_t dpa_size_show(struct device dev, struct device_attribute attr,
	229	+ char *buf)
	230	+{
	231	+ struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
	232	+ resource_size_t size = cxl_dpa_size(cxled);
	233	+
	234	+ return sysfs_emit(buf, "%pa\n", &size);
	235	+}
	236	+
	237	+static ssize_t dpa_size_store(struct device dev, struct device_attribute attr,
	238	+ const char *buf, size_t len)
	239	+{
	240	+ struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
	241	+ unsigned long long size;
	242	+ ssize_t rc;
	243	+
	244	+ rc = kstrtoull(buf, 0, &size);
	245	+ if (rc)
	246	+ return rc;
	247	+
	248	+ if (!IS_ALIGNED(size, SZ_256M))
	249	+ return -EINVAL;
	250	+
	251	+ rc = cxl_dpa_free(cxled);
	252	+ if (rc)
	253	+ return rc;
	254	+
	255	+ if (size == 0)
	256	+ return len;
	257	+
	258	+ rc = cxl_dpa_alloc(cxled, size);
	259	+ if (rc)
	260	+ return rc;
	261	+
	262	+ return len;
	263	+}
	264	+static DEVICE_ATTR_RW(dpa_size);
	265	+
	266	+static ssize_t interleave_granularity_show(struct device *dev,
	267	+ struct device_attribute *attr,
	268	+ char *buf)
	269	+{
	270	+ struct cxl_decoder *cxld = to_cxl_decoder(dev);
	271	+
	272	+ return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
	273	+}
	274	+
	275	+static DEVICE_ATTR_RO(interleave_granularity);
	276	+
	277	+static ssize_t interleave_ways_show(struct device *dev,
	278	+ struct device_attribute attr, char buf)
	279	+{
	280	+ struct cxl_decoder *cxld = to_cxl_decoder(dev);
	281	+
	282	+ return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
	283	+}
	284	+
	285	+static DEVICE_ATTR_RO(interleave_ways);
	286	+
183	287	static struct attribute *cxl_decoder_base_attrs[] = {
184	288	&dev_attr_start.attr,
185	289	&dev_attr_size.attr,
186	290	&dev_attr_locked.attr,
	291	+ &dev_attr_interleave_granularity.attr,
	292	+ &dev_attr_interleave_ways.attr,
187	293	NULL,
188	294	};
189	295
@@ -197,11 +303,35 @@ static struct attribute *cxl_decoder_root_attrs[] = { · 7 unmodified lines
197	303	&dev_attr_cap_type2.attr,
198	304	&dev_attr_cap_type3.attr,
199	305	&dev_attr_target_list.attr,
	306	+ SET_CXL_REGION_ATTR(create_pmem_region)
	307	+ SET_CXL_REGION_ATTR(delete_region)
200	308	NULL,
201	309	};
202	310
	311	+static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
	312	+{
	313	+ unsigned long flags = CXL_DECODER_F_TYPE3 \| CXL_DECODER_F_PMEM;
	314	+
	315	+ return (cxlrd->cxlsd.cxld.flags & flags) == flags;
	316	+}
	317	+
	318	+static umode_t cxl_root_decoder_visible(struct kobject kobj, struct attribute a, int n)
	319	+{
	320	+ struct device *dev = kobj_to_dev(kobj);
	321	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
	322	+
	323	+ if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
	324	+ return 0;
	325	+
	326	+ if (a == CXL_REGION_ATTR(delete_region) && !can_create_pmem(cxlrd))
	327	+ return 0;
	328	+
	329	+ return a->mode;
	330	+}
	331	+
203	332	static struct attribute_group cxl_decoder_root_attribute_group = {
204	333	.attrs = cxl_decoder_root_attrs,
	334	+ .is_visible = cxl_root_decoder_visible,
205	335	};
206	336
207	337	static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
@@ -214,6 +344,7 @@ static const struct attribute_group *cxl_decoder_root_attribute_groups[] = { · 6 unmodified lines
214	344	static struct attribute *cxl_decoder_switch_attrs[] = {
215	345	&dev_attr_target_type.attr,
216	346	&dev_attr_target_list.attr,
	347	+ SET_CXL_REGION_ATTR(region)
217	348	NULL,
218	349	};
219	350
@@ -230,6 +361,10 @@ static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = { · 10 unmodified lines
230	361
231	362	static struct attribute *cxl_decoder_endpoint_attrs[] = {
232	363	&dev_attr_target_type.attr,
	364	+ &dev_attr_mode.attr,
	365	+ &dev_attr_dpa_size.attr,
	366	+ &dev_attr_dpa_resource.attr,
	367	+ SET_CXL_REGION_ATTR(region)
233	368	NULL,
234	369	};
235	370
@@ -244,31 +379,64 @@ static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = { · 8 unmodified lines
244	379	NULL,
245	380	};
246	381
247		−static void cxl_decoder_release(struct device *dev)
	382	+static void __cxl_decoder_release(struct cxl_decoder *cxld)
248	383	{
249		− struct cxl_decoder *cxld = to_cxl_decoder(dev);
250		− struct cxl_port *port = to_cxl_port(dev->parent);
	384	+ struct cxl_port *port = to_cxl_port(cxld->dev.parent);
251	385
252	386	ida_free(&port->decoder_ida, cxld->id);
253		− kfree(cxld);
254	387	put_device(&port->dev);
255	388	}
256	389
	390	+static void cxl_endpoint_decoder_release(struct device *dev)
	391	+{
	392	+ struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
	393	+
	394	+ __cxl_decoder_release(&cxled->cxld);
	395	+ kfree(cxled);
	396	+}
	397	+
	398	+static void cxl_switch_decoder_release(struct device *dev)
	399	+{
	400	+ struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
	401	+
	402	+ __cxl_decoder_release(&cxlsd->cxld);
	403	+ kfree(cxlsd);
	404	+}
	405	+
	406	+struct cxl_root_decoder to_cxl_root_decoder(struct device dev)
	407	+{
	408	+ if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
	409	+ "not a cxl_root_decoder device\n"))
	410	+ return NULL;
	411	+ return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
	412	+}
	413	+EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
	414	+
	415	+static void cxl_root_decoder_release(struct device *dev)
	416	+{
	417	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
	418	+
	419	+ if (atomic_read(&cxlrd->region_id) >= 0)
	420	+ memregion_free(atomic_read(&cxlrd->region_id));
	421	+ __cxl_decoder_release(&cxlrd->cxlsd.cxld);
	422	+ kfree(cxlrd);
	423	+}
	424	+
257	425	static const struct device_type cxl_decoder_endpoint_type = {
258	426	.name = "cxl_decoder_endpoint",
259		− .release = cxl_decoder_release,
	427	+ .release = cxl_endpoint_decoder_release,
260	428	.groups = cxl_decoder_endpoint_attribute_groups,
261	429	};
262	430
263	431	static const struct device_type cxl_decoder_switch_type = {
264	432	.name = "cxl_decoder_switch",
265		− .release = cxl_decoder_release,
	433	+ .release = cxl_switch_decoder_release,
266	434	.groups = cxl_decoder_switch_attribute_groups,
267	435	};
268	436
269	437	static const struct device_type cxl_decoder_root_type = {
270	438	.name = "cxl_decoder_root",
271		− .release = cxl_decoder_release,
	439	+ .release = cxl_root_decoder_release,
272	440	.groups = cxl_decoder_root_attribute_groups,
273	441	};
274	442
@@ -283,39 +451,63 @@ bool is_root_decoder(struct device *dev) · 8 unmodified lines
283	451	}
284	452	EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
285	453
286		−bool is_cxl_decoder(struct device *dev)
	454	+bool is_switch_decoder(struct device *dev)
287	455	{
288		− return dev->type && dev->type->release == cxl_decoder_release;
	456	+ return is_root_decoder(dev) \|\| dev->type == &cxl_decoder_switch_type;
289	457	}
290		−EXPORT_SYMBOL_NS_GPL(is_cxl_decoder, CXL);
291	458
292	459	struct cxl_decoder to_cxl_decoder(struct device dev)
293	460	{
294		− if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release,
	461	+ if (dev_WARN_ONCE(dev,
	462	+ !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
295	463	"not a cxl_decoder device\n"))
296	464	return NULL;
297	465	return container_of(dev, struct cxl_decoder, dev);
298	466	}
299	467	EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
300	468
	469	+struct cxl_endpoint_decoder to_cxl_endpoint_decoder(struct device dev)
	470	+{
	471	+ if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
	472	+ "not a cxl_endpoint_decoder device\n"))
	473	+ return NULL;
	474	+ return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
	475	+}
	476	+EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
	477	+
	478	+struct cxl_switch_decoder to_cxl_switch_decoder(struct device dev)
	479	+{
	480	+ if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
	481	+ "not a cxl_switch_decoder device\n"))
	482	+ return NULL;
	483	+ return container_of(dev, struct cxl_switch_decoder, cxld.dev);
	484	+}
	485	+
301	486	static void cxl_ep_release(struct cxl_ep *ep)
302	487	{
303		− if (!ep)
304		− return;
305		− list_del(&ep->list);
306	488	put_device(ep->ep);
307	489	kfree(ep);
308	490	}
309	491
	492	+static void cxl_ep_remove(struct cxl_port port, struct cxl_ep ep)
	493	+{
	494	+ if (!ep)
	495	+ return;
	496	+ xa_erase(&port->endpoints, (unsigned long) ep->ep);
	497	+ cxl_ep_release(ep);
	498	+}
	499	+
310	500	static void cxl_port_release(struct device *dev)
311	501	{
312	502	struct cxl_port *port = to_cxl_port(dev);
313		− struct cxl_ep ep, _e;
	503	+ unsigned long index;
	504	+ struct cxl_ep *ep;
314	505
315		− device_lock(dev);
316		− list_for_each_entry_safe(ep, _e, &port->endpoints, list)
317		− cxl_ep_release(ep);
318		− device_unlock(dev);
	506	+ xa_for_each(&port->endpoints, index, ep)
	507	+ cxl_ep_remove(port, ep);
	508	+ xa_destroy(&port->endpoints);
	509	+ xa_destroy(&port->dports);
	510	+ xa_destroy(&port->regions);
319	511	ida_free(&cxl_port_ida, port->id);
320	512	kfree(port);
321	513	}
@@ -370,7 +562,7 @@ static void unregister_port(void *_port) · 48 unmodified lines
370	562	lock_dev = &parent->dev;
371	563
372	564	device_lock_assert(lock_dev);
373		− port->uport = NULL;
	565	+ port->dead = true;
374	566	device_unregister(&port->dev);
375	567	}
376	568
@@ -395,7 +587,7 @@ static struct lock_class_key cxl_port_key; · 18 unmodified lines
395	587
396	588	static struct cxl_port cxl_port_alloc(struct device uport,
397	589	resource_size_t component_reg_phys,
398		− struct cxl_port *parent_port)
	590	+ struct cxl_dport *parent_dport)
399	591	{
400	592	struct cxl_port *port;
401	593	struct device *dev;
@@ -409,6 +601,7 @@ static struct cxl_port cxl_port_alloc(struct device uport, · 7 unmodified lines
409	601	if (rc < 0)
410	602	goto err;
411	603	port->id = rc;
	604	+ port->uport = uport;
412	605
413	606	/*
414	607	* The top-level cxl_port "cxl_root" does not have a cxl_port as
@@ -417,17 +610,37 @@ static struct cxl_port cxl_port_alloc(struct device uport, · 2 unmodified lines
417	610	* description.
418	611	*/
419	612	dev = &port->dev;
420		− if (parent_port) {
	613	+ if (parent_dport) {
	614	+ struct cxl_port *parent_port = parent_dport->port;
	615	+ struct cxl_port *iter;
	616	+
421	617	dev->parent = &parent_port->dev;
422	618	port->depth = parent_port->depth + 1;
	619	+ port->parent_dport = parent_dport;
	620	+
	621	+ /*
	622	+ * walk to the host bridge, or the first ancestor that knows
	623	+ * the host bridge
	624	+ */
	625	+ iter = port;
	626	+ while (!iter->host_bridge &&
	627	+ !is_cxl_root(to_cxl_port(iter->dev.parent)))
	628	+ iter = to_cxl_port(iter->dev.parent);
	629	+ if (iter->host_bridge)
	630	+ port->host_bridge = iter->host_bridge;
	631	+ else
	632	+ port->host_bridge = iter->uport;
	633	+ dev_dbg(uport, "host-bridge: %s\n", dev_name(port->host_bridge));
423	634	} else
424	635	dev->parent = uport;
425	636
426		− port->uport = uport;
427	637	port->component_reg_phys = component_reg_phys;
428	638	ida_init(&port->decoder_ida);
429		− INIT_LIST_HEAD(&port->dports);
430		− INIT_LIST_HEAD(&port->endpoints);
	639	+ port->hdm_end = -1;
	640	+ port->commit_end = -1;
	641	+ xa_init(&port->dports);
	642	+ xa_init(&port->endpoints);
	643	+ xa_init(&port->regions);
431	644
432	645	device_initialize(dev);
433	646	lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
@@ -447,24 +660,24 @@ err: · 13 unmodified lines
447	660	* @host: host device for devm operations
448	661	* @uport: "physical" device implementing this upstream port
449	662	* @component_reg_phys: (optional) for configurable cxl_port instances
450		− * @parent_port: next hop up in the CXL memory decode hierarchy
	663	+ * @parent_dport: next hop up in the CXL memory decode hierarchy
451	664	*/
452	665	struct cxl_port devm_cxl_add_port(struct device host, struct device *uport,
453	666	resource_size_t component_reg_phys,
454		− struct cxl_port *parent_port)
	667	+ struct cxl_dport *parent_dport)
455	668	{
456	669	struct cxl_port *port;
457	670	struct device *dev;
458	671	int rc;
459	672
460		− port = cxl_port_alloc(uport, component_reg_phys, parent_port);
	673	+ port = cxl_port_alloc(uport, component_reg_phys, parent_dport);
461	674	if (IS_ERR(port))
462	675	return port;
463	676
464	677	dev = &port->dev;
465	678	if (is_cxl_memdev(uport))
466	679	rc = dev_set_name(dev, "endpoint%d", port->id);
467		− else if (parent_port)
	680	+ else if (parent_dport)
468	681	rc = dev_set_name(dev, "port%d", port->id);
469	682	else
470	683	rc = dev_set_name(dev, "root%d", port->id);
@@ -556,17 +769,13 @@ static int match_root_child(struct device dev, const void match) · 85 unmodified lines
556	769	return 0;
557	770
558	771	port = to_cxl_port(dev);
559		− device_lock(dev);
560		− list_for_each_entry(dport, &port->dports, list) {
561		− iter = match;
562		− while (iter) {
563		− if (iter == dport->dport)
564		− goto out;
565		− iter = iter->parent;
566		− }
	772	+ iter = match;
	773	+ while (iter) {
	774	+ dport = cxl_find_dport_by_dev(port, iter);
	775	+ if (dport)
	776	+ break;
	777	+ iter = iter->parent;
567	778	}
568		−out:
569		− device_unlock(dev);
570	779
571	780	return !!iter;
572	781	}
@@ -590,9 +799,10 @@ EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL); · 17 unmodified lines
590	799	static struct cxl_dport find_dport(struct cxl_port port, int id)
591	800	{
592	801	struct cxl_dport *dport;
	802	+ unsigned long index;
593	803
594	804	device_lock_assert(&port->dev);
595		− list_for_each_entry (dport, &port->dports, list)
	805	+ xa_for_each(&port->dports, index, dport)
596	806	if (dport->port_id == id)
597	807	return dport;
598	808	return NULL;
@@ -604,15 +814,15 @@ static int add_dport(struct cxl_port port, struct cxl_dport new) · 5 unmodified lines
604	814
605	815	device_lock_assert(&port->dev);
606	816	dup = find_dport(port, new->port_id);
607		− if (dup)
	817	+ if (dup) {
608	818	dev_err(&port->dev,
609	819	"unable to add dport%d-%s non-unique port id (%s)\n",
610	820	new->port_id, dev_name(new->dport),
611	821	dev_name(dup->dport));
612		− else
613		− list_add_tail(&new->list, &port->dports);
614		−
615		− return dup ? -EEXIST : 0;
	822	+ return -EBUSY;
	823	+ }
	824	+ return xa_insert(&port->dports, (unsigned long)new->dport, new,
	825	+ GFP_KERNEL);
616	826	}
617	827
618	828	/*
@@ -639,10 +849,8 @@ static void cxl_dport_remove(void *data) · 20 unmodified lines
639	849	struct cxl_dport *dport = data;
640	850	struct cxl_port *port = dport->port;
641	851
	852	+ xa_erase(&port->dports, (unsigned long) dport->dport);
642	853	put_device(dport->dport);
643		− cond_cxl_root_lock(port);
644		− list_del(&dport->list);
645		− cond_cxl_root_unlock(port);
646	854	}
647	855
648	856	static void cxl_dport_unlink(void *data)
@@ -694,7 +902,6 @@ struct cxl_dport devm_cxl_add_dport(struct cxl_port port, · 45 unmodified lines
694	902	if (!dport)
695	903	return ERR_PTR(-ENOMEM);
696	904
697		− INIT_LIST_HEAD(&dport->list);
698	905	dport->dport = dport_dev;
699	906	dport->port_id = port_id;
700	907	dport->component_reg_phys = component_reg_phys;
@@ -723,44 +930,33 @@ struct cxl_dport devm_cxl_add_dport(struct cxl_port port, · 22 unmodified lines
723	930	}
724	931	EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
725	932
726		−static struct cxl_ep find_ep(struct cxl_port port, struct device *ep_dev)
727		−{
728		− struct cxl_ep *ep;
729		−
730		− device_lock_assert(&port->dev);
731		− list_for_each_entry(ep, &port->endpoints, list)
732		− if (ep->ep == ep_dev)
733		− return ep;
734		− return NULL;
735		−}
736		−
737		−static int add_ep(struct cxl_port port, struct cxl_ep new)
	933	+static int add_ep(struct cxl_ep *new)
738	934	{
739		− struct cxl_ep *dup;
	935	+ struct cxl_port *port = new->dport->port;
	936	+ int rc;
740	937
741	938	device_lock(&port->dev);
742	939	if (port->dead) {
743	940	device_unlock(&port->dev);
744	941	return -ENXIO;
745	942	}
746		− dup = find_ep(port, new->ep);
747		− if (!dup)
748		− list_add_tail(&new->list, &port->endpoints);
	943	+ rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
	944	+ GFP_KERNEL);
749	945	device_unlock(&port->dev);
750	946
751		− return dup ? -EEXIST : 0;
	947	+ return rc;
752	948	}
753	949
754	950	/**
755	951	* cxl_add_ep - register an endpoint's interest in a port
756		− * @port: a port in the endpoint's topology ancestry
	952	+ * @dport: the dport that routes to @ep_dev
757	953	* @ep_dev: device representing the endpoint
758	954	*
759	955	* Intermediate CXL ports are scanned based on the arrival of endpoints.
760	956	* When those endpoints depart the port can be destroyed once all
761	957	* endpoints that care about that port have been removed.
762	958	*/
763		−static int cxl_add_ep(struct cxl_port port, struct device ep_dev)
	959	+static int cxl_add_ep(struct cxl_dport dport, struct device ep_dev)
764	960	{
765	961	struct cxl_ep *ep;
766	962	int rc;
@@ -769,10 +965,10 @@ static int cxl_add_ep(struct cxl_port port, struct device ep_dev) · 2 unmodified lines
769	965	if (!ep)
770	966	return -ENOMEM;
771	967
772		− INIT_LIST_HEAD(&ep->list);
773	968	ep->ep = get_device(ep_dev);
	969	+ ep->dport = dport;
774	970
775		− rc = add_ep(port, ep);
	971	+ rc = add_ep(ep);
776	972	if (rc)
777	973	cxl_ep_release(ep);
778	974	return rc;
@@ -781,11 +977,13 @@ static int cxl_add_ep(struct cxl_port port, struct device ep_dev) · 2 unmodified lines
781	977	struct cxl_find_port_ctx {
782	978	const struct device *dport_dev;
783	979	const struct cxl_port *parent_port;
	980	+ struct cxl_dport **dport;
784	981	};
785	982
786	983	static int match_port_by_dport(struct device dev, const void data)
787	984	{
788	985	const struct cxl_find_port_ctx *ctx = data;
	986	+ struct cxl_dport *dport;
789	987	struct cxl_port *port;
790	988
791	989	if (!is_cxl_port(dev))
@@ -794,7 +992,10 @@ static int match_port_by_dport(struct device dev, const void data) · 2 unmodified lines
794	992	return 0;
795	993
796	994	port = to_cxl_port(dev);
797		− return cxl_find_dport_by_dev(port, ctx->dport_dev) != NULL;
	995	+ dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
	996	+ if (ctx->dport)
	997	+ *ctx->dport = dport;
	998	+ return dport != NULL;
798	999	}
799	1000
800	1001	static struct cxl_port __find_cxl_port(struct cxl_find_port_ctx ctx)
@@ -810,24 +1011,32 @@ static struct cxl_port __find_cxl_port(struct cxl_find_port_ctx ctx) · 9 unmodified lines
810	1011	return NULL;
811	1012	}
812	1013
813		−static struct cxl_port find_cxl_port(struct device dport_dev)
	1014	+static struct cxl_port find_cxl_port(struct device dport_dev,
	1015	+ struct cxl_dport **dport)
814	1016	{
815	1017	struct cxl_find_port_ctx ctx = {
816	1018	.dport_dev = dport_dev,
	1019	+ .dport = dport,
817	1020	};
	1021	+ struct cxl_port *port;
818	1022
819		− return __find_cxl_port(&ctx);
	1023	+ port = __find_cxl_port(&ctx);
	1024	+ return port;
820	1025	}
821	1026
822	1027	static struct cxl_port find_cxl_port_at(struct cxl_port parent_port,
823		− struct device *dport_dev)
	1028	+ struct device *dport_dev,
	1029	+ struct cxl_dport **dport)
824	1030	{
825	1031	struct cxl_find_port_ctx ctx = {
826	1032	.dport_dev = dport_dev,
827	1033	.parent_port = parent_port,
	1034	+ .dport = dport,
828	1035	};
	1036	+ struct cxl_port *port;
829	1037
830		− return __find_cxl_port(&ctx);
	1038	+ port = __find_cxl_port(&ctx);
	1039	+ return port;
831	1040	}
832	1041
833	1042	/*
@@ -851,13 +1060,13 @@ static void delete_endpoint(void *data) · 17 unmodified lines
851	1060	struct cxl_port *parent_port;
852	1061	struct device *parent;
853	1062
854		− parent_port = cxl_mem_find_port(cxlmd);
	1063	+ parent_port = cxl_mem_find_port(cxlmd, NULL);
855	1064	if (!parent_port)
856	1065	goto out;
857	1066	parent = &parent_port->dev;
858	1067
859	1068	device_lock(parent);
860		− if (parent->driver && endpoint->uport) {
	1069	+ if (parent->driver && !endpoint->dead) {
861	1070	devm_release_action(parent, cxl_unlink_uport, endpoint);
862	1071	devm_release_action(parent, unregister_port, endpoint);
863	1072	}
@@ -883,21 +1092,70 @@ EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL); · 19 unmodified lines
883	1092	* for a port to be unregistered is when all memdevs beneath that port have gone
884	1093	* through ->remove(). This "bottom-up" removal selectively removes individual
885	1094	* child ports manually. This depends on devm_cxl_add_port() to not change is
886		− * devm action registration order.
	1095	+ * devm action registration order, and for dports to have already been
	1096	+ * destroyed by reap_dports().
887	1097	*/
888		−static void delete_switch_port(struct cxl_port port, struct list_head dports)
	1098	+static void delete_switch_port(struct cxl_port *port)
	1099	+{
	1100	+ devm_release_action(port->dev.parent, cxl_unlink_uport, port);
	1101	+ devm_release_action(port->dev.parent, unregister_port, port);
	1102	+}
	1103	+
	1104	+static void reap_dports(struct cxl_port *port)
889	1105	{
890		− struct cxl_dport dport, _d;
	1106	+ struct cxl_dport *dport;
	1107	+ unsigned long index;
	1108	+
	1109	+ device_lock_assert(&port->dev);
891	1110
892		− list_for_each_entry_safe(dport, _d, dports, list) {
	1111	+ xa_for_each(&port->dports, index, dport) {
893	1112	devm_release_action(&port->dev, cxl_dport_unlink, dport);
894	1113	devm_release_action(&port->dev, cxl_dport_remove, dport);
895	1114	devm_kfree(&port->dev, dport);
896	1115	}
897		− devm_release_action(port->dev.parent, cxl_unlink_uport, port);
898		− devm_release_action(port->dev.parent, unregister_port, port);
899	1116	}
900	1117
	1118	+int devm_cxl_add_endpoint(struct cxl_memdev *cxlmd,
	1119	+ struct cxl_dport *parent_dport)
	1120	+{
	1121	+ struct cxl_port *parent_port = parent_dport->port;
	1122	+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
	1123	+ struct cxl_port endpoint, iter, *down;
	1124	+ int rc;
	1125	+
	1126	+ /*
	1127	+ * Now that the path to the root is established record all the
	1128	+ * intervening ports in the chain.
	1129	+ */
	1130	+ for (iter = parent_port, down = NULL; !is_cxl_root(iter);
	1131	+ down = iter, iter = to_cxl_port(iter->dev.parent)) {
	1132	+ struct cxl_ep *ep;
	1133	+
	1134	+ ep = cxl_ep_load(iter, cxlmd);
	1135	+ ep->next = down;
	1136	+ }
	1137	+
	1138	+ endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev,
	1139	+ cxlds->component_reg_phys, parent_dport);
	1140	+ if (IS_ERR(endpoint))
	1141	+ return PTR_ERR(endpoint);
	1142	+
	1143	+ dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev));
	1144	+
	1145	+ rc = cxl_endpoint_autoremove(cxlmd, endpoint);
	1146	+ if (rc)
	1147	+ return rc;
	1148	+
	1149	+ if (!endpoint->dev.driver) {
	1150	+ dev_err(&cxlmd->dev, "%s failed probe\n",
	1151	+ dev_name(&endpoint->dev));
	1152	+ return -ENXIO;
	1153	+ }
	1154	+
	1155	+ return 0;
	1156	+}
	1157	+EXPORT_SYMBOL_NS_GPL(devm_cxl_add_endpoint, CXL);
	1158	+
901	1159	static void cxl_detach_ep(void *data)
902	1160	{
903	1161	struct cxl_memdev *cxlmd = data;
@@ -906,13 +1164,13 @@ static void cxl_detach_ep(void *data) · 2 unmodified lines
906	1164	for (iter = &cxlmd->dev; iter; iter = grandparent(iter)) {
907	1165	struct device *dport_dev = grandparent(iter);
908	1166	struct cxl_port port, parent_port;
909		− LIST_HEAD(reap_dports);
910	1167	struct cxl_ep *ep;
	1168	+ bool died = false;
911	1169
912	1170	if (!dport_dev)
913	1171	break;
914	1172
915		− port = find_cxl_port(dport_dev);
	1173	+ port = find_cxl_port(dport_dev, NULL);
916	1174	if (!port)
917	1175	continue;
918	1176
@@ -936,26 +1194,27 @@ static void cxl_detach_ep(void *data) · 17 unmodified lines
936	1194	}
937	1195
938	1196	device_lock(&port->dev);
939		− ep = find_ep(port, &cxlmd->dev);
	1197	+ ep = cxl_ep_load(port, cxlmd);
940	1198	dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
941	1199	ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
942		− cxl_ep_release(ep);
943		− if (ep && !port->dead && list_empty(&port->endpoints) &&
	1200	+ cxl_ep_remove(port, ep);
	1201	+ if (ep && !port->dead && xa_empty(&port->endpoints) &&
944	1202	!is_cxl_root(parent_port)) {
945	1203	/*
946	1204	* This was the last ep attached to a dynamically
947	1205	* enumerated port. Block new cxl_add_ep() and garbage
948	1206	* collect the port.
949	1207	*/
	1208	+ died = true;
950	1209	port->dead = true;
951		− list_splice_init(&port->dports, &reap_dports);
	1210	+ reap_dports(port);
952	1211	}
953	1212	device_unlock(&port->dev);
954	1213
955		− if (!list_empty(&reap_dports)) {
	1214	+ if (died) {
956	1215	dev_dbg(&cxlmd->dev, "delete %s\n",
957	1216	dev_name(&port->dev));
958		− delete_switch_port(port, &reap_dports);
	1217	+ delete_switch_port(port);
959	1218	}
960	1219	put_device(&port->dev);
961	1220	device_unlock(&parent_port->dev);
@@ -986,6 +1245,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, · 24 unmodified lines
986	1245	{
987	1246	struct device *dparent = grandparent(dport_dev);
988	1247	struct cxl_port port, parent_port = NULL;
	1248	+ struct cxl_dport dport, parent_dport;
989	1249	resource_size_t component_reg_phys;
990	1250	int rc;
991	1251
@@ -1000,7 +1260,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, · 8 unmodified lines
1000	1260	return -ENXIO;
1001	1261	}
1002	1262
1003		− parent_port = find_cxl_port(dparent);
	1263	+ parent_port = find_cxl_port(dparent, &parent_dport);
1004	1264	if (!parent_port) {
1005	1265	/* iterate to create this parent_port */
1006	1266	return -EAGAIN;
@@ -1015,13 +1275,14 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, · 8 unmodified lines
1015	1275	goto out;
1016	1276	}
1017	1277
1018		− port = find_cxl_port_at(parent_port, dport_dev);
	1278	+ port = find_cxl_port_at(parent_port, dport_dev, &dport);
1019	1279	if (!port) {
1020	1280	component_reg_phys = find_component_registers(uport_dev);
1021	1281	port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1022		− component_reg_phys, parent_port);
	1282	+ component_reg_phys, parent_dport);
	1283	+ /* retry find to pick up the new dport information */
1023	1284	if (!IS_ERR(port))
1024		− get_device(&port->dev);
	1285	+ port = find_cxl_port_at(parent_port, dport_dev, &dport);
1025	1286	}
1026	1287	out:
1027	1288	device_unlock(&parent_port->dev);
@@ -1031,8 +1292,8 @@ out: · 3 unmodified lines
1031	1292	else {
1032	1293	dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1033	1294	dev_name(&port->dev), dev_name(port->uport));
1034		− rc = cxl_add_ep(port, &cxlmd->dev);
1035		− if (rc == -EEXIST) {
	1295	+ rc = cxl_add_ep(dport, &cxlmd->dev);
	1296	+ if (rc == -EBUSY) {
1036	1297	/*
1037	1298	* "can't" happen, but this error code means
1038	1299	* something to the caller, so translate it.
@@ -1065,6 +1326,7 @@ retry: · 26 unmodified lines
1065	1326	for (iter = dev; iter; iter = grandparent(iter)) {
1066	1327	struct device *dport_dev = grandparent(iter);
1067	1328	struct device *uport_dev;
	1329	+ struct cxl_dport *dport;
1068	1330	struct cxl_port *port;
1069	1331
1070	1332	if (!dport_dev)
@@ -1080,12 +1342,12 @@ retry: · 9 unmodified lines
1080	1342	dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1081	1343	dev_name(iter), dev_name(dport_dev),
1082	1344	dev_name(uport_dev));
1083		− port = find_cxl_port(dport_dev);
	1345	+ port = find_cxl_port(dport_dev, &dport);
1084	1346	if (port) {
1085	1347	dev_dbg(&cxlmd->dev,
1086	1348	"found already registered port %s:%s\n",
1087	1349	dev_name(&port->dev), dev_name(port->uport));
1088		− rc = cxl_add_ep(port, &cxlmd->dev);
	1350	+ rc = cxl_add_ep(dport, &cxlmd->dev);
1089	1351
1090	1352	/*
1091	1353	* If the endpoint already exists in the port's list,
@@ -1094,7 +1356,7 @@ retry: · 2 unmodified lines
1094	1356	* the parent_port lock as the current port may be being
1095	1357	* reaped.
1096	1358	*/
1097		− if (rc && rc != -EEXIST) {
	1359	+ if (rc && rc != -EBUSY) {
1098	1360	put_device(&port->dev);
1099	1361	return rc;
1100	1362	}
@@ -1124,30 +1386,14 @@ retry: · 23 unmodified lines
1124	1386	}
1125	1387	EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1126	1388
1127		−struct cxl_port cxl_mem_find_port(struct cxl_memdev cxlmd)
	1389	+struct cxl_port cxl_mem_find_port(struct cxl_memdev cxlmd,
	1390	+ struct cxl_dport **dport)
1128	1391	{
1129		− return find_cxl_port(grandparent(&cxlmd->dev));
	1392	+ return find_cxl_port(grandparent(&cxlmd->dev), dport);
1130	1393	}
1131	1394	EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1132	1395
1133		−struct cxl_dport cxl_find_dport_by_dev(struct cxl_port port,
1134		− const struct device *dev)
1135		−{
1136		− struct cxl_dport *dport;
1137		−
1138		− device_lock(&port->dev);
1139		− list_for_each_entry(dport, &port->dports, list)
1140		− if (dport->dport == dev) {
1141		− device_unlock(&port->dev);
1142		− return dport;
1143		− }
1144		−
1145		− device_unlock(&port->dev);
1146		− return NULL;
1147		−}
1148		−EXPORT_SYMBOL_NS_GPL(cxl_find_dport_by_dev, CXL);
1149		−
1150		−static int decoder_populate_targets(struct cxl_decoder *cxld,
	1396	+static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1151	1397	struct cxl_port port, int target_map)
1152	1398	{
1153	1399	int i, rc = 0;
@@ -1157,88 +1403,92 @@ static int decoder_populate_targets(struct cxl_decoder *cxld, · 3 unmodified lines
1157	1403
1158	1404	device_lock_assert(&port->dev);
1159	1405
1160		− if (list_empty(&port->dports))
	1406	+ if (xa_empty(&port->dports))
1161	1407	return -EINVAL;
1162	1408
1163		− write_seqlock(&cxld->target_lock);
1164		− for (i = 0; i < cxld->nr_targets; i++) {
	1409	+ write_seqlock(&cxlsd->target_lock);
	1410	+ for (i = 0; i < cxlsd->nr_targets; i++) {
1165	1411	struct cxl_dport *dport = find_dport(port, target_map[i]);
1166	1412
1167	1413	if (!dport) {
1168	1414	rc = -ENXIO;
1169	1415	break;
1170	1416	}
1171		− cxld->target[i] = dport;
	1417	+ cxlsd->target[i] = dport;
1172	1418	}
1173		− write_sequnlock(&cxld->target_lock);
	1419	+ write_sequnlock(&cxlsd->target_lock);
1174	1420
1175	1421	return rc;
1176	1422	}
1177	1423
	1424	+static struct cxl_dport cxl_hb_modulo(struct cxl_root_decoder cxlrd, int pos)
	1425	+{
	1426	+ struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
	1427	+ struct cxl_decoder *cxld = &cxlsd->cxld;
	1428	+ int iw;
	1429	+
	1430	+ iw = cxld->interleave_ways;
	1431	+ if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
	1432	+ "misconfigured root decoder\n"))
	1433	+ return NULL;
	1434	+
	1435	+ return cxlrd->cxlsd.target[pos % iw];
	1436	+}
	1437	+
1178	1438	static struct lock_class_key cxl_decoder_key;
1179	1439
1180	1440	/**
1181		− * cxl_decoder_alloc - Allocate a new CXL decoder
	1441	+ * cxl_decoder_init - Common decoder setup / initialization
1182	1442	* @port: owning port of this decoder
1183		− * @nr_targets: downstream targets accessible by this decoder. All upstream
1184		− * ports and root ports must have at least 1 target. Endpoint
1185		− * devices will have 0 targets. Callers wishing to register an
1186		− * endpoint device should specify 0.
1187		− *
1188		− * A port should contain one or more decoders. Each of those decoders enable
1189		− * some address space for CXL.mem utilization. A decoder is expected to be
1190		− * configured by the caller before registering.
	1443	+ * @cxld: common decoder properties to initialize
1191	1444	*
1192		− * Return: A new cxl decoder to be registered by cxl_decoder_add(). The decoder
1193		− * is initialized to be a "passthrough" decoder.
	1445	+ * A port may contain one or more decoders. Each of those decoders
	1446	+ * enable some address space for CXL.mem utilization. A decoder is
	1447	+ * expected to be configured by the caller before registering via
	1448	+ * cxl_decoder_add()
1194	1449	*/
1195		−static struct cxl_decoder cxl_decoder_alloc(struct cxl_port port,
1196		− unsigned int nr_targets)
	1450	+static int cxl_decoder_init(struct cxl_port port, struct cxl_decoder cxld)
1197	1451	{
1198		− struct cxl_decoder *cxld;
1199	1452	struct device *dev;
1200		− int rc = 0;
1201		−
1202		− if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1203		− return ERR_PTR(-EINVAL);
1204		−
1205		− cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);
1206		− if (!cxld)
1207		− return ERR_PTR(-ENOMEM);
	1453	+ int rc;
1208	1454
1209	1455	rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1210	1456	if (rc < 0)
1211		− goto err;
	1457	+ return rc;
1212	1458
1213	1459	/* need parent to stick around to release the id */
1214	1460	get_device(&port->dev);
1215	1461	cxld->id = rc;
1216	1462
1217		− cxld->nr_targets = nr_targets;
1218		− seqlock_init(&cxld->target_lock);
1219	1463	dev = &cxld->dev;
1220	1464	device_initialize(dev);
1221	1465	lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1222	1466	device_set_pm_not_required(dev);
1223	1467	dev->parent = &port->dev;
1224	1468	dev->bus = &cxl_bus_type;
1225		− if (is_cxl_root(port))
1226		− cxld->dev.type = &cxl_decoder_root_type;
1227		− else if (is_cxl_endpoint(port))
1228		− cxld->dev.type = &cxl_decoder_endpoint_type;
1229		− else
1230		− cxld->dev.type = &cxl_decoder_switch_type;
1231	1469
1232	1470	/* Pre initialize an "empty" decoder */
1233	1471	cxld->interleave_ways = 1;
1234	1472	cxld->interleave_granularity = PAGE_SIZE;
1235	1473	cxld->target_type = CXL_DECODER_EXPANDER;
1236		− cxld->platform_res = (struct resource)DEFINE_RES_MEM(0, 0);
	1474	+ cxld->hpa_range = (struct range) {
	1475	+ .start = 0,
	1476	+ .end = -1,
	1477	+ };
1237	1478
1238		− return cxld;
1239		−err:
1240		− kfree(cxld);
1241		− return ERR_PTR(rc);
	1479	+ return 0;
	1480	+}
	1481	+
	1482	+static int cxl_switch_decoder_init(struct cxl_port *port,
	1483	+ struct cxl_switch_decoder *cxlsd,
	1484	+ int nr_targets)
	1485	+{
	1486	+ if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
	1487	+ return -EINVAL;
	1488	+
	1489	+ cxlsd->nr_targets = nr_targets;
	1490	+ seqlock_init(&cxlsd->target_lock);
	1491	+ return cxl_decoder_init(port, &cxlsd->cxld);
1242	1492	}
1243	1493
1244	1494	/**
@@ -1251,13 +1501,46 @@ err: · 6 unmodified lines
1251	1501	* firmware description of CXL resources into a CXL standard decode
1252	1502	* topology.
1253	1503	*/
1254		−struct cxl_decoder cxl_root_decoder_alloc(struct cxl_port port,
1255		− unsigned int nr_targets)
	1504	+struct cxl_root_decoder cxl_root_decoder_alloc(struct cxl_port port,
	1505	+ unsigned int nr_targets)
1256	1506	{
	1507	+ struct cxl_root_decoder *cxlrd;
	1508	+ struct cxl_switch_decoder *cxlsd;
	1509	+ struct cxl_decoder *cxld;
	1510	+ int rc;
	1511	+
1257	1512	if (!is_cxl_root(port))
1258	1513	return ERR_PTR(-EINVAL);
1259	1514
1260		− return cxl_decoder_alloc(port, nr_targets);
	1515	+ cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
	1516	+ GFP_KERNEL);
	1517	+ if (!cxlrd)
	1518	+ return ERR_PTR(-ENOMEM);
	1519	+
	1520	+ cxlsd = &cxlrd->cxlsd;
	1521	+ rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
	1522	+ if (rc) {
	1523	+ kfree(cxlrd);
	1524	+ return ERR_PTR(rc);
	1525	+ }
	1526	+
	1527	+ cxlrd->calc_hb = cxl_hb_modulo;
	1528	+
	1529	+ cxld = &cxlsd->cxld;
	1530	+ cxld->dev.type = &cxl_decoder_root_type;
	1531	+ /*
	1532	+ * cxl_root_decoder_release() special cases negative ids to
	1533	+ * detect memregion_alloc() failures.
	1534	+ */
	1535	+ atomic_set(&cxlrd->region_id, -1);
	1536	+ rc = memregion_alloc(GFP_KERNEL);
	1537	+ if (rc < 0) {
	1538	+ put_device(&cxld->dev);
	1539	+ return ERR_PTR(rc);
	1540	+ }
	1541	+
	1542	+ atomic_set(&cxlrd->region_id, rc);
	1543	+ return cxlrd;
1261	1544	}
1262	1545	EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1263	1546
@@ -1272,13 +1555,29 @@ EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL); · 8 unmodified lines
1272	1555	* that sit between Switch Upstream Ports / Switch Downstream Ports and
1273	1556	* Host Bridges / Root Ports.
1274	1557	*/
1275		−struct cxl_decoder cxl_switch_decoder_alloc(struct cxl_port port,
1276		− unsigned int nr_targets)
	1558	+struct cxl_switch_decoder cxl_switch_decoder_alloc(struct cxl_port port,
	1559	+ unsigned int nr_targets)
1277	1560	{
	1561	+ struct cxl_switch_decoder *cxlsd;
	1562	+ struct cxl_decoder *cxld;
	1563	+ int rc;
	1564	+
1278	1565	if (is_cxl_root(port) \|\| is_cxl_endpoint(port))
1279	1566	return ERR_PTR(-EINVAL);
1280	1567
1281		− return cxl_decoder_alloc(port, nr_targets);
	1568	+ cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
	1569	+ if (!cxlsd)
	1570	+ return ERR_PTR(-ENOMEM);
	1571	+
	1572	+ rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
	1573	+ if (rc) {
	1574	+ kfree(cxlsd);
	1575	+ return ERR_PTR(rc);
	1576	+ }
	1577	+
	1578	+ cxld = &cxlsd->cxld;
	1579	+ cxld->dev.type = &cxl_decoder_switch_type;
	1580	+ return cxlsd;
1282	1581	}
1283	1582	EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1284	1583
@@ -1288,18 +1587,35 @@ EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL); · 3 unmodified lines
1288	1587	*
1289	1588	* Return: A new cxl decoder to be registered by cxl_decoder_add()
1290	1589	*/
1291		−struct cxl_decoder cxl_endpoint_decoder_alloc(struct cxl_port port)
	1590	+struct cxl_endpoint_decoder cxl_endpoint_decoder_alloc(struct cxl_port port)
1292	1591	{
	1592	+ struct cxl_endpoint_decoder *cxled;
	1593	+ struct cxl_decoder *cxld;
	1594	+ int rc;
	1595	+
1293	1596	if (!is_cxl_endpoint(port))
1294	1597	return ERR_PTR(-EINVAL);
1295	1598
1296		− return cxl_decoder_alloc(port, 0);
	1599	+ cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
	1600	+ if (!cxled)
	1601	+ return ERR_PTR(-ENOMEM);
	1602	+
	1603	+ cxled->pos = -1;
	1604	+ cxld = &cxled->cxld;
	1605	+ rc = cxl_decoder_init(port, cxld);
	1606	+ if (rc) {
	1607	+ kfree(cxled);
	1608	+ return ERR_PTR(rc);
	1609	+ }
	1610	+
	1611	+ cxld->dev.type = &cxl_decoder_endpoint_type;
	1612	+ return cxled;
1297	1613	}
1298	1614	EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1299	1615
1300	1616	/**
1301	1617	* cxl_decoder_add_locked - Add a decoder with targets
1302		− * @cxld: The cxl decoder allocated by cxl_decoder_alloc()
	1618	+ * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1303	1619	* @target_map: A list of downstream ports that this decoder can direct memory
1304	1620	* traffic to. These numbers should correspond with the port number
1305	1621	* in the PCIe Link Capabilities structure.
@@ -1335,7 +1651,9 @@ int cxl_decoder_add_locked(struct cxl_decoder cxld, int target_map) · 29 unmodified lines
1335	1651
1336	1652	port = to_cxl_port(cxld->dev.parent);
1337	1653	if (!is_endpoint_decoder(dev)) {
1338		− rc = decoder_populate_targets(cxld, port, target_map);
	1654	+ struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
	1655	+
	1656	+ rc = decoder_populate_targets(cxlsd, port, target_map);
1339	1657	if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1340	1658	dev_err(&port->dev,
1341	1659	"Failed to populate active decoder targets\n");
@@ -1347,20 +1665,13 @@ int cxl_decoder_add_locked(struct cxl_decoder cxld, int target_map) · 5 unmodified lines
1347	1665	if (rc)
1348	1666	return rc;
1349	1667
1350		− /*
1351		− * Platform decoder resources should show up with a reasonable name. All
1352		− * other resources are just sub ranges within the main decoder resource.
1353		− */
1354		− if (is_root_decoder(dev))
1355		− cxld->platform_res.name = dev_name(dev);
1356		−
1357	1668	return device_add(dev);
1358	1669	}
1359	1670	EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1360	1671
1361	1672	/**
1362	1673	* cxl_decoder_add - Add a decoder with targets
1363		− * @cxld: The cxl decoder allocated by cxl_decoder_alloc()
	1674	+ * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1364	1675	* @target_map: A list of downstream ports that this decoder can direct memory
1365	1676	* traffic to. These numbers should correspond with the port number
1366	1677	* in the PCIe Link Capabilities structure.
@@ -1394,6 +1705,13 @@ EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL); · 27 unmodified lines
1394	1705
1395	1706	static void cxld_unregister(void *dev)
1396	1707	{
	1708	+ struct cxl_endpoint_decoder *cxled;
	1709	+
	1710	+ if (is_endpoint_decoder(dev)) {
	1711	+ cxled = to_cxl_endpoint_decoder(dev);
	1712	+ cxl_decoder_kill_region(cxled);
	1713	+ }
	1714	+
1397	1715	device_unregister(dev);
1398	1716	}
1399	1717
@@ -1521,10 +1839,20 @@ struct bus_type cxl_bus_type = { · 121 unmodified lines
1521	1839	};
1522	1840	EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
1523	1841
	1842	+static struct dentry *cxl_debugfs;
	1843	+
	1844	+struct dentry cxl_debugfs_create_dir(const char dir)
	1845	+{
	1846	+ return debugfs_create_dir(dir, cxl_debugfs);
	1847	+}
	1848	+EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
	1849	+
1524	1850	static __init int cxl_core_init(void)
1525	1851	{
1526	1852	int rc;
1527	1853
	1854	+ cxl_debugfs = debugfs_create_dir("cxl", NULL);
	1855	+
1528	1856	cxl_mbox_init();
1529	1857
1530	1858	rc = cxl_memdev_init();
@@ -1541,22 +1869,28 @@ static __init int cxl_core_init(void) · 10 unmodified lines
1541	1869	if (rc)
1542	1870	goto err_bus;
1543	1871
	1872	+ rc = cxl_region_init();
	1873	+ if (rc)
	1874	+ goto err_region;
	1875	+
1544	1876	return 0;
1545	1877
	1878	+err_region:
	1879	+ bus_unregister(&cxl_bus_type);
1546	1880	err_bus:
1547	1881	destroy_workqueue(cxl_bus_wq);
1548	1882	err_wq:
1549	1883	cxl_memdev_exit();
1550		− cxl_mbox_exit();
1551	1884	return rc;
1552	1885	}
1553	1886
1554	1887	static void cxl_core_exit(void)
1555	1888	{
	1889	+ cxl_region_exit();
1556	1890	bus_unregister(&cxl_bus_type);
1557	1891	destroy_workqueue(cxl_bus_wq);
1558	1892	cxl_memdev_exit();
1559		− cxl_mbox_exit();
	1893	+ debugfs_remove_recursive(cxl_debugfs);
1560	1894	}
1561	1895
1562	1896	module_init(cxl_core_init);

drivers/cxl/core/region.c+1896−0

@@ -0,0 +1,1896 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
	3	+#include <linux/memregion.h>
	4	+#include <linux/genalloc.h>
	5	+#include <linux/device.h>
	6	+#include <linux/module.h>
	7	+#include <linux/slab.h>
	8	+#include <linux/uuid.h>
	9	+#include <linux/idr.h>
	10	+#include <cxlmem.h>
	11	+#include <cxl.h>
	12	+#include "core.h"
	13	+
	14	+/**
	15	+ * DOC: cxl core region
	16	+ *
	17	+ * CXL Regions represent mapped memory capacity in system physical address
	18	+ * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
	19	+ * Memory ranges, Regions represent the active mapped capacity by the HDM
	20	+ * Decoder Capability structures throughout the Host Bridges, Switches, and
	21	+ * Endpoints in the topology.
	22	+ *
	23	+ * Region configuration has ordering constraints. UUID may be set at any time
	24	+ * but is only visible for persistent regions.
	25	+ * 1. Interleave granularity
	26	+ * 2. Interleave size
	27	+ * 3. Decoder targets
	28	+ */
	29	+
	30	+/*
	31	+ * All changes to the interleave configuration occur with this lock held
	32	+ * for write.
	33	+ */
	34	+static DECLARE_RWSEM(cxl_region_rwsem);
	35	+
	36	+static struct cxl_region to_cxl_region(struct device dev);
	37	+
	38	+static ssize_t uuid_show(struct device dev, struct device_attribute attr,
	39	+ char *buf)
	40	+{
	41	+ struct cxl_region *cxlr = to_cxl_region(dev);
	42	+ struct cxl_region_params *p = &cxlr->params;
	43	+ ssize_t rc;
	44	+
	45	+ rc = down_read_interruptible(&cxl_region_rwsem);
	46	+ if (rc)
	47	+ return rc;
	48	+ rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
	49	+ up_read(&cxl_region_rwsem);
	50	+
	51	+ return rc;
	52	+}
	53	+
	54	+static int is_dup(struct device match, void data)
	55	+{
	56	+ struct cxl_region_params *p;
	57	+ struct cxl_region *cxlr;
	58	+ uuid_t *uuid = data;
	59	+
	60	+ if (!is_cxl_region(match))
	61	+ return 0;
	62	+
	63	+ lockdep_assert_held(&cxl_region_rwsem);
	64	+ cxlr = to_cxl_region(match);
	65	+ p = &cxlr->params;
	66	+
	67	+ if (uuid_equal(&p->uuid, uuid)) {
	68	+ dev_dbg(match, "already has uuid: %pUb\n", uuid);
	69	+ return -EBUSY;
	70	+ }
	71	+
	72	+ return 0;
	73	+}
	74	+
	75	+static ssize_t uuid_store(struct device dev, struct device_attribute attr,
	76	+ const char *buf, size_t len)
	77	+{
	78	+ struct cxl_region *cxlr = to_cxl_region(dev);
	79	+ struct cxl_region_params *p = &cxlr->params;
	80	+ uuid_t temp;
	81	+ ssize_t rc;
	82	+
	83	+ if (len != UUID_STRING_LEN + 1)
	84	+ return -EINVAL;
	85	+
	86	+ rc = uuid_parse(buf, &temp);
	87	+ if (rc)
	88	+ return rc;
	89	+
	90	+ if (uuid_is_null(&temp))
	91	+ return -EINVAL;
	92	+
	93	+ rc = down_write_killable(&cxl_region_rwsem);
	94	+ if (rc)
	95	+ return rc;
	96	+
	97	+ if (uuid_equal(&p->uuid, &temp))
	98	+ goto out;
	99	+
	100	+ rc = -EBUSY;
	101	+ if (p->state >= CXL_CONFIG_ACTIVE)
	102	+ goto out;
	103	+
	104	+ rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
	105	+ if (rc < 0)
	106	+ goto out;
	107	+
	108	+ uuid_copy(&p->uuid, &temp);
	109	+out:
	110	+ up_write(&cxl_region_rwsem);
	111	+
	112	+ if (rc)
	113	+ return rc;
	114	+ return len;
	115	+}
	116	+static DEVICE_ATTR_RW(uuid);
	117	+
	118	+static struct cxl_region_ref cxl_rr_load(struct cxl_port port,
	119	+ struct cxl_region *cxlr)
	120	+{
	121	+ return xa_load(&port->regions, (unsigned long)cxlr);
	122	+}
	123	+
	124	+static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
	125	+{
	126	+ struct cxl_region_params *p = &cxlr->params;
	127	+ int i;
	128	+
	129	+ for (i = count - 1; i >= 0; i--) {
	130	+ struct cxl_endpoint_decoder *cxled = p->targets[i];
	131	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	132	+ struct cxl_port *iter = cxled_to_port(cxled);
	133	+ struct cxl_ep *ep;
	134	+ int rc;
	135	+
	136	+ while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
	137	+ iter = to_cxl_port(iter->dev.parent);
	138	+
	139	+ for (ep = cxl_ep_load(iter, cxlmd); iter;
	140	+ iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
	141	+ struct cxl_region_ref *cxl_rr;
	142	+ struct cxl_decoder *cxld;
	143	+
	144	+ cxl_rr = cxl_rr_load(iter, cxlr);
	145	+ cxld = cxl_rr->decoder;
	146	+ rc = cxld->reset(cxld);
	147	+ if (rc)
	148	+ return rc;
	149	+ }
	150	+
	151	+ rc = cxled->cxld.reset(&cxled->cxld);
	152	+ if (rc)
	153	+ return rc;
	154	+ }
	155	+
	156	+ return 0;
	157	+}
	158	+
	159	+static int cxl_region_decode_commit(struct cxl_region *cxlr)
	160	+{
	161	+ struct cxl_region_params *p = &cxlr->params;
	162	+ int i, rc = 0;
	163	+
	164	+ for (i = 0; i < p->nr_targets; i++) {
	165	+ struct cxl_endpoint_decoder *cxled = p->targets[i];
	166	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	167	+ struct cxl_region_ref *cxl_rr;
	168	+ struct cxl_decoder *cxld;
	169	+ struct cxl_port *iter;
	170	+ struct cxl_ep *ep;
	171	+
	172	+ /* commit bottom up */
	173	+ for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
	174	+ iter = to_cxl_port(iter->dev.parent)) {
	175	+ cxl_rr = cxl_rr_load(iter, cxlr);
	176	+ cxld = cxl_rr->decoder;
	177	+ rc = cxld->commit(cxld);
	178	+ if (rc)
	179	+ break;
	180	+ }
	181	+
	182	+ if (rc) {
	183	+ /* programming @iter failed, teardown */
	184	+ for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
	185	+ iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
	186	+ cxl_rr = cxl_rr_load(iter, cxlr);
	187	+ cxld = cxl_rr->decoder;
	188	+ cxld->reset(cxld);
	189	+ }
	190	+
	191	+ cxled->cxld.reset(&cxled->cxld);
	192	+ goto err;
	193	+ }
	194	+ }
	195	+
	196	+ return 0;
	197	+
	198	+err:
	199	+ /* undo the targets that were successfully committed */
	200	+ cxl_region_decode_reset(cxlr, i);
	201	+ return rc;
	202	+}
	203	+
	204	+static ssize_t commit_store(struct device dev, struct device_attribute attr,
	205	+ const char *buf, size_t len)
	206	+{
	207	+ struct cxl_region *cxlr = to_cxl_region(dev);
	208	+ struct cxl_region_params *p = &cxlr->params;
	209	+ bool commit;
	210	+ ssize_t rc;
	211	+
	212	+ rc = kstrtobool(buf, &commit);
	213	+ if (rc)
	214	+ return rc;
	215	+
	216	+ rc = down_write_killable(&cxl_region_rwsem);
	217	+ if (rc)
	218	+ return rc;
	219	+
	220	+ /* Already in the requested state? */
	221	+ if (commit && p->state >= CXL_CONFIG_COMMIT)
	222	+ goto out;
	223	+ if (!commit && p->state < CXL_CONFIG_COMMIT)
	224	+ goto out;
	225	+
	226	+ /* Not ready to commit? */
	227	+ if (commit && p->state < CXL_CONFIG_ACTIVE) {
	228	+ rc = -ENXIO;
	229	+ goto out;
	230	+ }
	231	+
	232	+ if (commit)
	233	+ rc = cxl_region_decode_commit(cxlr);
	234	+ else {
	235	+ p->state = CXL_CONFIG_RESET_PENDING;
	236	+ up_write(&cxl_region_rwsem);
	237	+ device_release_driver(&cxlr->dev);
	238	+ down_write(&cxl_region_rwsem);
	239	+
	240	+ /*
	241	+ * The lock was dropped, so need to revalidate that the reset is
	242	+ * still pending.
	243	+ */
	244	+ if (p->state == CXL_CONFIG_RESET_PENDING)
	245	+ rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
	246	+ }
	247	+
	248	+ if (rc)
	249	+ goto out;
	250	+
	251	+ if (commit)
	252	+ p->state = CXL_CONFIG_COMMIT;
	253	+ else if (p->state == CXL_CONFIG_RESET_PENDING)
	254	+ p->state = CXL_CONFIG_ACTIVE;
	255	+
	256	+out:
	257	+ up_write(&cxl_region_rwsem);
	258	+
	259	+ if (rc)
	260	+ return rc;
	261	+ return len;
	262	+}
	263	+
	264	+static ssize_t commit_show(struct device dev, struct device_attribute attr,
	265	+ char *buf)
	266	+{
	267	+ struct cxl_region *cxlr = to_cxl_region(dev);
	268	+ struct cxl_region_params *p = &cxlr->params;
	269	+ ssize_t rc;
	270	+
	271	+ rc = down_read_interruptible(&cxl_region_rwsem);
	272	+ if (rc)
	273	+ return rc;
	274	+ rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
	275	+ up_read(&cxl_region_rwsem);
	276	+
	277	+ return rc;
	278	+}
	279	+static DEVICE_ATTR_RW(commit);
	280	+
	281	+static umode_t cxl_region_visible(struct kobject kobj, struct attribute a,
	282	+ int n)
	283	+{
	284	+ struct device *dev = kobj_to_dev(kobj);
	285	+ struct cxl_region *cxlr = to_cxl_region(dev);
	286	+
	287	+ if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
	288	+ return 0;
	289	+ return a->mode;
	290	+}
	291	+
	292	+static ssize_t interleave_ways_show(struct device *dev,
	293	+ struct device_attribute attr, char buf)
	294	+{
	295	+ struct cxl_region *cxlr = to_cxl_region(dev);
	296	+ struct cxl_region_params *p = &cxlr->params;
	297	+ ssize_t rc;
	298	+
	299	+ rc = down_read_interruptible(&cxl_region_rwsem);
	300	+ if (rc)
	301	+ return rc;
	302	+ rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
	303	+ up_read(&cxl_region_rwsem);
	304	+
	305	+ return rc;
	306	+}
	307	+
	308	+static const struct attribute_group *get_cxl_region_target_group(void);
	309	+
	310	+static ssize_t interleave_ways_store(struct device *dev,
	311	+ struct device_attribute *attr,
	312	+ const char *buf, size_t len)
	313	+{
	314	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
	315	+ struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
	316	+ struct cxl_region *cxlr = to_cxl_region(dev);
	317	+ struct cxl_region_params *p = &cxlr->params;
	318	+ unsigned int val, save;
	319	+ int rc;
	320	+ u8 iw;
	321	+
	322	+ rc = kstrtouint(buf, 0, &val);
	323	+ if (rc)
	324	+ return rc;
	325	+
	326	+ rc = ways_to_cxl(val, &iw);
	327	+ if (rc)
	328	+ return rc;
	329	+
	330	+ /*
	331	+ * Even for x3, x9, and x12 interleaves the region interleave must be a
	332	+ * power of 2 multiple of the host bridge interleave.
	333	+ */
	334	+ if (!is_power_of_2(val / cxld->interleave_ways) \|\|
	335	+ (val % cxld->interleave_ways)) {
	336	+ dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
	337	+ return -EINVAL;
	338	+ }
	339	+
	340	+ rc = down_write_killable(&cxl_region_rwsem);
	341	+ if (rc)
	342	+ return rc;
	343	+ if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
	344	+ rc = -EBUSY;
	345	+ goto out;
	346	+ }
	347	+
	348	+ save = p->interleave_ways;
	349	+ p->interleave_ways = val;
	350	+ rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
	351	+ if (rc)
	352	+ p->interleave_ways = save;
	353	+out:
	354	+ up_write(&cxl_region_rwsem);
	355	+ if (rc)
	356	+ return rc;
	357	+ return len;
	358	+}
	359	+static DEVICE_ATTR_RW(interleave_ways);
	360	+
	361	+static ssize_t interleave_granularity_show(struct device *dev,
	362	+ struct device_attribute *attr,
	363	+ char *buf)
	364	+{
	365	+ struct cxl_region *cxlr = to_cxl_region(dev);
	366	+ struct cxl_region_params *p = &cxlr->params;
	367	+ ssize_t rc;
	368	+
	369	+ rc = down_read_interruptible(&cxl_region_rwsem);
	370	+ if (rc)
	371	+ return rc;
	372	+ rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
	373	+ up_read(&cxl_region_rwsem);
	374	+
	375	+ return rc;
	376	+}
	377	+
	378	+static ssize_t interleave_granularity_store(struct device *dev,
	379	+ struct device_attribute *attr,
	380	+ const char *buf, size_t len)
	381	+{
	382	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
	383	+ struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
	384	+ struct cxl_region *cxlr = to_cxl_region(dev);
	385	+ struct cxl_region_params *p = &cxlr->params;
	386	+ int rc, val;
	387	+ u16 ig;
	388	+
	389	+ rc = kstrtoint(buf, 0, &val);
	390	+ if (rc)
	391	+ return rc;
	392	+
	393	+ rc = granularity_to_cxl(val, &ig);
	394	+ if (rc)
	395	+ return rc;
	396	+
	397	+ /*
	398	+ * When the host-bridge is interleaved, disallow region granularity !=
	399	+ * root granularity. Regions with a granularity less than the root
	400	+ * interleave result in needing multiple endpoints to support a single
	401	+ * slot in the interleave (possible to suport in the future). Regions
	402	+ * with a granularity greater than the root interleave result in invalid
	403	+ * DPA translations (invalid to support).
	404	+ */
	405	+ if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
	406	+ return -EINVAL;
	407	+
	408	+ rc = down_write_killable(&cxl_region_rwsem);
	409	+ if (rc)
	410	+ return rc;
	411	+ if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
	412	+ rc = -EBUSY;
	413	+ goto out;
	414	+ }
	415	+
	416	+ p->interleave_granularity = val;
	417	+out:
	418	+ up_write(&cxl_region_rwsem);
	419	+ if (rc)
	420	+ return rc;
	421	+ return len;
	422	+}
	423	+static DEVICE_ATTR_RW(interleave_granularity);
	424	+
	425	+static ssize_t resource_show(struct device dev, struct device_attribute attr,
	426	+ char *buf)
	427	+{
	428	+ struct cxl_region *cxlr = to_cxl_region(dev);
	429	+ struct cxl_region_params *p = &cxlr->params;
	430	+ u64 resource = -1ULL;
	431	+ ssize_t rc;
	432	+
	433	+ rc = down_read_interruptible(&cxl_region_rwsem);
	434	+ if (rc)
	435	+ return rc;
	436	+ if (p->res)
	437	+ resource = p->res->start;
	438	+ rc = sysfs_emit(buf, "%#llx\n", resource);
	439	+ up_read(&cxl_region_rwsem);
	440	+
	441	+ return rc;
	442	+}
	443	+static DEVICE_ATTR_RO(resource);
	444	+
	445	+static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
	446	+{
	447	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
	448	+ struct cxl_region_params *p = &cxlr->params;
	449	+ struct resource *res;
	450	+ u32 remainder = 0;
	451	+
	452	+ lockdep_assert_held_write(&cxl_region_rwsem);
	453	+
	454	+ /* Nothing to do... */
	455	+ if (p->res && resource_size(p->res) == size)
	456	+ return 0;
	457	+
	458	+ /* To change size the old size must be freed first */
	459	+ if (p->res)
	460	+ return -EBUSY;
	461	+
	462	+ if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
	463	+ return -EBUSY;
	464	+
	465	+ /* ways, granularity and uuid (if PMEM) need to be set before HPA */
	466	+ if (!p->interleave_ways \|\| !p->interleave_granularity \|\|
	467	+ (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
	468	+ return -ENXIO;
	469	+
	470	+ div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
	471	+ if (remainder)
	472	+ return -EINVAL;
	473	+
	474	+ res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
	475	+ dev_name(&cxlr->dev));
	476	+ if (IS_ERR(res)) {
	477	+ dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
	478	+ PTR_ERR(res));
	479	+ return PTR_ERR(res);
	480	+ }
	481	+
	482	+ p->res = res;
	483	+ p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
	484	+
	485	+ return 0;
	486	+}
	487	+
	488	+static void cxl_region_iomem_release(struct cxl_region *cxlr)
	489	+{
	490	+ struct cxl_region_params *p = &cxlr->params;
	491	+
	492	+ if (device_is_registered(&cxlr->dev))
	493	+ lockdep_assert_held_write(&cxl_region_rwsem);
	494	+ if (p->res) {
	495	+ remove_resource(p->res);
	496	+ kfree(p->res);
	497	+ p->res = NULL;
	498	+ }
	499	+}
	500	+
	501	+static int free_hpa(struct cxl_region *cxlr)
	502	+{
	503	+ struct cxl_region_params *p = &cxlr->params;
	504	+
	505	+ lockdep_assert_held_write(&cxl_region_rwsem);
	506	+
	507	+ if (!p->res)
	508	+ return 0;
	509	+
	510	+ if (p->state >= CXL_CONFIG_ACTIVE)
	511	+ return -EBUSY;
	512	+
	513	+ cxl_region_iomem_release(cxlr);
	514	+ p->state = CXL_CONFIG_IDLE;
	515	+ return 0;
	516	+}
	517	+
	518	+static ssize_t size_store(struct device dev, struct device_attribute attr,
	519	+ const char *buf, size_t len)
	520	+{
	521	+ struct cxl_region *cxlr = to_cxl_region(dev);
	522	+ u64 val;
	523	+ int rc;
	524	+
	525	+ rc = kstrtou64(buf, 0, &val);
	526	+ if (rc)
	527	+ return rc;
	528	+
	529	+ rc = down_write_killable(&cxl_region_rwsem);
	530	+ if (rc)
	531	+ return rc;
	532	+
	533	+ if (val)
	534	+ rc = alloc_hpa(cxlr, val);
	535	+ else
	536	+ rc = free_hpa(cxlr);
	537	+ up_write(&cxl_region_rwsem);
	538	+
	539	+ if (rc)
	540	+ return rc;
	541	+
	542	+ return len;
	543	+}
	544	+
	545	+static ssize_t size_show(struct device dev, struct device_attribute attr,
	546	+ char *buf)
	547	+{
	548	+ struct cxl_region *cxlr = to_cxl_region(dev);
	549	+ struct cxl_region_params *p = &cxlr->params;
	550	+ u64 size = 0;
	551	+ ssize_t rc;
	552	+
	553	+ rc = down_read_interruptible(&cxl_region_rwsem);
	554	+ if (rc)
	555	+ return rc;
	556	+ if (p->res)
	557	+ size = resource_size(p->res);
	558	+ rc = sysfs_emit(buf, "%#llx\n", size);
	559	+ up_read(&cxl_region_rwsem);
	560	+
	561	+ return rc;
	562	+}
	563	+static DEVICE_ATTR_RW(size);
	564	+
	565	+static struct attribute *cxl_region_attrs[] = {
	566	+ &dev_attr_uuid.attr,
	567	+ &dev_attr_commit.attr,
	568	+ &dev_attr_interleave_ways.attr,
	569	+ &dev_attr_interleave_granularity.attr,
	570	+ &dev_attr_resource.attr,
	571	+ &dev_attr_size.attr,
	572	+ NULL,
	573	+};
	574	+
	575	+static const struct attribute_group cxl_region_group = {
	576	+ .attrs = cxl_region_attrs,
	577	+ .is_visible = cxl_region_visible,
	578	+};
	579	+
	580	+static size_t show_targetN(struct cxl_region cxlr, char buf, int pos)
	581	+{
	582	+ struct cxl_region_params *p = &cxlr->params;
	583	+ struct cxl_endpoint_decoder *cxled;
	584	+ int rc;
	585	+
	586	+ rc = down_read_interruptible(&cxl_region_rwsem);
	587	+ if (rc)
	588	+ return rc;
	589	+
	590	+ if (pos >= p->interleave_ways) {
	591	+ dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
	592	+ p->interleave_ways);
	593	+ rc = -ENXIO;
	594	+ goto out;
	595	+ }
	596	+
	597	+ cxled = p->targets[pos];
	598	+ if (!cxled)
	599	+ rc = sysfs_emit(buf, "\n");
	600	+ else
	601	+ rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
	602	+out:
	603	+ up_read(&cxl_region_rwsem);
	604	+
	605	+ return rc;
	606	+}
	607	+
	608	+static int match_free_decoder(struct device dev, void data)
	609	+{
	610	+ struct cxl_decoder *cxld;
	611	+ int *id = data;
	612	+
	613	+ if (!is_switch_decoder(dev))
	614	+ return 0;
	615	+
	616	+ cxld = to_cxl_decoder(dev);
	617	+
	618	+ /* enforce ordered allocation */
	619	+ if (cxld->id != *id)
	620	+ return 0;
	621	+
	622	+ if (!cxld->region)
	623	+ return 1;
	624	+
	625	+ (*id)++;
	626	+
	627	+ return 0;
	628	+}
	629	+
	630	+static struct cxl_decoder cxl_region_find_decoder(struct cxl_port port,
	631	+ struct cxl_region *cxlr)
	632	+{
	633	+ struct device *dev;
	634	+ int id = 0;
	635	+
	636	+ dev = device_find_child(&port->dev, &id, match_free_decoder);
	637	+ if (!dev)
	638	+ return NULL;
	639	+ /*
	640	+ * This decoder is pinned registered as long as the endpoint decoder is
	641	+ * registered, and endpoint decoder unregistration holds the
	642	+ * cxl_region_rwsem over unregister events, so no need to hold on to
	643	+ * this extra reference.
	644	+ */
	645	+ put_device(dev);
	646	+ return to_cxl_decoder(dev);
	647	+}
	648	+
	649	+static struct cxl_region_ref alloc_region_ref(struct cxl_port port,
	650	+ struct cxl_region *cxlr)
	651	+{
	652	+ struct cxl_region_params *p = &cxlr->params;
	653	+ struct cxl_region_ref cxl_rr, iter;
	654	+ unsigned long index;
	655	+ int rc;
	656	+
	657	+ xa_for_each(&port->regions, index, iter) {
	658	+ struct cxl_region_params *ip = &iter->region->params;
	659	+
	660	+ if (ip->res->start > p->res->start) {
	661	+ dev_dbg(&cxlr->dev,
	662	+ "%s: HPA order violation %s:%pr vs %pr\n",
	663	+ dev_name(&port->dev),
	664	+ dev_name(&iter->region->dev), ip->res, p->res);
	665	+ return ERR_PTR(-EBUSY);
	666	+ }
	667	+ }
	668	+
	669	+ cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
	670	+ if (!cxl_rr)
	671	+ return ERR_PTR(-ENOMEM);
	672	+ cxl_rr->port = port;
	673	+ cxl_rr->region = cxlr;
	674	+ cxl_rr->nr_targets = 1;
	675	+ xa_init(&cxl_rr->endpoints);
	676	+
	677	+ rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
	678	+ if (rc) {
	679	+ dev_dbg(&cxlr->dev,
	680	+ "%s: failed to track region reference: %d\n",
	681	+ dev_name(&port->dev), rc);
	682	+ kfree(cxl_rr);
	683	+ return ERR_PTR(rc);
	684	+ }
	685	+
	686	+ return cxl_rr;
	687	+}
	688	+
	689	+static void free_region_ref(struct cxl_region_ref *cxl_rr)
	690	+{
	691	+ struct cxl_port *port = cxl_rr->port;
	692	+ struct cxl_region *cxlr = cxl_rr->region;
	693	+ struct cxl_decoder *cxld = cxl_rr->decoder;
	694	+
	695	+ dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
	696	+ if (cxld->region == cxlr) {
	697	+ cxld->region = NULL;
	698	+ put_device(&cxlr->dev);
	699	+ }
	700	+
	701	+ xa_erase(&port->regions, (unsigned long)cxlr);
	702	+ xa_destroy(&cxl_rr->endpoints);
	703	+ kfree(cxl_rr);
	704	+}
	705	+
	706	+static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
	707	+ struct cxl_endpoint_decoder *cxled)
	708	+{
	709	+ int rc;
	710	+ struct cxl_port *port = cxl_rr->port;
	711	+ struct cxl_region *cxlr = cxl_rr->region;
	712	+ struct cxl_decoder *cxld = cxl_rr->decoder;
	713	+ struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
	714	+
	715	+ if (ep) {
	716	+ rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
	717	+ GFP_KERNEL);
	718	+ if (rc)
	719	+ return rc;
	720	+ }
	721	+ cxl_rr->nr_eps++;
	722	+
	723	+ if (!cxld->region) {
	724	+ cxld->region = cxlr;
	725	+ get_device(&cxlr->dev);
	726	+ }
	727	+
	728	+ return 0;
	729	+}
	730	+
	731	+/**
	732	+ * cxl_port_attach_region() - track a region's interest in a port by endpoint
	733	+ * @port: port to add a new region reference 'struct cxl_region_ref'
	734	+ * @cxlr: region to attach to @port
	735	+ * @cxled: endpoint decoder used to create or further pin a region reference
	736	+ * @pos: interleave position of @cxled in @cxlr
	737	+ *
	738	+ * The attach event is an opportunity to validate CXL decode setup
	739	+ * constraints and record metadata needed for programming HDM decoders,
	740	+ * in particular decoder target lists.
	741	+ *
	742	+ * The steps are:
	743	+ *
	744	+ * - validate that there are no other regions with a higher HPA already
	745	+ * associated with @port
	746	+ * - establish a region reference if one is not already present
	747	+ *
	748	+ * - additionally allocate a decoder instance that will host @cxlr on
	749	+ * @port
	750	+ *
	751	+ * - pin the region reference by the endpoint
	752	+ * - account for how many entries in @port's target list are needed to
	753	+ * cover all of the added endpoints.
	754	+ */
	755	+static int cxl_port_attach_region(struct cxl_port *port,
	756	+ struct cxl_region *cxlr,
	757	+ struct cxl_endpoint_decoder *cxled, int pos)
	758	+{
	759	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	760	+ struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
	761	+ struct cxl_region_ref *cxl_rr;
	762	+ bool nr_targets_inc = false;
	763	+ struct cxl_decoder *cxld;
	764	+ unsigned long index;
	765	+ int rc = -EBUSY;
	766	+
	767	+ lockdep_assert_held_write(&cxl_region_rwsem);
	768	+
	769	+ cxl_rr = cxl_rr_load(port, cxlr);
	770	+ if (cxl_rr) {
	771	+ struct cxl_ep *ep_iter;
	772	+ int found = 0;
	773	+
	774	+ /*
	775	+ * Walk the existing endpoints that have been attached to
	776	+ * @cxlr at @port and see if they share the same 'next' port
	777	+ * in the downstream direction. I.e. endpoints that share common
	778	+ * upstream switch.
	779	+ */
	780	+ xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
	781	+ if (ep_iter == ep)
	782	+ continue;
	783	+ if (ep_iter->next == ep->next) {
	784	+ found++;
	785	+ break;
	786	+ }
	787	+ }
	788	+
	789	+ /*
	790	+ * New target port, or @port is an endpoint port that always
	791	+ * accounts its own local decode as a target.
	792	+ */
	793	+ if (!found \|\| !ep->next) {
	794	+ cxl_rr->nr_targets++;
	795	+ nr_targets_inc = true;
	796	+ }
	797	+
	798	+ /*
	799	+ * The decoder for @cxlr was allocated when the region was first
	800	+ * attached to @port.
	801	+ */
	802	+ cxld = cxl_rr->decoder;
	803	+ } else {
	804	+ cxl_rr = alloc_region_ref(port, cxlr);
	805	+ if (IS_ERR(cxl_rr)) {
	806	+ dev_dbg(&cxlr->dev,
	807	+ "%s: failed to allocate region reference\n",
	808	+ dev_name(&port->dev));
	809	+ return PTR_ERR(cxl_rr);
	810	+ }
	811	+ nr_targets_inc = true;
	812	+
	813	+ if (port == cxled_to_port(cxled))
	814	+ cxld = &cxled->cxld;
	815	+ else
	816	+ cxld = cxl_region_find_decoder(port, cxlr);
	817	+ if (!cxld) {
	818	+ dev_dbg(&cxlr->dev, "%s: no decoder available\n",
	819	+ dev_name(&port->dev));
	820	+ goto out_erase;
	821	+ }
	822	+
	823	+ if (cxld->region) {
	824	+ dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
	825	+ dev_name(&port->dev), dev_name(&cxld->dev),
	826	+ dev_name(&cxld->region->dev));
	827	+ rc = -EBUSY;
	828	+ goto out_erase;
	829	+ }
	830	+
	831	+ cxl_rr->decoder = cxld;
	832	+ }
	833	+
	834	+ rc = cxl_rr_ep_add(cxl_rr, cxled);
	835	+ if (rc) {
	836	+ dev_dbg(&cxlr->dev,
	837	+ "%s: failed to track endpoint %s:%s reference\n",
	838	+ dev_name(&port->dev), dev_name(&cxlmd->dev),
	839	+ dev_name(&cxld->dev));
	840	+ goto out_erase;
	841	+ }
	842	+
	843	+ dev_dbg(&cxlr->dev,
	844	+ "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
	845	+ dev_name(port->uport), dev_name(&port->dev),
	846	+ dev_name(&cxld->dev), dev_name(&cxlmd->dev),
	847	+ dev_name(&cxled->cxld.dev), pos,
	848	+ ep ? ep->next ? dev_name(ep->next->uport) :
	849	+ dev_name(&cxlmd->dev) :
	850	+ "none",
	851	+ cxl_rr->nr_eps, cxl_rr->nr_targets);
	852	+
	853	+ return 0;
	854	+out_erase:
	855	+ if (nr_targets_inc)
	856	+ cxl_rr->nr_targets--;
	857	+ if (cxl_rr->nr_eps == 0)
	858	+ free_region_ref(cxl_rr);
	859	+ return rc;
	860	+}
	861	+
	862	+static void cxl_port_detach_region(struct cxl_port *port,
	863	+ struct cxl_region *cxlr,
	864	+ struct cxl_endpoint_decoder *cxled)
	865	+{
	866	+ struct cxl_region_ref *cxl_rr;
	867	+ struct cxl_ep *ep = NULL;
	868	+
	869	+ lockdep_assert_held_write(&cxl_region_rwsem);
	870	+
	871	+ cxl_rr = cxl_rr_load(port, cxlr);
	872	+ if (!cxl_rr)
	873	+ return;
	874	+
	875	+ /*
	876	+ * Endpoint ports do not carry cxl_ep references, and they
	877	+ * never target more than one endpoint by definition
	878	+ */
	879	+ if (cxl_rr->decoder == &cxled->cxld)
	880	+ cxl_rr->nr_eps--;
	881	+ else
	882	+ ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
	883	+ if (ep) {
	884	+ struct cxl_ep *ep_iter;
	885	+ unsigned long index;
	886	+ int found = 0;
	887	+
	888	+ cxl_rr->nr_eps--;
	889	+ xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
	890	+ if (ep_iter->next == ep->next) {
	891	+ found++;
	892	+ break;
	893	+ }
	894	+ }
	895	+ if (!found)
	896	+ cxl_rr->nr_targets--;
	897	+ }
	898	+
	899	+ if (cxl_rr->nr_eps == 0)
	900	+ free_region_ref(cxl_rr);
	901	+}
	902	+
	903	+static int check_last_peer(struct cxl_endpoint_decoder *cxled,
	904	+ struct cxl_ep ep, struct cxl_region_ref cxl_rr,
	905	+ int distance)
	906	+{
	907	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	908	+ struct cxl_region *cxlr = cxl_rr->region;
	909	+ struct cxl_region_params *p = &cxlr->params;
	910	+ struct cxl_endpoint_decoder *cxled_peer;
	911	+ struct cxl_port *port = cxl_rr->port;
	912	+ struct cxl_memdev *cxlmd_peer;
	913	+ struct cxl_ep *ep_peer;
	914	+ int pos = cxled->pos;
	915	+
	916	+ /*
	917	+ * If this position wants to share a dport with the last endpoint mapped
	918	+ * then that endpoint, at index 'position - distance', must also be
	919	+ * mapped by this dport.
	920	+ */
	921	+ if (pos < distance) {
	922	+ dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
	923	+ dev_name(port->uport), dev_name(&port->dev),
	924	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
	925	+ return -ENXIO;
	926	+ }
	927	+ cxled_peer = p->targets[pos - distance];
	928	+ cxlmd_peer = cxled_to_memdev(cxled_peer);
	929	+ ep_peer = cxl_ep_load(port, cxlmd_peer);
	930	+ if (ep->dport != ep_peer->dport) {
	931	+ dev_dbg(&cxlr->dev,
	932	+ "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
	933	+ dev_name(port->uport), dev_name(&port->dev),
	934	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
	935	+ dev_name(&cxlmd_peer->dev),
	936	+ dev_name(&cxled_peer->cxld.dev));
	937	+ return -ENXIO;
	938	+ }
	939	+
	940	+ return 0;
	941	+}
	942	+
	943	+static int cxl_port_setup_targets(struct cxl_port *port,
	944	+ struct cxl_region *cxlr,
	945	+ struct cxl_endpoint_decoder *cxled)
	946	+{
	947	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
	948	+ int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
	949	+ struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
	950	+ struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
	951	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	952	+ struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
	953	+ struct cxl_region_params *p = &cxlr->params;
	954	+ struct cxl_decoder *cxld = cxl_rr->decoder;
	955	+ struct cxl_switch_decoder *cxlsd;
	956	+ u16 eig, peig;
	957	+ u8 eiw, peiw;
	958	+
	959	+ /*
	960	+ * While root level decoders support x3, x6, x12, switch level
	961	+ * decoders only support powers of 2 up to x16.
	962	+ */
	963	+ if (!is_power_of_2(cxl_rr->nr_targets)) {
	964	+ dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
	965	+ dev_name(port->uport), dev_name(&port->dev),
	966	+ cxl_rr->nr_targets);
	967	+ return -EINVAL;
	968	+ }
	969	+
	970	+ cxlsd = to_cxl_switch_decoder(&cxld->dev);
	971	+ if (cxl_rr->nr_targets_set) {
	972	+ int i, distance;
	973	+
	974	+ distance = p->nr_targets / cxl_rr->nr_targets;
	975	+ for (i = 0; i < cxl_rr->nr_targets_set; i++)
	976	+ if (ep->dport == cxlsd->target[i]) {
	977	+ rc = check_last_peer(cxled, ep, cxl_rr,
	978	+ distance);
	979	+ if (rc)
	980	+ return rc;
	981	+ goto out_target_set;
	982	+ }
	983	+ goto add_target;
	984	+ }
	985	+
	986	+ if (is_cxl_root(parent_port)) {
	987	+ parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
	988	+ parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
	989	+ /*
	990	+ * For purposes of address bit routing, use power-of-2 math for
	991	+ * switch ports.
	992	+ */
	993	+ if (!is_power_of_2(parent_iw))
	994	+ parent_iw /= 3;
	995	+ } else {
	996	+ struct cxl_region_ref *parent_rr;
	997	+ struct cxl_decoder *parent_cxld;
	998	+
	999	+ parent_rr = cxl_rr_load(parent_port, cxlr);
	1000	+ parent_cxld = parent_rr->decoder;
	1001	+ parent_ig = parent_cxld->interleave_granularity;
	1002	+ parent_iw = parent_cxld->interleave_ways;
	1003	+ }
	1004	+
	1005	+ rc = granularity_to_cxl(parent_ig, &peig);
	1006	+ if (rc) {
	1007	+ dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
	1008	+ dev_name(parent_port->uport),
	1009	+ dev_name(&parent_port->dev), parent_ig);
	1010	+ return rc;
	1011	+ }
	1012	+
	1013	+ rc = ways_to_cxl(parent_iw, &peiw);
	1014	+ if (rc) {
	1015	+ dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
	1016	+ dev_name(parent_port->uport),
	1017	+ dev_name(&parent_port->dev), parent_iw);
	1018	+ return rc;
	1019	+ }
	1020	+
	1021	+ iw = cxl_rr->nr_targets;
	1022	+ rc = ways_to_cxl(iw, &eiw);
	1023	+ if (rc) {
	1024	+ dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
	1025	+ dev_name(port->uport), dev_name(&port->dev), iw);
	1026	+ return rc;
	1027	+ }
	1028	+
	1029	+ /*
	1030	+ * If @parent_port is masking address bits, pick the next unused address
	1031	+ * bit to route @port's targets.
	1032	+ */
	1033	+ if (parent_iw > 1 && cxl_rr->nr_targets > 1) {
	1034	+ u32 address_bit = max(peig + peiw, eiw + peig);
	1035	+
	1036	+ eig = address_bit - eiw + 1;
	1037	+ } else {
	1038	+ eiw = peiw;
	1039	+ eig = peig;
	1040	+ }
	1041	+
	1042	+ rc = cxl_to_granularity(eig, &ig);
	1043	+ if (rc) {
	1044	+ dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
	1045	+ dev_name(port->uport), dev_name(&port->dev),
	1046	+ 256 << eig);
	1047	+ return rc;
	1048	+ }
	1049	+
	1050	+ cxld->interleave_ways = iw;
	1051	+ cxld->interleave_granularity = ig;
	1052	+ cxld->hpa_range = (struct range) {
	1053	+ .start = p->res->start,
	1054	+ .end = p->res->end,
	1055	+ };
	1056	+ dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport),
	1057	+ dev_name(&port->dev), iw, ig);
	1058	+add_target:
	1059	+ if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
	1060	+ dev_dbg(&cxlr->dev,
	1061	+ "%s:%s: targets full trying to add %s:%s at %d\n",
	1062	+ dev_name(port->uport), dev_name(&port->dev),
	1063	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
	1064	+ return -ENXIO;
	1065	+ }
	1066	+ cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
	1067	+ inc = 1;
	1068	+out_target_set:
	1069	+ cxl_rr->nr_targets_set += inc;
	1070	+ dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
	1071	+ dev_name(port->uport), dev_name(&port->dev),
	1072	+ cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport),
	1073	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
	1074	+
	1075	+ return 0;
	1076	+}
	1077	+
	1078	+static void cxl_port_reset_targets(struct cxl_port *port,
	1079	+ struct cxl_region *cxlr)
	1080	+{
	1081	+ struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
	1082	+ struct cxl_decoder *cxld;
	1083	+
	1084	+ /*
	1085	+ * After the last endpoint has been detached the entire cxl_rr may now
	1086	+ * be gone.
	1087	+ */
	1088	+ if (!cxl_rr)
	1089	+ return;
	1090	+ cxl_rr->nr_targets_set = 0;
	1091	+
	1092	+ cxld = cxl_rr->decoder;
	1093	+ cxld->hpa_range = (struct range) {
	1094	+ .start = 0,
	1095	+ .end = -1,
	1096	+ };
	1097	+}
	1098	+
	1099	+static void cxl_region_teardown_targets(struct cxl_region *cxlr)
	1100	+{
	1101	+ struct cxl_region_params *p = &cxlr->params;
	1102	+ struct cxl_endpoint_decoder *cxled;
	1103	+ struct cxl_memdev *cxlmd;
	1104	+ struct cxl_port *iter;
	1105	+ struct cxl_ep *ep;
	1106	+ int i;
	1107	+
	1108	+ for (i = 0; i < p->nr_targets; i++) {
	1109	+ cxled = p->targets[i];
	1110	+ cxlmd = cxled_to_memdev(cxled);
	1111	+
	1112	+ iter = cxled_to_port(cxled);
	1113	+ while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
	1114	+ iter = to_cxl_port(iter->dev.parent);
	1115	+
	1116	+ for (ep = cxl_ep_load(iter, cxlmd); iter;
	1117	+ iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
	1118	+ cxl_port_reset_targets(iter, cxlr);
	1119	+ }
	1120	+}
	1121	+
	1122	+static int cxl_region_setup_targets(struct cxl_region *cxlr)
	1123	+{
	1124	+ struct cxl_region_params *p = &cxlr->params;
	1125	+ struct cxl_endpoint_decoder *cxled;
	1126	+ struct cxl_memdev *cxlmd;
	1127	+ struct cxl_port *iter;
	1128	+ struct cxl_ep *ep;
	1129	+ int i, rc;
	1130	+
	1131	+ for (i = 0; i < p->nr_targets; i++) {
	1132	+ cxled = p->targets[i];
	1133	+ cxlmd = cxled_to_memdev(cxled);
	1134	+
	1135	+ iter = cxled_to_port(cxled);
	1136	+ while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
	1137	+ iter = to_cxl_port(iter->dev.parent);
	1138	+
	1139	+ /*
	1140	+ * Descend the topology tree programming targets while
	1141	+ * looking for conflicts.
	1142	+ */
	1143	+ for (ep = cxl_ep_load(iter, cxlmd); iter;
	1144	+ iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
	1145	+ rc = cxl_port_setup_targets(iter, cxlr, cxled);
	1146	+ if (rc) {
	1147	+ cxl_region_teardown_targets(cxlr);
	1148	+ return rc;
	1149	+ }
	1150	+ }
	1151	+ }
	1152	+
	1153	+ return 0;
	1154	+}
	1155	+
	1156	+static int cxl_region_attach(struct cxl_region *cxlr,
	1157	+ struct cxl_endpoint_decoder *cxled, int pos)
	1158	+{
	1159	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
	1160	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	1161	+ struct cxl_port ep_port, root_port, *iter;
	1162	+ struct cxl_region_params *p = &cxlr->params;
	1163	+ struct cxl_dport *dport;
	1164	+ int i, rc = -ENXIO;
	1165	+
	1166	+ if (cxled->mode == CXL_DECODER_DEAD) {
	1167	+ dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
	1168	+ return -ENODEV;
	1169	+ }
	1170	+
	1171	+ /* all full of members, or interleave config not established? */
	1172	+ if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
	1173	+ dev_dbg(&cxlr->dev, "region already active\n");
	1174	+ return -EBUSY;
	1175	+ } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
	1176	+ dev_dbg(&cxlr->dev, "interleave config missing\n");
	1177	+ return -ENXIO;
	1178	+ }
	1179	+
	1180	+ if (pos < 0 \|\| pos >= p->interleave_ways) {
	1181	+ dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
	1182	+ p->interleave_ways);
	1183	+ return -ENXIO;
	1184	+ }
	1185	+
	1186	+ if (p->targets[pos] == cxled)
	1187	+ return 0;
	1188	+
	1189	+ if (p->targets[pos]) {
	1190	+ struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
	1191	+ struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
	1192	+
	1193	+ dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
	1194	+ pos, dev_name(&cxlmd_target->dev),
	1195	+ dev_name(&cxled_target->cxld.dev));
	1196	+ return -EBUSY;
	1197	+ }
	1198	+
	1199	+ for (i = 0; i < p->interleave_ways; i++) {
	1200	+ struct cxl_endpoint_decoder *cxled_target;
	1201	+ struct cxl_memdev *cxlmd_target;
	1202	+
	1203	+ cxled_target = p->targets[pos];
	1204	+ if (!cxled_target)
	1205	+ continue;
	1206	+
	1207	+ cxlmd_target = cxled_to_memdev(cxled_target);
	1208	+ if (cxlmd_target == cxlmd) {
	1209	+ dev_dbg(&cxlr->dev,
	1210	+ "%s already specified at position %d via: %s\n",
	1211	+ dev_name(&cxlmd->dev), pos,
	1212	+ dev_name(&cxled_target->cxld.dev));
	1213	+ return -EBUSY;
	1214	+ }
	1215	+ }
	1216	+
	1217	+ ep_port = cxled_to_port(cxled);
	1218	+ root_port = cxlrd_to_port(cxlrd);
	1219	+ dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
	1220	+ if (!dport) {
	1221	+ dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
	1222	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
	1223	+ dev_name(cxlr->dev.parent));
	1224	+ return -ENXIO;
	1225	+ }
	1226	+
	1227	+ if (cxlrd->calc_hb(cxlrd, pos) != dport) {
	1228	+ dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
	1229	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
	1230	+ dev_name(&cxlrd->cxlsd.cxld.dev));
	1231	+ return -ENXIO;
	1232	+ }
	1233	+
	1234	+ if (cxled->cxld.target_type != cxlr->type) {
	1235	+ dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
	1236	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
	1237	+ cxled->cxld.target_type, cxlr->type);
	1238	+ return -ENXIO;
	1239	+ }
	1240	+
	1241	+ if (!cxled->dpa_res) {
	1242	+ dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
	1243	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
	1244	+ return -ENXIO;
	1245	+ }
	1246	+
	1247	+ if (resource_size(cxled->dpa_res) * p->interleave_ways !=
	1248	+ resource_size(p->res)) {
	1249	+ dev_dbg(&cxlr->dev,
	1250	+ "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
	1251	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
	1252	+ (u64)resource_size(cxled->dpa_res), p->interleave_ways,
	1253	+ (u64)resource_size(p->res));
	1254	+ return -EINVAL;
	1255	+ }
	1256	+
	1257	+ for (iter = ep_port; !is_cxl_root(iter);
	1258	+ iter = to_cxl_port(iter->dev.parent)) {
	1259	+ rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
	1260	+ if (rc)
	1261	+ goto err;
	1262	+ }
	1263	+
	1264	+ p->targets[pos] = cxled;
	1265	+ cxled->pos = pos;
	1266	+ p->nr_targets++;
	1267	+
	1268	+ if (p->nr_targets == p->interleave_ways) {
	1269	+ rc = cxl_region_setup_targets(cxlr);
	1270	+ if (rc)
	1271	+ goto err_decrement;
	1272	+ p->state = CXL_CONFIG_ACTIVE;
	1273	+ }
	1274	+
	1275	+ cxled->cxld.interleave_ways = p->interleave_ways;
	1276	+ cxled->cxld.interleave_granularity = p->interleave_granularity;
	1277	+ cxled->cxld.hpa_range = (struct range) {
	1278	+ .start = p->res->start,
	1279	+ .end = p->res->end,
	1280	+ };
	1281	+
	1282	+ return 0;
	1283	+
	1284	+err_decrement:
	1285	+ p->nr_targets--;
	1286	+err:
	1287	+ for (iter = ep_port; !is_cxl_root(iter);
	1288	+ iter = to_cxl_port(iter->dev.parent))
	1289	+ cxl_port_detach_region(iter, cxlr, cxled);
	1290	+ return rc;
	1291	+}
	1292	+
	1293	+static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
	1294	+{
	1295	+ struct cxl_port iter, ep_port = cxled_to_port(cxled);
	1296	+ struct cxl_region *cxlr = cxled->cxld.region;
	1297	+ struct cxl_region_params *p;
	1298	+ int rc = 0;
	1299	+
	1300	+ lockdep_assert_held_write(&cxl_region_rwsem);
	1301	+
	1302	+ if (!cxlr)
	1303	+ return 0;
	1304	+
	1305	+ p = &cxlr->params;
	1306	+ get_device(&cxlr->dev);
	1307	+
	1308	+ if (p->state > CXL_CONFIG_ACTIVE) {
	1309	+ /*
	1310	+ * TODO: tear down all impacted regions if a device is
	1311	+ * removed out of order
	1312	+ */
	1313	+ rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
	1314	+ if (rc)
	1315	+ goto out;
	1316	+ p->state = CXL_CONFIG_ACTIVE;
	1317	+ }
	1318	+
	1319	+ for (iter = ep_port; !is_cxl_root(iter);
	1320	+ iter = to_cxl_port(iter->dev.parent))
	1321	+ cxl_port_detach_region(iter, cxlr, cxled);
	1322	+
	1323	+ if (cxled->pos < 0 \|\| cxled->pos >= p->interleave_ways \|\|
	1324	+ p->targets[cxled->pos] != cxled) {
	1325	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	1326	+
	1327	+ dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
	1328	+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
	1329	+ cxled->pos);
	1330	+ goto out;
	1331	+ }
	1332	+
	1333	+ if (p->state == CXL_CONFIG_ACTIVE) {
	1334	+ p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
	1335	+ cxl_region_teardown_targets(cxlr);
	1336	+ }
	1337	+ p->targets[cxled->pos] = NULL;
	1338	+ p->nr_targets--;
	1339	+ cxled->cxld.hpa_range = (struct range) {
	1340	+ .start = 0,
	1341	+ .end = -1,
	1342	+ };
	1343	+
	1344	+ /* notify the region driver that one of its targets has departed */
	1345	+ up_write(&cxl_region_rwsem);
	1346	+ device_release_driver(&cxlr->dev);
	1347	+ down_write(&cxl_region_rwsem);
	1348	+out:
	1349	+ put_device(&cxlr->dev);
	1350	+ return rc;
	1351	+}
	1352	+
	1353	+void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
	1354	+{
	1355	+ down_write(&cxl_region_rwsem);
	1356	+ cxled->mode = CXL_DECODER_DEAD;
	1357	+ cxl_region_detach(cxled);
	1358	+ up_write(&cxl_region_rwsem);
	1359	+}
	1360	+
	1361	+static int attach_target(struct cxl_region cxlr, const char decoder, int pos)
	1362	+{
	1363	+ struct device *dev;
	1364	+ int rc;
	1365	+
	1366	+ dev = bus_find_device_by_name(&cxl_bus_type, NULL, decoder);
	1367	+ if (!dev)
	1368	+ return -ENODEV;
	1369	+
	1370	+ if (!is_endpoint_decoder(dev)) {
	1371	+ put_device(dev);
	1372	+ return -EINVAL;
	1373	+ }
	1374	+
	1375	+ rc = down_write_killable(&cxl_region_rwsem);
	1376	+ if (rc)
	1377	+ goto out;
	1378	+ down_read(&cxl_dpa_rwsem);
	1379	+ rc = cxl_region_attach(cxlr, to_cxl_endpoint_decoder(dev), pos);
	1380	+ up_read(&cxl_dpa_rwsem);
	1381	+ up_write(&cxl_region_rwsem);
	1382	+out:
	1383	+ put_device(dev);
	1384	+ return rc;
	1385	+}
	1386	+
	1387	+static int detach_target(struct cxl_region *cxlr, int pos)
	1388	+{
	1389	+ struct cxl_region_params *p = &cxlr->params;
	1390	+ int rc;
	1391	+
	1392	+ rc = down_write_killable(&cxl_region_rwsem);
	1393	+ if (rc)
	1394	+ return rc;
	1395	+
	1396	+ if (pos >= p->interleave_ways) {
	1397	+ dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
	1398	+ p->interleave_ways);
	1399	+ rc = -ENXIO;
	1400	+ goto out;
	1401	+ }
	1402	+
	1403	+ if (!p->targets[pos]) {
	1404	+ rc = 0;
	1405	+ goto out;
	1406	+ }
	1407	+
	1408	+ rc = cxl_region_detach(p->targets[pos]);
	1409	+out:
	1410	+ up_write(&cxl_region_rwsem);
	1411	+ return rc;
	1412	+}
	1413	+
	1414	+static size_t store_targetN(struct cxl_region cxlr, const char buf, int pos,
	1415	+ size_t len)
	1416	+{
	1417	+ int rc;
	1418	+
	1419	+ if (sysfs_streq(buf, "\n"))
	1420	+ rc = detach_target(cxlr, pos);
	1421	+ else
	1422	+ rc = attach_target(cxlr, buf, pos);
	1423	+
	1424	+ if (rc < 0)
	1425	+ return rc;
	1426	+ return len;
	1427	+}
	1428	+
	1429	+#define TARGET_ATTR_RW(n) \
	1430	+static ssize_t target##n##_show( \
	1431	+ struct device dev, struct device_attribute attr, char *buf) \
	1432	+{ \
	1433	+ return show_targetN(to_cxl_region(dev), buf, (n)); \
	1434	+} \
	1435	+static ssize_t target##n##_store(struct device *dev, \
	1436	+ struct device_attribute *attr, \
	1437	+ const char *buf, size_t len) \
	1438	+{ \
	1439	+ return store_targetN(to_cxl_region(dev), buf, (n), len); \
	1440	+} \
	1441	+static DEVICE_ATTR_RW(target##n)
	1442	+
	1443	+TARGET_ATTR_RW(0);
	1444	+TARGET_ATTR_RW(1);
	1445	+TARGET_ATTR_RW(2);
	1446	+TARGET_ATTR_RW(3);
	1447	+TARGET_ATTR_RW(4);
	1448	+TARGET_ATTR_RW(5);
	1449	+TARGET_ATTR_RW(6);
	1450	+TARGET_ATTR_RW(7);
	1451	+TARGET_ATTR_RW(8);
	1452	+TARGET_ATTR_RW(9);
	1453	+TARGET_ATTR_RW(10);
	1454	+TARGET_ATTR_RW(11);
	1455	+TARGET_ATTR_RW(12);
	1456	+TARGET_ATTR_RW(13);
	1457	+TARGET_ATTR_RW(14);
	1458	+TARGET_ATTR_RW(15);
	1459	+
	1460	+static struct attribute *target_attrs[] = {
	1461	+ &dev_attr_target0.attr,
	1462	+ &dev_attr_target1.attr,
	1463	+ &dev_attr_target2.attr,
	1464	+ &dev_attr_target3.attr,
	1465	+ &dev_attr_target4.attr,
	1466	+ &dev_attr_target5.attr,
	1467	+ &dev_attr_target6.attr,
	1468	+ &dev_attr_target7.attr,
	1469	+ &dev_attr_target8.attr,
	1470	+ &dev_attr_target9.attr,
	1471	+ &dev_attr_target10.attr,
	1472	+ &dev_attr_target11.attr,
	1473	+ &dev_attr_target12.attr,
	1474	+ &dev_attr_target13.attr,
	1475	+ &dev_attr_target14.attr,
	1476	+ &dev_attr_target15.attr,
	1477	+ NULL,
	1478	+};
	1479	+
	1480	+static umode_t cxl_region_target_visible(struct kobject *kobj,
	1481	+ struct attribute *a, int n)
	1482	+{
	1483	+ struct device *dev = kobj_to_dev(kobj);
	1484	+ struct cxl_region *cxlr = to_cxl_region(dev);
	1485	+ struct cxl_region_params *p = &cxlr->params;
	1486	+
	1487	+ if (n < p->interleave_ways)
	1488	+ return a->mode;
	1489	+ return 0;
	1490	+}
	1491	+
	1492	+static const struct attribute_group cxl_region_target_group = {
	1493	+ .attrs = target_attrs,
	1494	+ .is_visible = cxl_region_target_visible,
	1495	+};
	1496	+
	1497	+static const struct attribute_group *get_cxl_region_target_group(void)
	1498	+{
	1499	+ return &cxl_region_target_group;
	1500	+}
	1501	+
	1502	+static const struct attribute_group *region_groups[] = {
	1503	+ &cxl_base_attribute_group,
	1504	+ &cxl_region_group,
	1505	+ &cxl_region_target_group,
	1506	+ NULL,
	1507	+};
	1508	+
	1509	+static void cxl_region_release(struct device *dev)
	1510	+{
	1511	+ struct cxl_region *cxlr = to_cxl_region(dev);
	1512	+
	1513	+ memregion_free(cxlr->id);
	1514	+ kfree(cxlr);
	1515	+}
	1516	+
	1517	+const struct device_type cxl_region_type = {
	1518	+ .name = "cxl_region",
	1519	+ .release = cxl_region_release,
	1520	+ .groups = region_groups
	1521	+};
	1522	+
	1523	+bool is_cxl_region(struct device *dev)
	1524	+{
	1525	+ return dev->type == &cxl_region_type;
	1526	+}
	1527	+EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
	1528	+
	1529	+static struct cxl_region to_cxl_region(struct device dev)
	1530	+{
	1531	+ if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
	1532	+ "not a cxl_region device\n"))
	1533	+ return NULL;
	1534	+
	1535	+ return container_of(dev, struct cxl_region, dev);
	1536	+}
	1537	+
	1538	+static void unregister_region(void *dev)
	1539	+{
	1540	+ struct cxl_region *cxlr = to_cxl_region(dev);
	1541	+
	1542	+ device_del(dev);
	1543	+ cxl_region_iomem_release(cxlr);
	1544	+ put_device(dev);
	1545	+}
	1546	+
	1547	+static struct lock_class_key cxl_region_key;
	1548	+
	1549	+static struct cxl_region cxl_region_alloc(struct cxl_root_decoder cxlrd, int id)
	1550	+{
	1551	+ struct cxl_region *cxlr;
	1552	+ struct device *dev;
	1553	+
	1554	+ cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
	1555	+ if (!cxlr) {
	1556	+ memregion_free(id);
	1557	+ return ERR_PTR(-ENOMEM);
	1558	+ }
	1559	+
	1560	+ dev = &cxlr->dev;
	1561	+ device_initialize(dev);
	1562	+ lockdep_set_class(&dev->mutex, &cxl_region_key);
	1563	+ dev->parent = &cxlrd->cxlsd.cxld.dev;
	1564	+ device_set_pm_not_required(dev);
	1565	+ dev->bus = &cxl_bus_type;
	1566	+ dev->type = &cxl_region_type;
	1567	+ cxlr->id = id;
	1568	+
	1569	+ return cxlr;
	1570	+}
	1571	+
	1572	+/**
	1573	+ * devm_cxl_add_region - Adds a region to a decoder
	1574	+ * @cxlrd: root decoder
	1575	+ * @id: memregion id to create, or memregion_free() on failure
	1576	+ * @mode: mode for the endpoint decoders of this region
	1577	+ * @type: select whether this is an expander or accelerator (type-2 or type-3)
	1578	+ *
	1579	+ * This is the second step of region initialization. Regions exist within an
	1580	+ * address space which is mapped by a @cxlrd.
	1581	+ *
	1582	+ * Return: 0 if the region was added to the @cxlrd, else returns negative error
	1583	+ * code. The region will be named "regionZ" where Z is the unique region number.
	1584	+ */
	1585	+static struct cxl_region devm_cxl_add_region(struct cxl_root_decoder cxlrd,
	1586	+ int id,
	1587	+ enum cxl_decoder_mode mode,
	1588	+ enum cxl_decoder_type type)
	1589	+{
	1590	+ struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
	1591	+ struct cxl_region *cxlr;
	1592	+ struct device *dev;
	1593	+ int rc;
	1594	+
	1595	+ cxlr = cxl_region_alloc(cxlrd, id);
	1596	+ if (IS_ERR(cxlr))
	1597	+ return cxlr;
	1598	+ cxlr->mode = mode;
	1599	+ cxlr->type = type;
	1600	+
	1601	+ dev = &cxlr->dev;
	1602	+ rc = dev_set_name(dev, "region%d", id);
	1603	+ if (rc)
	1604	+ goto err;
	1605	+
	1606	+ rc = device_add(dev);
	1607	+ if (rc)
	1608	+ goto err;
	1609	+
	1610	+ rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr);
	1611	+ if (rc)
	1612	+ return ERR_PTR(rc);
	1613	+
	1614	+ dev_dbg(port->uport, "%s: created %s\n",
	1615	+ dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
	1616	+ return cxlr;
	1617	+
	1618	+err:
	1619	+ put_device(dev);
	1620	+ return ERR_PTR(rc);
	1621	+}
	1622	+
	1623	+static ssize_t create_pmem_region_show(struct device *dev,
	1624	+ struct device_attribute attr, char buf)
	1625	+{
	1626	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
	1627	+
	1628	+ return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
	1629	+}
	1630	+
	1631	+static ssize_t create_pmem_region_store(struct device *dev,
	1632	+ struct device_attribute *attr,
	1633	+ const char *buf, size_t len)
	1634	+{
	1635	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
	1636	+ struct cxl_region *cxlr;
	1637	+ int id, rc;
	1638	+
	1639	+ rc = sscanf(buf, "region%d\n", &id);
	1640	+ if (rc != 1)
	1641	+ return -EINVAL;
	1642	+
	1643	+ rc = memregion_alloc(GFP_KERNEL);
	1644	+ if (rc < 0)
	1645	+ return rc;
	1646	+
	1647	+ if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
	1648	+ memregion_free(rc);
	1649	+ return -EBUSY;
	1650	+ }
	1651	+
	1652	+ cxlr = devm_cxl_add_region(cxlrd, id, CXL_DECODER_PMEM,
	1653	+ CXL_DECODER_EXPANDER);
	1654	+ if (IS_ERR(cxlr))
	1655	+ return PTR_ERR(cxlr);
	1656	+
	1657	+ return len;
	1658	+}
	1659	+DEVICE_ATTR_RW(create_pmem_region);
	1660	+
	1661	+static ssize_t region_show(struct device dev, struct device_attribute attr,
	1662	+ char *buf)
	1663	+{
	1664	+ struct cxl_decoder *cxld = to_cxl_decoder(dev);
	1665	+ ssize_t rc;
	1666	+
	1667	+ rc = down_read_interruptible(&cxl_region_rwsem);
	1668	+ if (rc)
	1669	+ return rc;
	1670	+
	1671	+ if (cxld->region)
	1672	+ rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
	1673	+ else
	1674	+ rc = sysfs_emit(buf, "\n");
	1675	+ up_read(&cxl_region_rwsem);
	1676	+
	1677	+ return rc;
	1678	+}
	1679	+DEVICE_ATTR_RO(region);
	1680	+
	1681	+static struct cxl_region *
	1682	+cxl_find_region_by_name(struct cxl_root_decoder cxlrd, const char name)
	1683	+{
	1684	+ struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
	1685	+ struct device *region_dev;
	1686	+
	1687	+ region_dev = device_find_child_by_name(&cxld->dev, name);
	1688	+ if (!region_dev)
	1689	+ return ERR_PTR(-ENODEV);
	1690	+
	1691	+ return to_cxl_region(region_dev);
	1692	+}
	1693	+
	1694	+static ssize_t delete_region_store(struct device *dev,
	1695	+ struct device_attribute *attr,
	1696	+ const char *buf, size_t len)
	1697	+{
	1698	+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
	1699	+ struct cxl_port *port = to_cxl_port(dev->parent);
	1700	+ struct cxl_region *cxlr;
	1701	+
	1702	+ cxlr = cxl_find_region_by_name(cxlrd, buf);
	1703	+ if (IS_ERR(cxlr))
	1704	+ return PTR_ERR(cxlr);
	1705	+
	1706	+ devm_release_action(port->uport, unregister_region, cxlr);
	1707	+ put_device(&cxlr->dev);
	1708	+
	1709	+ return len;
	1710	+}
	1711	+DEVICE_ATTR_WO(delete_region);
	1712	+
	1713	+static void cxl_pmem_region_release(struct device *dev)
	1714	+{
	1715	+ struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
	1716	+ int i;
	1717	+
	1718	+ for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
	1719	+ struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
	1720	+
	1721	+ put_device(&cxlmd->dev);
	1722	+ }
	1723	+
	1724	+ kfree(cxlr_pmem);
	1725	+}
	1726	+
	1727	+static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
	1728	+ &cxl_base_attribute_group,
	1729	+ NULL,
	1730	+};
	1731	+
	1732	+const struct device_type cxl_pmem_region_type = {
	1733	+ .name = "cxl_pmem_region",
	1734	+ .release = cxl_pmem_region_release,
	1735	+ .groups = cxl_pmem_region_attribute_groups,
	1736	+};
	1737	+
	1738	+bool is_cxl_pmem_region(struct device *dev)
	1739	+{
	1740	+ return dev->type == &cxl_pmem_region_type;
	1741	+}
	1742	+EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
	1743	+
	1744	+struct cxl_pmem_region to_cxl_pmem_region(struct device dev)
	1745	+{
	1746	+ if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
	1747	+ "not a cxl_pmem_region device\n"))
	1748	+ return NULL;
	1749	+ return container_of(dev, struct cxl_pmem_region, dev);
	1750	+}
	1751	+EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
	1752	+
	1753	+static struct lock_class_key cxl_pmem_region_key;
	1754	+
	1755	+static struct cxl_pmem_region cxl_pmem_region_alloc(struct cxl_region cxlr)
	1756	+{
	1757	+ struct cxl_region_params *p = &cxlr->params;
	1758	+ struct cxl_pmem_region *cxlr_pmem;
	1759	+ struct device *dev;
	1760	+ int i;
	1761	+
	1762	+ down_read(&cxl_region_rwsem);
	1763	+ if (p->state != CXL_CONFIG_COMMIT) {
	1764	+ cxlr_pmem = ERR_PTR(-ENXIO);
	1765	+ goto out;
	1766	+ }
	1767	+
	1768	+ cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
	1769	+ GFP_KERNEL);
	1770	+ if (!cxlr_pmem) {
	1771	+ cxlr_pmem = ERR_PTR(-ENOMEM);
	1772	+ goto out;
	1773	+ }
	1774	+
	1775	+ cxlr_pmem->hpa_range.start = p->res->start;
	1776	+ cxlr_pmem->hpa_range.end = p->res->end;
	1777	+
	1778	+ /* Snapshot the region configuration underneath the cxl_region_rwsem */
	1779	+ cxlr_pmem->nr_mappings = p->nr_targets;
	1780	+ for (i = 0; i < p->nr_targets; i++) {
	1781	+ struct cxl_endpoint_decoder *cxled = p->targets[i];
	1782	+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
	1783	+ struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
	1784	+
	1785	+ m->cxlmd = cxlmd;
	1786	+ get_device(&cxlmd->dev);
	1787	+ m->start = cxled->dpa_res->start;
	1788	+ m->size = resource_size(cxled->dpa_res);
	1789	+ m->position = i;
	1790	+ }
	1791	+
	1792	+ dev = &cxlr_pmem->dev;
	1793	+ cxlr_pmem->cxlr = cxlr;
	1794	+ device_initialize(dev);
	1795	+ lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
	1796	+ device_set_pm_not_required(dev);
	1797	+ dev->parent = &cxlr->dev;
	1798	+ dev->bus = &cxl_bus_type;
	1799	+ dev->type = &cxl_pmem_region_type;
	1800	+out:
	1801	+ up_read(&cxl_region_rwsem);
	1802	+
	1803	+ return cxlr_pmem;
	1804	+}
	1805	+
	1806	+static void cxlr_pmem_unregister(void *dev)
	1807	+{
	1808	+ device_unregister(dev);
	1809	+}
	1810	+
	1811	+/**
	1812	+ * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
	1813	+ * @cxlr: parent CXL region for this pmem region bridge device
	1814	+ *
	1815	+ * Return: 0 on success negative error code on failure.
	1816	+ */
	1817	+static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
	1818	+{
	1819	+ struct cxl_pmem_region *cxlr_pmem;
	1820	+ struct device *dev;
	1821	+ int rc;
	1822	+
	1823	+ cxlr_pmem = cxl_pmem_region_alloc(cxlr);
	1824	+ if (IS_ERR(cxlr_pmem))
	1825	+ return PTR_ERR(cxlr_pmem);
	1826	+
	1827	+ dev = &cxlr_pmem->dev;
	1828	+ rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
	1829	+ if (rc)
	1830	+ goto err;
	1831	+
	1832	+ rc = device_add(dev);
	1833	+ if (rc)
	1834	+ goto err;
	1835	+
	1836	+ dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
	1837	+ dev_name(dev));
	1838	+
	1839	+ return devm_add_action_or_reset(&cxlr->dev, cxlr_pmem_unregister, dev);
	1840	+
	1841	+err:
	1842	+ put_device(dev);
	1843	+ return rc;
	1844	+}
	1845	+
	1846	+static int cxl_region_probe(struct device *dev)
	1847	+{
	1848	+ struct cxl_region *cxlr = to_cxl_region(dev);
	1849	+ struct cxl_region_params *p = &cxlr->params;
	1850	+ int rc;
	1851	+
	1852	+ rc = down_read_interruptible(&cxl_region_rwsem);
	1853	+ if (rc) {
	1854	+ dev_dbg(&cxlr->dev, "probe interrupted\n");
	1855	+ return rc;
	1856	+ }
	1857	+
	1858	+ if (p->state < CXL_CONFIG_COMMIT) {
	1859	+ dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
	1860	+ rc = -ENXIO;
	1861	+ }
	1862	+
	1863	+ /*
	1864	+ * From this point on any path that changes the region's state away from
	1865	+ * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
	1866	+ */
	1867	+ up_read(&cxl_region_rwsem);
	1868	+
	1869	+ switch (cxlr->mode) {
	1870	+ case CXL_DECODER_PMEM:
	1871	+ return devm_cxl_add_pmem_region(cxlr);
	1872	+ default:
	1873	+ dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
	1874	+ cxlr->mode);
	1875	+ return -ENXIO;
	1876	+ }
	1877	+}
	1878	+
	1879	+static struct cxl_driver cxl_region_driver = {
	1880	+ .name = "cxl_region",
	1881	+ .probe = cxl_region_probe,
	1882	+ .id = CXL_DEVICE_REGION,
	1883	+};
	1884	+
	1885	+int cxl_region_init(void)
	1886	+{
	1887	+ return cxl_driver_register(&cxl_region_driver);
	1888	+}
	1889	+
	1890	+void cxl_region_exit(void)
	1891	+{
	1892	+ cxl_driver_unregister(&cxl_region_driver);
	1893	+}
	1894	+
	1895	+MODULE_IMPORT_NS(CXL);
	1896	+MODULE_ALIAS_CXL(CXL_DEVICE_REGION);

drivers/cxl/cxl.h+283−29

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/libnvdimm.h>
8	8	#include <linux/bitfield.h>
9	9	#include <linux/bitops.h>
	10	+#include <linux/log2.h>
10	11	#include <linux/io.h>
11	12
12	13	/**
@@ -53,9 +54,12 @@ · 40 unmodified lines
53	54	#define CXL_HDM_DECODER0_CTRL_LOCK BIT(8)
54	55	#define CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)
55	56	#define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
	57	+#define CXL_HDM_DECODER0_CTRL_COMMIT_ERROR BIT(11)
56	58	#define CXL_HDM_DECODER0_CTRL_TYPE BIT(12)
57	59	#define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
58	60	#define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
	61	+#define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)
	62	+#define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i)
59	63
60	64	static inline int cxl_hdm_decoder_count(u32 cap_hdr)
61	65	{
@@ -64,6 +68,57 @@ static inline int cxl_hdm_decoder_count(u32 cap_hdr) · 2 unmodified lines
64	68	return val ? val * 2 : 1;
65	69	}
66	70
	71	+/* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */
	72	+static inline int cxl_to_granularity(u16 ig, unsigned int *val)
	73	+{
	74	+ if (ig > 6)
	75	+ return -EINVAL;
	76	+ *val = 256 << ig;
	77	+ return 0;
	78	+}
	79	+
	80	+/* Encode defined in CXL ECN "3, 6, 12 and 16-way memory Interleaving" */
	81	+static inline int cxl_to_ways(u8 eniw, unsigned int *val)
	82	+{
	83	+ switch (eniw) {
	84	+ case 0 ... 4:
	85	+ *val = 1 << eniw;
	86	+ break;
	87	+ case 8 ... 10:
	88	+ *val = 3 << (eniw - 8);
	89	+ break;
	90	+ default:
	91	+ return -EINVAL;
	92	+ }
	93	+
	94	+ return 0;
	95	+}
	96	+
	97	+static inline int granularity_to_cxl(int g, u16 *ig)
	98	+{
	99	+ if (g > SZ_16K \|\| g < 256 \|\| !is_power_of_2(g))
	100	+ return -EINVAL;
	101	+ *ig = ilog2(g) - 8;
	102	+ return 0;
	103	+}
	104	+
	105	+static inline int ways_to_cxl(unsigned int ways, u8 *iw)
	106	+{
	107	+ if (ways > 16)
	108	+ return -EINVAL;
	109	+ if (is_power_of_2(ways)) {
	110	+ *iw = ilog2(ways);
	111	+ return 0;
	112	+ }
	113	+ if (ways % 3)
	114	+ return -EINVAL;
	115	+ ways /= 3;
	116	+ if (!is_power_of_2(ways))
	117	+ return -EINVAL;
	118	+ *iw = ilog2(ways) + 8;
	119	+ return 0;
	120	+}
	121	+
67	122	/* CXL 2.0 8.2.8.1 Device Capabilities Array Register */
68	123	#define CXLDEV_CAP_ARRAY_OFFSET 0x0
69	124	#define CXLDEV_CAP_ARRAY_CAP_ID 0
@@ -193,37 +248,153 @@ enum cxl_decoder_type { · 123 unmodified lines
193	248	*/
194	249	#define CXL_DECODER_MAX_INTERLEAVE 16
195	250
	251	+#define CXL_DECODER_MIN_GRANULARITY 256
	252	+
196	253	/**
197		− * struct cxl_decoder - CXL address range decode configuration
	254	+ * struct cxl_decoder - Common CXL HDM Decoder Attributes
198	255	* @dev: this decoder's device
199	256	* @id: kernel device name id
200		− * @platform_res: address space resources considered by root decoder
201		− * @decoder_range: address space resources considered by midlevel decoder
	257	+ * @hpa_range: Host physical address range mapped by this decoder
202	258	* @interleave_ways: number of cxl_dports in this decode
203	259	* @interleave_granularity: data stride per dport
204	260	* @target_type: accelerator vs expander (type2 vs type3) selector
	261	+ * @region: currently assigned region for this decoder
205	262	* @flags: memory type capabilities and locking
206		− * @target_lock: coordinate coherent reads of the target list
207		− * @nr_targets: number of elements in @target
208		− * @target: active ordered target list in current decoder configuration
209		− */
	263	+ * @commit: device/decoder-type specific callback to commit settings to hw
	264	+ * @reset: device/decoder-type specific callback to reset hw settings
	265	+*/
210	266	struct cxl_decoder {
211	267	struct device dev;
212	268	int id;
213		− union {
214		− struct resource platform_res;
215		− struct range decoder_range;
216		− };
	269	+ struct range hpa_range;
217	270	int interleave_ways;
218	271	int interleave_granularity;
219	272	enum cxl_decoder_type target_type;
	273	+ struct cxl_region *region;
220	274	unsigned long flags;
	275	+ int (commit)(struct cxl_decoder cxld);
	276	+ int (reset)(struct cxl_decoder cxld);
	277	+};
	278	+
	279	+/*
	280	+ * CXL_DECODER_DEAD prevents endpoints from being reattached to regions
	281	+ * while cxld_unregister() is running
	282	+ */
	283	+enum cxl_decoder_mode {
	284	+ CXL_DECODER_NONE,
	285	+ CXL_DECODER_RAM,
	286	+ CXL_DECODER_PMEM,
	287	+ CXL_DECODER_MIXED,
	288	+ CXL_DECODER_DEAD,
	289	+};
	290	+
	291	+/**
	292	+ * struct cxl_endpoint_decoder - Endpoint / SPA to DPA decoder
	293	+ * @cxld: base cxl_decoder_object
	294	+ * @dpa_res: actively claimed DPA span of this decoder
	295	+ * @skip: offset into @dpa_res where @cxld.hpa_range maps
	296	+ * @mode: which memory type / access-mode-partition this decoder targets
	297	+ * @pos: interleave position in @cxld.region
	298	+ */
	299	+struct cxl_endpoint_decoder {
	300	+ struct cxl_decoder cxld;
	301	+ struct resource *dpa_res;
	302	+ resource_size_t skip;
	303	+ enum cxl_decoder_mode mode;
	304	+ int pos;
	305	+};
	306	+
	307	+/**
	308	+ * struct cxl_switch_decoder - Switch specific CXL HDM Decoder
	309	+ * @cxld: base cxl_decoder object
	310	+ * @target_lock: coordinate coherent reads of the target list
	311	+ * @nr_targets: number of elements in @target
	312	+ * @target: active ordered target list in current decoder configuration
	313	+ *
	314	+ * The 'switch' decoder type represents the decoder instances of cxl_port's that
	315	+ * route from the root of a CXL memory decode topology to the endpoints. They
	316	+ * come in two flavors, root-level decoders, statically defined by platform
	317	+ * firmware, and mid-level decoders, where interleave-granularity,
	318	+ * interleave-width, and the target list are mutable.
	319	+ */
	320	+struct cxl_switch_decoder {
	321	+ struct cxl_decoder cxld;
221	322	seqlock_t target_lock;
222	323	int nr_targets;
223	324	struct cxl_dport *target[];
224	325	};
225	326
226	327
	328	+/**
	329	+ * struct cxl_root_decoder - Static platform CXL address decoder
	330	+ * @res: host / parent resource for region allocations
	331	+ * @region_id: region id for next region provisioning event
	332	+ * @calc_hb: which host bridge covers the n'th position by granularity
	333	+ * @cxlsd: base cxl switch decoder
	334	+ */
	335	+struct cxl_root_decoder {
	336	+ struct resource *res;
	337	+ atomic_t region_id;
	338	+ struct cxl_dport (calc_hb)(struct cxl_root_decoder *cxlrd, int pos);
	339	+ struct cxl_switch_decoder cxlsd;
	340	+};
	341	+
	342	+/*
	343	+ * enum cxl_config_state - State machine for region configuration
	344	+ * @CXL_CONFIG_IDLE: Any sysfs attribute can be written freely
	345	+ * @CXL_CONFIG_INTERLEAVE_ACTIVE: region size has been set, no more
	346	+ * changes to interleave_ways or interleave_granularity
	347	+ * @CXL_CONFIG_ACTIVE: All targets have been added the region is now
	348	+ * active
	349	+ * @CXL_CONFIG_RESET_PENDING: see commit_store()
	350	+ * @CXL_CONFIG_COMMIT: Soft-config has been committed to hardware
	351	+ */
	352	+enum cxl_config_state {
	353	+ CXL_CONFIG_IDLE,
	354	+ CXL_CONFIG_INTERLEAVE_ACTIVE,
	355	+ CXL_CONFIG_ACTIVE,
	356	+ CXL_CONFIG_RESET_PENDING,
	357	+ CXL_CONFIG_COMMIT,
	358	+};
	359	+
	360	+/**
	361	+ * struct cxl_region_params - region settings
	362	+ * @state: allow the driver to lockdown further parameter changes
	363	+ * @uuid: unique id for persistent regions
	364	+ * @interleave_ways: number of endpoints in the region
	365	+ * @interleave_granularity: capacity each endpoint contributes to a stripe
	366	+ * @res: allocated iomem capacity for this region
	367	+ * @targets: active ordered targets in current decoder configuration
	368	+ * @nr_targets: number of targets
	369	+ *
	370	+ * State transitions are protected by the cxl_region_rwsem
	371	+ */
	372	+struct cxl_region_params {
	373	+ enum cxl_config_state state;
	374	+ uuid_t uuid;
	375	+ int interleave_ways;
	376	+ int interleave_granularity;
	377	+ struct resource *res;
	378	+ struct cxl_endpoint_decoder *targets[CXL_DECODER_MAX_INTERLEAVE];
	379	+ int nr_targets;
	380	+};
	381	+
	382	+/**
	383	+ * struct cxl_region - CXL region
	384	+ * @dev: This region's device
	385	+ * @id: This region's id. Id is globally unique across all regions
	386	+ * @mode: Endpoint decoder allocation / access mode
	387	+ * @type: Endpoint decoder target type
	388	+ * @params: active + config params for the region
	389	+ */
	390	+struct cxl_region {
	391	+ struct device dev;
	392	+ int id;
	393	+ enum cxl_decoder_mode mode;
	394	+ enum cxl_decoder_type type;
	395	+ struct cxl_region_params params;
	396	+};
	397	+
227	398	/**
228	399	* enum cxl_nvdimm_brige_state - state machine for managing bus rescans
229	400	* @CXL_NVB_NEW: Set at bridge create and after cxl_pmem_wq is destroyed
@@ -251,7 +422,26 @@ struct cxl_nvdimm_bridge { · 21 unmodified lines
251	422	struct cxl_nvdimm {
252	423	struct device dev;
253	424	struct cxl_memdev *cxlmd;
254		− struct nvdimm *nvdimm;
	425	+ struct cxl_nvdimm_bridge *bridge;
	426	+ struct cxl_pmem_region *region;
	427	+};
	428	+
	429	+struct cxl_pmem_region_mapping {
	430	+ struct cxl_memdev *cxlmd;
	431	+ struct cxl_nvdimm *cxl_nvd;
	432	+ u64 start;
	433	+ u64 size;
	434	+ int position;
	435	+};
	436	+
	437	+struct cxl_pmem_region {
	438	+ struct device dev;
	439	+ struct cxl_region *cxlr;
	440	+ struct nd_region *nd_region;
	441	+ struct cxl_nvdimm_bridge *bridge;
	442	+ struct range hpa_range;
	443	+ int nr_mappings;
	444	+ struct cxl_pmem_region_mapping mapping[];
255	445	};
256	446
257	447	/**
@@ -260,50 +450,94 @@ struct cxl_nvdimm { · 2 unmodified lines
260	450	* decode hierarchy.
261	451	* @dev: this port's device
262	452	* @uport: PCI or platform device implementing the upstream port capability
	453	+ * @host_bridge: Shortcut to the platform attach point for this port
263	454	* @id: id for port device-name
264	455	* @dports: cxl_dport instances referenced by decoders
265	456	* @endpoints: cxl_ep instances, endpoints that are a descendant of this port
	457	+ * @regions: cxl_region_ref instances, regions mapped by this port
	458	+ * @parent_dport: dport that points to this port in the parent
266	459	* @decoder_ida: allocator for decoder ids
	460	+ * @hdm_end: track last allocated HDM decoder instance for allocation ordering
	461	+ * @commit_end: cursor to track highest committed decoder for commit ordering
267	462	* @component_reg_phys: component register capability base address (optional)
268	463	* @dead: last ep has been removed, force port re-creation
269	464	* @depth: How deep this port is relative to the root. depth 0 is the root.
	465	+ * @cdat: Cached CDAT data
	466	+ * @cdat_available: Should a CDAT attribute be available in sysfs
270	467	*/
271	468	struct cxl_port {
272	469	struct device dev;
273	470	struct device *uport;
	471	+ struct device *host_bridge;
274	472	int id;
275		− struct list_head dports;
276		− struct list_head endpoints;
	473	+ struct xarray dports;
	474	+ struct xarray endpoints;
	475	+ struct xarray regions;
	476	+ struct cxl_dport *parent_dport;
277	477	struct ida decoder_ida;
	478	+ int hdm_end;
	479	+ int commit_end;
278	480	resource_size_t component_reg_phys;
279	481	bool dead;
280	482	unsigned int depth;
	483	+ struct cxl_cdat {
	484	+ void *table;
	485	+ size_t length;
	486	+ } cdat;
	487	+ bool cdat_available;
281	488	};
282	489
	490	+static inline struct cxl_dport *
	491	+cxl_find_dport_by_dev(struct cxl_port port, const struct device dport_dev)
	492	+{
	493	+ return xa_load(&port->dports, (unsigned long)dport_dev);
	494	+}
	495	+
283	496	/**
284	497	* struct cxl_dport - CXL downstream port
285	498	* @dport: PCI bridge or firmware device representing the downstream link
286	499	* @port_id: unique hardware identifier for dport in decoder target list
287	500	* @component_reg_phys: downstream port component registers
288	501	* @port: reference to cxl_port that contains this downstream port
289		− * @list: node for a cxl_port's list of cxl_dport instances
290	502	*/
291	503	struct cxl_dport {
292	504	struct device *dport;
293	505	int port_id;
294	506	resource_size_t component_reg_phys;
295	507	struct cxl_port *port;
296		− struct list_head list;
297	508	};
298	509
299	510	/**
300	511	* struct cxl_ep - track an endpoint's interest in a port
301	512	* @ep: device that hosts a generic CXL endpoint (expander or accelerator)
302		− * @list: node on port->endpoints list
	513	+ * @dport: which dport routes to this endpoint on @port
	514	+ * @next: cxl switch port across the link attached to @dport NULL if
	515	+ * attached to an endpoint
303	516	*/
304	517	struct cxl_ep {
305	518	struct device *ep;
306		− struct list_head list;
	519	+ struct cxl_dport *dport;
	520	+ struct cxl_port *next;
	521	+};
	522	+
	523	+/**
	524	+ * struct cxl_region_ref - track a region's interest in a port
	525	+ * @port: point in topology to install this reference
	526	+ * @decoder: decoder assigned for @region in @port
	527	+ * @region: region for this reference
	528	+ * @endpoints: cxl_ep references for region members beneath @port
	529	+ * @nr_targets_set: track how many targets have been programmed during setup
	530	+ * @nr_eps: number of endpoints beneath @port
	531	+ * @nr_targets: number of distinct targets needed to reach @nr_eps
	532	+ */
	533	+struct cxl_region_ref {
	534	+ struct cxl_port *port;
	535	+ struct cxl_decoder *decoder;
	536	+ struct cxl_region *region;
	537	+ struct xarray endpoints;
	538	+ int nr_targets_set;
	539	+ int nr_eps;
	540	+ int nr_targets;
307	541	};
308	542
309	543	/*
@@ -325,29 +559,31 @@ int devm_cxl_register_pci_bus(struct device host, struct device uport, · 15 unmodified lines
325	559	struct pci_bus cxl_port_to_pci_bus(struct cxl_port port);
326	560	struct cxl_port devm_cxl_add_port(struct device host, struct device *uport,
327	561	resource_size_t component_reg_phys,
328		− struct cxl_port *parent_port);
	562	+ struct cxl_dport *parent_dport);
	563	+int devm_cxl_add_endpoint(struct cxl_memdev *cxlmd,
	564	+ struct cxl_dport *parent_dport);
329	565	struct cxl_port find_cxl_root(struct device dev);
330	566	int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
331	567	int cxl_bus_rescan(void);
332		−struct cxl_port cxl_mem_find_port(struct cxl_memdev cxlmd);
	568	+struct cxl_port cxl_mem_find_port(struct cxl_memdev cxlmd,
	569	+ struct cxl_dport **dport);
333	570	bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);
334	571
335	572	struct cxl_dport devm_cxl_add_dport(struct cxl_port port,
336	573	struct device *dport, int port_id,
337	574	resource_size_t component_reg_phys);
338		−struct cxl_dport cxl_find_dport_by_dev(struct cxl_port port,
339		− const struct device *dev);
340	575
341	576	struct cxl_decoder to_cxl_decoder(struct device dev);
	577	+struct cxl_root_decoder to_cxl_root_decoder(struct device dev);
	578	+struct cxl_endpoint_decoder to_cxl_endpoint_decoder(struct device dev);
342	579	bool is_root_decoder(struct device *dev);
343	580	bool is_endpoint_decoder(struct device *dev);
344		−bool is_cxl_decoder(struct device *dev);
345		−struct cxl_decoder cxl_root_decoder_alloc(struct cxl_port port,
346		− unsigned int nr_targets);
347		−struct cxl_decoder cxl_switch_decoder_alloc(struct cxl_port port,
348		− unsigned int nr_targets);
	581	+struct cxl_root_decoder cxl_root_decoder_alloc(struct cxl_port port,
	582	+ unsigned int nr_targets);
	583	+struct cxl_switch_decoder cxl_switch_decoder_alloc(struct cxl_port port,
	584	+ unsigned int nr_targets);
349	585	int cxl_decoder_add(struct cxl_decoder cxld, int target_map);
350		−struct cxl_decoder cxl_endpoint_decoder_alloc(struct cxl_port port);
	586	+struct cxl_endpoint_decoder cxl_endpoint_decoder_alloc(struct cxl_port port);
351	587	int cxl_decoder_add_locked(struct cxl_decoder cxld, int target_map);
352	588	int cxl_decoder_autoremove(struct device host, struct cxl_decoder cxld);
353	589	int cxl_endpoint_autoremove(struct cxl_memdev cxlmd, struct cxl_port endpoint);
@@ -357,6 +593,8 @@ struct cxl_hdm devm_cxl_setup_hdm(struct cxl_port port); · 3 unmodified lines
357	593	int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm);
358	594	int devm_cxl_add_passthrough_decoder(struct cxl_port *port);
359	595
	596	+bool is_cxl_region(struct device *dev);
	597	+
360	598	extern struct bus_type cxl_bus_type;
361	599
362	600	struct cxl_driver {
@@ -385,6 +623,8 @@ void cxl_driver_unregister(struct cxl_driver *cxl_drv); · 22 unmodified lines
385	623	#define CXL_DEVICE_PORT 3
386	624	#define CXL_DEVICE_ROOT 4
387	625	#define CXL_DEVICE_MEMORY_EXPANDER 5
	626	+#define CXL_DEVICE_REGION 6
	627	+#define CXL_DEVICE_PMEM_REGION 7
388	628
389	629	#define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")
390	630	#define CXL_MODALIAS_FMT "cxl:t%d"
@@ -396,7 +636,21 @@ struct cxl_nvdimm to_cxl_nvdimm(struct device dev); · 5 unmodified lines
396	636	bool is_cxl_nvdimm(struct device *dev);
397	637	bool is_cxl_nvdimm_bridge(struct device *dev);
398	638	int devm_cxl_add_nvdimm(struct device host, struct cxl_memdev cxlmd);
399		−struct cxl_nvdimm_bridge cxl_find_nvdimm_bridge(struct cxl_nvdimm cxl_nvd);
	639	+struct cxl_nvdimm_bridge cxl_find_nvdimm_bridge(struct device dev);
	640	+
	641	+#ifdef CONFIG_CXL_REGION
	642	+bool is_cxl_pmem_region(struct device *dev);
	643	+struct cxl_pmem_region to_cxl_pmem_region(struct device dev);
	644	+#else
	645	+static inline bool is_cxl_pmem_region(struct device *dev)
	646	+{
	647	+ return false;
	648	+}
	649	+static inline struct cxl_pmem_region to_cxl_pmem_region(struct device dev)
	650	+{
	651	+ return NULL;
	652	+}
	653	+#endif
400	654
401	655	/*
402	656	* Unit test builds overrides this to __weak, find the 'strong' version

drivers/cxl/cxlmem.h+38−4

@@ -50,6 +50,24 @@ static inline struct cxl_memdev to_cxl_memdev(struct device dev) · 49 unmodified lines
50	50	return container_of(dev, struct cxl_memdev, dev);
51	51	}
52	52
	53	+static inline struct cxl_port cxled_to_port(struct cxl_endpoint_decoder cxled)
	54	+{
	55	+ return to_cxl_port(cxled->cxld.dev.parent);
	56	+}
	57	+
	58	+static inline struct cxl_port cxlrd_to_port(struct cxl_root_decoder cxlrd)
	59	+{
	60	+ return to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
	61	+}
	62	+
	63	+static inline struct cxl_memdev *
	64	+cxled_to_memdev(struct cxl_endpoint_decoder *cxled)
	65	+{
	66	+ struct cxl_port *port = to_cxl_port(cxled->cxld.dev.parent);
	67	+
	68	+ return to_cxl_memdev(port->uport);
	69	+}
	70	+
53	71	bool is_cxl_memdev(struct device *dev);
54	72	static inline bool is_cxl_endpoint(struct cxl_port *port)
55	73	{
@@ -178,8 +196,9 @@ struct cxl_endpoint_dvsec_info { · 122 unmodified lines
178	196	* @firmware_version: Firmware version for the memory device.
179	197	* @enabled_cmds: Hardware commands found enabled in CEL.
180	198	* @exclusive_cmds: Commands that are kernel-internal only
181		− * @pmem_range: Active Persistent memory capacity configuration
182		− * @ram_range: Active Volatile memory capacity configuration
	199	+ * @dpa_res: Overall DPA resource tree for the device
	200	+ * @pmem_res: Active Persistent memory capacity configuration
	201	+ * @ram_res: Active Volatile memory capacity configuration
183	202	* @total_bytes: sum of all possible capacities
184	203	* @volatile_only_bytes: hard volatile capacity
185	204	* @persistent_only_bytes: hard persistent capacity
@@ -191,6 +210,7 @@ struct cxl_endpoint_dvsec_info { · 5 unmodified lines
191	210	* @component_reg_phys: register base of component registers
192	211	* @info: Cached DVSEC information about the device.
193	212	* @serial: PCIe Device Serial Number
	213	+ * @doe_mbs: PCI DOE mailbox array
194	214	* @mbox_send: @dev specific transport for transmitting mailbox commands
195	215	*
196	216	* See section 8.2.9.5.2 Capacity Configuration and Label Storage for
@@ -209,8 +229,9 @@ struct cxl_dev_state { · 12 unmodified lines
209	229	DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
210	230	DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
211	231
212		− struct range pmem_range;
213		− struct range ram_range;
	232	+ struct resource dpa_res;
	233	+ struct resource pmem_res;
	234	+ struct resource ram_res;
214	235	u64 total_bytes;
215	236	u64 volatile_only_bytes;
216	237	u64 persistent_only_bytes;
@@ -224,6 +245,8 @@ struct cxl_dev_state { · 7 unmodified lines
224	245	resource_size_t component_reg_phys;
225	246	u64 serial;
226	247
	248	+ struct xarray doe_mbs;
	249	+
227	250	int (mbox_send)(struct cxl_dev_state cxlds, struct cxl_mbox_cmd *cmd);
228	251	};
229	252
@@ -299,6 +322,13 @@ struct cxl_mbox_identify { · 69 unmodified lines
299	322	u8 qos_telemetry_caps;
300	323	} __packed;
301	324
	325	+struct cxl_mbox_get_partition_info {
	326	+ __le64 active_volatile_cap;
	327	+ __le64 active_persistent_cap;
	328	+ __le64 next_volatile_cap;
	329	+ __le64 next_persistent_cap;
	330	+} __packed;
	331	+
302	332	struct cxl_mbox_get_lsa {
303	333	__le32 offset;
304	334	__le32 length;
@@ -370,4 +400,8 @@ struct cxl_hdm { · 65 unmodified lines
370	400	unsigned int interleave_mask;
371	401	struct cxl_port *port;
372	402	};
	403	+
	404	+struct seq_file;
	405	+struct dentry cxl_debugfs_create_dir(const char dir);
	406	+void cxl_dpa_debug(struct seq_file file, struct cxl_dev_state cxlds);
373	407	#endif /* __CXL_MEM_H__ */

drivers/cxl/cxlpci.h+1−0

@@ -74,4 +74,5 @@ static inline resource_size_t cxl_regmap_to_base(struct pci_dev *pdev, · 73 unmodified lines
74	74	int devm_cxl_port_enumerate_dports(struct cxl_port *port);
75	75	struct cxl_dev_state;
76	76	int cxl_hdm_decode_init(struct cxl_dev_state cxlds, struct cxl_hdm cxlhdm);
	77	+void read_cdat_data(struct cxl_port *port);
77	78	#endif /* __CXL_PCI_H__ */

drivers/cxl/mem.c+23−26

@@ -1,5 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0-only
2	2	/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
	3	+#include <linux/debugfs.h>
3	4	#include <linux/device.h>
4	5	#include <linux/module.h>
5	6	#include <linux/pci.h>
@@ -24,42 +25,32 @@ · 18 unmodified lines
24	25	* in higher level operations.
25	26	*/
26	27
27		−static int create_endpoint(struct cxl_memdev *cxlmd,
28		− struct cxl_port *parent_port)
	28	+static void enable_suspend(void *data)
29	29	{
30		− struct cxl_dev_state *cxlds = cxlmd->cxlds;
31		− struct cxl_port *endpoint;
32		− int rc;
	30	+ cxl_mem_active_dec();
	31	+}
33	32
34		− endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev,
35		− cxlds->component_reg_phys, parent_port);
36		− if (IS_ERR(endpoint))
37		− return PTR_ERR(endpoint);
	33	+static void remove_debugfs(void *dentry)
	34	+{
	35	+ debugfs_remove_recursive(dentry);
	36	+}
38	37
39		− dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev));
	38	+static int cxl_mem_dpa_show(struct seq_file file, void data)
	39	+{
	40	+ struct device *dev = file->private;
	41	+ struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
40	42
41		− rc = cxl_endpoint_autoremove(cxlmd, endpoint);
42		− if (rc)
43		− return rc;
44		−
45		− if (!endpoint->dev.driver) {
46		− dev_err(&cxlmd->dev, "%s failed probe\n",
47		− dev_name(&endpoint->dev));
48		− return -ENXIO;
49		− }
	43	+ cxl_dpa_debug(file, cxlmd->cxlds);
50	44
51	45	return 0;
52	46	}
53	47
54		−static void enable_suspend(void *data)
55		−{
56		− cxl_mem_active_dec();
57		−}
58		−
59	48	static int cxl_mem_probe(struct device *dev)
60	49	{
61	50	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
62	51	struct cxl_port *parent_port;
	52	+ struct cxl_dport *dport;
	53	+ struct dentry *dentry;
63	54	int rc;
64	55
65	56	/*
@@ -73,11 +64,17 @@ static int cxl_mem_probe(struct device *dev) · 7 unmodified lines
73	64	if (work_pending(&cxlmd->detach_work))
74	65	return -EBUSY;
75	66
	67	+ dentry = cxl_debugfs_create_dir(dev_name(dev));
	68	+ debugfs_create_devm_seqfile(dev, "dpamem", dentry, cxl_mem_dpa_show);
	69	+ rc = devm_add_action_or_reset(dev, remove_debugfs, dentry);
	70	+ if (rc)
	71	+ return rc;
	72	+
76	73	rc = devm_cxl_enumerate_ports(cxlmd);
77	74	if (rc)
78	75	return rc;
79	76
80		− parent_port = cxl_mem_find_port(cxlmd);
	77	+ parent_port = cxl_mem_find_port(cxlmd, &dport);
81	78	if (!parent_port) {
82	79	dev_err(dev, "CXL port topology not found\n");
83	80	return -ENXIO;
@@ -91,7 +88,7 @@ static int cxl_mem_probe(struct device *dev) · 7 unmodified lines
91	88	goto unlock;
92	89	}
93	90
94		− rc = create_endpoint(cxlmd, parent_port);
	91	+ rc = devm_cxl_add_endpoint(cxlmd, dport);
95	92	unlock:
96	93	device_unlock(&parent_port->dev);
97	94	put_device(&parent_port->dev);

drivers/cxl/pci.c+45−1

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#include <linux/mutex.h>
9	9	#include <linux/list.h>
10	10	#include <linux/pci.h>
	11	+#include <linux/pci-doe.h>
11	12	#include <linux/io.h>
12	13	#include "cxlmem.h"
13	14	#include "cxlpci.h"
@@ -386,6 +387,47 @@ static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, · 372 unmodified lines
386	387	return rc;
387	388	}
388	389
	390	+static void cxl_pci_destroy_doe(void *mbs)
	391	+{
	392	+ xa_destroy(mbs);
	393	+}
	394	+
	395	+static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds)
	396	+{
	397	+ struct device *dev = cxlds->dev;
	398	+ struct pci_dev *pdev = to_pci_dev(dev);
	399	+ u16 off = 0;
	400	+
	401	+ xa_init(&cxlds->doe_mbs);
	402	+ if (devm_add_action(&pdev->dev, cxl_pci_destroy_doe, &cxlds->doe_mbs)) {
	403	+ dev_err(dev, "Failed to create XArray for DOE's\n");
	404	+ return;
	405	+ }
	406	+
	407	+ /*
	408	+ * Mailbox creation is best effort. Higher layers must determine if
	409	+ * the lack of a mailbox for their protocol is a device failure or not.
	410	+ */
	411	+ pci_doe_for_each_off(pdev, off) {
	412	+ struct pci_doe_mb *doe_mb;
	413	+
	414	+ doe_mb = pcim_doe_create_mb(pdev, off);
	415	+ if (IS_ERR(doe_mb)) {
	416	+ dev_err(dev, "Failed to create MB object for MB @ %x\n",
	417	+ off);
	418	+ continue;
	419	+ }
	420	+
	421	+ if (xa_insert(&cxlds->doe_mbs, off, doe_mb, GFP_KERNEL)) {
	422	+ dev_err(dev, "xa_insert failed to insert MB @ %x\n",
	423	+ off);
	424	+ continue;
	425	+ }
	426	+
	427	+ dev_dbg(dev, "Created DOE mailbox @%x\n", off);
	428	+ }
	429	+}
	430	+
389	431	static int cxl_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
390	432	{
391	433	struct cxl_register_map map;
@@ -434,6 +476,8 @@ static int cxl_pci_probe(struct pci_dev pdev, const struct pci_device_id id) · 42 unmodified lines
434	476
435	477	cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);
436	478
	479	+ devm_cxl_pci_create_doe(cxlds);
	480	+
437	481	rc = cxl_pci_setup_mailbox(cxlds);
438	482	if (rc)
439	483	return rc;
@@ -454,7 +498,7 @@ static int cxl_pci_probe(struct pci_dev pdev, const struct pci_device_id id) · 14 unmodified lines
454	498	if (IS_ERR(cxlmd))
455	499	return PTR_ERR(cxlmd);
456	500
457		− if (range_len(&cxlds->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))
	501	+ if (resource_size(&cxlds->pmem_res) && IS_ENABLED(CONFIG_CXL_PMEM))
458	502	rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd);
459	503
460	504	return rc;

drivers/cxl/pmem.c+254−5

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/ndctl.h>
8	8	#include <linux/async.h>
9	9	#include <linux/slab.h>
	10	+#include <linux/nd.h>
10	11	#include "cxlmem.h"
11	12	#include "cxl.h"
12	13
@@ -26,7 +27,23 @@ static void clear_exclusive(void *cxlds) · 13 unmodified lines
26	27
27	28	static void unregister_nvdimm(void *nvdimm)
28	29	{
	30	+ struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	31	+ struct cxl_nvdimm_bridge *cxl_nvb = cxl_nvd->bridge;
	32	+ struct cxl_pmem_region *cxlr_pmem;
	33	+
	34	+ device_lock(&cxl_nvb->dev);
	35	+ cxlr_pmem = cxl_nvd->region;
	36	+ dev_set_drvdata(&cxl_nvd->dev, NULL);
	37	+ cxl_nvd->region = NULL;
	38	+ device_unlock(&cxl_nvb->dev);
	39	+
	40	+ if (cxlr_pmem) {
	41	+ device_release_driver(&cxlr_pmem->dev);
	42	+ put_device(&cxlr_pmem->dev);
	43	+ }
	44	+
29	45	nvdimm_delete(nvdimm);
	46	+ cxl_nvd->bridge = NULL;
30	47	}
31	48
32	49	static int cxl_nvdimm_probe(struct device *dev)
@@ -39,7 +56,7 @@ static int cxl_nvdimm_probe(struct device *dev) · 6 unmodified lines
39	56	struct nvdimm *nvdimm;
40	57	int rc;
41	58
42		− cxl_nvb = cxl_find_nvdimm_bridge(cxl_nvd);
	59	+ cxl_nvb = cxl_find_nvdimm_bridge(dev);
43	60	if (!cxl_nvb)
44	61	return -ENXIO;
45	62
@@ -66,6 +83,7 @@ static int cxl_nvdimm_probe(struct device *dev) · 20 unmodified lines
66	83	}
67	84
68	85	dev_set_drvdata(dev, nvdimm);
	86	+ cxl_nvd->bridge = cxl_nvb;
69	87	rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
70	88	out:
71	89	device_unlock(&cxl_nvb->dev);
@@ -204,15 +222,38 @@ static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb) · 132 unmodified lines
204	222	return cxl_nvb->nvdimm_bus != NULL;
205	223	}
206	224
207		−static int cxl_nvdimm_release_driver(struct device dev, void data)
	225	+static int cxl_nvdimm_release_driver(struct device dev, void cxl_nvb)
208	226	{
	227	+ struct cxl_nvdimm *cxl_nvd;
	228	+
209	229	if (!is_cxl_nvdimm(dev))
210	230	return 0;
	231	+
	232	+ cxl_nvd = to_cxl_nvdimm(dev);
	233	+ if (cxl_nvd->bridge != cxl_nvb)
	234	+ return 0;
	235	+
211	236	device_release_driver(dev);
212	237	return 0;
213	238	}
214	239
215		−static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus)
	240	+static int cxl_pmem_region_release_driver(struct device dev, void cxl_nvb)
	241	+{
	242	+ struct cxl_pmem_region *cxlr_pmem;
	243	+
	244	+ if (!is_cxl_pmem_region(dev))
	245	+ return 0;
	246	+
	247	+ cxlr_pmem = to_cxl_pmem_region(dev);
	248	+ if (cxlr_pmem->bridge != cxl_nvb)
	249	+ return 0;
	250	+
	251	+ device_release_driver(dev);
	252	+ return 0;
	253	+}
	254	+
	255	+static void offline_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb,
	256	+ struct nvdimm_bus *nvdimm_bus)
216	257	{
217	258	if (!nvdimm_bus)
218	259	return;
@@ -222,7 +263,10 @@ static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus) · 3 unmodified lines
222	263	* nvdimm_bus_unregister() rips the nvdimm objects out from
223	264	* underneath them.
224	265	*/
225		− bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_release_driver);
	266	+ bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb,
	267	+ cxl_pmem_region_release_driver);
	268	+ bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb,
	269	+ cxl_nvdimm_release_driver);
226	270	nvdimm_bus_unregister(nvdimm_bus);
227	271	}
228	272
@@ -260,7 +304,7 @@ static void cxl_nvb_update_state(struct work_struct *work) · 31 unmodified lines
260	304
261	305	dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc);
262	306	}
263		− offline_nvdimm_bus(victim_bus);
	307	+ offline_nvdimm_bus(cxl_nvb, victim_bus);
264	308
265	309	put_device(&cxl_nvb->dev);
266	310	}
@@ -315,6 +359,203 @@ static struct cxl_driver cxl_nvdimm_bridge_driver = { · 48 unmodified lines
315	359	.id = CXL_DEVICE_NVDIMM_BRIDGE,
316	360	};
317	361
	362	+static int match_cxl_nvdimm(struct device dev, void data)
	363	+{
	364	+ return is_cxl_nvdimm(dev);
	365	+}
	366	+
	367	+static void unregister_nvdimm_region(void *nd_region)
	368	+{
	369	+ struct cxl_nvdimm_bridge *cxl_nvb;
	370	+ struct cxl_pmem_region *cxlr_pmem;
	371	+ int i;
	372	+
	373	+ cxlr_pmem = nd_region_provider_data(nd_region);
	374	+ cxl_nvb = cxlr_pmem->bridge;
	375	+ device_lock(&cxl_nvb->dev);
	376	+ for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
	377	+ struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
	378	+ struct cxl_nvdimm *cxl_nvd = m->cxl_nvd;
	379	+
	380	+ if (cxl_nvd->region) {
	381	+ put_device(&cxlr_pmem->dev);
	382	+ cxl_nvd->region = NULL;
	383	+ }
	384	+ }
	385	+ device_unlock(&cxl_nvb->dev);
	386	+
	387	+ nvdimm_region_delete(nd_region);
	388	+}
	389	+
	390	+static void cxlr_pmem_remove_resource(void *res)
	391	+{
	392	+ remove_resource(res);
	393	+}
	394	+
	395	+struct cxl_pmem_region_info {
	396	+ u64 offset;
	397	+ u64 serial;
	398	+};
	399	+
	400	+static int cxl_pmem_region_probe(struct device *dev)
	401	+{
	402	+ struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
	403	+ struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
	404	+ struct cxl_region *cxlr = cxlr_pmem->cxlr;
	405	+ struct cxl_pmem_region_info *info = NULL;
	406	+ struct cxl_nvdimm_bridge *cxl_nvb;
	407	+ struct nd_interleave_set *nd_set;
	408	+ struct nd_region_desc ndr_desc;
	409	+ struct cxl_nvdimm *cxl_nvd;
	410	+ struct nvdimm *nvdimm;
	411	+ struct resource *res;
	412	+ int rc, i = 0;
	413	+
	414	+ cxl_nvb = cxl_find_nvdimm_bridge(&cxlr_pmem->mapping[0].cxlmd->dev);
	415	+ if (!cxl_nvb) {
	416	+ dev_dbg(dev, "bridge not found\n");
	417	+ return -ENXIO;
	418	+ }
	419	+ cxlr_pmem->bridge = cxl_nvb;
	420	+
	421	+ device_lock(&cxl_nvb->dev);
	422	+ if (!cxl_nvb->nvdimm_bus) {
	423	+ dev_dbg(dev, "nvdimm bus not found\n");
	424	+ rc = -ENXIO;
	425	+ goto err;
	426	+ }
	427	+
	428	+ memset(&mappings, 0, sizeof(mappings));
	429	+ memset(&ndr_desc, 0, sizeof(ndr_desc));
	430	+
	431	+ res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
	432	+ if (!res) {
	433	+ rc = -ENOMEM;
	434	+ goto err;
	435	+ }
	436	+
	437	+ res->name = "Persistent Memory";
	438	+ res->start = cxlr_pmem->hpa_range.start;
	439	+ res->end = cxlr_pmem->hpa_range.end;
	440	+ res->flags = IORESOURCE_MEM;
	441	+ res->desc = IORES_DESC_PERSISTENT_MEMORY;
	442	+
	443	+ rc = insert_resource(&iomem_resource, res);
	444	+ if (rc)
	445	+ goto err;
	446	+
	447	+ rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
	448	+ if (rc)
	449	+ goto err;
	450	+
	451	+ ndr_desc.res = res;
	452	+ ndr_desc.provider_data = cxlr_pmem;
	453	+
	454	+ ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
	455	+ ndr_desc.target_node = phys_to_target_node(res->start);
	456	+ if (ndr_desc.target_node == NUMA_NO_NODE) {
	457	+ ndr_desc.target_node = ndr_desc.numa_node;
	458	+ dev_dbg(&cxlr->dev, "changing target node from %d to %d",
	459	+ NUMA_NO_NODE, ndr_desc.target_node);
	460	+ }
	461	+
	462	+ nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
	463	+ if (!nd_set) {
	464	+ rc = -ENOMEM;
	465	+ goto err;
	466	+ }
	467	+
	468	+ ndr_desc.memregion = cxlr->id;
	469	+ set_bit(ND_REGION_CXL, &ndr_desc.flags);
	470	+ set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
	471	+
	472	+ info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
	473	+ if (!info) {
	474	+ rc = -ENOMEM;
	475	+ goto err;
	476	+ }
	477	+
	478	+ for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
	479	+ struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
	480	+ struct cxl_memdev *cxlmd = m->cxlmd;
	481	+ struct cxl_dev_state *cxlds = cxlmd->cxlds;
	482	+ struct device *d;
	483	+
	484	+ d = device_find_child(&cxlmd->dev, NULL, match_cxl_nvdimm);
	485	+ if (!d) {
	486	+ dev_dbg(dev, "[%d]: %s: no cxl_nvdimm found\n", i,
	487	+ dev_name(&cxlmd->dev));
	488	+ rc = -ENODEV;
	489	+ goto err;
	490	+ }
	491	+
	492	+ /* safe to drop ref now with bridge lock held */
	493	+ put_device(d);
	494	+
	495	+ cxl_nvd = to_cxl_nvdimm(d);
	496	+ nvdimm = dev_get_drvdata(&cxl_nvd->dev);
	497	+ if (!nvdimm) {
	498	+ dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
	499	+ dev_name(&cxlmd->dev));
	500	+ rc = -ENODEV;
	501	+ goto err;
	502	+ }
	503	+ cxl_nvd->region = cxlr_pmem;
	504	+ get_device(&cxlr_pmem->dev);
	505	+ m->cxl_nvd = cxl_nvd;
	506	+ mappings[i] = (struct nd_mapping_desc) {
	507	+ .nvdimm = nvdimm,
	508	+ .start = m->start,
	509	+ .size = m->size,
	510	+ .position = i,
	511	+ };
	512	+ info[i].offset = m->start;
	513	+ info[i].serial = cxlds->serial;
	514	+ }
	515	+ ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
	516	+ ndr_desc.mapping = mappings;
	517	+
	518	+ /*
	519	+ * TODO enable CXL labels which skip the need for 'interleave-set cookie'
	520	+ */
	521	+ nd_set->cookie1 =
	522	+ nd_fletcher64(info, sizeof(info) cxlr_pmem->nr_mappings, 0);
	523	+ nd_set->cookie2 = nd_set->cookie1;
	524	+ ndr_desc.nd_set = nd_set;
	525	+
	526	+ cxlr_pmem->nd_region =
	527	+ nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
	528	+ if (!cxlr_pmem->nd_region) {
	529	+ rc = -ENOMEM;
	530	+ goto err;
	531	+ }
	532	+
	533	+ rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
	534	+ cxlr_pmem->nd_region);
	535	+out:
	536	+ kfree(info);
	537	+ device_unlock(&cxl_nvb->dev);
	538	+ put_device(&cxl_nvb->dev);
	539	+
	540	+ return rc;
	541	+
	542	+err:
	543	+ dev_dbg(dev, "failed to create nvdimm region\n");
	544	+ for (i--; i >= 0; i--) {
	545	+ nvdimm = mappings[i].nvdimm;
	546	+ cxl_nvd = nvdimm_provider_data(nvdimm);
	547	+ put_device(&cxl_nvd->region->dev);
	548	+ cxl_nvd->region = NULL;
	549	+ }
	550	+ goto out;
	551	+}
	552	+
	553	+static struct cxl_driver cxl_pmem_region_driver = {
	554	+ .name = "cxl_pmem_region",
	555	+ .probe = cxl_pmem_region_probe,
	556	+ .id = CXL_DEVICE_PMEM_REGION,
	557	+};
	558	+
318	559	/*
319	560	* Return all bridges to the CXL_NVB_NEW state to invalidate any
320	561	* ->state_work referring to the now destroyed cxl_pmem_wq.
@@ -359,8 +600,14 @@ static __init int cxl_pmem_init(void) · 38 unmodified lines
359	600	if (rc)
360	601	goto err_nvdimm;
361	602
	603	+ rc = cxl_driver_register(&cxl_pmem_region_driver);
	604	+ if (rc)
	605	+ goto err_region;
	606	+
362	607	return 0;
363	608
	609	+err_region:
	610	+ cxl_driver_unregister(&cxl_nvdimm_driver);
364	611	err_nvdimm:
365	612	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
366	613	err_bridge:
@@ -370,6 +617,7 @@ err_bridge: · 3 unmodified lines
370	617
371	618	static __exit void cxl_pmem_exit(void)
372	619	{
	620	+ cxl_driver_unregister(&cxl_pmem_region_driver);
373	621	cxl_driver_unregister(&cxl_nvdimm_driver);
374	622	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
375	623	destroy_cxl_pmem_wq();
@@ -381,3 +629,4 @@ module_exit(cxl_pmem_exit); · 5 unmodified lines
381	629	MODULE_IMPORT_NS(CXL);
382	630	MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
383	631	MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
	632	+MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);

drivers/cxl/port.c+53−0

@@ -53,6 +53,9 @@ static int cxl_port_probe(struct device *dev) · 52 unmodified lines
53	53	struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport);
54	54	struct cxl_dev_state *cxlds = cxlmd->cxlds;
55	55
	56	+ /* Cache the data early to ensure is_visible() works */
	57	+ read_cdat_data(port);
	58	+
56	59	get_device(&cxlmd->dev);
57	60	rc = devm_add_action_or_reset(dev, schedule_detach, cxlmd);
58	61	if (rc)
@@ -78,10 +81,60 @@ static int cxl_port_probe(struct device *dev) · 19 unmodified lines
78	81	return 0;
79	82	}
80	83
	84	+static ssize_t CDAT_read(struct file filp, struct kobject kobj,
	85	+ struct bin_attribute bin_attr, char buf,
	86	+ loff_t offset, size_t count)
	87	+{
	88	+ struct device *dev = kobj_to_dev(kobj);
	89	+ struct cxl_port *port = to_cxl_port(dev);
	90	+
	91	+ if (!port->cdat_available)
	92	+ return -ENXIO;
	93	+
	94	+ if (!port->cdat.table)
	95	+ return 0;
	96	+
	97	+ return memory_read_from_buffer(buf, count, &offset,
	98	+ port->cdat.table,
	99	+ port->cdat.length);
	100	+}
	101	+
	102	+static BIN_ATTR_ADMIN_RO(CDAT, 0);
	103	+
	104	+static umode_t cxl_port_bin_attr_is_visible(struct kobject *kobj,
	105	+ struct bin_attribute *attr, int i)
	106	+{
	107	+ struct device *dev = kobj_to_dev(kobj);
	108	+ struct cxl_port *port = to_cxl_port(dev);
	109	+
	110	+ if ((attr == &bin_attr_CDAT) && port->cdat_available)
	111	+ return attr->attr.mode;
	112	+
	113	+ return 0;
	114	+}
	115	+
	116	+static struct bin_attribute *cxl_cdat_bin_attributes[] = {
	117	+ &bin_attr_CDAT,
	118	+ NULL,
	119	+};
	120	+
	121	+static struct attribute_group cxl_cdat_attribute_group = {
	122	+ .bin_attrs = cxl_cdat_bin_attributes,
	123	+ .is_bin_visible = cxl_port_bin_attr_is_visible,
	124	+};
	125	+
	126	+static const struct attribute_group *cxl_port_attribute_groups[] = {
	127	+ &cxl_cdat_attribute_group,
	128	+ NULL,
	129	+};
	130	+
81	131	static struct cxl_driver cxl_port_driver = {
82	132	.name = "cxl_port",
83	133	.probe = cxl_port_probe,
84	134	.id = CXL_DEVICE_PORT,
	135	+ .drv = {
	136	+ .dev_groups = cxl_port_attribute_groups,
	137	+ },
85	138	};
86	139
87	140	module_cxl_driver(cxl_port_driver);

drivers/nvdimm/region_devs.c+21−7

@@ -133,7 +133,8 @@ static void nd_region_release(struct device *dev) · 132 unmodified lines
133	133	put_device(&nvdimm->dev);
134	134	}
135	135	free_percpu(nd_region->lane);
136		− memregion_free(nd_region->id);
	136	+ if (!test_bit(ND_REGION_CXL, &nd_region->flags))
	137	+ memregion_free(nd_region->id);
137	138	kfree(nd_region);
138	139	}
139	140
@@ -982,9 +983,14 @@ static struct nd_region nd_region_create(struct nvdimm_bus nvdimm_bus, · 842 unmodified lines
982	983
983	984	if (!nd_region)
984	985	return NULL;
985		− nd_region->id = memregion_alloc(GFP_KERNEL);
986		− if (nd_region->id < 0)
987		− goto err_id;
	986	+ /* CXL pre-assigns memregion ids before creating nvdimm regions */
	987	+ if (test_bit(ND_REGION_CXL, &ndr_desc->flags)) {
	988	+ nd_region->id = ndr_desc->memregion;
	989	+ } else {
	990	+ nd_region->id = memregion_alloc(GFP_KERNEL);
	991	+ if (nd_region->id < 0)
	992	+ goto err_id;
	993	+ }
988	994
989	995	nd_region->lane = alloc_percpu(struct nd_percpu_lane);
990	996	if (!nd_region->lane)
@@ -1043,9 +1049,10 @@ static struct nd_region nd_region_create(struct nvdimm_bus nvdimm_bus, · 52 unmodified lines
1043	1049
1044	1050	return nd_region;
1045	1051
1046		− err_percpu:
1047		− memregion_free(nd_region->id);
1048		− err_id:
	1052	+err_percpu:
	1053	+ if (!test_bit(ND_REGION_CXL, &ndr_desc->flags))
	1054	+ memregion_free(nd_region->id);
	1055	+err_id:
1049	1056	kfree(nd_region);
1050	1057	return NULL;
1051	1058	}
@@ -1068,6 +1075,13 @@ struct nd_region nvdimm_volatile_region_create(struct nvdimm_bus nvdimm_bus, · 16 unmodified lines
1068	1075	}
1069	1076	EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);
1070	1077
	1078	+void nvdimm_region_delete(struct nd_region *nd_region)
	1079	+{
	1080	+ if (nd_region)
	1081	+ nd_device_unregister(&nd_region->dev, ND_SYNC);
	1082	+}
	1083	+EXPORT_SYMBOL_GPL(nvdimm_region_delete);
	1084	+
1071	1085	int nvdimm_flush(struct nd_region nd_region, struct bio bio)
1072	1086	{
1073	1087	int rc = 0;

drivers/pci/Kconfig+3−0

@@ -121,6 +121,9 @@ config XEN_PCIDEV_FRONTEND · 120 unmodified lines
121	121	config PCI_ATS
122	122	bool
123	123
	124	+config PCI_DOE
	125	+ bool
	126	+
124	127	config PCI_ECAM
125	128	bool
126	129

drivers/pci/Makefile+1−0

@@ -31,6 +31,7 @@ obj-$(CONFIG_PCI_ECAM) += ecam.o · 30 unmodified lines
31	31	obj-$(CONFIG_PCI_P2PDMA) += p2pdma.o
32	32	obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o
33	33	obj-$(CONFIG_VGA_ARB) += vgaarb.o
	34	+obj-$(CONFIG_PCI_DOE) += doe.o
34	35
35	36	# Endpoint library must be initialized before its users
36	37	obj-$(CONFIG_PCI_ENDPOINT) += endpoint/

drivers/pci/doe.c+536−0

@@ -0,0 +1,536 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Data Object Exchange
	4	+ * PCIe r6.0, sec 6.30 DOE
	5	+ *
	6	+ * Copyright (C) 2021 Huawei
	7	+ * Jonathan Cameron <Jonathan.Cameron@huawei.com>
	8	+ *
	9	+ * Copyright (C) 2022 Intel Corporation
	10	+ * Ira Weiny <ira.weiny@intel.com>
	11	+ */
	12	+
	13	+#define dev_fmt(fmt) "DOE: " fmt
	14	+
	15	+#include <linux/bitfield.h>
	16	+#include <linux/delay.h>
	17	+#include <linux/jiffies.h>
	18	+#include <linux/mutex.h>
	19	+#include <linux/pci.h>
	20	+#include <linux/pci-doe.h>
	21	+#include <linux/workqueue.h>
	22	+
	23	+#define PCI_DOE_PROTOCOL_DISCOVERY 0
	24	+
	25	+/* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
	26	+#define PCI_DOE_TIMEOUT HZ
	27	+#define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128)
	28	+
	29	+#define PCI_DOE_FLAG_CANCEL 0
	30	+#define PCI_DOE_FLAG_DEAD 1
	31	+
	32	+/**
	33	+ * struct pci_doe_mb - State for a single DOE mailbox
	34	+ *
	35	+ * This state is used to manage a single DOE mailbox capability. All fields
	36	+ * should be considered opaque to the consumers and the structure passed into
	37	+ * the helpers below after being created by devm_pci_doe_create()
	38	+ *
	39	+ * @pdev: PCI device this mailbox belongs to
	40	+ * @cap_offset: Capability offset
	41	+ * @prots: Array of protocols supported (encoded as long values)
	42	+ * @wq: Wait queue for work item
	43	+ * @work_queue: Queue of pci_doe_work items
	44	+ * @flags: Bit array of PCI_DOE_FLAG_* flags
	45	+ */
	46	+struct pci_doe_mb {
	47	+ struct pci_dev *pdev;
	48	+ u16 cap_offset;
	49	+ struct xarray prots;
	50	+
	51	+ wait_queue_head_t wq;
	52	+ struct workqueue_struct *work_queue;
	53	+ unsigned long flags;
	54	+};
	55	+
	56	+static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
	57	+{
	58	+ if (wait_event_timeout(doe_mb->wq,
	59	+ test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
	60	+ timeout))
	61	+ return -EIO;
	62	+ return 0;
	63	+}
	64	+
	65	+static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
	66	+{
	67	+ struct pci_dev *pdev = doe_mb->pdev;
	68	+ int offset = doe_mb->cap_offset;
	69	+
	70	+ pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
	71	+}
	72	+
	73	+static int pci_doe_abort(struct pci_doe_mb *doe_mb)
	74	+{
	75	+ struct pci_dev *pdev = doe_mb->pdev;
	76	+ int offset = doe_mb->cap_offset;
	77	+ unsigned long timeout_jiffies;
	78	+
	79	+ pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
	80	+
	81	+ timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
	82	+ pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
	83	+
	84	+ do {
	85	+ int rc;
	86	+ u32 val;
	87	+
	88	+ rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
	89	+ if (rc)
	90	+ return rc;
	91	+ pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
	92	+
	93	+ /* Abort success! */
	94	+ if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
	95	+ !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
	96	+ return 0;
	97	+
	98	+ } while (!time_after(jiffies, timeout_jiffies));
	99	+
	100	+ /* Abort has timed out and the MB is dead */
	101	+ pci_err(pdev, "[%x] ABORT timed out\n", offset);
	102	+ return -EIO;
	103	+}
	104	+
	105	+static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
	106	+ struct pci_doe_task *task)
	107	+{
	108	+ struct pci_dev *pdev = doe_mb->pdev;
	109	+ int offset = doe_mb->cap_offset;
	110	+ u32 val;
	111	+ int i;
	112	+
	113	+ /*
	114	+ * Check the DOE busy bit is not set. If it is set, this could indicate
	115	+ * someone other than Linux (e.g. firmware) is using the mailbox. Note
	116	+ * it is expected that firmware and OS will negotiate access rights via
	117	+ * an, as yet to be defined, method.
	118	+ */
	119	+ pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
	120	+ if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
	121	+ return -EBUSY;
	122	+
	123	+ if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
	124	+ return -EIO;
	125	+
	126	+ /* Write DOE Header */
	127	+ val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) \|
	128	+ FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type);
	129	+ pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
	130	+ /* Length is 2 DW of header + length of payload in DW */
	131	+ pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
	132	+ FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
	133	+ 2 + task->request_pl_sz /
	134	+ sizeof(u32)));
	135	+ for (i = 0; i < task->request_pl_sz / sizeof(u32); i++)
	136	+ pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
	137	+ task->request_pl[i]);
	138	+
	139	+ pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
	140	+
	141	+ return 0;
	142	+}
	143	+
	144	+static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
	145	+{
	146	+ struct pci_dev *pdev = doe_mb->pdev;
	147	+ int offset = doe_mb->cap_offset;
	148	+ u32 val;
	149	+
	150	+ pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
	151	+ if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
	152	+ return true;
	153	+ return false;
	154	+}
	155	+
	156	+static int pci_doe_recv_resp(struct pci_doe_mb doe_mb, struct pci_doe_task task)
	157	+{
	158	+ struct pci_dev *pdev = doe_mb->pdev;
	159	+ int offset = doe_mb->cap_offset;
	160	+ size_t length, payload_length;
	161	+ u32 val;
	162	+ int i;
	163	+
	164	+ /* Read the first dword to get the protocol */
	165	+ pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
	166	+ if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) \|\|
	167	+ (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) {
	168	+ dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n",
	169	+ doe_mb->cap_offset, task->prot.vid, task->prot.type,
	170	+ FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
	171	+ FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
	172	+ return -EIO;
	173	+ }
	174	+
	175	+ pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
	176	+ /* Read the second dword to get the length */
	177	+ pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
	178	+ pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
	179	+
	180	+ length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
	181	+ if (length > SZ_1M \|\| length < 2)
	182	+ return -EIO;
	183	+
	184	+ /* First 2 dwords have already been read */
	185	+ length -= 2;
	186	+ payload_length = min(length, task->response_pl_sz / sizeof(u32));
	187	+ /* Read the rest of the response payload */
	188	+ for (i = 0; i < payload_length; i++) {
	189	+ pci_read_config_dword(pdev, offset + PCI_DOE_READ,
	190	+ &task->response_pl[i]);
	191	+ /* Prior to the last ack, ensure Data Object Ready */
	192	+ if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb))
	193	+ return -EIO;
	194	+ pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
	195	+ }
	196	+
	197	+ /* Flush excess length */
	198	+ for (; i < length; i++) {
	199	+ pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
	200	+ pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
	201	+ }
	202	+
	203	+ /* Final error check to pick up on any since Data Object Ready */
	204	+ pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
	205	+ if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
	206	+ return -EIO;
	207	+
	208	+ return min(length, task->response_pl_sz / sizeof(u32)) * sizeof(u32);
	209	+}
	210	+
	211	+static void signal_task_complete(struct pci_doe_task *task, int rv)
	212	+{
	213	+ task->rv = rv;
	214	+ task->complete(task);
	215	+}
	216	+
	217	+static void signal_task_abort(struct pci_doe_task *task, int rv)
	218	+{
	219	+ struct pci_doe_mb *doe_mb = task->doe_mb;
	220	+ struct pci_dev *pdev = doe_mb->pdev;
	221	+
	222	+ if (pci_doe_abort(doe_mb)) {
	223	+ /*
	224	+ * If the device can't process an abort; set the mailbox dead
	225	+ * - no more submissions
	226	+ */
	227	+ pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
	228	+ doe_mb->cap_offset);
	229	+ set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
	230	+ }
	231	+ signal_task_complete(task, rv);
	232	+}
	233	+
	234	+static void doe_statemachine_work(struct work_struct *work)
	235	+{
	236	+ struct pci_doe_task *task = container_of(work, struct pci_doe_task,
	237	+ work);
	238	+ struct pci_doe_mb *doe_mb = task->doe_mb;
	239	+ struct pci_dev *pdev = doe_mb->pdev;
	240	+ int offset = doe_mb->cap_offset;
	241	+ unsigned long timeout_jiffies;
	242	+ u32 val;
	243	+ int rc;
	244	+
	245	+ if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
	246	+ signal_task_complete(task, -EIO);
	247	+ return;
	248	+ }
	249	+
	250	+ /* Send request */
	251	+ rc = pci_doe_send_req(doe_mb, task);
	252	+ if (rc) {
	253	+ /*
	254	+ * The specification does not provide any guidance on how to
	255	+ * resolve conflicting requests from other entities.
	256	+ * Furthermore, it is likely that busy will not be detected
	257	+ * most of the time. Flag any detection of status busy with an
	258	+ * error.
	259	+ */
	260	+ if (rc == -EBUSY)
	261	+ dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
	262	+ offset);
	263	+ signal_task_abort(task, rc);
	264	+ return;
	265	+ }
	266	+
	267	+ timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
	268	+ /* Poll for response */
	269	+retry_resp:
	270	+ pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
	271	+ if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
	272	+ signal_task_abort(task, -EIO);
	273	+ return;
	274	+ }
	275	+
	276	+ if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
	277	+ if (time_after(jiffies, timeout_jiffies)) {
	278	+ signal_task_abort(task, -EIO);
	279	+ return;
	280	+ }
	281	+ rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
	282	+ if (rc) {
	283	+ signal_task_abort(task, rc);
	284	+ return;
	285	+ }
	286	+ goto retry_resp;
	287	+ }
	288	+
	289	+ rc = pci_doe_recv_resp(doe_mb, task);
	290	+ if (rc < 0) {
	291	+ signal_task_abort(task, rc);
	292	+ return;
	293	+ }
	294	+
	295	+ signal_task_complete(task, rc);
	296	+}
	297	+
	298	+static void pci_doe_task_complete(struct pci_doe_task *task)
	299	+{
	300	+ complete(task->private);
	301	+}
	302	+
	303	+static int pci_doe_discovery(struct pci_doe_mb doe_mb, u8 index, u16 *vid,
	304	+ u8 *protocol)
	305	+{
	306	+ u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
	307	+ *index);
	308	+ u32 response_pl;
	309	+ DECLARE_COMPLETION_ONSTACK(c);
	310	+ struct pci_doe_task task = {
	311	+ .prot.vid = PCI_VENDOR_ID_PCI_SIG,
	312	+ .prot.type = PCI_DOE_PROTOCOL_DISCOVERY,
	313	+ .request_pl = &request_pl,
	314	+ .request_pl_sz = sizeof(request_pl),
	315	+ .response_pl = &response_pl,
	316	+ .response_pl_sz = sizeof(response_pl),
	317	+ .complete = pci_doe_task_complete,
	318	+ .private = &c,
	319	+ };
	320	+ int rc;
	321	+
	322	+ rc = pci_doe_submit_task(doe_mb, &task);
	323	+ if (rc < 0)
	324	+ return rc;
	325	+
	326	+ wait_for_completion(&c);
	327	+
	328	+ if (task.rv != sizeof(response_pl))
	329	+ return -EIO;
	330	+
	331	+ *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
	332	+ *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL,
	333	+ response_pl);
	334	+ *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
	335	+ response_pl);
	336	+
	337	+ return 0;
	338	+}
	339	+
	340	+static void *pci_doe_xa_prot_entry(u16 vid, u8 prot)
	341	+{
	342	+ return xa_mk_value((vid << 8) \| prot);
	343	+}
	344	+
	345	+static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb)
	346	+{
	347	+ u8 index = 0;
	348	+ u8 xa_idx = 0;
	349	+
	350	+ do {
	351	+ int rc;
	352	+ u16 vid;
	353	+ u8 prot;
	354	+
	355	+ rc = pci_doe_discovery(doe_mb, &index, &vid, &prot);
	356	+ if (rc)
	357	+ return rc;
	358	+
	359	+ pci_dbg(doe_mb->pdev,
	360	+ "[%x] Found protocol %d vid: %x prot: %x\n",
	361	+ doe_mb->cap_offset, xa_idx, vid, prot);
	362	+
	363	+ rc = xa_insert(&doe_mb->prots, xa_idx++,
	364	+ pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL);
	365	+ if (rc)
	366	+ return rc;
	367	+ } while (index);
	368	+
	369	+ return 0;
	370	+}
	371	+
	372	+static void pci_doe_xa_destroy(void *mb)
	373	+{
	374	+ struct pci_doe_mb *doe_mb = mb;
	375	+
	376	+ xa_destroy(&doe_mb->prots);
	377	+}
	378	+
	379	+static void pci_doe_destroy_workqueue(void *mb)
	380	+{
	381	+ struct pci_doe_mb *doe_mb = mb;
	382	+
	383	+ destroy_workqueue(doe_mb->work_queue);
	384	+}
	385	+
	386	+static void pci_doe_flush_mb(void *mb)
	387	+{
	388	+ struct pci_doe_mb *doe_mb = mb;
	389	+
	390	+ /* Stop all pending work items from starting */
	391	+ set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
	392	+
	393	+ /* Cancel an in progress work item, if necessary */
	394	+ set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
	395	+ wake_up(&doe_mb->wq);
	396	+
	397	+ /* Flush all work items */
	398	+ flush_workqueue(doe_mb->work_queue);
	399	+}
	400	+
	401	+/**
	402	+ * pcim_doe_create_mb() - Create a DOE mailbox object
	403	+ *
	404	+ * @pdev: PCI device to create the DOE mailbox for
	405	+ * @cap_offset: Offset of the DOE mailbox
	406	+ *
	407	+ * Create a single mailbox object to manage the mailbox protocol at the
	408	+ * cap_offset specified.
	409	+ *
	410	+ * RETURNS: created mailbox object on success
	411	+ * ERR_PTR(-errno) on failure
	412	+ */
	413	+struct pci_doe_mb pcim_doe_create_mb(struct pci_dev pdev, u16 cap_offset)
	414	+{
	415	+ struct pci_doe_mb *doe_mb;
	416	+ struct device *dev = &pdev->dev;
	417	+ int rc;
	418	+
	419	+ doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL);
	420	+ if (!doe_mb)
	421	+ return ERR_PTR(-ENOMEM);
	422	+
	423	+ doe_mb->pdev = pdev;
	424	+ doe_mb->cap_offset = cap_offset;
	425	+ init_waitqueue_head(&doe_mb->wq);
	426	+
	427	+ xa_init(&doe_mb->prots);
	428	+ rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb);
	429	+ if (rc)
	430	+ return ERR_PTR(rc);
	431	+
	432	+ doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
	433	+ dev_driver_string(&pdev->dev),
	434	+ pci_name(pdev),
	435	+ doe_mb->cap_offset);
	436	+ if (!doe_mb->work_queue) {
	437	+ pci_err(pdev, "[%x] failed to allocate work queue\n",
	438	+ doe_mb->cap_offset);
	439	+ return ERR_PTR(-ENOMEM);
	440	+ }
	441	+ rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb);
	442	+ if (rc)
	443	+ return ERR_PTR(rc);
	444	+
	445	+ /* Reset the mailbox by issuing an abort */
	446	+ rc = pci_doe_abort(doe_mb);
	447	+ if (rc) {
	448	+ pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
	449	+ doe_mb->cap_offset, rc);
	450	+ return ERR_PTR(rc);
	451	+ }
	452	+
	453	+ /*
	454	+ * The state machine and the mailbox should be in sync now;
	455	+ * Set up mailbox flush prior to using the mailbox to query protocols.
	456	+ */
	457	+ rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
	458	+ if (rc)
	459	+ return ERR_PTR(rc);
	460	+
	461	+ rc = pci_doe_cache_protocols(doe_mb);
	462	+ if (rc) {
	463	+ pci_err(pdev, "[%x] failed to cache protocols : %d\n",
	464	+ doe_mb->cap_offset, rc);
	465	+ return ERR_PTR(rc);
	466	+ }
	467	+
	468	+ return doe_mb;
	469	+}
	470	+EXPORT_SYMBOL_GPL(pcim_doe_create_mb);
	471	+
	472	+/**
	473	+ * pci_doe_supports_prot() - Return if the DOE instance supports the given
	474	+ * protocol
	475	+ * @doe_mb: DOE mailbox capability to query
	476	+ * @vid: Protocol Vendor ID
	477	+ * @type: Protocol type
	478	+ *
	479	+ * RETURNS: True if the DOE mailbox supports the protocol specified
	480	+ */
	481	+bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
	482	+{
	483	+ unsigned long index;
	484	+ void *entry;
	485	+
	486	+ /* The discovery protocol must always be supported */
	487	+ if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY)
	488	+ return true;
	489	+
	490	+ xa_for_each(&doe_mb->prots, index, entry)
	491	+ if (entry == pci_doe_xa_prot_entry(vid, type))
	492	+ return true;
	493	+
	494	+ return false;
	495	+}
	496	+EXPORT_SYMBOL_GPL(pci_doe_supports_prot);
	497	+
	498	+/**
	499	+ * pci_doe_submit_task() - Submit a task to be processed by the state machine
	500	+ *
	501	+ * @doe_mb: DOE mailbox capability to submit to
	502	+ * @task: task to be queued
	503	+ *
	504	+ * Submit a DOE task (request/response) to the DOE mailbox to be processed.
	505	+ * Returns upon queueing the task object. If the queue is full this function
	506	+ * will sleep until there is room in the queue.
	507	+ *
	508	+ * task->complete will be called when the state machine is done processing this
	509	+ * task.
	510	+ *
	511	+ * Excess data will be discarded.
	512	+ *
	513	+ * RETURNS: 0 when task has been successfully queued, -ERRNO on error
	514	+ */
	515	+int pci_doe_submit_task(struct pci_doe_mb doe_mb, struct pci_doe_task task)
	516	+{
	517	+ if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type))
	518	+ return -EINVAL;
	519	+
	520	+ /*
	521	+ * DOE requests must be a whole number of DW and the response needs to
	522	+ * be big enough for at least 1 DW
	523	+ */
	524	+ if (task->request_pl_sz % sizeof(u32) \|\|
	525	+ task->response_pl_sz < sizeof(u32))
	526	+ return -EINVAL;
	527	+
	528	+ if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
	529	+ return -EIO;
	530	+
	531	+ task->doe_mb = doe_mb;
	532	+ INIT_WORK(&task->work, doe_statemachine_work);
	533	+ queue_work(doe_mb->work_queue, &task->work);
	534	+ return 0;
	535	+}
	536	+EXPORT_SYMBOL_GPL(pci_doe_submit_task);

drivers/pci/probe.c+1−1

@@ -2315,7 +2315,7 @@ EXPORT_SYMBOL(pci_alloc_dev); · 2314 unmodified lines
2315	2315
2316	2316	static bool pci_bus_crs_vendor_id(u32 l)
2317	2317	{
2318		− return (l & 0xffff) == 0x0001;
	2318	+ return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG;
2319	2319	}
2320	2320
2321	2321	static bool pci_bus_wait_crs(struct pci_bus bus, int devfn, u32 l,

include/linux/ioport.h+3−0

@@ -141,6 +141,7 @@ enum { · 140 unmodified lines
141	141	IORES_DESC_DEVICE_PRIVATE_MEMORY = 6,
142	142	IORES_DESC_RESERVED = 7,
143	143	IORES_DESC_SOFT_RESERVED = 8,
	144	+ IORES_DESC_CXL = 9,
144	145	};
145	146
146	147	/*
@@ -329,6 +330,8 @@ struct resource devm_request_free_mem_region(struct device dev, · 182 unmodified lines
329	330	struct resource *base, unsigned long size);
330	331	struct resource request_free_mem_region(struct resource base,
331	332	unsigned long size, const char *name);
	333	+struct resource alloc_free_mem_region(struct resource base,
	334	+ unsigned long size, unsigned long align, const char *name);
332	335
333	336	static inline void irqresource_disabled(struct resource *res, u32 irq)
334	337	{

include/linux/libnvdimm.h+5−0

@@ -59,6 +59,9 @@ enum { · 58 unmodified lines
59	59	/* Platform provides asynchronous flush mechanism */
60	60	ND_REGION_ASYNC = 3,
61	61
	62	+ /* Region was created by CXL subsystem */
	63	+ ND_REGION_CXL = 4,
	64	+
62	65	/* mark newly adjusted resources as requiring a label update */
63	66	DPA_RESOURCE_ADJUSTED = 1 << 0,
64	67	};
@@ -122,6 +125,7 @@ struct nd_region_desc { · 57 unmodified lines
122	125	int numa_node;
123	126	int target_node;
124	127	unsigned long flags;
	128	+ int memregion;
125	129	struct device_node *of_node;
126	130	int (flush)(struct nd_region nd_region, struct bio *bio);
127	131	};
@@ -259,6 +263,7 @@ static inline struct nvdimm nvdimm_create(struct nvdimm_bus nvdimm_bus, · 131 unmodified lines
259	263	cmd_mask, num_flush, flush_wpq, NULL, NULL, NULL);
260	264	}
261	265	void nvdimm_delete(struct nvdimm *nvdimm);
	266	+void nvdimm_region_delete(struct nd_region *nd_region);
262	267
263	268	const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd);
264	269	const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd);

include/linux/pci-doe.h+77−0

@@ -0,0 +1,77 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * Data Object Exchange
	4	+ * PCIe r6.0, sec 6.30 DOE
	5	+ *
	6	+ * Copyright (C) 2021 Huawei
	7	+ * Jonathan Cameron <Jonathan.Cameron@huawei.com>
	8	+ *
	9	+ * Copyright (C) 2022 Intel Corporation
	10	+ * Ira Weiny <ira.weiny@intel.com>
	11	+ */
	12	+
	13	+#ifndef LINUX_PCI_DOE_H
	14	+#define LINUX_PCI_DOE_H
	15	+
	16	+struct pci_doe_protocol {
	17	+ u16 vid;
	18	+ u8 type;
	19	+};
	20	+
	21	+struct pci_doe_mb;
	22	+
	23	+/**
	24	+ * struct pci_doe_task - represents a single query/response
	25	+ *
	26	+ * @prot: DOE Protocol
	27	+ * @request_pl: The request payload
	28	+ * @request_pl_sz: Size of the request payload (bytes)
	29	+ * @response_pl: The response payload
	30	+ * @response_pl_sz: Size of the response payload (bytes)
	31	+ * @rv: Return value. Length of received response or error (bytes)
	32	+ * @complete: Called when task is complete
	33	+ * @private: Private data for the consumer
	34	+ * @work: Used internally by the mailbox
	35	+ * @doe_mb: Used internally by the mailbox
	36	+ *
	37	+ * The payload sizes and rv are specified in bytes with the following
	38	+ * restrictions concerning the protocol.
	39	+ *
	40	+ * 1) The request_pl_sz must be a multiple of double words (4 bytes)
	41	+ * 2) The response_pl_sz must be >= a single double word (4 bytes)
	42	+ * 3) rv is returned as bytes but it will be a multiple of double words
	43	+ *
	44	+ * NOTE there is no need for the caller to initialize work or doe_mb.
	45	+ */
	46	+struct pci_doe_task {
	47	+ struct pci_doe_protocol prot;
	48	+ u32 *request_pl;
	49	+ size_t request_pl_sz;
	50	+ u32 *response_pl;
	51	+ size_t response_pl_sz;
	52	+ int rv;
	53	+ void (complete)(struct pci_doe_task task);
	54	+ void *private;
	55	+
	56	+ /* No need for the user to initialize these fields */
	57	+ struct work_struct work;
	58	+ struct pci_doe_mb *doe_mb;
	59	+};
	60	+
	61	+/**
	62	+ * pci_doe_for_each_off - Iterate each DOE capability
	63	+ * @pdev: struct pci_dev to iterate
	64	+ * @off: u16 of config space offset of each mailbox capability found
	65	+ */
	66	+#define pci_doe_for_each_off(pdev, off) \
	67	+ for (off = pci_find_next_ext_capability(pdev, off, \
	68	+ PCI_EXT_CAP_ID_DOE); \
	69	+ off > 0; \
	70	+ off = pci_find_next_ext_capability(pdev, off, \
	71	+ PCI_EXT_CAP_ID_DOE))
	72	+
	73	+struct pci_doe_mb pcim_doe_create_mb(struct pci_dev pdev, u16 cap_offset);
	74	+bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type);
	75	+int pci_doe_submit_task(struct pci_doe_mb doe_mb, struct pci_doe_task task);
	76	+
	77	+#endif

include/linux/pci_ids.h+1−0

@@ -151,6 +151,7 @@ · 150 unmodified lines
151	151	#define PCI_CLASS_OTHERS 0xff
152	152
153	153	/* Vendors and devices. Sort key: vendor first, device next. */
	154	+#define PCI_VENDOR_ID_PCI_SIG 0x0001
154	155
155	156	#define PCI_VENDOR_ID_LOONGSON 0x0014
156	157

include/linux/sysfs.h+16−0

@@ -235,6 +235,22 @@ struct bin_attribute bin_attr_##_name = __BIN_ATTR_WO(_name, _size) · 234 unmodified lines
235	235	#define BIN_ATTR_RW(_name, _size) \
236	236	struct bin_attribute bin_attr_##_name = __BIN_ATTR_RW(_name, _size)
237	237
	238	+
	239	+#define __BIN_ATTR_ADMIN_RO(_name, _size) { \
	240	+ .attr = { .name = __stringify(_name), .mode = 0400 }, \
	241	+ .read = _name##_read, \
	242	+ .size = _size, \
	243	+}
	244	+
	245	+#define __BIN_ATTR_ADMIN_RW(_name, _size) \
	246	+ __BIN_ATTR(_name, 0600, _name##_read, _name##_write, _size)
	247	+
	248	+#define BIN_ATTR_ADMIN_RO(_name, _size) \
	249	+struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RO(_name, _size)
	250	+
	251	+#define BIN_ATTR_ADMIN_RW(_name, _size) \
	252	+struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RW(_name, _size)
	253	+
238	254	struct sysfs_ops {
239	255	ssize_t (show)(struct kobject , struct attribute , char );
240	256	ssize_t (store)(struct kobject , struct attribute , const char , size_t);

include/uapi/linux/pci_regs.h+28−1

@@ -737,7 +737,8 @@ · 736 unmodified lines
737	737	#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
738	738	#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
739	739	#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
740		−#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_PL_16GT
	740	+#define PCI_EXT_CAP_ID_DOE 0x2E /* Data Object Exchange */
	741	+#define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_DOE
741	742
742	743	#define PCI_EXT_CAP_DSN_SIZEOF 12
743	744	#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
@@ -1103,4 +1104,30 @@ · 359 unmodified lines
1103	1104	#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK 0x000000F0
1104	1105	#define PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT 4
1105	1106
	1107	+/* Data Object Exchange */
	1108	+#define PCI_DOE_CAP 0x04 /* DOE Capabilities Register */
	1109	+#define PCI_DOE_CAP_INT_SUP 0x00000001 /* Interrupt Support */
	1110	+#define PCI_DOE_CAP_INT_MSG_NUM 0x00000ffe /* Interrupt Message Number */
	1111	+#define PCI_DOE_CTRL 0x08 /* DOE Control Register */
	1112	+#define PCI_DOE_CTRL_ABORT 0x00000001 /* DOE Abort */
	1113	+#define PCI_DOE_CTRL_INT_EN 0x00000002 /* DOE Interrupt Enable */
	1114	+#define PCI_DOE_CTRL_GO 0x80000000 /* DOE Go */
	1115	+#define PCI_DOE_STATUS 0x0c /* DOE Status Register */
	1116	+#define PCI_DOE_STATUS_BUSY 0x00000001 /* DOE Busy */
	1117	+#define PCI_DOE_STATUS_INT_STATUS 0x00000002 /* DOE Interrupt Status */
	1118	+#define PCI_DOE_STATUS_ERROR 0x00000004 /* DOE Error */
	1119	+#define PCI_DOE_STATUS_DATA_OBJECT_READY 0x80000000 /* Data Object Ready */
	1120	+#define PCI_DOE_WRITE 0x10 /* DOE Write Data Mailbox Register */
	1121	+#define PCI_DOE_READ 0x14 /* DOE Read Data Mailbox Register */
	1122	+
	1123	+/* DOE Data Object - note not actually registers */
	1124	+#define PCI_DOE_DATA_OBJECT_HEADER_1_VID 0x0000ffff
	1125	+#define PCI_DOE_DATA_OBJECT_HEADER_1_TYPE 0x00ff0000
	1126	+#define PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH 0x0003ffff
	1127	+
	1128	+#define PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX 0x000000ff
	1129	+#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID 0x0000ffff
	1130	+#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL 0x00ff0000
	1131	+#define PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX 0xff000000
	1132	+
1106	1133	#endif /* LINUX_PCI_REGS_H */

kernel/resource.c+150−35

@@ -489,8 +489,9 @@ int __weak page_is_ram(unsigned long pfn) · 488 unmodified lines
489	489	}
490	490	EXPORT_SYMBOL_GPL(page_is_ram);
491	491
492		−static int __region_intersects(resource_size_t start, size_t size,
493		− unsigned long flags, unsigned long desc)
	492	+static int __region_intersects(struct resource *parent, resource_size_t start,
	493	+ size_t size, unsigned long flags,
	494	+ unsigned long desc)
494	495	{
495	496	struct resource res;
496	497	int type = 0; int other = 0;
@@ -499,7 +500,7 @@ static int __region_intersects(resource_size_t start, size_t size, · 2 unmodified lines
499	500	res.start = start;
500	501	res.end = start + size - 1;
501	502
502		− for (p = iomem_resource.child; p ; p = p->sibling) {
	503	+ for (p = parent->child; p ; p = p->sibling) {
503	504	bool is_type = (((p->flags & flags) == flags) &&
504	505	((desc == IORES_DESC_NONE) \|\|
505	506	(desc == p->desc)));
@@ -543,7 +544,7 @@ int region_intersects(resource_size_t start, size_t size, unsigned long flags, · 37 unmodified lines
543	544	int ret;
544	545
545	546	read_lock(&resource_lock);
546		− ret = __region_intersects(start, size, flags, desc);
	547	+ ret = __region_intersects(&iomem_resource, start, size, flags, desc);
547	548	read_unlock(&resource_lock);
548	549
549	550	return ret;
@@ -891,6 +892,13 @@ void insert_resource_expand_to_fit(struct resource root, struct resource new) · 341 unmodified lines
891	892	}
892	893	write_unlock(&resource_lock);
893	894	}
	895	+/*
	896	+ * Not for general consumption, only early boot memory map parsing, PCI
	897	+ * resource discovery, and late discovery of CXL resources are expected
	898	+ * to use this interface. The former are built-in and only the latter,
	899	+ * CXL, is a module.
	900	+ */
	901	+EXPORT_SYMBOL_NS_GPL(insert_resource_expand_to_fit, CXL);
894	902
895	903	/**
896	904	* remove_resource - Remove a resource in the resource tree
@@ -1773,62 +1781,139 @@ void resource_list_free(struct list_head *head) · 876 unmodified lines
1773	1781	}
1774	1782	EXPORT_SYMBOL(resource_list_free);
1775	1783
1776		−#ifdef CONFIG_DEVICE_PRIVATE
1777		−static struct resource __request_free_mem_region(struct device dev,
1778		− struct resource base, unsigned long size, const char name)
	1784	+#ifdef CONFIG_GET_FREE_REGION
	1785	+#define GFR_DESCENDING (1UL << 0)
	1786	+#define GFR_REQUEST_REGION (1UL << 1)
	1787	+#define GFR_DEFAULT_ALIGN (1UL << PA_SECTION_SHIFT)
	1788	+
	1789	+static resource_size_t gfr_start(struct resource *base, resource_size_t size,
	1790	+ resource_size_t align, unsigned long flags)
	1791	+{
	1792	+ if (flags & GFR_DESCENDING) {
	1793	+ resource_size_t end;
	1794	+
	1795	+ end = min_t(resource_size_t, base->end,
	1796	+ (1ULL << MAX_PHYSMEM_BITS) - 1);
	1797	+ return end - size + 1;
	1798	+ }
	1799	+
	1800	+ return ALIGN(base->start, align);
	1801	+}
	1802	+
	1803	+static bool gfr_continue(struct resource *base, resource_size_t addr,
	1804	+ resource_size_t size, unsigned long flags)
	1805	+{
	1806	+ if (flags & GFR_DESCENDING)
	1807	+ return addr > size && addr >= base->start;
	1808	+ /*
	1809	+ * In the ascend case be careful that the last increment by
	1810	+ * @size did not wrap 0.
	1811	+ */
	1812	+ return addr > addr - size &&
	1813	+ addr <= min_t(resource_size_t, base->end,
	1814	+ (1ULL << MAX_PHYSMEM_BITS) - 1);
	1815	+}
	1816	+
	1817	+static resource_size_t gfr_next(resource_size_t addr, resource_size_t size,
	1818	+ unsigned long flags)
1779	1819	{
1780		− resource_size_t end, addr;
	1820	+ if (flags & GFR_DESCENDING)
	1821	+ return addr - size;
	1822	+ return addr + size;
	1823	+}
	1824	+
	1825	+static void remove_free_mem_region(void *_res)
	1826	+{
	1827	+ struct resource *res = _res;
	1828	+
	1829	+ if (res->parent)
	1830	+ remove_resource(res);
	1831	+ free_resource(res);
	1832	+}
	1833	+
	1834	+static struct resource *
	1835	+get_free_mem_region(struct device dev, struct resource base,
	1836	+ resource_size_t size, const unsigned long align,
	1837	+ const char *name, const unsigned long desc,
	1838	+ const unsigned long flags)
	1839	+{
	1840	+ resource_size_t addr;
1781	1841	struct resource *res;
1782	1842	struct region_devres *dr = NULL;
1783	1843
1784		− size = ALIGN(size, 1UL << PA_SECTION_SHIFT);
1785		− end = min_t(unsigned long, base->end, (1UL << MAX_PHYSMEM_BITS) - 1);
1786		− addr = end - size + 1UL;
	1844	+ size = ALIGN(size, align);
1787	1845
1788	1846	res = alloc_resource(GFP_KERNEL);
1789	1847	if (!res)
1790	1848	return ERR_PTR(-ENOMEM);
1791	1849
1792		− if (dev) {
	1850	+ if (dev && (flags & GFR_REQUEST_REGION)) {
1793	1851	dr = devres_alloc(devm_region_release,
1794	1852	sizeof(struct region_devres), GFP_KERNEL);
1795	1853	if (!dr) {
1796	1854	free_resource(res);
1797	1855	return ERR_PTR(-ENOMEM);
1798	1856	}
	1857	+ } else if (dev) {
	1858	+ if (devm_add_action_or_reset(dev, remove_free_mem_region, res))
	1859	+ return ERR_PTR(-ENOMEM);
1799	1860	}
1800	1861
1801	1862	write_lock(&resource_lock);
1802		− for (; addr > size && addr >= base->start; addr -= size) {
1803		− if (__region_intersects(addr, size, 0, IORES_DESC_NONE) !=
1804		− REGION_DISJOINT)
	1863	+ for (addr = gfr_start(base, size, align, flags);
	1864	+ gfr_continue(base, addr, size, flags);
	1865	+ addr = gfr_next(addr, size, flags)) {
	1866	+ if (__region_intersects(base, addr, size, 0, IORES_DESC_NONE) !=
	1867	+ REGION_DISJOINT)
1805	1868	continue;
1806	1869
1807		− if (__request_region_locked(res, &iomem_resource, addr, size,
1808		− name, 0))
1809		− break;
	1870	+ if (flags & GFR_REQUEST_REGION) {
	1871	+ if (__request_region_locked(res, &iomem_resource, addr,
	1872	+ size, name, 0))
	1873	+ break;
1810	1874
1811		− if (dev) {
1812		− dr->parent = &iomem_resource;
1813		− dr->start = addr;
1814		− dr->n = size;
1815		− devres_add(dev, dr);
1816		− }
	1875	+ if (dev) {
	1876	+ dr->parent = &iomem_resource;
	1877	+ dr->start = addr;
	1878	+ dr->n = size;
	1879	+ devres_add(dev, dr);
	1880	+ }
1817	1881
1818		− res->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
1819		− write_unlock(&resource_lock);
	1882	+ res->desc = desc;
	1883	+ write_unlock(&resource_lock);
	1884	+
	1885	+
	1886	+ /*
	1887	+ * A driver is claiming this region so revoke any
	1888	+ * mappings.
	1889	+ */
	1890	+ revoke_iomem(res);
	1891	+ } else {
	1892	+ res->start = addr;
	1893	+ res->end = addr + size - 1;
	1894	+ res->name = name;
	1895	+ res->desc = desc;
	1896	+ res->flags = IORESOURCE_MEM;
	1897	+
	1898	+ /*
	1899	+ * Only succeed if the resource hosts an exclusive
	1900	+ * range after the insert
	1901	+ */
	1902	+ if (__insert_resource(base, res) \|\| res->child)
	1903	+ break;
	1904	+
	1905	+ write_unlock(&resource_lock);
	1906	+ }
1820	1907
1821		− /*
1822		− * A driver is claiming this region so revoke any mappings.
1823		− */
1824		− revoke_iomem(res);
1825	1908	return res;
1826	1909	}
1827	1910	write_unlock(&resource_lock);
1828	1911
1829		− free_resource(res);
1830		− if (dr)
	1912	+ if (flags & GFR_REQUEST_REGION) {
	1913	+ free_resource(res);
1831	1914	devres_free(dr);
	1915	+ } else if (dev)
	1916	+ devm_release_action(dev, remove_free_mem_region, res);
1832	1917
1833	1918	return ERR_PTR(-ERANGE);
1834	1919	}
@@ -1847,18 +1932,48 @@ static struct resource __request_free_mem_region(struct device dev, · 12 unmodified lines
1847	1932	struct resource devm_request_free_mem_region(struct device dev,
1848	1933	struct resource *base, unsigned long size)
1849	1934	{
1850		− return __request_free_mem_region(dev, base, size, dev_name(dev));
	1935	+ unsigned long flags = GFR_DESCENDING \| GFR_REQUEST_REGION;
	1936	+
	1937	+ return get_free_mem_region(dev, base, size, GFR_DEFAULT_ALIGN,
	1938	+ dev_name(dev),
	1939	+ IORES_DESC_DEVICE_PRIVATE_MEMORY, flags);
1851	1940	}
1852	1941	EXPORT_SYMBOL_GPL(devm_request_free_mem_region);
1853	1942
1854	1943	struct resource request_free_mem_region(struct resource base,
1855	1944	unsigned long size, const char *name)
1856	1945	{
1857		− return __request_free_mem_region(NULL, base, size, name);
	1946	+ unsigned long flags = GFR_DESCENDING \| GFR_REQUEST_REGION;
	1947	+
	1948	+ return get_free_mem_region(NULL, base, size, GFR_DEFAULT_ALIGN, name,
	1949	+ IORES_DESC_DEVICE_PRIVATE_MEMORY, flags);
1858	1950	}
1859	1951	EXPORT_SYMBOL_GPL(request_free_mem_region);
1860	1952
1861		−#endif /* CONFIG_DEVICE_PRIVATE */
	1953	+/**
	1954	+ * alloc_free_mem_region - find a free region relative to @base
	1955	+ * @base: resource that will parent the new resource
	1956	+ * @size: size in bytes of memory to allocate from @base
	1957	+ * @align: alignment requirements for the allocation
	1958	+ * @name: resource name
	1959	+ *
	1960	+ * Buses like CXL, that can dynamically instantiate new memory regions,
	1961	+ * need a method to allocate physical address space for those regions.
	1962	+ * Allocate and insert a new resource to cover a free, unclaimed by a
	1963	+ * descendant of @base, range in the span of @base.
	1964	+ */
	1965	+struct resource alloc_free_mem_region(struct resource base,
	1966	+ unsigned long size, unsigned long align,
	1967	+ const char *name)
	1968	+{
	1969	+ /* Default of ascending direction and insert resource */
	1970	+ unsigned long flags = 0;
	1971	+
	1972	+ return get_free_mem_region(NULL, base, size, align, name,
	1973	+ IORES_DESC_NONE, flags);
	1974	+}
	1975	+EXPORT_SYMBOL_NS_GPL(alloc_free_mem_region, CXL);
	1976	+#endif /* CONFIG_GET_FREE_REGION */
1862	1977
1863	1978	static int __init strict_iomem(char *str)
1864	1979	{

mm/Kconfig+5−0

@@ -983,9 +983,14 @@ config HMM_MIRROR · 982 unmodified lines
983	983	bool
984	984	depends on MMU
985	985
	986	+config GET_FREE_REGION
	987	+ depends on SPARSEMEM
	988	+ bool
	989	+
986	990	config DEVICE_PRIVATE
987	991	bool "Unaddressable device memory (GPU memory, ...)"
988	992	depends on ZONE_DEVICE
	993	+ select GET_FREE_REGION
989	994
990	995	help
991	996	Allows creation of struct pages to represent unaddressable device

tools/testing/cxl/Kbuild+1−0

@@ -47,6 +47,7 @@ cxl_core-y += $(CXL_CORE_SRC)/memdev.o · 46 unmodified lines
47	47	cxl_core-y += $(CXL_CORE_SRC)/mbox.o
48	48	cxl_core-y += $(CXL_CORE_SRC)/pci.o
49	49	cxl_core-y += $(CXL_CORE_SRC)/hdm.o
	50	+cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o
50	51	cxl_core-y += config_check.o
51	52
52	53	obj-m += test/

tools/testing/cxl/test/cxl.c+76−55

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14	#define NR_CXL_HOST_BRIDGES 2
15	15	#define NR_CXL_ROOT_PORTS 2
16	16	#define NR_CXL_SWITCH_PORTS 2
17		−#define NR_CXL_PORT_DECODERS 2
	17	+#define NR_CXL_PORT_DECODERS 8
18	18
19	19	static struct platform_device *cxl_acpi;
20	20	static struct platform_device *cxl_host_bridge[NR_CXL_HOST_BRIDGES];
@@ -118,7 +118,7 @@ static struct { · 97 unmodified lines
118	118	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
119	119	ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
120	120	.qtg_id = 0,
121		− .window_size = SZ_256M,
	121	+ .window_size = SZ_256M * 4UL,
122	122	},
123	123	.target = { 0 },
124	124	},
@@ -133,7 +133,7 @@ static struct { · 8 unmodified lines
133	133	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
134	134	ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
135	135	.qtg_id = 1,
136		− .window_size = SZ_256M * 2,
	136	+ .window_size = SZ_256M * 8UL,
137	137	},
138	138	.target = { 0, 1, },
139	139	},
@@ -148,7 +148,7 @@ static struct { · 8 unmodified lines
148	148	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
149	149	ACPI_CEDT_CFMWS_RESTRICT_PMEM,
150	150	.qtg_id = 2,
151		− .window_size = SZ_256M,
	151	+ .window_size = SZ_256M * 4UL,
152	152	},
153	153	.target = { 0 },
154	154	},
@@ -163,7 +163,7 @@ static struct { · 8 unmodified lines
163	163	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
164	164	ACPI_CEDT_CFMWS_RESTRICT_PMEM,
165	165	.qtg_id = 3,
166		− .window_size = SZ_256M * 2,
	166	+ .window_size = SZ_256M * 8UL,
167	167	},
168	168	.target = { 0, 1, },
169	169	},
@@ -429,6 +429,50 @@ static int map_targets(struct device dev, void data) · 259 unmodified lines
429	429	return 0;
430	430	}
431	431
	432	+static int mock_decoder_commit(struct cxl_decoder *cxld)
	433	+{
	434	+ struct cxl_port *port = to_cxl_port(cxld->dev.parent);
	435	+ int id = cxld->id;
	436	+
	437	+ if (cxld->flags & CXL_DECODER_F_ENABLE)
	438	+ return 0;
	439	+
	440	+ dev_dbg(&port->dev, "%s commit\n", dev_name(&cxld->dev));
	441	+ if (port->commit_end + 1 != id) {
	442	+ dev_dbg(&port->dev,
	443	+ "%s: out of order commit, expected decoder%d.%d\n",
	444	+ dev_name(&cxld->dev), port->id, port->commit_end + 1);
	445	+ return -EBUSY;
	446	+ }
	447	+
	448	+ port->commit_end++;
	449	+ cxld->flags \|= CXL_DECODER_F_ENABLE;
	450	+
	451	+ return 0;
	452	+}
	453	+
	454	+static int mock_decoder_reset(struct cxl_decoder *cxld)
	455	+{
	456	+ struct cxl_port *port = to_cxl_port(cxld->dev.parent);
	457	+ int id = cxld->id;
	458	+
	459	+ if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
	460	+ return 0;
	461	+
	462	+ dev_dbg(&port->dev, "%s reset\n", dev_name(&cxld->dev));
	463	+ if (port->commit_end != id) {
	464	+ dev_dbg(&port->dev,
	465	+ "%s: out of order reset, expected decoder%d.%d\n",
	466	+ dev_name(&cxld->dev), port->id, port->commit_end);
	467	+ return -EBUSY;
	468	+ }
	469	+
	470	+ port->commit_end--;
	471	+ cxld->flags &= ~CXL_DECODER_F_ENABLE;
	472	+
	473	+ return 0;
	474	+}
	475	+
432	476	static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
433	477	{
434	478	struct cxl_port *port = cxlhdm->port;
@@ -451,25 +495,39 @@ static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) · 16 unmodified lines
451	495	struct cxl_decoder *cxld;
452	496	int rc;
453	497
454		− if (target_count)
455		− cxld = cxl_switch_decoder_alloc(port, target_count);
456		− else
457		− cxld = cxl_endpoint_decoder_alloc(port);
458		− if (IS_ERR(cxld)) {
459		− dev_warn(&port->dev,
460		− "Failed to allocate the decoder\n");
461		− return PTR_ERR(cxld);
	498	+ if (target_count) {
	499	+ struct cxl_switch_decoder *cxlsd;
	500	+
	501	+ cxlsd = cxl_switch_decoder_alloc(port, target_count);
	502	+ if (IS_ERR(cxlsd)) {
	503	+ dev_warn(&port->dev,
	504	+ "Failed to allocate the decoder\n");
	505	+ return PTR_ERR(cxlsd);
	506	+ }
	507	+ cxld = &cxlsd->cxld;
	508	+ } else {
	509	+ struct cxl_endpoint_decoder *cxled;
	510	+
	511	+ cxled = cxl_endpoint_decoder_alloc(port);
	512	+
	513	+ if (IS_ERR(cxled)) {
	514	+ dev_warn(&port->dev,
	515	+ "Failed to allocate the decoder\n");
	516	+ return PTR_ERR(cxled);
	517	+ }
	518	+ cxld = &cxled->cxld;
462	519	}
463	520
464		− cxld->decoder_range = (struct range) {
	521	+ cxld->hpa_range = (struct range) {
465	522	.start = 0,
466	523	.end = -1,
467	524	};
468	525
469		− cxld->flags = CXL_DECODER_F_ENABLE;
470	526	cxld->interleave_ways = min_not_zero(target_count, 1);
471	527	cxld->interleave_granularity = SZ_4K;
472	528	cxld->target_type = CXL_DECODER_EXPANDER;
	529	+ cxld->commit = mock_decoder_commit;
	530	+ cxld->reset = mock_decoder_reset;
473	531
474	532	if (target_count) {
475	533	rc = device_for_each_child(port->uport, &ctx,
@@ -569,44 +627,6 @@ static void mock_companion(struct acpi_device adev, struct device dev) · 93 unmodified lines
569	627	#define SZ_512G (SZ_64G * 8)
570	628	#endif
571	629
572		−static struct platform_device *alloc_memdev(int id)
573		−{
574		− struct resource res[] = {
575		− [0] = {
576		− .flags = IORESOURCE_MEM,
577		− },
578		− [1] = {
579		− .flags = IORESOURCE_MEM,
580		− .desc = IORES_DESC_PERSISTENT_MEMORY,
581		− },
582		− };
583		− struct platform_device *pdev;
584		− int i, rc;
585		−
586		− for (i = 0; i < ARRAY_SIZE(res); i++) {
587		− struct cxl_mock_res *r = alloc_mock_res(SZ_256M);
588		−
589		− if (!r)
590		− return NULL;
591		− res[i].start = r->range.start;
592		− res[i].end = r->range.end;
593		− }
594		−
595		− pdev = platform_device_alloc("cxl_mem", id);
596		− if (!pdev)
597		− return NULL;
598		−
599		− rc = platform_device_add_resources(pdev, res, ARRAY_SIZE(res));
600		− if (rc)
601		− goto err;
602		−
603		− return pdev;
604		−
605		−err:
606		− platform_device_put(pdev);
607		− return NULL;
608		−}
609		−
610	630	static __init int cxl_test_init(void)
611	631	{
612	632	int rc, i;
@@ -619,7 +639,8 @@ static __init int cxl_test_init(void) · 6 unmodified lines
619	639	goto err_gen_pool_create;
620	640	}
621	641
622		− rc = gen_pool_add(cxl_mock_pool, SZ_512G, SZ_64G, NUMA_NO_NODE);
	642	+ rc = gen_pool_add(cxl_mock_pool, iomem_resource.end + 1 - SZ_64G,
	643	+ SZ_64G, NUMA_NO_NODE);
623	644	if (rc)
624	645	goto err_gen_pool_add;
625	646
@@ -708,7 +729,7 @@ static __init int cxl_test_init(void) · 82 unmodified lines
708	729	struct platform_device *dport = cxl_switch_dport[i];
709	730	struct platform_device *pdev;
710	731
711		− pdev = alloc_memdev(i);
	732	+ pdev = platform_device_alloc("cxl_mem", i);
712	733	if (!pdev)
713	734	goto err_mem;
714	735	pdev->dev.parent = &dport->dev;

tools/testing/cxl/test/mem.c+28−25

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <cxlmem.h>
11	11
12	12	#define LSA_SIZE SZ_128K
	13	+#define DEV_SIZE SZ_2G
13	14	#define EFFECT(x) (1U << x)
14	15
15	16	static struct cxl_cel_entry mock_cel[] = {
@@ -25,6 +26,10 @@ static struct cxl_cel_entry mock_cel[] = { · 9 unmodified lines
25	26	.opcode = cpu_to_le16(CXL_MBOX_OP_GET_LSA),
26	27	.effect = cpu_to_le16(0),
27	28	},
	29	+ {
	30	+ .opcode = cpu_to_le16(CXL_MBOX_OP_GET_PARTITION_INFO),
	31	+ .effect = cpu_to_le16(0),
	32	+ },
28	33	{
29	34	.opcode = cpu_to_le16(CXL_MBOX_OP_SET_LSA),
30	35	.effect = cpu_to_le16(EFFECT(1) \| EFFECT(2)),
@@ -97,42 +102,37 @@ static int mock_get_log(struct cxl_dev_state cxlds, struct cxl_mbox_cmd cmd) · 66 unmodified lines
97	102
98	103	static int mock_id(struct cxl_dev_state cxlds, struct cxl_mbox_cmd cmd)
99	104	{
100		− struct platform_device *pdev = to_platform_device(cxlds->dev);
101	105	struct cxl_mbox_identify id = {
102	106	.fw_revision = { "mock fw v1 " },
103	107	.lsa_size = cpu_to_le32(LSA_SIZE),
104		− /* FIXME: Add partition support */
105		− .partition_align = cpu_to_le64(0),
	108	+ .partition_align =
	109	+ cpu_to_le64(SZ_256M / CXL_CAPACITY_MULTIPLIER),
	110	+ .total_capacity =
	111	+ cpu_to_le64(DEV_SIZE / CXL_CAPACITY_MULTIPLIER),
106	112	};
107		− u64 capacity = 0;
108		− int i;
109	113
110	114	if (cmd->size_out < sizeof(id))
111	115	return -EINVAL;
112	116
113		− for (i = 0; i < 2; i++) {
114		− struct resource *res;
115		−
116		− res = platform_get_resource(pdev, IORESOURCE_MEM, i);
117		− if (!res)
118		− break;
119		−
120		− capacity += resource_size(res) / CXL_CAPACITY_MULTIPLIER;
	117	+ memcpy(cmd->payload_out, &id, sizeof(id));
121	118
122		− if (le64_to_cpu(id.partition_align))
123		− continue;
	119	+ return 0;
	120	+}
124	121
125		− if (res->desc == IORES_DESC_PERSISTENT_MEMORY)
126		− id.persistent_capacity = cpu_to_le64(
127		− resource_size(res) / CXL_CAPACITY_MULTIPLIER);
128		− else
129		− id.volatile_capacity = cpu_to_le64(
130		− resource_size(res) / CXL_CAPACITY_MULTIPLIER);
131		− }
	122	+static int mock_partition_info(struct cxl_dev_state *cxlds,
	123	+ struct cxl_mbox_cmd *cmd)
	124	+{
	125	+ struct cxl_mbox_get_partition_info pi = {
	126	+ .active_volatile_cap =
	127	+ cpu_to_le64(DEV_SIZE / 2 / CXL_CAPACITY_MULTIPLIER),
	128	+ .active_persistent_cap =
	129	+ cpu_to_le64(DEV_SIZE / 2 / CXL_CAPACITY_MULTIPLIER),
	130	+ };
132	131
133		− id.total_capacity = cpu_to_le64(capacity);
	132	+ if (cmd->size_out < sizeof(pi))
	133	+ return -EINVAL;
134	134
135		− memcpy(cmd->payload_out, &id, sizeof(id));
	135	+ memcpy(cmd->payload_out, &pi, sizeof(pi));
136	136
137	137	return 0;
138	138	}
@@ -221,6 +221,9 @@ static int cxl_mock_mbox_send(struct cxl_dev_state cxlds, struct cxl_mbox_cmd · 82 unmodified lines
221	221	case CXL_MBOX_OP_GET_LSA:
222	222	rc = mock_get_lsa(cxlds, cmd);
223	223	break;
	224	+ case CXL_MBOX_OP_GET_PARTITION_INFO:
	225	+ rc = mock_partition_info(cxlds, cmd);
	226	+ break;
224	227	case CXL_MBOX_OP_SET_LSA:
225	228	rc = mock_set_lsa(cxlds, cmd);
226	229	break;
@@ -282,7 +285,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) · 55 unmodified lines
282	285	if (IS_ERR(cxlmd))
283	286	return PTR_ERR(cxlmd);
284	287
285		− if (range_len(&cxlds->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))
	288	+ if (resource_size(&cxlds->pmem_res) && IS_ENABLED(CONFIG_CXL_PMEM))
286	289	rc = devm_cxl_add_nvdimm(dev, cxlmd);
287	290
288	291	return 0;

tools/testing/cxl/test/mock.c+5−3

@@ -208,13 +208,15 @@ int __wrap_cxl_await_media_ready(struct cxl_dev_state *cxlds) · 207 unmodified lines
208	208	}
209	209	EXPORT_SYMBOL_NS_GPL(__wrap_cxl_await_media_ready, CXL);
210	210
211		−bool __wrap_cxl_hdm_decode_init(struct cxl_dev_state *cxlds,
212		− struct cxl_hdm *cxlhdm)
	211	+int __wrap_cxl_hdm_decode_init(struct cxl_dev_state *cxlds,
	212	+ struct cxl_hdm *cxlhdm)
213	213	{
214	214	int rc = 0, index;
215	215	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
216	216
217		− if (!ops \|\| !ops->is_mock_dev(cxlds->dev))
	217	+ if (ops && ops->is_mock_dev(cxlds->dev))
	218	+ rc = 0;
	219	+ else
218	220	rc = cxl_hdm_decode_init(cxlds, cxlhdm);
219	221	put_cxl_mock_ops(index);
220	222