lael-labs/linux

Documentation/driver-api/cxl/memory-devices.rst+6−0

 .. kernel-doc:: drivers/cxl/core/bus.c
    :doc: cxl core
+.. kernel-doc:: drivers/cxl/core/bus.c
+   :identifiers:
 .. kernel-doc:: drivers/cxl/core/pmem.c
    :doc: cxl pmem
 .. kernel-doc:: drivers/cxl/core/regs.c
    :doc: cxl registers
+.. kernel-doc:: drivers/cxl/core/mbox.c
+   :doc: cxl mbox
 External Interfaces
 ===================

arch/powerpc/platforms/powernv/ocxl.c+2−1

@@ -107,7 +107,8 @@ static int get_max_afu_index(struct pci_dev dev, int afu_idx) · 106 unmodified lines
107	int pos;	107	int pos;
108	u32 val;	108	u32 val;
109		109
110	pos = find_dvsec_from_pos(dev, OCXL_DVSEC_FUNC_ID, 0);	110	pos = pci_find_dvsec_capability(dev, PCI_VENDOR_ID_IBM,
		111	OCXL_DVSEC_FUNC_ID);
111	if (!pos)	112	if (!pos)
112	return -ESRCH;	113	return -ESRCH;
113		114

drivers/cxl/acpi.c+97−42

@@ -52,6 +52,12 @@ static int cxl_acpi_cfmws_verify(struct device *dev, · 51 unmodified lines
52	return -EINVAL;	52	return -EINVAL;
53	}	53	}
54		54
		55	if (CFMWS_INTERLEAVE_WAYS(cfmws) > CXL_DECODER_MAX_INTERLEAVE) {
		56	dev_err(dev, "CFMWS Interleave Ways (%d) too large\n",
		57	CFMWS_INTERLEAVE_WAYS(cfmws));
		58	return -EINVAL;
		59	}
		60
55	expected_len = struct_size((cfmws), interleave_targets,	61	expected_len = struct_size((cfmws), interleave_targets,
56	CFMWS_INTERLEAVE_WAYS(cfmws));	62	CFMWS_INTERLEAVE_WAYS(cfmws));
57		63
@@ -71,11 +77,11 @@ static int cxl_acpi_cfmws_verify(struct device *dev, · 13 unmodified lines
71	static void cxl_add_cfmws_decoders(struct device *dev,	77	static void cxl_add_cfmws_decoders(struct device *dev,
72	struct cxl_port *root_port)	78	struct cxl_port *root_port)
73	{	79	{
		80	int target_map[CXL_DECODER_MAX_INTERLEAVE];
74	struct acpi_cedt_cfmws *cfmws;	81	struct acpi_cedt_cfmws *cfmws;
75	struct cxl_decoder *cxld;	82	struct cxl_decoder *cxld;
76	acpi_size len, cur = 0;	83	acpi_size len, cur = 0;
77	void *cedt_subtable;	84	void *cedt_subtable;
78	unsigned long flags;
79	int rc;	85	int rc;
80		86
81	len = acpi_cedt->length - sizeof(*acpi_cedt);	87	len = acpi_cedt->length - sizeof(*acpi_cedt);
@@ -83,6 +89,7 @@ static void cxl_add_cfmws_decoders(struct device *dev, · 1 unmodified line
83		89
84	while (cur < len) {	90	while (cur < len) {
85	struct acpi_cedt_header *c = cedt_subtable + cur;	91	struct acpi_cedt_header *c = cedt_subtable + cur;
		92	int i;
86		93
87	if (c->type != ACPI_CEDT_TYPE_CFMWS) {	94	if (c->type != ACPI_CEDT_TYPE_CFMWS) {
88	cur += c->length;	95	cur += c->length;
@@ -108,24 +115,39 @@ static void cxl_add_cfmws_decoders(struct device *dev, · 19 unmodified lines
108	continue;	115	continue;
109	}	116	}
110		117
111	flags = cfmws_to_decoder_flags(cfmws->restrictions);	118	for (i = 0; i < CFMWS_INTERLEAVE_WAYS(cfmws); i++)
112	cxld = devm_cxl_add_decoder(dev, root_port,	119	target_map[i] = cfmws->interleave_targets[i];
113	CFMWS_INTERLEAVE_WAYS(cfmws),
114	cfmws->base_hpa, cfmws->window_size,
115	CFMWS_INTERLEAVE_WAYS(cfmws),
116	CFMWS_INTERLEAVE_GRANULARITY(cfmws),
117	CXL_DECODER_EXPANDER,
118	flags);
119		120
120	if (IS_ERR(cxld)) {	121	cxld = cxl_decoder_alloc(root_port,
		122	CFMWS_INTERLEAVE_WAYS(cfmws));
		123	if (IS_ERR(cxld))
		124	goto next;
		125
		126	cxld->flags = cfmws_to_decoder_flags(cfmws->restrictions);
		127	cxld->target_type = CXL_DECODER_EXPANDER;
		128	cxld->range = (struct range) {
		129	.start = cfmws->base_hpa,
		130	.end = cfmws->base_hpa + cfmws->window_size - 1,
		131	};
		132	cxld->interleave_ways = CFMWS_INTERLEAVE_WAYS(cfmws);
		133	cxld->interleave_granularity =
		134	CFMWS_INTERLEAVE_GRANULARITY(cfmws);
		135
		136	rc = cxl_decoder_add(cxld, target_map);
		137	if (rc)
		138	put_device(&cxld->dev);
		139	else
		140	rc = cxl_decoder_autoremove(dev, cxld);
		141	if (rc) {
121	dev_err(dev, "Failed to add decoder for %#llx-%#llx\n",	142	dev_err(dev, "Failed to add decoder for %#llx-%#llx\n",
122	cfmws->base_hpa, cfmws->base_hpa +	143	cfmws->base_hpa, cfmws->base_hpa +
123	cfmws->window_size - 1);	144	cfmws->window_size - 1);
124	} else {	145	goto next;
125	dev_dbg(dev, "add: %s range %#llx-%#llx\n",
126	dev_name(&cxld->dev), cfmws->base_hpa,
127	cfmws->base_hpa + cfmws->window_size - 1);
128	}	146	}
		147	dev_dbg(dev, "add: %s range %#llx-%#llx\n",
		148	dev_name(&cxld->dev), cfmws->base_hpa,
		149	cfmws->base_hpa + cfmws->window_size - 1);
		150	next:
129	cur += c->length;	151	cur += c->length;
130	}	152	}
131	}	153	}
@@ -182,15 +204,7 @@ static resource_size_t get_chbcr(struct acpi_cedt_chbs *chbs) · 50 unmodified lines
182	return IS_ERR(chbs) ? CXL_RESOURCE_NONE : chbs->base;	204	return IS_ERR(chbs) ? CXL_RESOURCE_NONE : chbs->base;
183	}	205	}
184		206
185	struct cxl_walk_context {	207	__mock int match_add_root_ports(struct pci_dev pdev, void data)
186	struct device *dev;
187	struct pci_bus *root;
188	struct cxl_port *port;
189	int error;
190	int count;
191	};
192
193	static int match_add_root_ports(struct pci_dev pdev, void data)
194	{	208	{
195	struct cxl_walk_context *ctx = data;	209	struct cxl_walk_context *ctx = data;
196	struct pci_bus *root_bus = ctx->root;	210	struct pci_bus *root_bus = ctx->root;
@@ -239,7 +253,8 @@ static struct cxl_dport find_dport_by_dev(struct cxl_port port, struct device · 42 unmodified lines
239	return NULL;	253	return NULL;
240	}	254	}
241		255
242	static struct acpi_device to_cxl_host_bridge(struct device dev)	256	__mock struct acpi_device to_cxl_host_bridge(struct device host,
		257	struct device *dev)
243	{	258	{
244	struct acpi_device *adev = to_acpi_device(dev);	259	struct acpi_device *adev = to_acpi_device(dev);
245		260
@@ -257,11 +272,12 @@ static struct acpi_device to_cxl_host_bridge(struct device dev) · 11 unmodified lines
257	*/	272	*/
258	static int add_host_bridge_uport(struct device match, void arg)	273	static int add_host_bridge_uport(struct device match, void arg)
259	{	274	{
260	struct acpi_device *bridge = to_cxl_host_bridge(match);
261	struct cxl_port *root_port = arg;	275	struct cxl_port *root_port = arg;
262	struct device *host = root_port->dev.parent;	276	struct device *host = root_port->dev.parent;
		277	struct acpi_device *bridge = to_cxl_host_bridge(host, match);
263	struct acpi_pci_root *pci_root;	278	struct acpi_pci_root *pci_root;
264	struct cxl_walk_context ctx;	279	struct cxl_walk_context ctx;
		280	int single_port_map[1], rc;
265	struct cxl_decoder *cxld;	281	struct cxl_decoder *cxld;
266	struct cxl_dport *dport;	282	struct cxl_dport *dport;
267	struct cxl_port *port;	283	struct cxl_port *port;
@@ -272,7 +288,7 @@ static int add_host_bridge_uport(struct device match, void arg) · 4 unmodified lines
272	dport = find_dport_by_dev(root_port, match);	288	dport = find_dport_by_dev(root_port, match);
273	if (!dport) {	289	if (!dport) {
274	dev_dbg(host, "host bridge expected and not found\n");	290	dev_dbg(host, "host bridge expected and not found\n");
275	return -ENODEV;	291	return 0;
276	}	292	}
277		293
278	port = devm_cxl_add_port(host, match, dport->component_reg_phys,	294	port = devm_cxl_add_port(host, match, dport->component_reg_phys,
@@ -297,22 +313,46 @@ static int add_host_bridge_uport(struct device match, void arg) · 18 unmodified lines
297	return -ENODEV;	313	return -ENODEV;
298	if (ctx.error)	314	if (ctx.error)
299	return ctx.error;	315	return ctx.error;
		316	if (ctx.count > 1)
		317	return 0;
300		318
301	/* TODO: Scan CHBCR for HDM Decoder resources */	319	/* TODO: Scan CHBCR for HDM Decoder resources */
302		320
303	/*	321	/*
304	* In the single-port host-bridge case there are no HDM decoders	322	* Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability
305	* in the CHBCR and a 1:1 passthrough decode is implied.	323	* Structure) single ported host-bridges need not publish a decoder
		324	* capability when a passthrough decode can be assumed, i.e. all
		325	* transactions that the uport sees are claimed and passed to the single
		326	* dport. Disable the range until the first CXL region is enumerated /
		327	* activated.
306	*/	328	*/
307	if (ctx.count == 1) {	329	cxld = cxl_decoder_alloc(port, 1);
308	cxld = devm_cxl_add_passthrough_decoder(host, port);	330	if (IS_ERR(cxld))
309	if (IS_ERR(cxld))	331	return PTR_ERR(cxld);
310	return PTR_ERR(cxld);	332
		333	cxld->interleave_ways = 1;
		334	cxld->interleave_granularity = PAGE_SIZE;
		335	cxld->target_type = CXL_DECODER_EXPANDER;
		336	cxld->range = (struct range) {
		337	.start = 0,
		338	.end = -1,
		339	};
311		340
312	dev_dbg(host, "add: %s\n", dev_name(&cxld->dev));	341	device_lock(&port->dev);
313	}	342	dport = list_first_entry(&port->dports, typeof(*dport), list);
		343	device_unlock(&port->dev);
314		344
315	return 0;	345	single_port_map[0] = dport->port_id;
		346
		347	rc = cxl_decoder_add(cxld, single_port_map);
		348	if (rc)
		349	put_device(&cxld->dev);
		350	else
		351	rc = cxl_decoder_autoremove(host, cxld);
		352
		353	if (rc == 0)
		354	dev_dbg(host, "add: %s\n", dev_name(&cxld->dev));
		355	return rc;
316	}	356	}
317		357
318	static int add_host_bridge_dport(struct device match, void arg)	358	static int add_host_bridge_dport(struct device match, void arg)
@@ -323,7 +363,7 @@ static int add_host_bridge_dport(struct device match, void arg) · 4 unmodified lines
323	struct acpi_cedt_chbs *chbs;	363	struct acpi_cedt_chbs *chbs;
324	struct cxl_port *root_port = arg;	364	struct cxl_port *root_port = arg;
325	struct device *host = root_port->dev.parent;	365	struct device *host = root_port->dev.parent;
326	struct acpi_device *bridge = to_cxl_host_bridge(match);	366	struct acpi_device *bridge = to_cxl_host_bridge(host, match);
327		367
328	if (!bridge)	368	if (!bridge)
329	return 0;	369	return 0;
@@ -337,9 +377,11 @@ static int add_host_bridge_dport(struct device match, void arg) · 7 unmodified lines
337	}	377	}
338		378
339	chbs = cxl_acpi_match_chbs(host, uid);	379	chbs = cxl_acpi_match_chbs(host, uid);
340	if (IS_ERR(chbs))	380	if (IS_ERR(chbs)) {
341	dev_dbg(host, "No CHBS found for Host Bridge: %s\n",	381	dev_warn(host, "No CHBS found for Host Bridge: %s\n",
342	dev_name(match));	382	dev_name(match));
		383	return 0;
		384	}
343		385
344	rc = cxl_add_dport(root_port, match, uid, get_chbcr(chbs));	386	rc = cxl_add_dport(root_port, match, uid, get_chbcr(chbs));
345	if (rc) {	387	if (rc) {
@@ -375,6 +417,17 @@ static int add_root_nvdimm_bridge(struct device match, void data) · 29 unmodified lines
375	return 1;	417	return 1;
376	}	418	}
377		419
		420	static u32 cedt_instance(struct platform_device *pdev)
		421	{
		422	const bool *native_acpi0017 = acpi_device_get_match_data(&pdev->dev);
		423
		424	if (native_acpi0017 && *native_acpi0017)
		425	return 0;
		426
		427	/* for cxl_test request a non-canonical instance */
		428	return U32_MAX;
		429	}
		430
378	static int cxl_acpi_probe(struct platform_device *pdev)	431	static int cxl_acpi_probe(struct platform_device *pdev)
379	{	432	{
380	int rc;	433	int rc;
@@ -388,7 +441,7 @@ static int cxl_acpi_probe(struct platform_device *pdev) · 7 unmodified lines
388	return PTR_ERR(root_port);	441	return PTR_ERR(root_port);
389	dev_dbg(host, "add: %s\n", dev_name(&root_port->dev));	442	dev_dbg(host, "add: %s\n", dev_name(&root_port->dev));
390		443
391	status = acpi_get_table(ACPI_SIG_CEDT, 0, &acpi_cedt);	444	status = acpi_get_table(ACPI_SIG_CEDT, cedt_instance(pdev), &acpi_cedt);
392	if (ACPI_FAILURE(status))	445	if (ACPI_FAILURE(status))
393	return -ENXIO;	446	return -ENXIO;
394		447
@@ -419,9 +472,11 @@ out: · 24 unmodified lines
419	return 0;	472	return 0;
420	}	473	}
421		474
		475	static bool native_acpi0017 = true;
		476
422	static const struct acpi_device_id cxl_acpi_ids[] = {	477	static const struct acpi_device_id cxl_acpi_ids[] = {
423	{ "ACPI0017", 0 },	478	{ "ACPI0017", (unsigned long) &native_acpi0017 },
424	{ "", 0 },	479	{ },
425	};	480	};
426	MODULE_DEVICE_TABLE(acpi, cxl_acpi_ids);	481	MODULE_DEVICE_TABLE(acpi, cxl_acpi_ids);
427		482

drivers/cxl/core/Makefile+1−0

@@ -6,3 +6,4 @@ cxl_core-y := bus.o · 5 unmodified lines
6	cxl_core-y += pmem.o	6	cxl_core-y += pmem.o
7	cxl_core-y += regs.o	7	cxl_core-y += regs.o
8	cxl_core-y += memdev.o	8	cxl_core-y += memdev.o
		9	cxl_core-y += mbox.o

drivers/cxl/core/bus.c+68−51

@@ -453,50 +453,57 @@ err: · 452 unmodified lines
453	}	453	}
454	EXPORT_SYMBOL_GPL(cxl_add_dport);	454	EXPORT_SYMBOL_GPL(cxl_add_dport);
455		455
456	static struct cxl_decoder *	456	static int decoder_populate_targets(struct cxl_decoder *cxld,
457	cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,	457	struct cxl_port port, int target_map)
458	resource_size_t len, int interleave_ways,
459	int interleave_granularity, enum cxl_decoder_type type,
460	unsigned long flags)
461	{	458	{
462	struct cxl_decoder *cxld;	459	int rc = 0, i;
463	struct device *dev;
464	int rc = 0;
465		460
466	if (interleave_ways < 1)	461	if (!target_map)
467	return ERR_PTR(-EINVAL);	462	return 0;
468		463
469	device_lock(&port->dev);	464	device_lock(&port->dev);
470	if (list_empty(&port->dports))	465	if (list_empty(&port->dports)) {
471	rc = -EINVAL;	466	rc = -EINVAL;
		467	goto out_unlock;
		468	}
		469
		470	for (i = 0; i < cxld->nr_targets; i++) {
		471	struct cxl_dport *dport = find_dport(port, target_map[i]);
		472
		473	if (!dport) {
		474	rc = -ENXIO;
		475	goto out_unlock;
		476	}
		477	cxld->target[i] = dport;
		478	}
		479
		480	out_unlock:
472	device_unlock(&port->dev);	481	device_unlock(&port->dev);
473	if (rc)	482
474	return ERR_PTR(rc);	483	return rc;
		484	}
		485
		486	struct cxl_decoder cxl_decoder_alloc(struct cxl_port port, int nr_targets)
		487	{
		488	struct cxl_decoder *cxld, cxld_const_init = {
		489	.nr_targets = nr_targets,
		490	};
		491	struct device *dev;
		492	int rc = 0;
		493
		494	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE \|\| nr_targets < 1)
		495	return ERR_PTR(-EINVAL);
475		496
476	cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);	497	cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);
477	if (!cxld)	498	if (!cxld)
478	return ERR_PTR(-ENOMEM);	499	return ERR_PTR(-ENOMEM);
		500	memcpy(cxld, &cxld_const_init, sizeof(cxld_const_init));
479		501
480	rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);	502	rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
481	if (rc < 0)	503	if (rc < 0)
482	goto err;	504	goto err;
483		505
484	*cxld = (struct cxl_decoder) {	506	cxld->id = rc;
485	.id = rc,
486	.range = {
487	.start = base,
488	.end = base + len - 1,
489	},
490	.flags = flags,
491	.interleave_ways = interleave_ways,
492	.interleave_granularity = interleave_granularity,
493	.target_type = type,
494	};
495
496	/* handle implied target_list */
497	if (interleave_ways == 1)
498	cxld->target[0] =
499	list_first_entry(&port->dports, struct cxl_dport, list);
500	dev = &cxld->dev;	507	dev = &cxld->dev;
501	device_initialize(dev);	508	device_initialize(dev);
502	device_set_pm_not_required(dev);	509	device_set_pm_not_required(dev);
@@ -514,41 +521,47 @@ err: · 11 unmodified lines
514	kfree(cxld);	521	kfree(cxld);
515	return ERR_PTR(rc);	522	return ERR_PTR(rc);
516	}	523	}
		524	EXPORT_SYMBOL_GPL(cxl_decoder_alloc);
517		525
518	struct cxl_decoder *	526	int cxl_decoder_add(struct cxl_decoder cxld, int target_map)
519	devm_cxl_add_decoder(struct device host, struct cxl_port port, int nr_targets,
520	resource_size_t base, resource_size_t len,
521	int interleave_ways, int interleave_granularity,
522	enum cxl_decoder_type type, unsigned long flags)
523	{	527	{
524	struct cxl_decoder *cxld;	528	struct cxl_port *port;
525	struct device *dev;	529	struct device *dev;
526	int rc;	530	int rc;
527		531
528	cxld = cxl_decoder_alloc(port, nr_targets, base, len, interleave_ways,	532	if (WARN_ON_ONCE(!cxld))
529	interleave_granularity, type, flags);	533	return -EINVAL;
530	if (IS_ERR(cxld))	534
531	return cxld;	535	if (WARN_ON_ONCE(IS_ERR(cxld)))
		536	return PTR_ERR(cxld);
		537
		538	if (cxld->interleave_ways < 1)
		539	return -EINVAL;
		540
		541	port = to_cxl_port(cxld->dev.parent);
		542	rc = decoder_populate_targets(cxld, port, target_map);
		543	if (rc)
		544	return rc;
532		545
533	dev = &cxld->dev;	546	dev = &cxld->dev;
534	rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);	547	rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
535	if (rc)	548	if (rc)
536	goto err;	549	return rc;
537		550
538	rc = device_add(dev);	551	return device_add(dev);
539	if (rc)	552	}
540	goto err;	553	EXPORT_SYMBOL_GPL(cxl_decoder_add);
541		554
542	rc = devm_add_action_or_reset(host, unregister_cxl_dev, dev);	555	static void cxld_unregister(void *dev)
543	if (rc)	556	{
544	return ERR_PTR(rc);	557	device_unregister(dev);
545	return cxld;	558	}
546		559
547	err:	560	int cxl_decoder_autoremove(struct device host, struct cxl_decoder cxld)
548	put_device(dev);	561	{
549	return ERR_PTR(rc);	562	return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
550	}	563	}
551	EXPORT_SYMBOL_GPL(devm_cxl_add_decoder);	564	EXPORT_SYMBOL_GPL(cxl_decoder_autoremove);
552		565
553	/**	566	/**
554	* __cxl_driver_register - register a driver for the cxl bus	567	* __cxl_driver_register - register a driver for the cxl bus
@@ -635,6 +648,8 @@ static __init int cxl_core_init(void) · 80 unmodified lines
635	{	648	{
636	int rc;	649	int rc;
637		650
		651	cxl_mbox_init();
		652
638	rc = cxl_memdev_init();	653	rc = cxl_memdev_init();
639	if (rc)	654	if (rc)
640	return rc;	655	return rc;
@@ -646,6 +661,7 @@ static __init int cxl_core_init(void) · 5 unmodified lines
646		661
647	err:	662	err:
648	cxl_memdev_exit();	663	cxl_memdev_exit();
		664	cxl_mbox_exit();
649	return rc;	665	return rc;
650	}	666	}
651		667
@@ -653,6 +669,7 @@ static void cxl_core_exit(void) · 1 unmodified line
653	{	669	{
654	bus_unregister(&cxl_bus_type);	670	bus_unregister(&cxl_bus_type);
655	cxl_memdev_exit();	671	cxl_memdev_exit();
		672	cxl_mbox_exit();
656	}	673	}
657		674
658	module_init(cxl_core_init);	675	module_init(cxl_core_init);

drivers/cxl/core/core.h+7−4

 extern struct attribute_group cxl_base_attribute_group;
-static inline void unregister_cxl_dev(void *dev)
+struct cxl_send_command;
-{
+struct cxl_mem_query_commands;
-	device_unregister(dev);
+int cxl_query_cmd(struct cxl_memdev *cxlmd,
-}
+		  struct cxl_mem_query_commands __user *q);
+int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s);
 int cxl_memdev_init(void);
 void cxl_memdev_exit(void);
+void cxl_mbox_init(void);
+void cxl_mbox_exit(void);
 #endif /* __CXL_CORE_H__ */

drivers/cxl/core/mbox.c+787−0

@@ -0,0 +1,787 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
	3	#include <linux/io-64-nonatomic-lo-hi.h>
	4	#include <linux/security.h>
	5	#include <linux/debugfs.h>
	6	#include <linux/mutex.h>
	7	#include <cxlmem.h>
	8	#include <cxl.h>
	9
	10	#include "core.h"
	11
	12	static bool cxl_raw_allow_all;
	13
	14	/**
	15	* DOC: cxl mbox
	16	*
	17	* Core implementation of the CXL 2.0 Type-3 Memory Device Mailbox. The
	18	* implementation is used by the cxl_pci driver to initialize the device
	19	* and implement the cxl_mem.h IOCTL UAPI. It also implements the
	20	* backend of the cxl_pmem_ctl() transport for LIBNVDIMM.
	21	*/
	22
	23	#define cxl_for_each_cmd(cmd) \
	24	for ((cmd) = &cxl_mem_commands[0]; \
	25	((cmd) - cxl_mem_commands) < ARRAY_SIZE(cxl_mem_commands); (cmd)++)
	26
	27	#define CXL_CMD(_id, sin, sout, _flags) \
	28	[CXL_MEM_COMMAND_ID_##_id] = { \
	29	.info = { \
	30	.id = CXL_MEM_COMMAND_ID_##_id, \
	31	.size_in = sin, \
	32	.size_out = sout, \
	33	}, \
	34	.opcode = CXL_MBOX_OP_##_id, \
	35	.flags = _flags, \
	36	}
	37
	38	/*
	39	* This table defines the supported mailbox commands for the driver. This table
	40	* is made up of a UAPI structure. Non-negative values as parameters in the
	41	* table will be validated against the user's input. For example, if size_in is
	42	* 0, and the user passed in 1, it is an error.
	43	*/
	44	static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
	45	CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
	46	#ifdef CONFIG_CXL_MEM_RAW_COMMANDS
	47	CXL_CMD(RAW, ~0, ~0, 0),
	48	#endif
	49	CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE),
	50	CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
	51	CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
	52	CXL_CMD(GET_LSA, 0x8, ~0, 0),
	53	CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
	54	CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE),
	55	CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
	56	CXL_CMD(SET_LSA, ~0, 0, 0),
	57	CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
	58	CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
	59	CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
	60	CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
	61	CXL_CMD(GET_POISON, 0x10, ~0, 0),
	62	CXL_CMD(INJECT_POISON, 0x8, 0, 0),
	63	CXL_CMD(CLEAR_POISON, 0x48, 0, 0),
	64	CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
	65	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
	66	CXL_CMD(GET_SCAN_MEDIA, 0, ~0, 0),
	67	};
	68
	69	/*
	70	* Commands that RAW doesn't permit. The rationale for each:
	71	*
	72	* CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /
	73	* coordination of transaction timeout values at the root bridge level.
	74	*
	75	* CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live
	76	* and needs to be coordinated with HDM updates.
	77	*
	78	* CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the
	79	* driver and any writes from userspace invalidates those contents.
	80	*
	81	* CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes
	82	* to the device after it is marked clean, userspace can not make that
	83	* assertion.
	84	*
	85	* CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that
	86	* is kept up to date with patrol notifications and error management.
	87	*/
	88	static u16 cxl_disabled_raw_commands[] = {
	89	CXL_MBOX_OP_ACTIVATE_FW,
	90	CXL_MBOX_OP_SET_PARTITION_INFO,
	91	CXL_MBOX_OP_SET_LSA,
	92	CXL_MBOX_OP_SET_SHUTDOWN_STATE,
	93	CXL_MBOX_OP_SCAN_MEDIA,
	94	CXL_MBOX_OP_GET_SCAN_MEDIA,
	95	};
	96
	97	/*
	98	* Command sets that RAW doesn't permit. All opcodes in this set are
	99	* disabled because they pass plain text security payloads over the
	100	* user/kernel boundary. This functionality is intended to be wrapped
	101	* behind the keys ABI which allows for encrypted payloads in the UAPI
	102	*/
	103	static u8 security_command_sets[] = {
	104	0x44, /* Sanitize */
	105	0x45, /* Persistent Memory Data-at-rest Security */
	106	0x46, /* Security Passthrough */
	107	};
	108
	109	static bool cxl_is_security_command(u16 opcode)
	110	{
	111	int i;
	112
	113	for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)
	114	if (security_command_sets[i] == (opcode >> 8))
	115	return true;
	116	return false;
	117	}
	118
	119	static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
	120	{
	121	struct cxl_mem_command *c;
	122
	123	cxl_for_each_cmd(c)
	124	if (c->opcode == opcode)
	125	return c;
	126
	127	return NULL;
	128	}
	129
	130	/**
	131	* cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device.
	132	* @cxlm: The CXL memory device to communicate with.
	133	* @opcode: Opcode for the mailbox command.
	134	* @in: The input payload for the mailbox command.
	135	* @in_size: The length of the input payload
	136	* @out: Caller allocated buffer for the output.
	137	* @out_size: Expected size of output.
	138	*
	139	* Context: Any context. Will acquire and release mbox_mutex.
	140	* Return:
	141	* * %>=0 - Number of bytes returned in @out.
	142	* * %-E2BIG - Payload is too large for hardware.
	143	* * %-EBUSY - Couldn't acquire exclusive mailbox access.
	144	* * %-EFAULT - Hardware error occurred.
	145	* * %-ENXIO - Command completed, but device reported an error.
	146	* * %-EIO - Unexpected output size.
	147	*
	148	* Mailbox commands may execute successfully yet the device itself reported an
	149	* error. While this distinction can be useful for commands from userspace, the
	150	* kernel will only be able to use results when both are successful.
	151	*
	152	* See __cxl_mem_mbox_send_cmd()
	153	*/
	154	int cxl_mem_mbox_send_cmd(struct cxl_mem cxlm, u16 opcode, void in,
	155	size_t in_size, void *out, size_t out_size)
	156	{
	157	const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
	158	struct cxl_mbox_cmd mbox_cmd = {
	159	.opcode = opcode,
	160	.payload_in = in,
	161	.size_in = in_size,
	162	.size_out = out_size,
	163	.payload_out = out,
	164	};
	165	int rc;
	166
	167	if (out_size > cxlm->payload_size)
	168	return -E2BIG;
	169
	170	rc = cxlm->mbox_send(cxlm, &mbox_cmd);
	171	if (rc)
	172	return rc;
	173
	174	/* TODO: Map return code to proper kernel style errno */
	175	if (mbox_cmd.return_code != CXL_MBOX_SUCCESS)
	176	return -ENXIO;
	177
	178	/*
	179	* Variable sized commands can't be validated and so it's up to the
	180	* caller to do that if they wish.
	181	*/
	182	if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size)
	183	return -EIO;
	184
	185	return 0;
	186	}
	187	EXPORT_SYMBOL_GPL(cxl_mem_mbox_send_cmd);
	188
	189	static bool cxl_mem_raw_command_allowed(u16 opcode)
	190	{
	191	int i;
	192
	193	if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))
	194	return false;
	195
	196	if (security_locked_down(LOCKDOWN_PCI_ACCESS))
	197	return false;
	198
	199	if (cxl_raw_allow_all)
	200	return true;
	201
	202	if (cxl_is_security_command(opcode))
	203	return false;
	204
	205	for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)
	206	if (cxl_disabled_raw_commands[i] == opcode)
	207	return false;
	208
	209	return true;
	210	}
	211
	212	/**
	213	* cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
	214	* @cxlm: &struct cxl_mem device whose mailbox will be used.
	215	* @send_cmd: &struct cxl_send_command copied in from userspace.
	216	* @out_cmd: Sanitized and populated &struct cxl_mem_command.
	217	*
	218	* Return:
	219	* * %0 - @out_cmd is ready to send.
	220	* * %-ENOTTY - Invalid command specified.
	221	* * %-EINVAL - Reserved fields or invalid values were used.
	222	* * %-ENOMEM - Input or output buffer wasn't sized properly.
	223	* * %-EPERM - Attempted to use a protected command.
	224	* * %-EBUSY - Kernel has claimed exclusive access to this opcode
	225	*
	226	* The result of this command is a fully validated command in @out_cmd that is
	227	* safe to send to the hardware.
	228	*
	229	* See handle_mailbox_cmd_from_user()
	230	*/
	231	static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm,
	232	const struct cxl_send_command *send_cmd,
	233	struct cxl_mem_command *out_cmd)
	234	{
	235	const struct cxl_command_info *info;
	236	struct cxl_mem_command *c;
	237
	238	if (send_cmd->id == 0 \|\| send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
	239	return -ENOTTY;
	240
	241	/*
	242	* The user can never specify an input payload larger than what hardware
	243	* supports, but output can be arbitrarily large (simply write out as
	244	* much data as the hardware provides).
	245	*/
	246	if (send_cmd->in.size > cxlm->payload_size)
	247	return -EINVAL;
	248
	249	/*
	250	* Checks are bypassed for raw commands but a WARN/taint will occur
	251	* later in the callchain
	252	*/
	253	if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) {
	254	const struct cxl_mem_command temp = {
	255	.info = {
	256	.id = CXL_MEM_COMMAND_ID_RAW,
	257	.flags = 0,
	258	.size_in = send_cmd->in.size,
	259	.size_out = send_cmd->out.size,
	260	},
	261	.opcode = send_cmd->raw.opcode
	262	};
	263
	264	if (send_cmd->raw.rsvd)
	265	return -EINVAL;
	266
	267	/*
	268	* Unlike supported commands, the output size of RAW commands
	269	* gets passed along without further checking, so it must be
	270	* validated here.
	271	*/
	272	if (send_cmd->out.size > cxlm->payload_size)
	273	return -EINVAL;
	274
	275	if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
	276	return -EPERM;
	277
	278	memcpy(out_cmd, &temp, sizeof(temp));
	279
	280	return 0;
	281	}
	282
	283	if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
	284	return -EINVAL;
	285
	286	if (send_cmd->rsvd)
	287	return -EINVAL;
	288
	289	if (send_cmd->in.rsvd \|\| send_cmd->out.rsvd)
	290	return -EINVAL;
	291
	292	/* Convert user's command into the internal representation */
	293	c = &cxl_mem_commands[send_cmd->id];
	294	info = &c->info;
	295
	296	/* Check that the command is enabled for hardware */
	297	if (!test_bit(info->id, cxlm->enabled_cmds))
	298	return -ENOTTY;
	299
	300	/* Check that the command is not claimed for exclusive kernel use */
	301	if (test_bit(info->id, cxlm->exclusive_cmds))
	302	return -EBUSY;
	303
	304	/* Check the input buffer is the expected size */
	305	if (info->size_in >= 0 && info->size_in != send_cmd->in.size)
	306	return -ENOMEM;
	307
	308	/* Check the output buffer is at least large enough */
	309	if (info->size_out >= 0 && send_cmd->out.size < info->size_out)
	310	return -ENOMEM;
	311
	312	memcpy(out_cmd, c, sizeof(*c));
	313	out_cmd->info.size_in = send_cmd->in.size;
	314	/*
	315	* XXX: out_cmd->info.size_out will be controlled by the driver, and the
	316	* specified number of bytes @send_cmd->out.size will be copied back out
	317	* to userspace.
	318	*/
	319
	320	return 0;
	321	}
	322
	323	int cxl_query_cmd(struct cxl_memdev *cxlmd,
	324	struct cxl_mem_query_commands __user *q)
	325	{
	326	struct device *dev = &cxlmd->dev;
	327	struct cxl_mem_command *cmd;
	328	u32 n_commands;
	329	int j = 0;
	330
	331	dev_dbg(dev, "Query IOCTL\n");
	332
	333	if (get_user(n_commands, &q->n_commands))
	334	return -EFAULT;
	335
	336	/* returns the total number if 0 elements are requested. */
	337	if (n_commands == 0)
	338	return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
	339
	340	/*
	341	* otherwise, return max(n_commands, total commands) cxl_command_info
	342	* structures.
	343	*/
	344	cxl_for_each_cmd(cmd) {
	345	const struct cxl_command_info *info = &cmd->info;
	346
	347	if (copy_to_user(&q->commands[j++], info, sizeof(*info)))
	348	return -EFAULT;
	349
	350	if (j == n_commands)
	351	break;
	352	}
	353
	354	return 0;
	355	}
	356
	357	/**
	358	* handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
	359	* @cxlm: The CXL memory device to communicate with.
	360	* @cmd: The validated command.
	361	* @in_payload: Pointer to userspace's input payload.
	362	* @out_payload: Pointer to userspace's output payload.
	363	* @size_out: (Input) Max payload size to copy out.
	364	* (Output) Payload size hardware generated.
	365	* @retval: Hardware generated return code from the operation.
	366	*
	367	* Return:
	368	* * %0 - Mailbox transaction succeeded. This implies the mailbox
	369	* protocol completed successfully not that the operation itself
	370	* was successful.
	371	* * %-ENOMEM - Couldn't allocate a bounce buffer.
	372	* * %-EFAULT - Something happened with copy_to/from_user.
	373	* * %-EINTR - Mailbox acquisition interrupted.
	374	* * %-EXXX - Transaction level failures.
	375	*
	376	* Creates the appropriate mailbox command and dispatches it on behalf of a
	377	* userspace request. The input and output payloads are copied between
	378	* userspace.
	379	*
	380	* See cxl_send_cmd().
	381	*/
	382	static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm,
	383	const struct cxl_mem_command *cmd,
	384	u64 in_payload, u64 out_payload,
	385	s32 size_out, u32 retval)
	386	{
	387	struct device *dev = cxlm->dev;
	388	struct cxl_mbox_cmd mbox_cmd = {
	389	.opcode = cmd->opcode,
	390	.size_in = cmd->info.size_in,
	391	.size_out = cmd->info.size_out,
	392	};
	393	int rc;
	394
	395	if (cmd->info.size_out) {
	396	mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL);
	397	if (!mbox_cmd.payload_out)
	398	return -ENOMEM;
	399	}
	400
	401	if (cmd->info.size_in) {
	402	mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
	403	cmd->info.size_in);
	404	if (IS_ERR(mbox_cmd.payload_in)) {
	405	kvfree(mbox_cmd.payload_out);
	406	return PTR_ERR(mbox_cmd.payload_in);
	407	}
	408	}
	409
	410	dev_dbg(dev,
	411	"Submitting %s command for user\n"
	412	"\topcode: %x\n"
	413	"\tsize: %ub\n",
	414	cxl_command_names[cmd->info.id].name, mbox_cmd.opcode,
	415	cmd->info.size_in);
	416
	417	dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW,
	418	"raw command path used\n");
	419
	420	rc = cxlm->mbox_send(cxlm, &mbox_cmd);
	421	if (rc)
	422	goto out;
	423
	424	/*
	425	* @size_out contains the max size that's allowed to be written back out
	426	* to userspace. While the payload may have written more output than
	427	* this it will have to be ignored.
	428	*/
	429	if (mbox_cmd.size_out) {
	430	dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out,
	431	"Invalid return size\n");
	432	if (copy_to_user(u64_to_user_ptr(out_payload),
	433	mbox_cmd.payload_out, mbox_cmd.size_out)) {
	434	rc = -EFAULT;
	435	goto out;
	436	}
	437	}
	438
	439	*size_out = mbox_cmd.size_out;
	440	*retval = mbox_cmd.return_code;
	441
	442	out:
	443	kvfree(mbox_cmd.payload_in);
	444	kvfree(mbox_cmd.payload_out);
	445	return rc;
	446	}
	447
	448	int cxl_send_cmd(struct cxl_memdev cxlmd, struct cxl_send_command __user s)
	449	{
	450	struct cxl_mem *cxlm = cxlmd->cxlm;
	451	struct device *dev = &cxlmd->dev;
	452	struct cxl_send_command send;
	453	struct cxl_mem_command c;
	454	int rc;
	455
	456	dev_dbg(dev, "Send IOCTL\n");
	457
	458	if (copy_from_user(&send, s, sizeof(send)))
	459	return -EFAULT;
	460
	461	rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c);
	462	if (rc)
	463	return rc;
	464
	465	/* Prepare to handle a full payload for variable sized output */
	466	if (c.info.size_out < 0)
	467	c.info.size_out = cxlm->payload_size;
	468
	469	rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload,
	470	send.out.payload, &send.out.size,
	471	&send.retval);
	472	if (rc)
	473	return rc;
	474
	475	if (copy_to_user(s, &send, sizeof(send)))
	476	return -EFAULT;
	477
	478	return 0;
	479	}
	480
	481	static int cxl_xfer_log(struct cxl_mem cxlm, uuid_t uuid, u32 size, u8 *out)
	482	{
	483	u32 remaining = size;
	484	u32 offset = 0;
	485
	486	while (remaining) {
	487	u32 xfer_size = min_t(u32, remaining, cxlm->payload_size);
	488	struct cxl_mbox_get_log log = {
	489	.uuid = *uuid,
	490	.offset = cpu_to_le32(offset),
	491	.length = cpu_to_le32(xfer_size)
	492	};
	493	int rc;
	494
	495	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log,
	496	sizeof(log), out, xfer_size);
	497	if (rc < 0)
	498	return rc;
	499
	500	out += xfer_size;
	501	remaining -= xfer_size;
	502	offset += xfer_size;
	503	}
	504
	505	return 0;
	506	}
	507
	508	/**
	509	* cxl_walk_cel() - Walk through the Command Effects Log.
	510	* @cxlm: Device.
	511	* @size: Length of the Command Effects Log.
	512	* @cel: CEL
	513	*
	514	* Iterate over each entry in the CEL and determine if the driver supports the
	515	* command. If so, the command is enabled for the device and can be used later.
	516	*/
	517	static void cxl_walk_cel(struct cxl_mem cxlm, size_t size, u8 cel)
	518	{
	519	struct cxl_cel_entry *cel_entry;
	520	const int cel_entries = size / sizeof(*cel_entry);
	521	int i;
	522
	523	cel_entry = (struct cxl_cel_entry *) cel;
	524
	525	for (i = 0; i < cel_entries; i++) {
	526	u16 opcode = le16_to_cpu(cel_entry[i].opcode);
	527	struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
	528
	529	if (!cmd) {
	530	dev_dbg(cxlm->dev,
	531	"Opcode 0x%04x unsupported by driver", opcode);
	532	continue;
	533	}
	534
	535	set_bit(cmd->info.id, cxlm->enabled_cmds);
	536	}
	537	}
	538
	539	static struct cxl_mbox_get_supported_logs cxl_get_gsl(struct cxl_mem cxlm)
	540	{
	541	struct cxl_mbox_get_supported_logs *ret;
	542	int rc;
	543
	544	ret = kvmalloc(cxlm->payload_size, GFP_KERNEL);
	545	if (!ret)
	546	return ERR_PTR(-ENOMEM);
	547
	548	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL,
	549	0, ret, cxlm->payload_size);
	550	if (rc < 0) {
	551	kvfree(ret);
	552	return ERR_PTR(rc);
	553	}
	554
	555	return ret;
	556	}
	557
	558	enum {
	559	CEL_UUID,
	560	VENDOR_DEBUG_UUID,
	561	};
	562
	563	/* See CXL 2.0 Table 170. Get Log Input Payload */
	564	static const uuid_t log_uuid[] = {
	565	[CEL_UUID] = DEFINE_CXL_CEL_UUID,
	566	[VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,
	567	};
	568
	569	/**
	570	* cxl_mem_enumerate_cmds() - Enumerate commands for a device.
	571	* @cxlm: The device.
	572	*
	573	* Returns 0 if enumerate completed successfully.
	574	*
	575	* CXL devices have optional support for certain commands. This function will
	576	* determine the set of supported commands for the hardware and update the
	577	* enabled_cmds bitmap in the @cxlm.
	578	*/
	579	int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm)
	580	{
	581	struct cxl_mbox_get_supported_logs *gsl;
	582	struct device *dev = cxlm->dev;
	583	struct cxl_mem_command *cmd;
	584	int i, rc;
	585
	586	gsl = cxl_get_gsl(cxlm);
	587	if (IS_ERR(gsl))
	588	return PTR_ERR(gsl);
	589
	590	rc = -ENOENT;
	591	for (i = 0; i < le16_to_cpu(gsl->entries); i++) {
	592	u32 size = le32_to_cpu(gsl->entry[i].size);
	593	uuid_t uuid = gsl->entry[i].uuid;
	594	u8 *log;
	595
	596	dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);
	597
	598	if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))
	599	continue;
	600
	601	log = kvmalloc(size, GFP_KERNEL);
	602	if (!log) {
	603	rc = -ENOMEM;
	604	goto out;
	605	}
	606
	607	rc = cxl_xfer_log(cxlm, &uuid, size, log);
	608	if (rc) {
	609	kvfree(log);
	610	goto out;
	611	}
	612
	613	cxl_walk_cel(cxlm, size, log);
	614	kvfree(log);
	615
	616	/* In case CEL was bogus, enable some default commands. */
	617	cxl_for_each_cmd(cmd)
	618	if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
	619	set_bit(cmd->info.id, cxlm->enabled_cmds);
	620
	621	/* Found the required CEL */
	622	rc = 0;
	623	}
	624
	625	out:
	626	kvfree(gsl);
	627	return rc;
	628	}
	629	EXPORT_SYMBOL_GPL(cxl_mem_enumerate_cmds);
	630
	631	/**
	632	* cxl_mem_get_partition_info - Get partition info
	633	* @cxlm: cxl_mem instance to update partition info
	634	*
	635	* Retrieve the current partition info for the device specified. The active
	636	* values are the current capacity in bytes. If not 0, the 'next' values are
	637	* the pending values, in bytes, which take affect on next cold reset.
	638	*
	639	* Return: 0 if no error: or the result of the mailbox command.
	640	*
	641	* See CXL @8.2.9.5.2.1 Get Partition Info
	642	*/
	643	static int cxl_mem_get_partition_info(struct cxl_mem *cxlm)
	644	{
	645	struct cxl_mbox_get_partition_info {
	646	__le64 active_volatile_cap;
	647	__le64 active_persistent_cap;
	648	__le64 next_volatile_cap;
	649	__le64 next_persistent_cap;
	650	} __packed pi;
	651	int rc;
	652
	653	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_PARTITION_INFO,
	654	NULL, 0, &pi, sizeof(pi));
	655
	656	if (rc)
	657	return rc;
	658
	659	cxlm->active_volatile_bytes =
	660	le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
	661	cxlm->active_persistent_bytes =
	662	le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;
	663	cxlm->next_volatile_bytes =
	664	le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
	665	cxlm->next_persistent_bytes =
	666	le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
	667
	668	return 0;
	669	}
	670
	671	/**
	672	* cxl_mem_identify() - Send the IDENTIFY command to the device.
	673	* @cxlm: The device to identify.
	674	*
	675	* Return: 0 if identify was executed successfully.
	676	*
	677	* This will dispatch the identify command to the device and on success populate
	678	* structures to be exported to sysfs.
	679	*/
	680	int cxl_mem_identify(struct cxl_mem *cxlm)
	681	{
	682	/* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
	683	struct cxl_mbox_identify id;
	684	int rc;
	685
	686	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id,
	687	sizeof(id));
	688	if (rc < 0)
	689	return rc;
	690
	691	cxlm->total_bytes =
	692	le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;
	693	cxlm->volatile_only_bytes =
	694	le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;
	695	cxlm->persistent_only_bytes =
	696	le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;
	697	cxlm->partition_align_bytes =
	698	le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
	699
	700	dev_dbg(cxlm->dev,
	701	"Identify Memory Device\n"
	702	" total_bytes = %#llx\n"
	703	" volatile_only_bytes = %#llx\n"
	704	" persistent_only_bytes = %#llx\n"
	705	" partition_align_bytes = %#llx\n",
	706	cxlm->total_bytes, cxlm->volatile_only_bytes,
	707	cxlm->persistent_only_bytes, cxlm->partition_align_bytes);
	708
	709	cxlm->lsa_size = le32_to_cpu(id.lsa_size);
	710	memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision));
	711
	712	return 0;
	713	}
	714	EXPORT_SYMBOL_GPL(cxl_mem_identify);
	715
	716	int cxl_mem_create_range_info(struct cxl_mem *cxlm)
	717	{
	718	int rc;
	719
	720	if (cxlm->partition_align_bytes == 0) {
	721	cxlm->ram_range.start = 0;
	722	cxlm->ram_range.end = cxlm->volatile_only_bytes - 1;
	723	cxlm->pmem_range.start = cxlm->volatile_only_bytes;
	724	cxlm->pmem_range.end = cxlm->volatile_only_bytes +
	725	cxlm->persistent_only_bytes - 1;
	726	return 0;
	727	}
	728
	729	rc = cxl_mem_get_partition_info(cxlm);
	730	if (rc) {
	731	dev_err(cxlm->dev, "Failed to query partition information\n");
	732	return rc;
	733	}
	734
	735	dev_dbg(cxlm->dev,
	736	"Get Partition Info\n"
	737	" active_volatile_bytes = %#llx\n"
	738	" active_persistent_bytes = %#llx\n"
	739	" next_volatile_bytes = %#llx\n"
	740	" next_persistent_bytes = %#llx\n",
	741	cxlm->active_volatile_bytes, cxlm->active_persistent_bytes,
	742	cxlm->next_volatile_bytes, cxlm->next_persistent_bytes);
	743
	744	cxlm->ram_range.start = 0;
	745	cxlm->ram_range.end = cxlm->active_volatile_bytes - 1;
	746
	747	cxlm->pmem_range.start = cxlm->active_volatile_bytes;
	748	cxlm->pmem_range.end =
	749	cxlm->active_volatile_bytes + cxlm->active_persistent_bytes - 1;
	750
	751	return 0;
	752	}
	753	EXPORT_SYMBOL_GPL(cxl_mem_create_range_info);
	754
	755	struct cxl_mem cxl_mem_create(struct device dev)
	756	{
	757	struct cxl_mem *cxlm;
	758
	759	cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL);
	760	if (!cxlm) {
	761	dev_err(dev, "No memory available\n");
	762	return ERR_PTR(-ENOMEM);
	763	}
	764
	765	mutex_init(&cxlm->mbox_mutex);
	766	cxlm->dev = dev;
	767
	768	return cxlm;
	769	}
	770	EXPORT_SYMBOL_GPL(cxl_mem_create);
	771
	772	static struct dentry *cxl_debugfs;
	773
	774	void __init cxl_mbox_init(void)
	775	{
	776	struct dentry *mbox_debugfs;
	777
	778	cxl_debugfs = debugfs_create_dir("cxl", NULL);
	779	mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs);
	780	debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
	781	&cxl_raw_allow_all);
	782	}
	783
	784	void cxl_mbox_exit(void)
	785	{
	786	debugfs_remove_recursive(cxl_debugfs);
	787	}

drivers/cxl/core/memdev.c+106−12

@@ -8,6 +8,8 @@ · 7 unmodified lines
8	#include <cxlmem.h>	8	#include <cxlmem.h>
9	#include "core.h"	9	#include "core.h"
10		10
		11	static DECLARE_RWSEM(cxl_memdev_rwsem);
		12
11	/*	13	/*
12	* An entire PCI topology full of devices should be enough for any	14	* An entire PCI topology full of devices should be enough for any
13	* config	15	* config
@@ -132,16 +134,53 @@ static const struct device_type cxl_memdev_type = { · 118 unmodified lines
132	.groups = cxl_memdev_attribute_groups,	134	.groups = cxl_memdev_attribute_groups,
133	};	135	};
134		136
		137	/**
		138	* set_exclusive_cxl_commands() - atomically disable user cxl commands
		139	* @cxlm: cxl_mem instance to modify
		140	* @cmds: bitmap of commands to mark exclusive
		141	*
		142	* Grab the cxl_memdev_rwsem in write mode to flush in-flight
		143	* invocations of the ioctl path and then disable future execution of
		144	* commands with the command ids set in @cmds.
		145	*/
		146	void set_exclusive_cxl_commands(struct cxl_mem cxlm, unsigned long cmds)
		147	{
		148	down_write(&cxl_memdev_rwsem);
		149	bitmap_or(cxlm->exclusive_cmds, cxlm->exclusive_cmds, cmds,
		150	CXL_MEM_COMMAND_ID_MAX);
		151	up_write(&cxl_memdev_rwsem);
		152	}
		153	EXPORT_SYMBOL_GPL(set_exclusive_cxl_commands);
		154
		155	/**
		156	* clear_exclusive_cxl_commands() - atomically enable user cxl commands
		157	* @cxlm: cxl_mem instance to modify
		158	* @cmds: bitmap of commands to mark available for userspace
		159	*/
		160	void clear_exclusive_cxl_commands(struct cxl_mem cxlm, unsigned long cmds)
		161	{
		162	down_write(&cxl_memdev_rwsem);
		163	bitmap_andnot(cxlm->exclusive_cmds, cxlm->exclusive_cmds, cmds,
		164	CXL_MEM_COMMAND_ID_MAX);
		165	up_write(&cxl_memdev_rwsem);
		166	}
		167	EXPORT_SYMBOL_GPL(clear_exclusive_cxl_commands);
		168
		169	static void cxl_memdev_shutdown(struct device *dev)
		170	{
		171	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
		172
		173	down_write(&cxl_memdev_rwsem);
		174	cxlmd->cxlm = NULL;
		175	up_write(&cxl_memdev_rwsem);
		176	}
		177
135	static void cxl_memdev_unregister(void *_cxlmd)	178	static void cxl_memdev_unregister(void *_cxlmd)
136	{	179	{
137	struct cxl_memdev *cxlmd = _cxlmd;	180	struct cxl_memdev *cxlmd = _cxlmd;
138	struct device *dev = &cxlmd->dev;	181	struct device *dev = &cxlmd->dev;
139	struct cdev *cdev = &cxlmd->cdev;
140	const struct cdevm_file_operations *cdevm_fops;
141
142	cdevm_fops = container_of(cdev->ops, typeof(*cdevm_fops), fops);
143	cdevm_fops->shutdown(dev);
144		182
		183	cxl_memdev_shutdown(dev);
145	cdev_device_del(&cxlmd->cdev, dev);	184	cdev_device_del(&cxlmd->cdev, dev);
146	put_device(dev);	185	put_device(dev);
147	}	186	}
@@ -149,7 +188,6 @@ static void cxl_memdev_unregister(void *_cxlmd) · 1 unmodified line
149	static struct cxl_memdev cxl_memdev_alloc(struct cxl_mem cxlm,	188	static struct cxl_memdev cxl_memdev_alloc(struct cxl_mem cxlm,
150	const struct file_operations *fops)	189	const struct file_operations *fops)
151	{	190	{
152	struct pci_dev *pdev = cxlm->pdev;
153	struct cxl_memdev *cxlmd;	191	struct cxl_memdev *cxlmd;
154	struct device *dev;	192	struct device *dev;
155	struct cdev *cdev;	193	struct cdev *cdev;
@@ -166,7 +204,7 @@ static struct cxl_memdev cxl_memdev_alloc(struct cxl_mem cxlm, · 10 unmodified lines
166		204
167	dev = &cxlmd->dev;	205	dev = &cxlmd->dev;
168	device_initialize(dev);	206	device_initialize(dev);
169	dev->parent = &pdev->dev;	207	dev->parent = cxlm->dev;
170	dev->bus = &cxl_bus_type;	208	dev->bus = &cxl_bus_type;
171	dev->devt = MKDEV(cxl_mem_major, cxlmd->id);	209	dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
172	dev->type = &cxl_memdev_type;	210	dev->type = &cxl_memdev_type;
@@ -181,16 +219,72 @@ err: · 8 unmodified lines
181	return ERR_PTR(rc);	219	return ERR_PTR(rc);
182	}	220	}
183		221
		222	static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
		223	unsigned long arg)
		224	{
		225	switch (cmd) {
		226	case CXL_MEM_QUERY_COMMANDS:
		227	return cxl_query_cmd(cxlmd, (void __user *)arg);
		228	case CXL_MEM_SEND_COMMAND:
		229	return cxl_send_cmd(cxlmd, (void __user *)arg);
		230	default:
		231	return -ENOTTY;
		232	}
		233	}
		234
		235	static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
		236	unsigned long arg)
		237	{
		238	struct cxl_memdev *cxlmd = file->private_data;
		239	int rc = -ENXIO;
		240
		241	down_read(&cxl_memdev_rwsem);
		242	if (cxlmd->cxlm)
		243	rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
		244	up_read(&cxl_memdev_rwsem);
		245
		246	return rc;
		247	}
		248
		249	static int cxl_memdev_open(struct inode inode, struct file file)
		250	{
		251	struct cxl_memdev *cxlmd =
		252	container_of(inode->i_cdev, typeof(*cxlmd), cdev);
		253
		254	get_device(&cxlmd->dev);
		255	file->private_data = cxlmd;
		256
		257	return 0;
		258	}
		259
		260	static int cxl_memdev_release_file(struct inode inode, struct file file)
		261	{
		262	struct cxl_memdev *cxlmd =
		263	container_of(inode->i_cdev, typeof(*cxlmd), cdev);
		264
		265	put_device(&cxlmd->dev);
		266
		267	return 0;
		268	}
		269
		270	static const struct file_operations cxl_memdev_fops = {
		271	.owner = THIS_MODULE,
		272	.unlocked_ioctl = cxl_memdev_ioctl,
		273	.open = cxl_memdev_open,
		274	.release = cxl_memdev_release_file,
		275	.compat_ioctl = compat_ptr_ioctl,
		276	.llseek = noop_llseek,
		277	};
		278
184	struct cxl_memdev *	279	struct cxl_memdev *
185	devm_cxl_add_memdev(struct device host, struct cxl_mem cxlm,	280	devm_cxl_add_memdev(struct cxl_mem *cxlm)
186	const struct cdevm_file_operations *cdevm_fops)
187	{	281	{
188	struct cxl_memdev *cxlmd;	282	struct cxl_memdev *cxlmd;
189	struct device *dev;	283	struct device *dev;
190	struct cdev *cdev;	284	struct cdev *cdev;
191	int rc;	285	int rc;
192		286
193	cxlmd = cxl_memdev_alloc(cxlm, &cdevm_fops->fops);	287	cxlmd = cxl_memdev_alloc(cxlm, &cxl_memdev_fops);
194	if (IS_ERR(cxlmd))	288	if (IS_ERR(cxlmd))
195	return cxlmd;	289	return cxlmd;
196		290
@@ -210,7 +304,7 @@ devm_cxl_add_memdev(struct device host, struct cxl_mem cxlm, · 13 unmodified lines
210	if (rc)	304	if (rc)
211	goto err;	305	goto err;
212		306
213	rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);	307	rc = devm_add_action_or_reset(cxlm->dev, cxl_memdev_unregister, cxlmd);
214	if (rc)	308	if (rc)
215	return ERR_PTR(rc);	309	return ERR_PTR(rc);
216	return cxlmd;	310	return cxlmd;
@@ -220,7 +314,7 @@ err: · 3 unmodified lines
220	* The cdev was briefly live, shutdown any ioctl operations that	314	* The cdev was briefly live, shutdown any ioctl operations that
221	* saw that state.	315	* saw that state.
222	*/	316	*/
223	cdevm_fops->shutdown(dev);	317	cxl_memdev_shutdown(dev);
224	put_device(dev);	318	put_device(dev);
225	return ERR_PTR(rc);	319	return ERR_PTR(rc);
226	}	320	}

drivers/cxl/core/pmem.c+37−2

 /* Copyright(c) 2020 Intel Corporation. */
 #include <linux/device.h>
 #include <linux/slab.h>
+#include <linux/idr.h>
 #include <cxlmem.h>
 #include <cxl.h>
 #include "core.h"
  * operations, for example, namespace label access commands.
  */
+static DEFINE_IDA(cxl_nvdimm_bridge_ida);
 static void cxl_nvdimm_bridge_release(struct device *dev)
 {
 	struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
+	ida_free(&cxl_nvdimm_bridge_ida, cxl_nvb->id);
 	kfree(cxl_nvb);
 }
 }
 EXPORT_SYMBOL_GPL(to_cxl_nvdimm_bridge);
+__mock int match_nvdimm_bridge(struct device *dev, const void *data)
+{
+	return dev->type == &cxl_nvdimm_bridge_type;
+}
+struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd)
+{
+	struct device *dev;
+	dev = bus_find_device(&cxl_bus_type, NULL, cxl_nvd, match_nvdimm_bridge);
+	if (!dev)
+		return NULL;
+	return to_cxl_nvdimm_bridge(dev);
+}
+EXPORT_SYMBOL_GPL(cxl_find_nvdimm_bridge);
 static struct cxl_nvdimm_bridge *
 cxl_nvdimm_bridge_alloc(struct cxl_port *port)
 {
 	struct cxl_nvdimm_bridge *cxl_nvb;
 	struct device *dev;
+	int rc;
 	cxl_nvb = kzalloc(sizeof(*cxl_nvb), GFP_KERNEL);
 	if (!cxl_nvb)
 		return ERR_PTR(-ENOMEM);
+	rc = ida_alloc(&cxl_nvdimm_bridge_ida, GFP_KERNEL);
+	if (rc < 0)
+		goto err;
+	cxl_nvb->id = rc;
 	dev = &cxl_nvb->dev;
 	cxl_nvb->port = port;
 	cxl_nvb->state = CXL_NVB_NEW;
 	dev->type = &cxl_nvdimm_bridge_type;
 	return cxl_nvb;
+err:
+	kfree(cxl_nvb);
+	return ERR_PTR(rc);
 }
 static void unregister_nvb(void *_cxl_nvb)
 		return cxl_nvb;
 	dev = &cxl_nvb->dev;
-	rc = dev_set_name(dev, "nvdimm-bridge");
+	rc = dev_set_name(dev, "nvdimm-bridge%d", cxl_nvb->id);
 	if (rc)
 		goto err;
 	return cxl_nvd;
 }
+static void cxl_nvd_unregister(void *dev)
+{
+	device_unregister(dev);
+}
 /**
  * devm_cxl_add_nvdimm() - add a bridge between a cxl_memdev and an nvdimm
  * @host: same host as @cxlmd
 	dev_dbg(host, "%s: register %s\n", dev_name(dev->parent),
 		dev_name(dev));
-	return devm_add_action_or_reset(host, unregister_cxl_dev, dev);
+	return devm_add_action_or_reset(host, cxl_nvd_unregister, dev);
 err:
 	put_device(dev);

drivers/cxl/cxl.h+39−19

 	struct cxl_reg_map memdev;
 };
+/**
+ * struct cxl_register_map - DVSEC harvested register block mapping parameters
+ * @base: virtual base of the register-block-BAR + @block_offset
+ * @block_offset: offset to start of register block in @barno
+ * @reg_type: see enum cxl_regloc_type
+ * @barno: PCI BAR number containing the register block
+ * @component_map: cxl_reg_map for component registers
+ * @device_map: cxl_reg_maps for device registers
+ */
 struct cxl_register_map {
+	void __iomem *base;
 	u64 block_offset;
 	u8 reg_type;
 	u8 barno;
        CXL_DECODER_EXPANDER = 3,
 };
+/*
+ * Current specification goes up to 8, double that seems a reasonable
+ * software max for the foreseeable future
+ */
+#define CXL_DECODER_MAX_INTERLEAVE 16
 /**
  * struct cxl_decoder - CXL address range decode configuration
  * @dev: this decoder's device
  * @interleave_granularity: data stride per dport
  * @target_type: accelerator vs expander (type2 vs type3) selector
  * @flags: memory type capabilities and locking
+ * @nr_targets: number of elements in @target
  * @target: active ordered target list in current decoder configuration
  */
 struct cxl_decoder {
 	int interleave_granularity;
 	enum cxl_decoder_type target_type;
 	unsigned long flags;
+	const int nr_targets;
 	struct cxl_dport *target[];
 };
 };
 struct cxl_nvdimm_bridge {
+	int id;
 	struct device dev;
 	struct cxl_port *port;
 	struct nvdimm_bus *nvdimm_bus;
 	struct nvdimm *nvdimm;
 };
+struct cxl_walk_context {
+	struct device *dev;
+	struct pci_bus *root;
+	struct cxl_port *port;
+	int error;
+	int count;
+};
 /**
  * struct cxl_port - logical collection of upstream port devices and
  *		     downstream port devices to construct a CXL memory
 struct cxl_decoder *to_cxl_decoder(struct device *dev);
 bool is_root_decoder(struct device *dev);
-struct cxl_decoder *
+struct cxl_decoder *cxl_decoder_alloc(struct cxl_port *port, int nr_targets);
-devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
+int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map);
-		     resource_size_t base, resource_size_t len,
+int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);
-		     int interleave_ways, int interleave_granularity,
-		     enum cxl_decoder_type type, unsigned long flags);
-/*
- * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
- * single ported host-bridges need not publish a decoder capability when a
- * passthrough decode can be assumed, i.e. all transactions that the uport sees
- * are claimed and passed to the single dport. Default the range a 0-base
- * 0-length until the first CXL region is activated.
- */
-static inline struct cxl_decoder *
-devm_cxl_add_passthrough_decoder(struct device *host, struct cxl_port *port)
-{
-	return devm_cxl_add_decoder(host, port, 1, 0, 0, 1, PAGE_SIZE,
-				    CXL_DECODER_EXPANDER, 0);
-}
 extern struct bus_type cxl_bus_type;
 struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev);
 bool is_cxl_nvdimm(struct device *dev);
 int devm_cxl_add_nvdimm(struct device *host, struct cxl_memdev *cxlmd);
+struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd);
+/*
+ * Unit test builds overrides this to __weak, find the 'strong' version
+ * of these symbols in tools/testing/cxl/.
+ */
+#ifndef __mock
+#define __mock static
+#endif
 #endif /* __CXL_H__ */

drivers/cxl/cxlmem.h+179−23

@@ -2,6 +2,7 @@ · 1 unmodified line
2	/* Copyright(c) 2020-2021 Intel Corporation. */	2	/* Copyright(c) 2020-2021 Intel Corporation. */
3	#ifndef __CXL_MEM_H__	3	#ifndef __CXL_MEM_H__
4	#define __CXL_MEM_H__	4	#define __CXL_MEM_H__
		5	#include <uapi/linux/cxl_mem.h>
5	#include <linux/cdev.h>	6	#include <linux/cdev.h>
6	#include "cxl.h"	7	#include "cxl.h"
7		8
@@ -28,21 +29,6 @@ · 20 unmodified lines
28	(FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \	29	(FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \
29	CXLMDEV_RESET_NEEDED_NOT)	30	CXLMDEV_RESET_NEEDED_NOT)
30		31
31	/**
32	* struct cdevm_file_operations - devm coordinated cdev file operations
33	* @fops: file operations that are synchronized against @shutdown
34	* @shutdown: disconnect driver data
35	*
36	* @shutdown is invoked in the devres release path to disconnect any
37	* driver instance data from @dev. It assumes synchronization with any
38	* fops operation that requires driver data. After @shutdown an
39	* operation may only reference @device data.
40	*/
41	struct cdevm_file_operations {
42	struct file_operations fops;
43	void (shutdown)(struct device dev);
44	};
45
46	/**	32	/**
47	* struct cxl_memdev - CXL bus object representing a Type-3 Memory Device	33	* struct cxl_memdev - CXL bus object representing a Type-3 Memory Device
48	* @dev: driver core device object	34	* @dev: driver core device object
@@ -62,13 +48,50 @@ static inline struct cxl_memdev to_cxl_memdev(struct device dev) · 13 unmodified lines
62	return container_of(dev, struct cxl_memdev, dev);	48	return container_of(dev, struct cxl_memdev, dev);
63	}	49	}
64		50
65	struct cxl_memdev *	51	struct cxl_memdev devm_cxl_add_memdev(struct cxl_mem cxlm);
66	devm_cxl_add_memdev(struct device host, struct cxl_mem cxlm,	52
67	const struct cdevm_file_operations *cdevm_fops);	53	/**
		54	* struct cxl_mbox_cmd - A command to be submitted to hardware.
		55	* @opcode: (input) The command set and command submitted to hardware.
		56	* @payload_in: (input) Pointer to the input payload.
		57	* @payload_out: (output) Pointer to the output payload. Must be allocated by
		58	* the caller.
		59	* @size_in: (input) Number of bytes to load from @payload_in.
		60	* @size_out: (input) Max number of bytes loaded into @payload_out.
		61	* (output) Number of bytes generated by the device. For fixed size
		62	* outputs commands this is always expected to be deterministic. For
		63	* variable sized output commands, it tells the exact number of bytes
		64	* written.
		65	* @return_code: (output) Error code returned from hardware.
		66	*
		67	* This is the primary mechanism used to send commands to the hardware.
		68	* All the fields except @payload_* correspond exactly to the fields described in
		69	* Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and
		70	* @payload_out are written to, and read from the Command Payload Registers
		71	* defined in CXL 2.0 8.2.8.4.8.
		72	*/
		73	struct cxl_mbox_cmd {
		74	u16 opcode;
		75	void *payload_in;
		76	void *payload_out;
		77	size_t size_in;
		78	size_t size_out;
		79	u16 return_code;
		80	#define CXL_MBOX_SUCCESS 0
		81	};
		82
		83	/*
		84	* CXL 2.0 - Memory capacity multiplier
		85	* See Section 8.2.9.5
		86	*
		87	* Volatile, Persistent, and Partition capacities are specified to be in
		88	* multiples of 256MB - define a multiplier to convert to/from bytes.
		89	*/
		90	#define CXL_CAPACITY_MULTIPLIER SZ_256M
68		91
69	/**	92	/**
70	* struct cxl_mem - A CXL memory device	93	* struct cxl_mem - A CXL memory device
71	* @pdev: The PCI device associated with this CXL device.	94	* @dev: The device associated with this CXL device.
72	* @cxlmd: Logical memory device chardev / interface	95	* @cxlmd: Logical memory device chardev / interface
73	* @regs: Parsed register blocks	96	* @regs: Parsed register blocks
74	* @payload_size: Size of space for payload	97	* @payload_size: Size of space for payload
@@ -78,11 +101,24 @@ devm_cxl_add_memdev(struct device host, struct cxl_mem cxlm, · 3 unmodified lines
78	* @mbox_mutex: Mutex to synchronize mailbox access.	101	* @mbox_mutex: Mutex to synchronize mailbox access.
79	* @firmware_version: Firmware version for the memory device.	102	* @firmware_version: Firmware version for the memory device.
80	* @enabled_cmds: Hardware commands found enabled in CEL.	103	* @enabled_cmds: Hardware commands found enabled in CEL.
81	* @pmem_range: Persistent memory capacity information.	104	* @exclusive_cmds: Commands that are kernel-internal only
82	* @ram_range: Volatile memory capacity information.	105	* @pmem_range: Active Persistent memory capacity configuration
		106	* @ram_range: Active Volatile memory capacity configuration
		107	* @total_bytes: sum of all possible capacities
		108	* @volatile_only_bytes: hard volatile capacity
		109	* @persistent_only_bytes: hard persistent capacity
		110	* @partition_align_bytes: alignment size for partition-able capacity
		111	* @active_volatile_bytes: sum of hard + soft volatile
		112	* @active_persistent_bytes: sum of hard + soft persistent
		113	* @next_volatile_bytes: volatile capacity change pending device reset
		114	* @next_persistent_bytes: persistent capacity change pending device reset
		115	* @mbox_send: @dev specific transport for transmitting mailbox commands
		116	*
		117	* See section 8.2.9.5.2 Capacity Configuration and Label Storage for
		118	* details on capacity parameters.
83	*/	119	*/
84	struct cxl_mem {	120	struct cxl_mem {
85	struct pci_dev *pdev;	121	struct device *dev;
86	struct cxl_memdev *cxlmd;	122	struct cxl_memdev *cxlmd;
87		123
88	struct cxl_regs regs;	124	struct cxl_regs regs;
@@ -91,7 +127,8 @@ struct cxl_mem { · 2 unmodified lines
91	size_t lsa_size;	127	size_t lsa_size;
92	struct mutex mbox_mutex; /* Protects device mailbox and firmware */	128	struct mutex mbox_mutex; /* Protects device mailbox and firmware */
93	char firmware_version[0x10];	129	char firmware_version[0x10];
94	unsigned long *enabled_cmds;	130	DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
		131	DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
95		132
96	struct range pmem_range;	133	struct range pmem_range;
97	struct range ram_range;	134	struct range ram_range;
@@ -104,5 +141,124 @@ struct cxl_mem { · 6 unmodified lines
104	u64 active_persistent_bytes;	141	u64 active_persistent_bytes;
105	u64 next_volatile_bytes;	142	u64 next_volatile_bytes;
106	u64 next_persistent_bytes;	143	u64 next_persistent_bytes;
		144
		145	int (mbox_send)(struct cxl_mem cxlm, struct cxl_mbox_cmd *cmd);
		146	};
		147
		148	enum cxl_opcode {
		149	CXL_MBOX_OP_INVALID = 0x0000,
		150	CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID,
		151	CXL_MBOX_OP_GET_FW_INFO = 0x0200,
		152	CXL_MBOX_OP_ACTIVATE_FW = 0x0202,
		153	CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400,
		154	CXL_MBOX_OP_GET_LOG = 0x0401,
		155	CXL_MBOX_OP_IDENTIFY = 0x4000,
		156	CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
		157	CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
		158	CXL_MBOX_OP_GET_LSA = 0x4102,
		159	CXL_MBOX_OP_SET_LSA = 0x4103,
		160	CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200,
		161	CXL_MBOX_OP_GET_ALERT_CONFIG = 0x4201,
		162	CXL_MBOX_OP_SET_ALERT_CONFIG = 0x4202,
		163	CXL_MBOX_OP_GET_SHUTDOWN_STATE = 0x4203,
		164	CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204,
		165	CXL_MBOX_OP_GET_POISON = 0x4300,
		166	CXL_MBOX_OP_INJECT_POISON = 0x4301,
		167	CXL_MBOX_OP_CLEAR_POISON = 0x4302,
		168	CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303,
		169	CXL_MBOX_OP_SCAN_MEDIA = 0x4304,
		170	CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305,
		171	CXL_MBOX_OP_MAX = 0x10000
107	};	172	};
		173
		174	#define DEFINE_CXL_CEL_UUID \
		175	UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, 0xb1, 0x62, \
		176	0x3b, 0x3f, 0x17)
		177
		178	#define DEFINE_CXL_VENDOR_DEBUG_UUID \
		179	UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, 0xd6, 0x07, 0x19, \
		180	0x40, 0x3d, 0x86)
		181
		182	struct cxl_mbox_get_supported_logs {
		183	__le16 entries;
		184	u8 rsvd[6];
		185	struct cxl_gsl_entry {
		186	uuid_t uuid;
		187	__le32 size;
		188	} __packed entry[];
		189	} __packed;
		190
		191	struct cxl_cel_entry {
		192	__le16 opcode;
		193	__le16 effect;
		194	} __packed;
		195
		196	struct cxl_mbox_get_log {
		197	uuid_t uuid;
		198	__le32 offset;
		199	__le32 length;
		200	} __packed;
		201
		202	/* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
		203	struct cxl_mbox_identify {
		204	char fw_revision[0x10];
		205	__le64 total_capacity;
		206	__le64 volatile_capacity;
		207	__le64 persistent_capacity;
		208	__le64 partition_align;
		209	__le16 info_event_log_size;
		210	__le16 warning_event_log_size;
		211	__le16 failure_event_log_size;
		212	__le16 fatal_event_log_size;
		213	__le32 lsa_size;
		214	u8 poison_list_max_mer[3];
		215	__le16 inject_poison_limit;
		216	u8 poison_caps;
		217	u8 qos_telemetry_caps;
		218	} __packed;
		219
		220	struct cxl_mbox_get_lsa {
		221	u32 offset;
		222	u32 length;
		223	} __packed;
		224
		225	struct cxl_mbox_set_lsa {
		226	u32 offset;
		227	u32 reserved;
		228	u8 data[];
		229	} __packed;
		230
		231	/**
		232	* struct cxl_mem_command - Driver representation of a memory device command
		233	* @info: Command information as it exists for the UAPI
		234	* @opcode: The actual bits used for the mailbox protocol
		235	* @flags: Set of flags effecting driver behavior.
		236	*
		237	* * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag
		238	* will be enabled by the driver regardless of what hardware may have
		239	* advertised.
		240	*
		241	* The cxl_mem_command is the driver's internal representation of commands that
		242	* are supported by the driver. Some of these commands may not be supported by
		243	* the hardware. The driver will use @info to validate the fields passed in by
		244	* the user then submit the @opcode to the hardware.
		245	*
		246	* See struct cxl_command_info.
		247	*/
		248	struct cxl_mem_command {
		249	struct cxl_command_info info;
		250	enum cxl_opcode opcode;
		251	u32 flags;
		252	#define CXL_CMD_FLAG_NONE 0
		253	#define CXL_CMD_FLAG_FORCE_ENABLE BIT(0)
		254	};
		255
		256	int cxl_mem_mbox_send_cmd(struct cxl_mem cxlm, u16 opcode, void in,
		257	size_t in_size, void *out, size_t out_size);
		258	int cxl_mem_identify(struct cxl_mem *cxlm);
		259	int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm);
		260	int cxl_mem_create_range_info(struct cxl_mem *cxlm);
		261	struct cxl_mem cxl_mem_create(struct device dev);
		262	void set_exclusive_cxl_commands(struct cxl_mem cxlm, unsigned long cmds);
		263	void clear_exclusive_cxl_commands(struct cxl_mem cxlm, unsigned long cmds);
108	#endif /* __CXL_MEM_H__ */	264	#endif /* __CXL_MEM_H__ */

drivers/cxl/pci.c+109−1131

@@ -1,17 +1,12 @@
1	// SPDX-License-Identifier: GPL-2.0-only	1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright(c) 2020 Intel Corporation. All rights reserved. */	2	/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3	#include <uapi/linux/cxl_mem.h>	3	#include <linux/io-64-nonatomic-lo-hi.h>
4	#include <linux/security.h>
5	#include <linux/debugfs.h>
6	#include <linux/module.h>	4	#include <linux/module.h>
7	#include <linux/sizes.h>	5	#include <linux/sizes.h>
8	#include <linux/mutex.h>	6	#include <linux/mutex.h>
9	#include <linux/list.h>	7	#include <linux/list.h>
10	#include <linux/cdev.h>
11	#include <linux/idr.h>
12	#include <linux/pci.h>	8	#include <linux/pci.h>
13	#include <linux/io.h>	9	#include <linux/io.h>
14	#include <linux/io-64-nonatomic-lo-hi.h>
15	#include "cxlmem.h"	10	#include "cxlmem.h"
16	#include "pci.h"	11	#include "pci.h"
17	#include "cxl.h"	12	#include "cxl.h"
@@ -21,14 +16,16 @@ · 3 unmodified lines
21	*	16	*
22	* This implements the PCI exclusive functionality for a CXL device as it is	17	* This implements the PCI exclusive functionality for a CXL device as it is
23	* defined by the Compute Express Link specification. CXL devices may surface	18	* defined by the Compute Express Link specification. CXL devices may surface
24	* certain functionality even if it isn't CXL enabled.	19	* certain functionality even if it isn't CXL enabled. While this driver is
		20	* focused around the PCI specific aspects of a CXL device, it binds to the
		21	* specific CXL memory device class code, and therefore the implementation of
		22	* cxl_pci is focused around CXL memory devices.
25	*	23	*
26	* The driver has several responsibilities, mainly:	24	* The driver has several responsibilities, mainly:
27	* - Create the memX device and register on the CXL bus.	25	* - Create the memX device and register on the CXL bus.
28	* - Enumerate device's register interface and map them.	26	* - Enumerate device's register interface and map them.
29	* - Probe the device attributes to establish sysfs interface.	27	* - Registers nvdimm bridge device with cxl_core.
30	* - Provide an IOCTL interface to userspace to communicate with the device for	28	* - Registers a CXL mailbox with cxl_core.
31	* things like firmware update.
32	*/	29	*/
33		30
34	#define cxl_doorbell_busy(cxlm) \	31	#define cxl_doorbell_busy(cxlm) \
@@ -38,202 +35,7 @@ · 3 unmodified lines
38	/* CXL 2.0 - 8.2.8.4 */	35	/* CXL 2.0 - 8.2.8.4 */
39	#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)	36	#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)
40		37
41	enum opcode {	38	static int cxl_pci_mbox_wait_for_doorbell(struct cxl_mem *cxlm)
42	CXL_MBOX_OP_INVALID = 0x0000,
43	CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID,
44	CXL_MBOX_OP_GET_FW_INFO = 0x0200,
45	CXL_MBOX_OP_ACTIVATE_FW = 0x0202,
46	CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400,
47	CXL_MBOX_OP_GET_LOG = 0x0401,
48	CXL_MBOX_OP_IDENTIFY = 0x4000,
49	CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100,
50	CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101,
51	CXL_MBOX_OP_GET_LSA = 0x4102,
52	CXL_MBOX_OP_SET_LSA = 0x4103,
53	CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200,
54	CXL_MBOX_OP_GET_ALERT_CONFIG = 0x4201,
55	CXL_MBOX_OP_SET_ALERT_CONFIG = 0x4202,
56	CXL_MBOX_OP_GET_SHUTDOWN_STATE = 0x4203,
57	CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204,
58	CXL_MBOX_OP_GET_POISON = 0x4300,
59	CXL_MBOX_OP_INJECT_POISON = 0x4301,
60	CXL_MBOX_OP_CLEAR_POISON = 0x4302,
61	CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS = 0x4303,
62	CXL_MBOX_OP_SCAN_MEDIA = 0x4304,
63	CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305,
64	CXL_MBOX_OP_MAX = 0x10000
65	};
66
67	/*
68	* CXL 2.0 - Memory capacity multiplier
69	* See Section 8.2.9.5
70	*
71	* Volatile, Persistent, and Partition capacities are specified to be in
72	* multiples of 256MB - define a multiplier to convert to/from bytes.
73	*/
74	#define CXL_CAPACITY_MULTIPLIER SZ_256M
75
76	/**
77	* struct mbox_cmd - A command to be submitted to hardware.
78	* @opcode: (input) The command set and command submitted to hardware.
79	* @payload_in: (input) Pointer to the input payload.
80	* @payload_out: (output) Pointer to the output payload. Must be allocated by
81	* the caller.
82	* @size_in: (input) Number of bytes to load from @payload_in.
83	* @size_out: (input) Max number of bytes loaded into @payload_out.
84	* (output) Number of bytes generated by the device. For fixed size
85	* outputs commands this is always expected to be deterministic. For
86	* variable sized output commands, it tells the exact number of bytes
87	* written.
88	* @return_code: (output) Error code returned from hardware.
89	*
90	* This is the primary mechanism used to send commands to the hardware.
91	* All the fields except @payload_* correspond exactly to the fields described in
92	* Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and
93	* @payload_out are written to, and read from the Command Payload Registers
94	* defined in CXL 2.0 8.2.8.4.8.
95	*/
96	struct mbox_cmd {
97	u16 opcode;
98	void *payload_in;
99	void *payload_out;
100	size_t size_in;
101	size_t size_out;
102	u16 return_code;
103	#define CXL_MBOX_SUCCESS 0
104	};
105
106	static DECLARE_RWSEM(cxl_memdev_rwsem);
107	static struct dentry *cxl_debugfs;
108	static bool cxl_raw_allow_all;
109
110	enum {
111	CEL_UUID,
112	VENDOR_DEBUG_UUID,
113	};
114
115	/* See CXL 2.0 Table 170. Get Log Input Payload */
116	static const uuid_t log_uuid[] = {
117	[CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96,
118	0xb1, 0x62, 0x3b, 0x3f, 0x17),
119	[VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f,
120	0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86),
121	};
122
123	/**
124	* struct cxl_mem_command - Driver representation of a memory device command
125	* @info: Command information as it exists for the UAPI
126	* @opcode: The actual bits used for the mailbox protocol
127	* @flags: Set of flags effecting driver behavior.
128	*
129	* * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag
130	* will be enabled by the driver regardless of what hardware may have
131	* advertised.
132	*
133	* The cxl_mem_command is the driver's internal representation of commands that
134	* are supported by the driver. Some of these commands may not be supported by
135	* the hardware. The driver will use @info to validate the fields passed in by
136	* the user then submit the @opcode to the hardware.
137	*
138	* See struct cxl_command_info.
139	*/
140	struct cxl_mem_command {
141	struct cxl_command_info info;
142	enum opcode opcode;
143	u32 flags;
144	#define CXL_CMD_FLAG_NONE 0
145	#define CXL_CMD_FLAG_FORCE_ENABLE BIT(0)
146	};
147
148	#define CXL_CMD(_id, sin, sout, _flags) \
149	[CXL_MEM_COMMAND_ID_##_id] = { \
150	.info = { \
151	.id = CXL_MEM_COMMAND_ID_##_id, \
152	.size_in = sin, \
153	.size_out = sout, \
154	}, \
155	.opcode = CXL_MBOX_OP_##_id, \
156	.flags = _flags, \
157	}
158
159	/*
160	* This table defines the supported mailbox commands for the driver. This table
161	* is made up of a UAPI structure. Non-negative values as parameters in the
162	* table will be validated against the user's input. For example, if size_in is
163	* 0, and the user passed in 1, it is an error.
164	*/
165	static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
166	CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
167	#ifdef CONFIG_CXL_MEM_RAW_COMMANDS
168	CXL_CMD(RAW, ~0, ~0, 0),
169	#endif
170	CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE),
171	CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
172	CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
173	CXL_CMD(GET_LSA, 0x8, ~0, 0),
174	CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
175	CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE),
176	CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
177	CXL_CMD(SET_LSA, ~0, 0, 0),
178	CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
179	CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
180	CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
181	CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
182	CXL_CMD(GET_POISON, 0x10, ~0, 0),
183	CXL_CMD(INJECT_POISON, 0x8, 0, 0),
184	CXL_CMD(CLEAR_POISON, 0x48, 0, 0),
185	CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
186	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
187	CXL_CMD(GET_SCAN_MEDIA, 0, ~0, 0),
188	};
189
190	/*
191	* Commands that RAW doesn't permit. The rationale for each:
192	*
193	* CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /
194	* coordination of transaction timeout values at the root bridge level.
195	*
196	* CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live
197	* and needs to be coordinated with HDM updates.
198	*
199	* CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the
200	* driver and any writes from userspace invalidates those contents.
201	*
202	* CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes
203	* to the device after it is marked clean, userspace can not make that
204	* assertion.
205	*
206	* CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that
207	* is kept up to date with patrol notifications and error management.
208	*/
209	static u16 cxl_disabled_raw_commands[] = {
210	CXL_MBOX_OP_ACTIVATE_FW,
211	CXL_MBOX_OP_SET_PARTITION_INFO,
212	CXL_MBOX_OP_SET_LSA,
213	CXL_MBOX_OP_SET_SHUTDOWN_STATE,
214	CXL_MBOX_OP_SCAN_MEDIA,
215	CXL_MBOX_OP_GET_SCAN_MEDIA,
216	};
217
218	/*
219	* Command sets that RAW doesn't permit. All opcodes in this set are
220	* disabled because they pass plain text security payloads over the
221	* user/kernel boundary. This functionality is intended to be wrapped
222	* behind the keys ABI which allows for encrypted payloads in the UAPI
223	*/
224	static u8 security_command_sets[] = {
225	0x44, /* Sanitize */
226	0x45, /* Persistent Memory Data-at-rest Security */
227	0x46, /* Security Passthrough */
228	};
229
230	#define cxl_for_each_cmd(cmd) \
231	for ((cmd) = &mem_commands[0]; \
232	((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++)
233
234	#define cxl_cmd_count ARRAY_SIZE(mem_commands)
235
236	static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm)
237	{	39	{
238	const unsigned long start = jiffies;	40	const unsigned long start = jiffies;
239	unsigned long end = start;	41	unsigned long end = start;
@@ -250,32 +52,22 @@ static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) · 10 unmodified lines
250	cpu_relax();	52	cpu_relax();
251	}	53	}
252		54
253	dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms",	55	dev_dbg(cxlm->dev, "Doorbell wait took %dms",
254	jiffies_to_msecs(end) - jiffies_to_msecs(start));	56	jiffies_to_msecs(end) - jiffies_to_msecs(start));
255	return 0;	57	return 0;
256	}	58	}
257		59
258	static bool cxl_is_security_command(u16 opcode)	60	static void cxl_pci_mbox_timeout(struct cxl_mem *cxlm,
		61	struct cxl_mbox_cmd *mbox_cmd)
259	{	62	{
260	int i;	63	struct device *dev = cxlm->dev;
261
262	for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)
263	if (security_command_sets[i] == (opcode >> 8))
264	return true;
265	return false;
266	}
267
268	static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm,
269	struct mbox_cmd *mbox_cmd)
270	{
271	struct device *dev = &cxlm->pdev->dev;
272		64
273	dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n",	65	dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n",
274	mbox_cmd->opcode, mbox_cmd->size_in);	66	mbox_cmd->opcode, mbox_cmd->size_in);
275	}	67	}
276		68
277	/**	69	/**
278	* __cxl_mem_mbox_send_cmd() - Execute a mailbox command	70	* __cxl_pci_mbox_send_cmd() - Execute a mailbox command
279	* @cxlm: The CXL memory device to communicate with.	71	* @cxlm: The CXL memory device to communicate with.
280	* @mbox_cmd: Command to send to the memory device.	72	* @mbox_cmd: Command to send to the memory device.
281	*	73	*
@@ -296,10 +88,11 @@ static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, · 14 unmodified lines
296	* not need to coordinate with each other. The driver only uses the primary	88	* not need to coordinate with each other. The driver only uses the primary
297	* mailbox.	89	* mailbox.
298	*/	90	*/
299	static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm,	91	static int __cxl_pci_mbox_send_cmd(struct cxl_mem *cxlm,
300	struct mbox_cmd *mbox_cmd)	92	struct cxl_mbox_cmd *mbox_cmd)
301	{	93	{
302	void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;	94	void __iomem *payload = cxlm->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
		95	struct device *dev = cxlm->dev;
303	u64 cmd_reg, status_reg;	96	u64 cmd_reg, status_reg;
304	size_t out_len;	97	size_t out_len;
305	int rc;	98	int rc;
@@ -325,8 +118,7 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, · 19 unmodified lines
325		118
326	/* #1 */	119	/* #1 */
327	if (cxl_doorbell_busy(cxlm)) {	120	if (cxl_doorbell_busy(cxlm)) {
328	dev_err_ratelimited(&cxlm->pdev->dev,	121	dev_err_ratelimited(dev, "Mailbox re-busy after acquiring\n");
329	"Mailbox re-busy after acquiring\n");
330	return -EBUSY;	122	return -EBUSY;
331	}	123	}
332		124
@@ -345,14 +137,14 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, · 12 unmodified lines
345	writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);	137	writeq(cmd_reg, cxlm->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
346		138
347	/* #4 */	139	/* #4 */
348	dev_dbg(&cxlm->pdev->dev, "Sending command\n");	140	dev_dbg(dev, "Sending command\n");
349	writel(CXLDEV_MBOX_CTRL_DOORBELL,	141	writel(CXLDEV_MBOX_CTRL_DOORBELL,
350	cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);	142	cxlm->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
351		143
352	/* #5 */	144	/* #5 */
353	rc = cxl_mem_wait_for_doorbell(cxlm);	145	rc = cxl_pci_mbox_wait_for_doorbell(cxlm);
354	if (rc == -ETIMEDOUT) {	146	if (rc == -ETIMEDOUT) {
355	cxl_mem_mbox_timeout(cxlm, mbox_cmd);	147	cxl_pci_mbox_timeout(cxlm, mbox_cmd);
356	return rc;	148	return rc;
357	}	149	}
358		150
@@ -362,7 +154,7 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, · 3 unmodified lines
362	FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);	154	FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);
363		155
364	if (mbox_cmd->return_code != 0) {	156	if (mbox_cmd->return_code != 0) {
365	dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n");	157	dev_dbg(dev, "Mailbox operation had an error\n");
366	return 0;	158	return 0;
367	}	159	}
368		160
@@ -391,15 +183,15 @@ static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, · 22 unmodified lines
391	}	183	}
392		184
393	/**	185	/**
394	* cxl_mem_mbox_get() - Acquire exclusive access to the mailbox.	186	* cxl_pci_mbox_get() - Acquire exclusive access to the mailbox.
395	* @cxlm: The memory device to gain access to.	187	* @cxlm: The memory device to gain access to.
396	*	188	*
397	* Context: Any context. Takes the mbox_mutex.	189	* Context: Any context. Takes the mbox_mutex.
398	* Return: 0 if exclusive access was acquired.	190	* Return: 0 if exclusive access was acquired.
399	*/	191	*/
400	static int cxl_mem_mbox_get(struct cxl_mem *cxlm)	192	static int cxl_pci_mbox_get(struct cxl_mem *cxlm)
401	{	193	{
402	struct device *dev = &cxlm->pdev->dev;	194	struct device *dev = cxlm->dev;
403	u64 md_status;	195	u64 md_status;
404	int rc;	196	int rc;
405		197
@@ -422,7 +214,7 @@ static int cxl_mem_mbox_get(struct cxl_mem *cxlm) · 16 unmodified lines
422	* Mailbox Interface Ready bit. Therefore, waiting for the doorbell	214	* Mailbox Interface Ready bit. Therefore, waiting for the doorbell
423	* to be ready is sufficient.	215	* to be ready is sufficient.
424	*/	216	*/
425	rc = cxl_mem_wait_for_doorbell(cxlm);	217	rc = cxl_pci_mbox_wait_for_doorbell(cxlm);
426	if (rc) {	218	if (rc) {
427	dev_warn(dev, "Mailbox interface not ready\n");	219	dev_warn(dev, "Mailbox interface not ready\n");
428	goto out;	220	goto out;
@@ -462,457 +254,35 @@ out: · 33 unmodified lines
462	}	254	}
463		255
464	/**	256	/**
465	* cxl_mem_mbox_put() - Release exclusive access to the mailbox.	257	* cxl_pci_mbox_put() - Release exclusive access to the mailbox.
466	* @cxlm: The CXL memory device to communicate with.	258	* @cxlm: The CXL memory device to communicate with.
467	*	259	*
468	* Context: Any context. Expects mbox_mutex to be held.	260	* Context: Any context. Expects mbox_mutex to be held.
469	*/	261	*/
470	static void cxl_mem_mbox_put(struct cxl_mem *cxlm)	262	static void cxl_pci_mbox_put(struct cxl_mem *cxlm)
471	{	263	{
472	mutex_unlock(&cxlm->mbox_mutex);	264	mutex_unlock(&cxlm->mbox_mutex);
473	}	265	}
474		266
475	/**	267	static int cxl_pci_mbox_send(struct cxl_mem cxlm, struct cxl_mbox_cmd cmd)
476	* handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
477	* @cxlm: The CXL memory device to communicate with.
478	* @cmd: The validated command.
479	* @in_payload: Pointer to userspace's input payload.
480	* @out_payload: Pointer to userspace's output payload.
481	* @size_out: (Input) Max payload size to copy out.
482	* (Output) Payload size hardware generated.
483	* @retval: Hardware generated return code from the operation.
484	*
485	* Return:
486	* * %0 - Mailbox transaction succeeded. This implies the mailbox
487	* protocol completed successfully not that the operation itself
488	* was successful.
489	* * %-ENOMEM - Couldn't allocate a bounce buffer.
490	* * %-EFAULT - Something happened with copy_to/from_user.
491	* * %-EINTR - Mailbox acquisition interrupted.
492	* * %-EXXX - Transaction level failures.
493	*
494	* Creates the appropriate mailbox command and dispatches it on behalf of a
495	* userspace request. The input and output payloads are copied between
496	* userspace.
497	*
498	* See cxl_send_cmd().
499	*/
500	static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm,
501	const struct cxl_mem_command *cmd,
502	u64 in_payload, u64 out_payload,
503	s32 size_out, u32 retval)
504	{
505	struct device *dev = &cxlm->pdev->dev;
506	struct mbox_cmd mbox_cmd = {
507	.opcode = cmd->opcode,
508	.size_in = cmd->info.size_in,
509	.size_out = cmd->info.size_out,
510	};
511	int rc;
512
513	if (cmd->info.size_out) {
514	mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL);
515	if (!mbox_cmd.payload_out)
516	return -ENOMEM;
517	}
518
519	if (cmd->info.size_in) {
520	mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
521	cmd->info.size_in);
522	if (IS_ERR(mbox_cmd.payload_in)) {
523	kvfree(mbox_cmd.payload_out);
524	return PTR_ERR(mbox_cmd.payload_in);
525	}
526	}
527
528	rc = cxl_mem_mbox_get(cxlm);
529	if (rc)
530	goto out;
531
532	dev_dbg(dev,
533	"Submitting %s command for user\n"
534	"\topcode: %x\n"
535	"\tsize: %ub\n",
536	cxl_command_names[cmd->info.id].name, mbox_cmd.opcode,
537	cmd->info.size_in);
538
539	dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW,
540	"raw command path used\n");
541
542	rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd);
543	cxl_mem_mbox_put(cxlm);
544	if (rc)
545	goto out;
546
547	/*
548	* @size_out contains the max size that's allowed to be written back out
549	* to userspace. While the payload may have written more output than
550	* this it will have to be ignored.
551	*/
552	if (mbox_cmd.size_out) {
553	dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out,
554	"Invalid return size\n");
555	if (copy_to_user(u64_to_user_ptr(out_payload),
556	mbox_cmd.payload_out, mbox_cmd.size_out)) {
557	rc = -EFAULT;
558	goto out;
559	}
560	}
561
562	*size_out = mbox_cmd.size_out;
563	*retval = mbox_cmd.return_code;
564
565	out:
566	kvfree(mbox_cmd.payload_in);
567	kvfree(mbox_cmd.payload_out);
568	return rc;
569	}
570
571	static bool cxl_mem_raw_command_allowed(u16 opcode)
572	{
573	int i;
574
575	if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))
576	return false;
577
578	if (security_locked_down(LOCKDOWN_PCI_ACCESS))
579	return false;
580
581	if (cxl_raw_allow_all)
582	return true;
583
584	if (cxl_is_security_command(opcode))
585	return false;
586
587	for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)
588	if (cxl_disabled_raw_commands[i] == opcode)
589	return false;
590
591	return true;
592	}
593
594	/**
595	* cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
596	* @cxlm: &struct cxl_mem device whose mailbox will be used.
597	* @send_cmd: &struct cxl_send_command copied in from userspace.
598	* @out_cmd: Sanitized and populated &struct cxl_mem_command.
599	*
600	* Return:
601	* * %0 - @out_cmd is ready to send.
602	* * %-ENOTTY - Invalid command specified.
603	* * %-EINVAL - Reserved fields or invalid values were used.
604	* * %-ENOMEM - Input or output buffer wasn't sized properly.
605	* * %-EPERM - Attempted to use a protected command.
606	*
607	* The result of this command is a fully validated command in @out_cmd that is
608	* safe to send to the hardware.
609	*
610	* See handle_mailbox_cmd_from_user()
611	*/
612	static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm,
613	const struct cxl_send_command *send_cmd,
614	struct cxl_mem_command *out_cmd)
615	{
616	const struct cxl_command_info *info;
617	struct cxl_mem_command *c;
618
619	if (send_cmd->id == 0 \|\| send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
620	return -ENOTTY;
621
622	/*
623	* The user can never specify an input payload larger than what hardware
624	* supports, but output can be arbitrarily large (simply write out as
625	* much data as the hardware provides).
626	*/
627	if (send_cmd->in.size > cxlm->payload_size)
628	return -EINVAL;
629
630	/*
631	* Checks are bypassed for raw commands but a WARN/taint will occur
632	* later in the callchain
633	*/
634	if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) {
635	const struct cxl_mem_command temp = {
636	.info = {
637	.id = CXL_MEM_COMMAND_ID_RAW,
638	.flags = 0,
639	.size_in = send_cmd->in.size,
640	.size_out = send_cmd->out.size,
641	},
642	.opcode = send_cmd->raw.opcode
643	};
644
645	if (send_cmd->raw.rsvd)
646	return -EINVAL;
647
648	/*
649	* Unlike supported commands, the output size of RAW commands
650	* gets passed along without further checking, so it must be
651	* validated here.
652	*/
653	if (send_cmd->out.size > cxlm->payload_size)
654	return -EINVAL;
655
656	if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
657	return -EPERM;
658
659	memcpy(out_cmd, &temp, sizeof(temp));
660
661	return 0;
662	}
663
664	if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
665	return -EINVAL;
666
667	if (send_cmd->rsvd)
668	return -EINVAL;
669
670	if (send_cmd->in.rsvd \|\| send_cmd->out.rsvd)
671	return -EINVAL;
672
673	/* Convert user's command into the internal representation */
674	c = &mem_commands[send_cmd->id];
675	info = &c->info;
676
677	/* Check that the command is enabled for hardware */
678	if (!test_bit(info->id, cxlm->enabled_cmds))
679	return -ENOTTY;
680
681	/* Check the input buffer is the expected size */
682	if (info->size_in >= 0 && info->size_in != send_cmd->in.size)
683	return -ENOMEM;
684
685	/* Check the output buffer is at least large enough */
686	if (info->size_out >= 0 && send_cmd->out.size < info->size_out)
687	return -ENOMEM;
688
689	memcpy(out_cmd, c, sizeof(*c));
690	out_cmd->info.size_in = send_cmd->in.size;
691	/*
692	* XXX: out_cmd->info.size_out will be controlled by the driver, and the
693	* specified number of bytes @send_cmd->out.size will be copied back out
694	* to userspace.
695	*/
696
697	return 0;
698	}
699
700	static int cxl_query_cmd(struct cxl_memdev *cxlmd,
701	struct cxl_mem_query_commands __user *q)
702	{	268	{
703	struct device *dev = &cxlmd->dev;
704	struct cxl_mem_command *cmd;
705	u32 n_commands;
706	int j = 0;
707
708	dev_dbg(dev, "Query IOCTL\n");
709
710	if (get_user(n_commands, &q->n_commands))
711	return -EFAULT;
712
713	/* returns the total number if 0 elements are requested. */
714	if (n_commands == 0)
715	return put_user(cxl_cmd_count, &q->n_commands);
716
717	/*
718	* otherwise, return max(n_commands, total commands) cxl_command_info
719	* structures.
720	*/
721	cxl_for_each_cmd(cmd) {
722	const struct cxl_command_info *info = &cmd->info;
723
724	if (copy_to_user(&q->commands[j++], info, sizeof(*info)))
725	return -EFAULT;
726
727	if (j == n_commands)
728	break;
729	}
730
731	return 0;
732	}
733
734	static int cxl_send_cmd(struct cxl_memdev *cxlmd,
735	struct cxl_send_command __user *s)
736	{
737	struct cxl_mem *cxlm = cxlmd->cxlm;
738	struct device *dev = &cxlmd->dev;
739	struct cxl_send_command send;
740	struct cxl_mem_command c;
741	int rc;	269	int rc;
742		270
743	dev_dbg(dev, "Send IOCTL\n");	271	rc = cxl_pci_mbox_get(cxlm);
744
745	if (copy_from_user(&send, s, sizeof(send)))
746	return -EFAULT;
747
748	rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c);
749	if (rc)
750	return rc;
751
752	/* Prepare to handle a full payload for variable sized output */
753	if (c.info.size_out < 0)
754	c.info.size_out = cxlm->payload_size;
755
756	rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload,
757	send.out.payload, &send.out.size,
758	&send.retval);
759	if (rc)	272	if (rc)
760	return rc;	273	return rc;
761		274
762	if (copy_to_user(s, &send, sizeof(send)))	275	rc = __cxl_pci_mbox_send_cmd(cxlm, cmd);
763	return -EFAULT;	276	cxl_pci_mbox_put(cxlm);
764
765	return 0;
766	}
767
768	static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
769	unsigned long arg)
770	{
771	switch (cmd) {
772	case CXL_MEM_QUERY_COMMANDS:
773	return cxl_query_cmd(cxlmd, (void __user *)arg);
774	case CXL_MEM_SEND_COMMAND:
775	return cxl_send_cmd(cxlmd, (void __user *)arg);
776	default:
777	return -ENOTTY;
778	}
779	}
780
781	static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
782	unsigned long arg)
783	{
784	struct cxl_memdev *cxlmd = file->private_data;
785	int rc = -ENXIO;
786
787	down_read(&cxl_memdev_rwsem);
788	if (cxlmd->cxlm)
789	rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
790	up_read(&cxl_memdev_rwsem);
791		277
792	return rc;	278	return rc;
793	}	279	}
794		280
795	static int cxl_memdev_open(struct inode inode, struct file file)	281	static int cxl_pci_setup_mailbox(struct cxl_mem *cxlm)
796	{
797	struct cxl_memdev *cxlmd =
798	container_of(inode->i_cdev, typeof(*cxlmd), cdev);
799
800	get_device(&cxlmd->dev);
801	file->private_data = cxlmd;
802
803	return 0;
804	}
805
806	static int cxl_memdev_release_file(struct inode inode, struct file file)
807	{
808	struct cxl_memdev *cxlmd =
809	container_of(inode->i_cdev, typeof(*cxlmd), cdev);
810
811	put_device(&cxlmd->dev);
812
813	return 0;
814	}
815
816	static void cxl_memdev_shutdown(struct device *dev)
817	{
818	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
819
820	down_write(&cxl_memdev_rwsem);
821	cxlmd->cxlm = NULL;
822	up_write(&cxl_memdev_rwsem);
823	}
824
825	static const struct cdevm_file_operations cxl_memdev_fops = {
826	.fops = {
827	.owner = THIS_MODULE,
828	.unlocked_ioctl = cxl_memdev_ioctl,
829	.open = cxl_memdev_open,
830	.release = cxl_memdev_release_file,
831	.compat_ioctl = compat_ptr_ioctl,
832	.llseek = noop_llseek,
833	},
834	.shutdown = cxl_memdev_shutdown,
835	};
836
837	static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
838	{
839	struct cxl_mem_command *c;
840
841	cxl_for_each_cmd(c)
842	if (c->opcode == opcode)
843	return c;
844
845	return NULL;
846	}
847
848	/**
849	* cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device.
850	* @cxlm: The CXL memory device to communicate with.
851	* @opcode: Opcode for the mailbox command.
852	* @in: The input payload for the mailbox command.
853	* @in_size: The length of the input payload
854	* @out: Caller allocated buffer for the output.
855	* @out_size: Expected size of output.
856	*
857	* Context: Any context. Will acquire and release mbox_mutex.
858	* Return:
859	* * %>=0 - Number of bytes returned in @out.
860	* * %-E2BIG - Payload is too large for hardware.
861	* * %-EBUSY - Couldn't acquire exclusive mailbox access.
862	* * %-EFAULT - Hardware error occurred.
863	* * %-ENXIO - Command completed, but device reported an error.
864	* * %-EIO - Unexpected output size.
865	*
866	* Mailbox commands may execute successfully yet the device itself reported an
867	* error. While this distinction can be useful for commands from userspace, the
868	* kernel will only be able to use results when both are successful.
869	*
870	* See __cxl_mem_mbox_send_cmd()
871	*/
872	static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode,
873	void *in, size_t in_size,
874	void *out, size_t out_size)
875	{
876	const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
877	struct mbox_cmd mbox_cmd = {
878	.opcode = opcode,
879	.payload_in = in,
880	.size_in = in_size,
881	.size_out = out_size,
882	.payload_out = out,
883	};
884	int rc;
885
886	if (out_size > cxlm->payload_size)
887	return -E2BIG;
888
889	rc = cxl_mem_mbox_get(cxlm);
890	if (rc)
891	return rc;
892
893	rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd);
894	cxl_mem_mbox_put(cxlm);
895	if (rc)
896	return rc;
897
898	/* TODO: Map return code to proper kernel style errno */
899	if (mbox_cmd.return_code != CXL_MBOX_SUCCESS)
900	return -ENXIO;
901
902	/*
903	* Variable sized commands can't be validated and so it's up to the
904	* caller to do that if they wish.
905	*/
906	if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size)
907	return -EIO;
908
909	return 0;
910	}
911
912	static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm)
913	{	282	{
914	const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);	283	const int cap = readl(cxlm->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
915		284
		285	cxlm->mbox_send = cxl_pci_mbox_send;
916	cxlm->payload_size =	286	cxlm->payload_size =
917	1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);	287	1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
918		288
@@ -925,103 +295,57 @@ static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) · 6 unmodified lines
925	*/	295	*/
926	cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M);	296	cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M);
927	if (cxlm->payload_size < 256) {	297	if (cxlm->payload_size < 256) {
928	dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)",	298	dev_err(cxlm->dev, "Mailbox is too small (%zub)",
929	cxlm->payload_size);	299	cxlm->payload_size);
930	return -ENXIO;	300	return -ENXIO;
931	}	301	}
932		302
933	dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu",	303	dev_dbg(cxlm->dev, "Mailbox payload sized %zu",
934	cxlm->payload_size);	304	cxlm->payload_size);
935		305
936	return 0;	306	return 0;
937	}	307	}
938		308
939	static struct cxl_mem cxl_mem_create(struct pci_dev pdev)	309	static int cxl_map_regblock(struct pci_dev pdev, struct cxl_register_map map)
940	{
941	struct device *dev = &pdev->dev;
942	struct cxl_mem *cxlm;
943
944	cxlm = devm_kzalloc(dev, sizeof(*cxlm), GFP_KERNEL);
945	if (!cxlm) {
946	dev_err(dev, "No memory available\n");
947	return ERR_PTR(-ENOMEM);
948	}
949
950	mutex_init(&cxlm->mbox_mutex);
951	cxlm->pdev = pdev;
952	cxlm->enabled_cmds =
953	devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count),
954	sizeof(unsigned long),
955	GFP_KERNEL \| __GFP_ZERO);
956	if (!cxlm->enabled_cmds) {
957	dev_err(dev, "No memory available for bitmap\n");
958	return ERR_PTR(-ENOMEM);
959	}
960
961	return cxlm;
962	}
963
964	static void __iomem cxl_mem_map_regblock(struct cxl_mem cxlm,
965	u8 bar, u64 offset)
966	{	310	{
967	struct pci_dev *pdev = cxlm->pdev;
968	struct device *dev = &pdev->dev;
969	void __iomem *addr;	311	void __iomem *addr;
		312	int bar = map->barno;
		313	struct device *dev = &pdev->dev;
		314	resource_size_t offset = map->block_offset;
970		315
971	/* Basic sanity check that BAR is big enough */	316	/* Basic sanity check that BAR is big enough */
972	if (pci_resource_len(pdev, bar) < offset) {	317	if (pci_resource_len(pdev, bar) < offset) {
973	dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar,	318	dev_err(dev, "BAR%d: %pr: too small (offset: %pa)\n", bar,
974	&pdev->resource[bar], (unsigned long long)offset);	319	&pdev->resource[bar], &offset);
975	return IOMEM_ERR_PTR(-ENXIO);	320	return -ENXIO;
976	}	321	}
977		322
978	addr = pci_iomap(pdev, bar, 0);	323	addr = pci_iomap(pdev, bar, 0);
979	if (!addr) {	324	if (!addr) {
980	dev_err(dev, "failed to map registers\n");	325	dev_err(dev, "failed to map registers\n");
981	return addr;	326	return -ENOMEM;
982	}	327	}
983		328
984	dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %#llx\n",	329	dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %pa\n",
985	bar, offset);	330	bar, &offset);
986		331
987	return addr;	332	map->base = addr + map->block_offset;
		333	return 0;
988	}	334	}
989		335
990	static void cxl_mem_unmap_regblock(struct cxl_mem cxlm, void __iomem base)	336	static void cxl_unmap_regblock(struct pci_dev *pdev,
		337	struct cxl_register_map *map)
991	{	338	{
992	pci_iounmap(cxlm->pdev, base);	339	pci_iounmap(pdev, map->base - map->block_offset);
		340	map->base = NULL;
993	}	341	}
994		342
995	static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec)	343	static int cxl_probe_regs(struct pci_dev pdev, struct cxl_register_map map)
996	{	344	{
997	int pos;
998
999	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC);
1000	if (!pos)
1001	return 0;
1002
1003	while (pos) {
1004	u16 vendor, id;
1005
1006	pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor);
1007	pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id);
1008	if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id)
1009	return pos;
1010
1011	pos = pci_find_next_ext_capability(pdev, pos,
1012	PCI_EXT_CAP_ID_DVSEC);
1013	}
1014
1015	return 0;
1016	}
1017
1018	static int cxl_probe_regs(struct cxl_mem cxlm, void __iomem base,
1019	struct cxl_register_map *map)
1020	{
1021	struct pci_dev *pdev = cxlm->pdev;
1022	struct device *dev = &pdev->dev;
1023	struct cxl_component_reg_map *comp_map;	345	struct cxl_component_reg_map *comp_map;
1024	struct cxl_device_reg_map *dev_map;	346	struct cxl_device_reg_map *dev_map;
		347	struct device *dev = &pdev->dev;
		348	void __iomem *base = map->base;
1025		349
1026	switch (map->reg_type) {	350	switch (map->reg_type) {
1027	case CXL_REGLOC_RBI_COMPONENT:	351	case CXL_REGLOC_RBI_COMPONENT:
@@ -1057,8 +381,8 @@ static int cxl_probe_regs(struct cxl_mem cxlm, void __iomem base, · 29 unmodified lines
1057		381
1058	static int cxl_map_regs(struct cxl_mem cxlm, struct cxl_register_map map)	382	static int cxl_map_regs(struct cxl_mem cxlm, struct cxl_register_map map)
1059	{	383	{
1060	struct pci_dev *pdev = cxlm->pdev;	384	struct device *dev = cxlm->dev;
1061	struct device *dev = &pdev->dev;	385	struct pci_dev *pdev = to_pci_dev(dev);
1062		386
1063	switch (map->reg_type) {	387	switch (map->reg_type) {
1064	case CXL_REGLOC_RBI_COMPONENT:	388	case CXL_REGLOC_RBI_COMPONENT:
@@ -1076,426 +400,108 @@ static int cxl_map_regs(struct cxl_mem cxlm, struct cxl_register_map map) · 11 unmodified lines
1076	return 0;	400	return 0;
1077	}	401	}
1078		402
1079	static void cxl_decode_register_block(u32 reg_lo, u32 reg_hi,	403	static void cxl_decode_regblock(u32 reg_lo, u32 reg_hi,
1080	u8 bar, u64 offset, u8 *reg_type)	404	struct cxl_register_map *map)
1081	{	405	{
1082	*offset = ((u64)reg_hi << 32) \| (reg_lo & CXL_REGLOC_ADDR_MASK);	406	map->block_offset =
1083	*bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo);	407	((u64)reg_hi << 32) \| (reg_lo & CXL_REGLOC_ADDR_MASK);
1084	*reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo);	408	map->barno = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo);
		409	map->reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo);
1085	}	410	}
1086		411
1087	/**	412	/**
1088	* cxl_mem_setup_regs() - Setup necessary MMIO.	413	* cxl_find_regblock() - Locate register blocks by type
1089	* @cxlm: The CXL memory device to communicate with.	414	* @pdev: The CXL PCI device to enumerate.
		415	* @type: Register Block Indicator id
		416	* @map: Enumeration output, clobbered on error
1090	*	417	*
1091	* Return: 0 if all necessary registers mapped.	418	* Return: 0 if register block enumerated, negative error code otherwise
1092	*	419	*
1093	* A memory device is required by spec to implement a certain set of MMIO	420	* A CXL DVSEC may point to one or more register blocks, search for them
1094	* regions. The purpose of this function is to enumerate and map those	421	* by @type.
1095	* registers.
1096	*/	422	*/
1097	static int cxl_mem_setup_regs(struct cxl_mem *cxlm)	423	static int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
		424	struct cxl_register_map *map)
1098	{	425	{
1099	struct pci_dev *pdev = cxlm->pdev;
1100	struct device *dev = &pdev->dev;
1101	u32 regloc_size, regblocks;	426	u32 regloc_size, regblocks;
1102	void __iomem *base;	427	int regloc, i;
1103	int regloc, i, n_maps;
1104	struct cxl_register_map *map, maps[CXL_REGLOC_RBI_TYPES];
1105	int ret = 0;
1106
1107	regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID);
1108	if (!regloc) {
1109	dev_err(dev, "register location dvsec not found\n");
1110	return -ENXIO;
1111	}
1112		428
1113	if (pci_request_mem_regions(pdev, pci_name(pdev)))	429	regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL,
1114	return -ENODEV;	430	PCI_DVSEC_ID_CXL_REGLOC_DVSEC_ID);
		431	if (!regloc)
		432	return -ENXIO;
1115		433
1116	/* Get the size of the Register Locator DVSEC */
1117	pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, &regloc_size);	434	pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, &regloc_size);
1118	regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size);	435	regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size);
1119		436
1120	regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET;	437	regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET;
1121	regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8;	438	regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8;
1122		439
1123	for (i = 0, n_maps = 0; i < regblocks; i++, regloc += 8) {	440	for (i = 0; i < regblocks; i++, regloc += 8) {
1124	u32 reg_lo, reg_hi;	441	u32 reg_lo, reg_hi;
1125	u8 reg_type;
1126	u64 offset;
1127	u8 bar;
1128		442
1129	pci_read_config_dword(pdev, regloc, &reg_lo);	443	pci_read_config_dword(pdev, regloc, &reg_lo);
1130	pci_read_config_dword(pdev, regloc + 4, &reg_hi);	444	pci_read_config_dword(pdev, regloc + 4, &reg_hi);
1131		445
1132	cxl_decode_register_block(reg_lo, reg_hi, &bar, &offset,	446	cxl_decode_regblock(reg_lo, reg_hi, map);
1133	&reg_type);
1134
1135	dev_dbg(dev, "Found register block in bar %u @ 0x%llx of type %u\n",
1136	bar, offset, reg_type);
1137
1138	/* Ignore unknown register block types */
1139	if (reg_type > CXL_REGLOC_RBI_MEMDEV)
1140	continue;
1141
1142	base = cxl_mem_map_regblock(cxlm, bar, offset);
1143	if (!base)
1144	return -ENOMEM;
1145
1146	map = &maps[n_maps];
1147	map->barno = bar;
1148	map->block_offset = offset;
1149	map->reg_type = reg_type;
1150
1151	ret = cxl_probe_regs(cxlm, base + offset, map);
1152
1153	/* Always unmap the regblock regardless of probe success */
1154	cxl_mem_unmap_regblock(cxlm, base);
1155
1156	if (ret)
1157	return ret;
1158
1159	n_maps++;
1160	}
1161
1162	pci_release_mem_regions(pdev);
1163
1164	for (i = 0; i < n_maps; i++) {
1165	ret = cxl_map_regs(cxlm, &maps[i]);
1166	if (ret)
1167	break;
1168	}
1169
1170	return ret;
1171	}
1172
1173	static int cxl_xfer_log(struct cxl_mem cxlm, uuid_t uuid, u32 size, u8 *out)
1174	{
1175	u32 remaining = size;
1176	u32 offset = 0;
1177
1178	while (remaining) {
1179	u32 xfer_size = min_t(u32, remaining, cxlm->payload_size);
1180	struct cxl_mbox_get_log {
1181	uuid_t uuid;
1182	__le32 offset;
1183	__le32 length;
1184	} __packed log = {
1185	.uuid = *uuid,
1186	.offset = cpu_to_le32(offset),
1187	.length = cpu_to_le32(xfer_size)
1188	};
1189	int rc;
1190
1191	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log,
1192	sizeof(log), out, xfer_size);
1193	if (rc < 0)
1194	return rc;
1195
1196	out += xfer_size;
1197	remaining -= xfer_size;
1198	offset += xfer_size;
1199	}
1200
1201	return 0;
1202	}
1203
1204	/**
1205	* cxl_walk_cel() - Walk through the Command Effects Log.
1206	* @cxlm: Device.
1207	* @size: Length of the Command Effects Log.
1208	* @cel: CEL
1209	*
1210	* Iterate over each entry in the CEL and determine if the driver supports the
1211	* command. If so, the command is enabled for the device and can be used later.
1212	*/
1213	static void cxl_walk_cel(struct cxl_mem cxlm, size_t size, u8 cel)
1214	{
1215	struct cel_entry {
1216	__le16 opcode;
1217	__le16 effect;
1218	} __packed * cel_entry;
1219	const int cel_entries = size / sizeof(*cel_entry);
1220	int i;
1221
1222	cel_entry = (struct cel_entry *)cel;
1223
1224	for (i = 0; i < cel_entries; i++) {
1225	u16 opcode = le16_to_cpu(cel_entry[i].opcode);
1226	struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
1227
1228	if (!cmd) {
1229	dev_dbg(&cxlm->pdev->dev,
1230	"Opcode 0x%04x unsupported by driver", opcode);
1231	continue;
1232	}
1233
1234	set_bit(cmd->info.id, cxlm->enabled_cmds);
1235	}
1236	}
1237
1238	struct cxl_mbox_get_supported_logs {
1239	__le16 entries;
1240	u8 rsvd[6];
1241	struct gsl_entry {
1242	uuid_t uuid;
1243	__le32 size;
1244	} __packed entry[];
1245	} __packed;
1246
1247	static struct cxl_mbox_get_supported_logs cxl_get_gsl(struct cxl_mem cxlm)
1248	{
1249	struct cxl_mbox_get_supported_logs *ret;
1250	int rc;
1251		447
1252	ret = kvmalloc(cxlm->payload_size, GFP_KERNEL);	448	if (map->reg_type == type)
1253	if (!ret)	449	return 0;
1254	return ERR_PTR(-ENOMEM);
1255
1256	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL,
1257	0, ret, cxlm->payload_size);
1258	if (rc < 0) {
1259	kvfree(ret);
1260	return ERR_PTR(rc);
1261	}	450	}
1262		451
1263	return ret;	452	return -ENODEV;
1264	}	453	}
1265		454
1266	/**	455	static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
1267	* cxl_mem_get_partition_info - Get partition info	456	struct cxl_register_map *map)
1268	* @cxlm: The device to act on
1269	* @active_volatile_bytes: returned active volatile capacity
1270	* @active_persistent_bytes: returned active persistent capacity
1271	* @next_volatile_bytes: return next volatile capacity
1272	* @next_persistent_bytes: return next persistent capacity
1273	*
1274	* Retrieve the current partition info for the device specified. If not 0, the
1275	* 'next' values are pending and take affect on next cold reset.
1276	*
1277	* Return: 0 if no error: or the result of the mailbox command.
1278	*
1279	* See CXL @8.2.9.5.2.1 Get Partition Info
1280	*/
1281	static int cxl_mem_get_partition_info(struct cxl_mem *cxlm,
1282	u64 *active_volatile_bytes,
1283	u64 *active_persistent_bytes,
1284	u64 *next_volatile_bytes,
1285	u64 *next_persistent_bytes)
1286	{	457	{
1287	struct cxl_mbox_get_partition_info {
1288	__le64 active_volatile_cap;
1289	__le64 active_persistent_cap;
1290	__le64 next_volatile_cap;
1291	__le64 next_persistent_cap;
1292	} __packed pi;
1293	int rc;	458	int rc;
1294		459
1295	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_PARTITION_INFO,	460	rc = cxl_find_regblock(pdev, type, map);
1296	NULL, 0, &pi, sizeof(pi));
1297	if (rc)	461	if (rc)
1298	return rc;	462	return rc;
1299		463
1300	*active_volatile_bytes = le64_to_cpu(pi.active_volatile_cap);	464	rc = cxl_map_regblock(pdev, map);
1301	*active_persistent_bytes = le64_to_cpu(pi.active_persistent_cap);	465	if (rc)
1302	*next_volatile_bytes = le64_to_cpu(pi.next_volatile_cap);
1303	*next_persistent_bytes = le64_to_cpu(pi.next_volatile_cap);
1304
1305	active_volatile_bytes = CXL_CAPACITY_MULTIPLIER;
1306	active_persistent_bytes = CXL_CAPACITY_MULTIPLIER;
1307	next_volatile_bytes = CXL_CAPACITY_MULTIPLIER;
1308	next_persistent_bytes = CXL_CAPACITY_MULTIPLIER;
1309
1310	return 0;
1311	}
1312
1313	/**
1314	* cxl_mem_enumerate_cmds() - Enumerate commands for a device.
1315	* @cxlm: The device.
1316	*
1317	* Returns 0 if enumerate completed successfully.
1318	*
1319	* CXL devices have optional support for certain commands. This function will
1320	* determine the set of supported commands for the hardware and update the
1321	* enabled_cmds bitmap in the @cxlm.
1322	*/
1323	static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm)
1324	{
1325	struct cxl_mbox_get_supported_logs *gsl;
1326	struct device *dev = &cxlm->pdev->dev;
1327	struct cxl_mem_command *cmd;
1328	int i, rc;
1329
1330	gsl = cxl_get_gsl(cxlm);
1331	if (IS_ERR(gsl))
1332	return PTR_ERR(gsl);
1333
1334	rc = -ENOENT;
1335	for (i = 0; i < le16_to_cpu(gsl->entries); i++) {
1336	u32 size = le32_to_cpu(gsl->entry[i].size);
1337	uuid_t uuid = gsl->entry[i].uuid;
1338	u8 *log;
1339
1340	dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);
1341
1342	if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))
1343	continue;
1344
1345	log = kvmalloc(size, GFP_KERNEL);
1346	if (!log) {
1347	rc = -ENOMEM;
1348	goto out;
1349	}
1350
1351	rc = cxl_xfer_log(cxlm, &uuid, size, log);
1352	if (rc) {
1353	kvfree(log);
1354	goto out;
1355	}
1356
1357	cxl_walk_cel(cxlm, size, log);
1358	kvfree(log);
1359
1360	/* In case CEL was bogus, enable some default commands. */
1361	cxl_for_each_cmd(cmd)
1362	if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
1363	set_bit(cmd->info.id, cxlm->enabled_cmds);
1364
1365	/* Found the required CEL */
1366	rc = 0;
1367	}
1368
1369	out:
1370	kvfree(gsl);
1371	return rc;
1372	}
1373
1374	/**
1375	* cxl_mem_identify() - Send the IDENTIFY command to the device.
1376	* @cxlm: The device to identify.
1377	*
1378	* Return: 0 if identify was executed successfully.
1379	*
1380	* This will dispatch the identify command to the device and on success populate
1381	* structures to be exported to sysfs.
1382	*/
1383	static int cxl_mem_identify(struct cxl_mem *cxlm)
1384	{
1385	/* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
1386	struct cxl_mbox_identify {
1387	char fw_revision[0x10];
1388	__le64 total_capacity;
1389	__le64 volatile_capacity;
1390	__le64 persistent_capacity;
1391	__le64 partition_align;
1392	__le16 info_event_log_size;
1393	__le16 warning_event_log_size;
1394	__le16 failure_event_log_size;
1395	__le16 fatal_event_log_size;
1396	__le32 lsa_size;
1397	u8 poison_list_max_mer[3];
1398	__le16 inject_poison_limit;
1399	u8 poison_caps;
1400	u8 qos_telemetry_caps;
1401	} __packed id;
1402	int rc;
1403
1404	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id,
1405	sizeof(id));
1406	if (rc < 0)
1407	return rc;
1408
1409	cxlm->total_bytes = le64_to_cpu(id.total_capacity);
1410	cxlm->total_bytes *= CXL_CAPACITY_MULTIPLIER;
1411
1412	cxlm->volatile_only_bytes = le64_to_cpu(id.volatile_capacity);
1413	cxlm->volatile_only_bytes *= CXL_CAPACITY_MULTIPLIER;
1414
1415	cxlm->persistent_only_bytes = le64_to_cpu(id.persistent_capacity);
1416	cxlm->persistent_only_bytes *= CXL_CAPACITY_MULTIPLIER;
1417
1418	cxlm->partition_align_bytes = le64_to_cpu(id.partition_align);
1419	cxlm->partition_align_bytes *= CXL_CAPACITY_MULTIPLIER;
1420
1421	dev_dbg(&cxlm->pdev->dev, "Identify Memory Device\n"
1422	" total_bytes = %#llx\n"
1423	" volatile_only_bytes = %#llx\n"
1424	" persistent_only_bytes = %#llx\n"
1425	" partition_align_bytes = %#llx\n",
1426	cxlm->total_bytes,
1427	cxlm->volatile_only_bytes,
1428	cxlm->persistent_only_bytes,
1429	cxlm->partition_align_bytes);
1430
1431	cxlm->lsa_size = le32_to_cpu(id.lsa_size);
1432	memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision));
1433
1434	return 0;
1435	}
1436
1437	static int cxl_mem_create_range_info(struct cxl_mem *cxlm)
1438	{
1439	int rc;
1440
1441	if (cxlm->partition_align_bytes == 0) {
1442	cxlm->ram_range.start = 0;
1443	cxlm->ram_range.end = cxlm->volatile_only_bytes - 1;
1444	cxlm->pmem_range.start = cxlm->volatile_only_bytes;
1445	cxlm->pmem_range.end = cxlm->volatile_only_bytes +
1446	cxlm->persistent_only_bytes - 1;
1447	return 0;
1448	}
1449
1450	rc = cxl_mem_get_partition_info(cxlm,
1451	&cxlm->active_volatile_bytes,
1452	&cxlm->active_persistent_bytes,
1453	&cxlm->next_volatile_bytes,
1454	&cxlm->next_persistent_bytes);
1455	if (rc < 0) {
1456	dev_err(&cxlm->pdev->dev, "Failed to query partition information\n");
1457	return rc;	466	return rc;
1458	}
1459
1460	dev_dbg(&cxlm->pdev->dev, "Get Partition Info\n"
1461	" active_volatile_bytes = %#llx\n"
1462	" active_persistent_bytes = %#llx\n"
1463	" next_volatile_bytes = %#llx\n"
1464	" next_persistent_bytes = %#llx\n",
1465	cxlm->active_volatile_bytes,
1466	cxlm->active_persistent_bytes,
1467	cxlm->next_volatile_bytes,
1468	cxlm->next_persistent_bytes);
1469		467
1470	cxlm->ram_range.start = 0;	468	rc = cxl_probe_regs(pdev, map);
1471	cxlm->ram_range.end = cxlm->active_volatile_bytes - 1;	469	cxl_unmap_regblock(pdev, map);
1472		470
1473	cxlm->pmem_range.start = cxlm->active_volatile_bytes;	471	return rc;
1474	cxlm->pmem_range.end = cxlm->active_volatile_bytes +
1475	cxlm->active_persistent_bytes - 1;
1476
1477	return 0;
1478	}	472	}
1479		473
1480	static int cxl_mem_probe(struct pci_dev pdev, const struct pci_device_id id)	474	static int cxl_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
1481	{	475	{
		476	struct cxl_register_map map;
1482	struct cxl_memdev *cxlmd;	477	struct cxl_memdev *cxlmd;
1483	struct cxl_mem *cxlm;	478	struct cxl_mem *cxlm;
1484	int rc;	479	int rc;
1485		480
		481	/*
		482	* Double check the anonymous union trickery in struct cxl_regs
		483	* FIXME switch to struct_group()
		484	*/
		485	BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) !=
		486	offsetof(struct cxl_regs, device_regs.memdev));
		487
1486	rc = pcim_enable_device(pdev);	488	rc = pcim_enable_device(pdev);
1487	if (rc)	489	if (rc)
1488	return rc;	490	return rc;
1489		491
1490	cxlm = cxl_mem_create(pdev);	492	cxlm = cxl_mem_create(&pdev->dev);
1491	if (IS_ERR(cxlm))	493	if (IS_ERR(cxlm))
1492	return PTR_ERR(cxlm);	494	return PTR_ERR(cxlm);
1493		495
1494	rc = cxl_mem_setup_regs(cxlm);	496	rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
		497	if (rc)
		498	return rc;
		499
		500	rc = cxl_map_regs(cxlm, &map);
1495	if (rc)	501	if (rc)
1496	return rc;	502	return rc;
1497		503
1498	rc = cxl_mem_setup_mailbox(cxlm);	504	rc = cxl_pci_setup_mailbox(cxlm);
1499	if (rc)	505	if (rc)
1500	return rc;	506	return rc;
1501		507
@@ -1511,7 +517,7 @@ static int cxl_mem_probe(struct pci_dev pdev, const struct pci_device_id id) · 9 unmodified lines
1511	if (rc)	517	if (rc)
1512	return rc;	518	return rc;
1513		519
1514	cxlmd = devm_cxl_add_memdev(&pdev->dev, cxlm, &cxl_memdev_fops);	520	cxlmd = devm_cxl_add_memdev(cxlm);
1515	if (IS_ERR(cxlmd))	521	if (IS_ERR(cxlmd))
1516	return PTR_ERR(cxlmd);	522	return PTR_ERR(cxlmd);
1517		523
@@ -1528,43 +534,15 @@ static const struct pci_device_id cxl_mem_pci_tbl[] = { · 10 unmodified lines
1528	};	534	};
1529	MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);	535	MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
1530		536
1531	static struct pci_driver cxl_mem_driver = {	537	static struct pci_driver cxl_pci_driver = {
1532	.name = KBUILD_MODNAME,	538	.name = KBUILD_MODNAME,
1533	.id_table = cxl_mem_pci_tbl,	539	.id_table = cxl_mem_pci_tbl,
1534	.probe = cxl_mem_probe,	540	.probe = cxl_pci_probe,
1535	.driver = {	541	.driver = {
1536	.probe_type = PROBE_PREFER_ASYNCHRONOUS,	542	.probe_type = PROBE_PREFER_ASYNCHRONOUS,
1537	},	543	},
1538	};	544	};
1539		545
1540	static __init int cxl_mem_init(void)
1541	{
1542	struct dentry *mbox_debugfs;
1543	int rc;
1544
1545	/* Double check the anonymous union trickery in struct cxl_regs */
1546	BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) !=
1547	offsetof(struct cxl_regs, device_regs.memdev));
1548
1549	rc = pci_register_driver(&cxl_mem_driver);
1550	if (rc)
1551	return rc;
1552
1553	cxl_debugfs = debugfs_create_dir("cxl", NULL);
1554	mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs);
1555	debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
1556	&cxl_raw_allow_all);
1557
1558	return 0;
1559	}
1560
1561	static __exit void cxl_mem_exit(void)
1562	{
1563	debugfs_remove_recursive(cxl_debugfs);
1564	pci_unregister_driver(&cxl_mem_driver);
1565	}
1566
1567	MODULE_LICENSE("GPL v2");	546	MODULE_LICENSE("GPL v2");
1568	module_init(cxl_mem_init);	547	module_pci_driver(cxl_pci_driver);
1569	module_exit(cxl_mem_exit);
1570	MODULE_IMPORT_NS(CXL);	548	MODULE_IMPORT_NS(CXL);

drivers/cxl/pci.h+8−6

@@ -20,13 +20,15 @@ · 19 unmodified lines
20	#define CXL_REGLOC_BIR_MASK GENMASK(2, 0)	20	#define CXL_REGLOC_BIR_MASK GENMASK(2, 0)
21		21
22	/* Register Block Identifier (RBI) */	22	/* Register Block Identifier (RBI) */
23	#define CXL_REGLOC_RBI_MASK GENMASK(15, 8)	23	enum cxl_regloc_type {
24	#define CXL_REGLOC_RBI_EMPTY 0	24	CXL_REGLOC_RBI_EMPTY = 0,
25	#define CXL_REGLOC_RBI_COMPONENT 1	25	CXL_REGLOC_RBI_COMPONENT,
26	#define CXL_REGLOC_RBI_VIRT 2	26	CXL_REGLOC_RBI_VIRT,
27	#define CXL_REGLOC_RBI_MEMDEV 3	27	CXL_REGLOC_RBI_MEMDEV,
28	#define CXL_REGLOC_RBI_TYPES CXL_REGLOC_RBI_MEMDEV + 1	28	CXL_REGLOC_RBI_TYPES
		29	};
29		30
		31	#define CXL_REGLOC_RBI_MASK GENMASK(15, 8)
30	#define CXL_REGLOC_ADDR_MASK GENMASK(31, 16)	32	#define CXL_REGLOC_ADDR_MASK GENMASK(31, 16)
31		33
32	#endif /* __CXL_PCI_H__ */	34	#endif /* __CXL_PCI_H__ */

drivers/cxl/pmem.c+142−21

@@ -1,6 +1,7 @@
1	// SPDX-License-Identifier: GPL-2.0-only	1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright(c) 2021 Intel Corporation. All rights reserved. */	2	/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3	#include <linux/libnvdimm.h>	3	#include <linux/libnvdimm.h>
		4	#include <asm/unaligned.h>
4	#include <linux/device.h>	5	#include <linux/device.h>
5	#include <linux/module.h>	6	#include <linux/module.h>
6	#include <linux/ndctl.h>	7	#include <linux/ndctl.h>
@@ -16,48 +17,55 @@ · 9 unmodified lines
16	*/	17	*/
17	static struct workqueue_struct *cxl_pmem_wq;	18	static struct workqueue_struct *cxl_pmem_wq;
18		19
19	static void unregister_nvdimm(void *nvdimm)	20	static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
20	{
21	nvdimm_delete(nvdimm);
22	}
23		21
24	static int match_nvdimm_bridge(struct device dev, const void data)	22	static void clear_exclusive(void *cxlm)
25	{	23	{
26	return strcmp(dev_name(dev), "nvdimm-bridge") == 0;	24	clear_exclusive_cxl_commands(cxlm, exclusive_cmds);
27	}	25	}
28		26
29	static struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(void)	27	static void unregister_nvdimm(void *nvdimm)
30	{	28	{
31	struct device *dev;	29	nvdimm_delete(nvdimm);
32
33	dev = bus_find_device(&cxl_bus_type, NULL, NULL, match_nvdimm_bridge);
34	if (!dev)
35	return NULL;
36	return to_cxl_nvdimm_bridge(dev);
37	}	30	}
38		31
39	static int cxl_nvdimm_probe(struct device *dev)	32	static int cxl_nvdimm_probe(struct device *dev)
40	{	33	{
41	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);	34	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
		35	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
		36	unsigned long flags = 0, cmd_mask = 0;
		37	struct cxl_mem *cxlm = cxlmd->cxlm;
42	struct cxl_nvdimm_bridge *cxl_nvb;	38	struct cxl_nvdimm_bridge *cxl_nvb;
43	unsigned long flags = 0;
44	struct nvdimm *nvdimm;	39	struct nvdimm *nvdimm;
45	int rc = -ENXIO;	40	int rc;
46		41
47	cxl_nvb = cxl_find_nvdimm_bridge();	42	cxl_nvb = cxl_find_nvdimm_bridge(cxl_nvd);
48	if (!cxl_nvb)	43	if (!cxl_nvb)
49	return -ENXIO;	44	return -ENXIO;
50		45
51	device_lock(&cxl_nvb->dev);	46	device_lock(&cxl_nvb->dev);
52	if (!cxl_nvb->nvdimm_bus)	47	if (!cxl_nvb->nvdimm_bus) {
		48	rc = -ENXIO;
		49	goto out;
		50	}
		51
		52	set_exclusive_cxl_commands(cxlm, exclusive_cmds);
		53	rc = devm_add_action_or_reset(dev, clear_exclusive, cxlm);
		54	if (rc)
53	goto out;	55	goto out;
54		56
55	set_bit(NDD_LABELING, &flags);	57	set_bit(NDD_LABELING, &flags);
56	nvdimm = nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, NULL, flags, 0, 0,	58	set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
57	NULL);	59	set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
58	if (!nvdimm)	60	set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
		61	nvdimm = nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, NULL, flags,
		62	cmd_mask, 0, NULL);
		63	if (!nvdimm) {
		64	rc = -ENOMEM;
59	goto out;	65	goto out;
		66	}
60		67
		68	dev_set_drvdata(dev, nvdimm);
61	rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);	69	rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
62	out:	70	out:
63	device_unlock(&cxl_nvb->dev);	71	device_unlock(&cxl_nvb->dev);
@@ -72,11 +80,120 @@ static struct cxl_driver cxl_nvdimm_driver = { · 8 unmodified lines
72	.id = CXL_DEVICE_NVDIMM,	80	.id = CXL_DEVICE_NVDIMM,
73	};	81	};
74		82
		83	static int cxl_pmem_get_config_size(struct cxl_mem *cxlm,
		84	struct nd_cmd_get_config_size *cmd,
		85	unsigned int buf_len)
		86	{
		87	if (sizeof(*cmd) > buf_len)
		88	return -EINVAL;
		89
		90	*cmd = (struct nd_cmd_get_config_size) {
		91	.config_size = cxlm->lsa_size,
		92	.max_xfer = cxlm->payload_size,
		93	};
		94
		95	return 0;
		96	}
		97
		98	static int cxl_pmem_get_config_data(struct cxl_mem *cxlm,
		99	struct nd_cmd_get_config_data_hdr *cmd,
		100	unsigned int buf_len)
		101	{
		102	struct cxl_mbox_get_lsa get_lsa;
		103	int rc;
		104
		105	if (sizeof(*cmd) > buf_len)
		106	return -EINVAL;
		107	if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
		108	return -EINVAL;
		109
		110	get_lsa = (struct cxl_mbox_get_lsa) {
		111	.offset = cmd->in_offset,
		112	.length = cmd->in_length,
		113	};
		114
		115	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LSA, &get_lsa,
		116	sizeof(get_lsa), cmd->out_buf,
		117	cmd->in_length);
		118	cmd->status = 0;
		119
		120	return rc;
		121	}
		122
		123	static int cxl_pmem_set_config_data(struct cxl_mem *cxlm,
		124	struct nd_cmd_set_config_hdr *cmd,
		125	unsigned int buf_len)
		126	{
		127	struct cxl_mbox_set_lsa *set_lsa;
		128	int rc;
		129
		130	if (sizeof(*cmd) > buf_len)
		131	return -EINVAL;
		132
		133	/* 4-byte status follows the input data in the payload */
		134	if (struct_size(cmd, in_buf, cmd->in_length) + 4 > buf_len)
		135	return -EINVAL;
		136
		137	set_lsa =
		138	kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
		139	if (!set_lsa)
		140	return -ENOMEM;
		141
		142	*set_lsa = (struct cxl_mbox_set_lsa) {
		143	.offset = cmd->in_offset,
		144	};
		145	memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
		146
		147	rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_SET_LSA, set_lsa,
		148	struct_size(set_lsa, data, cmd->in_length),
		149	NULL, 0);
		150
		151	/*
		152	* Set "firmware" status (4-packed bytes at the end of the input
		153	* payload.
		154	*/
		155	put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
		156	kvfree(set_lsa);
		157
		158	return rc;
		159	}
		160
		161	static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
		162	void *buf, unsigned int buf_len)
		163	{
		164	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
		165	unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
		166	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
		167	struct cxl_mem *cxlm = cxlmd->cxlm;
		168
		169	if (!test_bit(cmd, &cmd_mask))
		170	return -ENOTTY;
		171
		172	switch (cmd) {
		173	case ND_CMD_GET_CONFIG_SIZE:
		174	return cxl_pmem_get_config_size(cxlm, buf, buf_len);
		175	case ND_CMD_GET_CONFIG_DATA:
		176	return cxl_pmem_get_config_data(cxlm, buf, buf_len);
		177	case ND_CMD_SET_CONFIG_DATA:
		178	return cxl_pmem_set_config_data(cxlm, buf, buf_len);
		179	default:
		180	return -ENOTTY;
		181	}
		182	}
		183
75	static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,	184	static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
76	struct nvdimm nvdimm, unsigned int cmd, void buf,	185	struct nvdimm nvdimm, unsigned int cmd, void buf,
77	unsigned int buf_len, int *cmd_rc)	186	unsigned int buf_len, int *cmd_rc)
78	{	187	{
79	return -ENOTTY;	188	/*
		189	* No firmware response to translate, let the transport error
		190	* code take precedence.
		191	*/
		192	*cmd_rc = 0;
		193
		194	if (!nvdimm)
		195	return -ENOTTY;
		196	return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
80	}	197	}
81		198
82	static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb)	199	static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb)
@@ -194,6 +311,10 @@ static __init int cxl_pmem_init(void) · 111 unmodified lines
194	{	311	{
195	int rc;	312	int rc;
196		313
		314	set_bit(CXL_MEM_COMMAND_ID_SET_PARTITION_INFO, exclusive_cmds);
		315	set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
		316	set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
		317
197	cxl_pmem_wq = alloc_ordered_workqueue("cxl_pmem", 0);	318	cxl_pmem_wq = alloc_ordered_workqueue("cxl_pmem", 0);
198	if (!cxl_pmem_wq)	319	if (!cxl_pmem_wq)
199	return -ENXIO;	320	return -ENXIO;

drivers/misc/ocxl/config.c+1−12

@@ -33,18 +33,7 @@ · 32 unmodified lines
33		33
34	static int find_dvsec(struct pci_dev *dev, int dvsec_id)	34	static int find_dvsec(struct pci_dev *dev, int dvsec_id)
35	{	35	{
36	int vsec = 0;	36	return pci_find_dvsec_capability(dev, PCI_VENDOR_ID_IBM, dvsec_id);
37	u16 vendor, id;
38
39	while ((vsec = pci_find_next_ext_capability(dev, vsec,
40	OCXL_EXT_CAP_ID_DVSEC))) {
41	pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
42	&vendor);
43	pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
44	if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
45	return vsec;
46	}
47	return 0;
48	}	37	}
49		38
50	static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)	39	static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)

drivers/nvdimm/btt.c+6−5

 	u64 sum;
 	struct btt_sb *super;
 	struct nd_btt *nd_btt = arena->nd_btt;
-	const u8 *parent_uuid = nd_dev_to_uuid(&nd_btt->ndns->dev);
+	const uuid_t *parent_uuid = nd_dev_to_uuid(&nd_btt->ndns->dev);
 	ret = btt_map_init(arena);
 	if (ret)
 		return -ENOMEM;
 	strncpy(super->signature, BTT_SIG, BTT_SIG_LEN);
-	memcpy(super->uuid, nd_btt->uuid, 16);
+	export_uuid(super->uuid, nd_btt->uuid);
-	memcpy(super->parent_uuid, parent_uuid, 16);
+	export_uuid(super->parent_uuid, parent_uuid);
 	super->flags = cpu_to_le32(arena->flags);
 	super->version_major = cpu_to_le16(arena->version_major);
 	super->version_minor = cpu_to_le16(arena->version_minor);
  * Pointer to a new struct btt on success, NULL on failure.
  */
 static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,
-		u32 lbasize, u8 *uuid, struct nd_region *nd_region)
+			    u32 lbasize, uuid_t *uuid,
+			    struct nd_region *nd_region)
 {
 	int ret;
 	struct btt *btt;
 	}
 	nd_region = to_nd_region(nd_btt->dev.parent);
 	btt = btt_init(nd_btt, rawsize, nd_btt->lbasize, nd_btt->uuid,
-			nd_region);
+		       nd_region);
 	if (!btt)
 		return -ENOMEM;
 	nd_btt->btt = btt;

drivers/nvdimm/btt_devs.c+8−6

 EXPORT_SYMBOL(is_nd_btt);
 static struct device *__nd_btt_create(struct nd_region *nd_region,
-		unsigned long lbasize, u8 *uuid,
+				      unsigned long lbasize, uuid_t *uuid,
-		struct nd_namespace_common *ndns)
+				      struct nd_namespace_common *ndns)
 {
 	struct nd_btt *nd_btt;
 	struct device *dev;
  */
 bool nd_btt_arena_is_valid(struct nd_btt *nd_btt, struct btt_sb *super)
 {
-	const u8 *parent_uuid = nd_dev_to_uuid(&nd_btt->ndns->dev);
+	const uuid_t *ns_uuid = nd_dev_to_uuid(&nd_btt->ndns->dev);
+	uuid_t parent_uuid;
 	u64 checksum;
 	if (memcmp(super->signature, BTT_SIG, BTT_SIG_LEN) != 0)
 		return false;
-	if (!guid_is_null((guid_t *)&super->parent_uuid))
+	import_uuid(&parent_uuid, super->parent_uuid);
-		if (memcmp(super->parent_uuid, parent_uuid, 16) != 0)
+	if (!uuid_is_null(&parent_uuid))
+		if (!uuid_equal(&parent_uuid, ns_uuid))
 			return false;
 	checksum = le64_to_cpu(super->checksum);
 		return rc;
 	nd_btt->lbasize = le32_to_cpu(btt_sb->external_lbasize);
-	nd_btt->uuid = kmemdup(btt_sb->uuid, 16, GFP_KERNEL);
+	nd_btt->uuid = kmemdup(&btt_sb->uuid, sizeof(uuid_t), GFP_KERNEL);
 	if (!nd_btt->uuid)
 		return -ENOMEM;

drivers/nvdimm/core.c+4−36

@@ -207,38 +207,6 @@ struct device to_nvdimm_bus_dev(struct nvdimm_bus nvdimm_bus) · 206 unmodified lines
207	}	207	}
208	EXPORT_SYMBOL_GPL(to_nvdimm_bus_dev);	208	EXPORT_SYMBOL_GPL(to_nvdimm_bus_dev);
209		209
210	static bool is_uuid_sep(char sep)
211	{
212	if (sep == '\n' \|\| sep == '-' \|\| sep == ':' \|\| sep == '\0')
213	return true;
214	return false;
215	}
216
217	static int nd_uuid_parse(struct device dev, u8 uuid_out, const char *buf,
218	size_t len)
219	{
220	const char *str = buf;
221	u8 uuid[16];
222	int i;
223
224	for (i = 0; i < 16; i++) {
225	if (!isxdigit(str[0]) \|\| !isxdigit(str[1])) {
226	dev_dbg(dev, "pos: %d buf[%zd]: %c buf[%zd]: %c\n",
227	i, str - buf, str[0],
228	str + 1 - buf, str[1]);
229	return -EINVAL;
230	}
231
232	uuid[i] = (hex_to_bin(str[0]) << 4) \| hex_to_bin(str[1]);
233	str += 2;
234	if (is_uuid_sep(*str))
235	str++;
236	}
237
238	memcpy(uuid_out, uuid, sizeof(uuid));
239	return 0;
240	}
241
242	/**	210	/**
243	* nd_uuid_store: common implementation for writing 'uuid' sysfs attributes	211	* nd_uuid_store: common implementation for writing 'uuid' sysfs attributes
244	* @dev: container device for the uuid property	212	* @dev: container device for the uuid property
@@ -249,21 +217,21 @@ static int nd_uuid_parse(struct device dev, u8 uuid_out, const char *buf, · 4 unmodified lines
249	* (driver detached)	217	* (driver detached)
250	* LOCKING: expects nd_device_lock() is held on entry	218	* LOCKING: expects nd_device_lock() is held on entry
251	*/	219	*/
252	int nd_uuid_store(struct device dev, u8 uuid_out, const char buf,	220	int nd_uuid_store(struct device dev, uuid_t uuid_out, const char buf,
253	size_t len)	221	size_t len)
254	{	222	{
255	u8 uuid[16];	223	uuid_t uuid;
256	int rc;	224	int rc;
257		225
258	if (dev->driver)	226	if (dev->driver)
259	return -EBUSY;	227	return -EBUSY;
260		228
261	rc = nd_uuid_parse(dev, uuid, buf, len);	229	rc = uuid_parse(buf, &uuid);
262	if (rc)	230	if (rc)
263	return rc;	231	return rc;
264		232
265	kfree(*uuid_out);	233	kfree(*uuid_out);
266	*uuid_out = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);	234	*uuid_out = kmemdup(&uuid, sizeof(uuid), GFP_KERNEL);
267	if (!(*uuid_out))	235	if (!(*uuid_out))
268	return -ENOMEM;	236	return -ENOMEM;
269		237

drivers/nvdimm/label.c+102−37

 static guid_t nvdimm_pfn_guid;
 static guid_t nvdimm_dax_guid;
+static uuid_t nvdimm_btt_uuid;
+static uuid_t nvdimm_btt2_uuid;
+static uuid_t nvdimm_pfn_uuid;
+static uuid_t nvdimm_dax_uuid;
+static uuid_t cxl_region_uuid;
+static uuid_t cxl_namespace_uuid;
 static const char NSINDEX_SIGNATURE[] = "NAMESPACE_INDEX\0";
 static u32 best_seq(u32 a, u32 b)
 	return true;
 }
-char *nd_label_gen_id(struct nd_label_id *label_id, u8 *uuid, u32 flags)
+char *nd_label_gen_id(struct nd_label_id *label_id, const uuid_t *uuid,
+		      u32 flags)
 {
 	if (!label_id || !uuid)
 		return NULL;
 {
 	u64 sum, sum_save;
-	if (!namespace_label_has(ndd, checksum))
+	if (!ndd->cxl && !efi_namespace_label_has(ndd, checksum))
 		return true;
 	sum_save = nsl_get_checksum(ndd, nd_label);
 {
 	u64 sum;
-	if (!namespace_label_has(ndd, checksum))
+	if (!ndd->cxl && !efi_namespace_label_has(ndd, checksum))
 		return;
 	nsl_set_checksum(ndd, nd_label, 0);
 	sum = nd_fletcher64(nd_label, sizeof_namespace_label(ndd), 1);
 		struct nvdimm *nvdimm = to_nvdimm(ndd->dev);
 		struct nd_namespace_label *nd_label;
 		struct nd_region *nd_region = NULL;
-		u8 label_uuid[NSLABEL_UUID_LEN];
 		struct nd_label_id label_id;
 		struct resource *res;
+		uuid_t label_uuid;
 		u32 flags;
 		nd_label = to_label(ndd, slot);
 		if (!slot_valid(ndd, nd_label, slot))
 			continue;
-		memcpy(label_uuid, nd_label->uuid, NSLABEL_UUID_LEN);
+		nsl_get_uuid(ndd, nd_label, &label_uuid);
 		flags = nsl_get_flags(ndd, nd_label);
 		if (test_bit(NDD_NOBLK, &nvdimm->flags))
 			flags &= ~NSLABEL_FLAG_LOCAL;
-		nd_label_gen_id(&label_id, label_uuid, flags);
+		nd_label_gen_id(&label_id, &label_uuid, flags);
 		res = nvdimm_allocate_dpa(ndd, &label_id,
 					  nsl_get_dpa(ndd, nd_label),
 					  nsl_get_rawsize(ndd, nd_label));
 		- (unsigned long) to_namespace_index(ndd, 0);
 }
-static enum nvdimm_claim_class to_nvdimm_cclass(guid_t *guid)
+static enum nvdimm_claim_class guid_to_nvdimm_cclass(guid_t *guid)
 {
 	if (guid_equal(guid, &nvdimm_btt_guid))
 		return NVDIMM_CCLASS_BTT;
 	return NVDIMM_CCLASS_UNKNOWN;
 }
+/* CXL labels store UUIDs instead of GUIDs for the same data */
+static enum nvdimm_claim_class uuid_to_nvdimm_cclass(uuid_t *uuid)
+{
+	if (uuid_equal(uuid, &nvdimm_btt_uuid))
+		return NVDIMM_CCLASS_BTT;
+	else if (uuid_equal(uuid, &nvdimm_btt2_uuid))
+		return NVDIMM_CCLASS_BTT2;
+	else if (uuid_equal(uuid, &nvdimm_pfn_uuid))
+		return NVDIMM_CCLASS_PFN;
+	else if (uuid_equal(uuid, &nvdimm_dax_uuid))
+		return NVDIMM_CCLASS_DAX;
+	else if (uuid_equal(uuid, &uuid_null))
+		return NVDIMM_CCLASS_NONE;
+	return NVDIMM_CCLASS_UNKNOWN;
+}
 static const guid_t *to_abstraction_guid(enum nvdimm_claim_class claim_class,
 	guid_t *target)
 {
 		return &guid_null;
 }
+/* CXL labels store UUIDs instead of GUIDs for the same data */
+static const uuid_t *to_abstraction_uuid(enum nvdimm_claim_class claim_class,
+					 uuid_t *target)
+{
+	if (claim_class == NVDIMM_CCLASS_BTT)
+		return &nvdimm_btt_uuid;
+	else if (claim_class == NVDIMM_CCLASS_BTT2)
+		return &nvdimm_btt2_uuid;
+	else if (claim_class == NVDIMM_CCLASS_PFN)
+		return &nvdimm_pfn_uuid;
+	else if (claim_class == NVDIMM_CCLASS_DAX)
+		return &nvdimm_dax_uuid;
+	else if (claim_class == NVDIMM_CCLASS_UNKNOWN) {
+		/*
+		 * If we're modifying a namespace for which we don't
+		 * know the claim_class, don't touch the existing uuid.
+		 */
+		return target;
+	} else
+		return &uuid_null;
+}
 static void reap_victim(struct nd_mapping *nd_mapping,
 		struct nd_label_ent *victim)
 {
 static void nsl_set_type_guid(struct nvdimm_drvdata *ndd,
 			      struct nd_namespace_label *nd_label, guid_t *guid)
 {
-	if (namespace_label_has(ndd, type_guid))
+	if (efi_namespace_label_has(ndd, type_guid))
-		guid_copy(&nd_label->type_guid, guid);
+		guid_copy(&nd_label->efi.type_guid, guid);
 }
 bool nsl_validate_type_guid(struct nvdimm_drvdata *ndd,
 			    struct nd_namespace_label *nd_label, guid_t *guid)
 {
-	if (!namespace_label_has(ndd, type_guid))
+	if (ndd->cxl || !efi_namespace_label_has(ndd, type_guid))
 		return true;
-	if (!guid_equal(&nd_label->type_guid, guid)) {
+	if (!guid_equal(&nd_label->efi.type_guid, guid)) {
 		dev_dbg(ndd->dev, "expect type_guid %pUb got %pUb\n", guid,
-			&nd_label->type_guid);
+			&nd_label->efi.type_guid);
 		return false;
 	}
 	return true;
 				struct nd_namespace_label *nd_label,
 				enum nvdimm_claim_class claim_class)
 {
-	if (!namespace_label_has(ndd, abstraction_guid))
+	if (ndd->cxl) {
+		uuid_t uuid;
+		import_uuid(&uuid, nd_label->cxl.abstraction_uuid);
+		export_uuid(nd_label->cxl.abstraction_uuid,
+			    to_abstraction_uuid(claim_class, &uuid));
+		return;
+	}
+	if (!efi_namespace_label_has(ndd, abstraction_guid))
 		return;
-	guid_copy(&nd_label->abstraction_guid,
+	guid_copy(&nd_label->efi.abstraction_guid,
 		  to_abstraction_guid(claim_class,
-				      &nd_label->abstraction_guid));
+				      &nd_label->efi.abstraction_guid));
 }
 enum nvdimm_claim_class nsl_get_claim_class(struct nvdimm_drvdata *ndd,
 					    struct nd_namespace_label *nd_label)
 {
-	if (!namespace_label_has(ndd, abstraction_guid))
+	if (ndd->cxl) {
+		uuid_t uuid;
+		import_uuid(&uuid, nd_label->cxl.abstraction_uuid);
+		return uuid_to_nvdimm_cclass(&uuid);
+	}
+	if (!efi_namespace_label_has(ndd, abstraction_guid))
 		return NVDIMM_CCLASS_NONE;
-	return to_nvdimm_cclass(&nd_label->abstraction_guid);
+	return guid_to_nvdimm_cclass(&nd_label->efi.abstraction_guid);
 }
 static int __pmem_label_update(struct nd_region *nd_region,
 	nd_label = to_label(ndd, slot);
 	memset(nd_label, 0, sizeof_namespace_label(ndd));
-	memcpy(nd_label->uuid, nspm->uuid, NSLABEL_UUID_LEN);
+	nsl_set_uuid(ndd, nd_label, nspm->uuid);
 	nsl_set_name(ndd, nd_label, nspm->alt_name);
 	nsl_set_flags(ndd, nd_label, flags);
 	nsl_set_nlabel(ndd, nd_label, nd_region->ndr_mappings);
+	nsl_set_nrange(ndd, nd_label, 1);
 	nsl_set_position(ndd, nd_label, pos);
 	nsl_set_isetcookie(ndd, nd_label, cookie);
 	nsl_set_rawsize(ndd, nd_label, resource_size(res));
 	list_for_each_entry(label_ent, &nd_mapping->labels, list) {
 		if (!label_ent->label)
 			continue;
-		if (test_and_clear_bit(ND_LABEL_REAP, &label_ent->flags)
+		if (test_and_clear_bit(ND_LABEL_REAP, &label_ent->flags) ||
-				|| memcmp(nspm->uuid, label_ent->label->uuid,
+		    nsl_uuid_equal(ndd, label_ent->label, nspm->uuid))
-					NSLABEL_UUID_LEN) == 0)
 			reap_victim(nd_mapping, label_ent);
 	}
 				   struct nd_namespace_label *nd_label,
 				   u64 isetcookie)
 {
-	if (namespace_label_has(ndd, type_guid)) {
+	if (efi_namespace_label_has(ndd, type_guid)) {
 		nsl_set_isetcookie(ndd, nd_label, isetcookie);
 		return;
 	}
 				 struct nd_namespace_label *nd_label,
 				 u64 isetcookie)
 {
-	if (!namespace_label_has(ndd, type_guid))
+	if (!efi_namespace_label_has(ndd, type_guid))
 		return true;
 	if (nsl_get_isetcookie(ndd, nd_label) != isetcookie) {
 			       struct nd_namespace_label *nd_label, int nlabel,
 			       bool first)
 {
-	if (!namespace_label_has(ndd, type_guid)) {
+	if (!efi_namespace_label_has(ndd, type_guid)) {
 		nsl_set_nlabel(ndd, nd_label, 0); /* N/A */
 		return;
 	}
 				 struct nd_namespace_label *nd_label,
 				 bool first)
 {
-	if (!namespace_label_has(ndd, type_guid)) {
+	if (!efi_namespace_label_has(ndd, type_guid)) {
 		nsl_set_position(ndd, nd_label, 0);
 		return;
 	}
 	unsigned long *free, *victim_map = NULL;
 	struct resource *res, **old_res_list;
 	struct nd_label_id label_id;
-	u8 uuid[NSLABEL_UUID_LEN];
 	int min_dpa_idx = 0;
 	LIST_HEAD(list);
 	u32 nslot, slot;
 		/* mark unused labels for garbage collection */
 		for_each_clear_bit_le(slot, free, nslot) {
 			nd_label = to_label(ndd, slot);
-			memcpy(uuid, nd_label->uuid, NSLABEL_UUID_LEN);
+			if (!nsl_uuid_equal(ndd, nd_label, nsblk->uuid))
-			if (memcmp(uuid, nsblk->uuid, NSLABEL_UUID_LEN) != 0)
 				continue;
 			res = to_resource(ndd, nd_label);
 			if (res && is_old_resource(res, old_res_list,
 		nd_label = to_label(ndd, slot);
 		memset(nd_label, 0, sizeof_namespace_label(ndd));
-		memcpy(nd_label->uuid, nsblk->uuid, NSLABEL_UUID_LEN);
+		nsl_set_uuid(ndd, nd_label, nsblk->uuid);
 		nsl_set_name(ndd, nd_label, nsblk->alt_name);
 		nsl_set_flags(ndd, nd_label, NSLABEL_FLAG_LOCAL);
 		if (!nd_label)
 			continue;
 		nlabel++;
-		memcpy(uuid, nd_label->uuid, NSLABEL_UUID_LEN);
+		if (!nsl_uuid_equal(ndd, nd_label, nsblk->uuid))
-		if (memcmp(uuid, nsblk->uuid, NSLABEL_UUID_LEN) != 0)
 			continue;
 		nlabel--;
 		list_move(&label_ent->list, &list);
 	}
 	for_each_clear_bit_le(slot, free, nslot) {
 		nd_label = to_label(ndd, slot);
-		memcpy(uuid, nd_label->uuid, NSLABEL_UUID_LEN);
+		if (!nsl_uuid_equal(ndd, nd_label, nsblk->uuid))
-		if (memcmp(uuid, nsblk->uuid, NSLABEL_UUID_LEN) != 0)
 			continue;
 		res = to_resource(ndd, nd_label);
 		res->flags &= ~DPA_RESOURCE_ADJUSTED;
 	return max(num_labels, old_num_labels);
 }
-static int del_labels(struct nd_mapping *nd_mapping, u8 *uuid)
+static int del_labels(struct nd_mapping *nd_mapping, uuid_t *uuid)
 {
 	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
 	struct nd_label_ent *label_ent, *e;
 	struct nd_namespace_index *nsindex;
-	u8 label_uuid[NSLABEL_UUID_LEN];
 	unsigned long *free;
 	LIST_HEAD(list);
 	u32 nslot, slot;
 		if (!nd_label)
 			continue;
 		active++;
-		memcpy(label_uuid, nd_label->uuid, NSLABEL_UUID_LEN);
+		if (!nsl_uuid_equal(ndd, nd_label, uuid))
-		if (memcmp(label_uuid, uuid, NSLABEL_UUID_LEN) != 0)
 			continue;
 		active--;
 		slot = to_slot(ndd, nd_label);
 	WARN_ON(guid_parse(NVDIMM_PFN_GUID, &nvdimm_pfn_guid));
 	WARN_ON(guid_parse(NVDIMM_DAX_GUID, &nvdimm_dax_guid));
+	WARN_ON(uuid_parse(NVDIMM_BTT_GUID, &nvdimm_btt_uuid));
+	WARN_ON(uuid_parse(NVDIMM_BTT2_GUID, &nvdimm_btt2_uuid));
+	WARN_ON(uuid_parse(NVDIMM_PFN_GUID, &nvdimm_pfn_uuid));
+	WARN_ON(uuid_parse(NVDIMM_DAX_GUID, &nvdimm_dax_uuid));
+	WARN_ON(uuid_parse(CXL_REGION_UUID, &cxl_region_uuid));
+	WARN_ON(uuid_parse(CXL_NAMESPACE_UUID, &cxl_namespace_uuid));
 	return 0;
 }

drivers/nvdimm/label.h+89−5

@@ -34,6 +34,7 @@ enum { · 33 unmodified lines
34	* struct nd_namespace_index - label set superblock	34	* struct nd_namespace_index - label set superblock
35	* @sig: NAMESPACE_INDEX\0	35	* @sig: NAMESPACE_INDEX\0
36	* @flags: placeholder	36	* @flags: placeholder
		37	* @labelsize: log2 size (v1 labels 128 bytes v2 labels 256 bytes)
37	* @seq: sequence number for this index	38	* @seq: sequence number for this index
38	* @myoff: offset of this index in label area	39	* @myoff: offset of this index in label area
39	* @mysize: size of this index struct	40	* @mysize: size of this index struct
@@ -43,7 +44,7 @@ enum { · 3 unmodified lines
43	* @major: label area major version	44	* @major: label area major version
44	* @minor: label area minor version	45	* @minor: label area minor version
45	* @checksum: fletcher64 of all fields	46	* @checksum: fletcher64 of all fields
46	* @free[0]: bitmap, nlabel bits	47	* @free: bitmap, nlabel bits
47	*	48	*
48	* The size of free[] is rounded up so the total struct size is a	49	* The size of free[] is rounded up so the total struct size is a
49	* multiple of NSINDEX_ALIGN bytes. Any bits this allocates beyond	50	* multiple of NSINDEX_ALIGN bytes. Any bits this allocates beyond
@@ -66,7 +67,39 @@ struct nd_namespace_index { · 16 unmodified lines
66	};	67	};
67		68
68	/**	69	/**
69	* struct nd_namespace_label - namespace superblock	70	* struct cxl_region_label - CXL 2.0 Table 211
		71	* @type: uuid identifying this label format (region)
		72	* @uuid: uuid for the region this label describes
		73	* @flags: NSLABEL_FLAG_UPDATING (all other flags reserved)
		74	* @nlabel: 1 per interleave-way in the region
		75	* @position: this label's position in the set
		76	* @dpa: start address in device-local capacity for this label
		77	* @rawsize: size of this label's contribution to region
		78	* @hpa: mandatory system physical address to map this region
		79	* @slot: slot id of this label in label area
		80	* @ig: interleave granularity (1 << @ig) * 256 bytes
		81	* @align: alignment in SZ_256M blocks
		82	* @reserved: reserved
		83	* @checksum: fletcher64 sum of this label
		84	*/
		85	struct cxl_region_label {
		86	u8 type[NSLABEL_UUID_LEN];
		87	u8 uuid[NSLABEL_UUID_LEN];
		88	__le32 flags;
		89	__le16 nlabel;
		90	__le16 position;
		91	__le64 dpa;
		92	__le64 rawsize;
		93	__le64 hpa;
		94	__le32 slot;
		95	__le32 ig;
		96	__le32 align;
		97	u8 reserved[0xac];
		98	__le64 checksum;
		99	};
		100
		101	/**
		102	* struct nvdimm_efi_label - namespace superblock
70	* @uuid: UUID per RFC 4122	103	* @uuid: UUID per RFC 4122
71	* @name: optional name (NULL-terminated)	104	* @name: optional name (NULL-terminated)
72	* @flags: see NSLABEL_FLAG_*	105	* @flags: see NSLABEL_FLAG_*
@@ -77,9 +110,14 @@ struct nd_namespace_index { · 4 unmodified lines
77	* @dpa: DPA of NVM range on this DIMM	110	* @dpa: DPA of NVM range on this DIMM
78	* @rawsize: size of namespace	111	* @rawsize: size of namespace
79	* @slot: slot of this label in label area	112	* @slot: slot of this label in label area
80	* @unused: must be zero	113	* @align: physical address alignment of the namespace
		114	* @reserved: reserved
		115	* @type_guid: copy of struct acpi_nfit_system_address.range_guid
		116	* @abstraction_guid: personality id (btt, btt2, fsdax, devdax....)
		117	* @reserved2: reserved
		118	* @checksum: fletcher64 sum of this object
81	*/	119	*/
82	struct nd_namespace_label {	120	struct nvdimm_efi_label {
83	u8 uuid[NSLABEL_UUID_LEN];	121	u8 uuid[NSLABEL_UUID_LEN];
84	u8 name[NSLABEL_NAME_LEN];	122	u8 name[NSLABEL_NAME_LEN];
85	__le32 flags;	123	__le32 flags;
@@ -92,7 +130,7 @@ struct nd_namespace_label { · 6 unmodified lines
92	__le32 slot;	130	__le32 slot;
93	/*	131	/*
94	* Accessing fields past this point should be gated by a	132	* Accessing fields past this point should be gated by a
95	* namespace_label_has() check.	133	* efi_namespace_label_has() check.
96	*/	134	*/
97	u8 align;	135	u8 align;
98	u8 reserved[3];	136	u8 reserved[3];
@@ -102,11 +140,57 @@ struct nd_namespace_label { · 3 unmodified lines
102	__le64 checksum;	140	__le64 checksum;
103	};	141	};
104		142
		143	/**
		144	* struct nvdimm_cxl_label - CXL 2.0 Table 212
		145	* @type: uuid identifying this label format (namespace)
		146	* @uuid: uuid for the namespace this label describes
		147	* @name: friendly name for the namespace
		148	* @flags: NSLABEL_FLAG_UPDATING (all other flags reserved)
		149	* @nrange: discontiguous namespace support
		150	* @position: this label's position in the set
		151	* @dpa: start address in device-local capacity for this label
		152	* @rawsize: size of this label's contribution to namespace
		153	* @slot: slot id of this label in label area
		154	* @align: alignment in SZ_256M blocks
		155	* @region_uuid: host interleave set identifier
		156	* @abstraction_uuid: personality driver for this namespace
		157	* @lbasize: address geometry for disk-like personalities
		158	* @reserved: reserved
		159	* @checksum: fletcher64 sum of this label
		160	*/
		161	struct nvdimm_cxl_label {
		162	u8 type[NSLABEL_UUID_LEN];
		163	u8 uuid[NSLABEL_UUID_LEN];
		164	u8 name[NSLABEL_NAME_LEN];
		165	__le32 flags;
		166	__le16 nrange;
		167	__le16 position;
		168	__le64 dpa;
		169	__le64 rawsize;
		170	__le32 slot;
		171	__le32 align;
		172	u8 region_uuid[16];
		173	u8 abstraction_uuid[16];
		174	__le16 lbasize;
		175	u8 reserved[0x56];
		176	__le64 checksum;
		177	};
		178
		179	struct nd_namespace_label {
		180	union {
		181	struct nvdimm_cxl_label cxl;
		182	struct nvdimm_efi_label efi;
		183	};
		184	};
		185
105	#define NVDIMM_BTT_GUID "8aed63a2-29a2-4c66-8b12-f05d15d3922a"	186	#define NVDIMM_BTT_GUID "8aed63a2-29a2-4c66-8b12-f05d15d3922a"
106	#define NVDIMM_BTT2_GUID "18633bfc-1735-4217-8ac9-17239282d3f8"	187	#define NVDIMM_BTT2_GUID "18633bfc-1735-4217-8ac9-17239282d3f8"
107	#define NVDIMM_PFN_GUID "266400ba-fb9f-4677-bcb0-968f11d0d225"	188	#define NVDIMM_PFN_GUID "266400ba-fb9f-4677-bcb0-968f11d0d225"
108	#define NVDIMM_DAX_GUID "97a86d9c-3cdd-4eda-986f-5068b4f80088"	189	#define NVDIMM_DAX_GUID "97a86d9c-3cdd-4eda-986f-5068b4f80088"
109		190
		191	#define CXL_REGION_UUID "529d7c61-da07-47c4-a93f-ecdf2c06f444"
		192	#define CXL_NAMESPACE_UUID "68bb2c0a-5a77-4937-9f85-3caf41a0f93c"
		193
110	/**	194	/**
111	* struct nd_label_id - identifier string for dpa allocation	195	* struct nd_label_id - identifier string for dpa allocation
112	* @id: "{blk\|pmem}-<namespace uuid>"	196	* @id: "{blk\|pmem}-<namespace uuid>"

drivers/nvdimm/namespace_devs.c+52−43

 static int is_uuid_busy(struct device *dev, void *data)
 {
-	u8 *uuid1 = data, *uuid2 = NULL;
+	uuid_t *uuid1 = data, *uuid2 = NULL;
 	if (is_namespace_pmem(dev)) {
 		struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
 		uuid2 = nd_pfn->uuid;
 	}
-	if (uuid2 && memcmp(uuid1, uuid2, NSLABEL_UUID_LEN) == 0)
+	if (uuid2 && uuid_equal(uuid1, uuid2))
 		return -EBUSY;
 	return 0;
  * @dev: any device on a nvdimm_bus
  * @uuid: uuid to check
  */
-bool nd_is_uuid_unique(struct device *dev, u8 *uuid)
+bool nd_is_uuid_unique(struct device *dev, uuid_t *uuid)
 {
 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 }
 EXPORT_SYMBOL(nvdimm_namespace_disk_name);
-const u8 *nd_dev_to_uuid(struct device *dev)
+const uuid_t *nd_dev_to_uuid(struct device *dev)
 {
-	static const u8 null_uuid[16];
 	if (!dev)
-		return null_uuid;
+		return &uuid_null;
 	if (is_namespace_pmem(dev)) {
 		struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
 		return nsblk->uuid;
 	} else
-		return null_uuid;
+		return &uuid_null;
 }
 EXPORT_SYMBOL(nd_dev_to_uuid);
 	res->end = res->start + size - 1;
 }
-static bool uuid_not_set(const u8 *uuid, struct device *dev, const char *where)
+static bool uuid_not_set(const uuid_t *uuid, struct device *dev,
+			 const char *where)
 {
 	if (!uuid) {
 		dev_dbg(dev, "%s: uuid not set\n", where);
 	struct nd_label_id label_id;
 	u32 flags = 0, remainder;
 	int rc, i, id = -1;
-	u8 *uuid = NULL;
+	uuid_t *uuid = NULL;
 	if (dev->driver || ndns->claim)
 		return -EBUSY;
 {
 	struct nd_region *nd_region = to_nd_region(dev->parent);
 	unsigned long long val;
-	u8 **uuid = NULL;
+	uuid_t **uuid = NULL;
 	int rc;
 	rc = kstrtoull(buf, 0, &val);
 }
 static DEVICE_ATTR(size, 0444, size_show, size_store);
-static u8 *namespace_to_uuid(struct device *dev)
+static uuid_t *namespace_to_uuid(struct device *dev)
 {
 	if (is_namespace_pmem(dev)) {
 		struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
 		return ERR_PTR(-ENXIO);
 }
-static ssize_t uuid_show(struct device *dev,
+static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
-		struct device_attribute *attr, char *buf)
+			 char *buf)
 {
-	u8 *uuid = namespace_to_uuid(dev);
+	uuid_t *uuid = namespace_to_uuid(dev);
 	if (IS_ERR(uuid))
 		return PTR_ERR(uuid);
  * @old_uuid: reference to the uuid storage location in the namespace object
  */
 static int namespace_update_uuid(struct nd_region *nd_region,
-		struct device *dev, u8 *new_uuid, u8 **old_uuid)
+				 struct device *dev, uuid_t *new_uuid,
+				 uuid_t **old_uuid)
 {
 	u32 flags = is_namespace_blk(dev) ? NSLABEL_FLAG_LOCAL : 0;
 	struct nd_label_id old_label_id;
 		list_for_each_entry(label_ent, &nd_mapping->labels, list) {
 			struct nd_namespace_label *nd_label = label_ent->label;
 			struct nd_label_id label_id;
+			uuid_t uuid;
 			if (!nd_label)
 				continue;
-			nd_label_gen_id(&label_id, nd_label->uuid,
+			nsl_get_uuid(ndd, nd_label, &uuid);
+			nd_label_gen_id(&label_id, &uuid,
 					nsl_get_flags(ndd, nd_label));
 			if (strcmp(old_label_id.id, label_id.id) == 0)
 				set_bit(ND_LABEL_REAP, &label_ent->flags);
 		struct device_attribute *attr, const char *buf, size_t len)
 {
 	struct nd_region *nd_region = to_nd_region(dev->parent);
-	u8 *uuid = NULL;
+	uuid_t *uuid = NULL;
+	uuid_t **ns_uuid;
 	ssize_t rc = 0;
-	u8 **ns_uuid;
 	if (is_namespace_pmem(dev)) {
 		struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
 {
 	struct nd_region *nd_region = to_nd_region(dev->parent);
 	struct nd_label_id label_id;
+	uuid_t *uuid = NULL;
 	int count = 0, i;
-	u8 *uuid = NULL;
 	u32 flags = 0;
 	nvdimm_bus_lock(dev);
 	return devs;
 }
-static bool has_uuid_at_pos(struct nd_region *nd_region, u8 *uuid,
+static bool has_uuid_at_pos(struct nd_region *nd_region, const uuid_t *uuid,
-		u64 cookie, u16 pos)
+			    u64 cookie, u16 pos)
 {
 	struct nd_namespace_label *found = NULL;
 	int i;
 		list_for_each_entry(label_ent, &nd_mapping->labels, list) {
 			struct nd_namespace_label *nd_label = label_ent->label;
-			u16 position, nlabel;
+			u16 position;
 			if (!nd_label)
 				continue;
 			position = nsl_get_position(ndd, nd_label);
-			nlabel = nsl_get_nlabel(ndd, nd_label);
 			if (!nsl_validate_isetcookie(ndd, nd_label, cookie))
 				continue;
-			if (memcmp(nd_label->uuid, uuid, NSLABEL_UUID_LEN) != 0)
+			if (!nsl_uuid_equal(ndd, nd_label, uuid))
 				continue;
 			if (!nsl_validate_type_guid(ndd, nd_label,
 				return false;
 			}
 			found_uuid = true;
-			if (nlabel != nd_region->ndr_mappings)
+			if (!nsl_validate_nlabel(nd_region, ndd, nd_label))
 				continue;
 			if (position != pos)
 				continue;
 	return found != NULL;
 }
-static int select_pmem_id(struct nd_region *nd_region, u8 *pmem_id)
+static int select_pmem_id(struct nd_region *nd_region, const uuid_t *pmem_id)
 {
 	int i;
 			nd_label = label_ent->label;
 			if (!nd_label)
 				continue;
-			if (memcmp(nd_label->uuid, pmem_id, NSLABEL_UUID_LEN) == 0)
+			if (nsl_uuid_equal(ndd, nd_label, pmem_id))
 				break;
 			nd_label = NULL;
 		}
 			/* pass */;
 		else {
 			dev_dbg(&nd_region->dev, "%s invalid label for %pUb\n",
-					dev_name(ndd->dev), nd_label->uuid);
+				dev_name(ndd->dev),
+				nsl_uuid_raw(ndd, nd_label));
 			return -EINVAL;
 		}
 	resource_size_t size = 0;
 	struct resource *res;
 	struct device *dev;
+	uuid_t uuid;
 	int rc = 0;
 	u16 i;
 	if (!nsl_validate_isetcookie(ndd, nd_label, cookie)) {
 		dev_dbg(&nd_region->dev, "invalid cookie in label: %pUb\n",
-				nd_label->uuid);
+			nsl_uuid_raw(ndd, nd_label));
 		if (!nsl_validate_isetcookie(ndd, nd_label, altcookie))
 			return ERR_PTR(-EAGAIN);
 		dev_dbg(&nd_region->dev, "valid altcookie in label: %pUb\n",
-				nd_label->uuid);
+			nsl_uuid_raw(ndd, nd_label));
 	}
 	nspm = kzalloc(sizeof(*nspm), GFP_KERNEL);
 	res->flags = IORESOURCE_MEM;
 	for (i = 0; i < nd_region->ndr_mappings; i++) {
-		if (has_uuid_at_pos(nd_region, nd_label->uuid, cookie, i))
+		uuid_t uuid;
+		nsl_get_uuid(ndd, nd_label, &uuid);
+		if (has_uuid_at_pos(nd_region, &uuid, cookie, i))
 			continue;
-		if (has_uuid_at_pos(nd_region, nd_label->uuid, altcookie, i))
+		if (has_uuid_at_pos(nd_region, &uuid, altcookie, i))
 			continue;
 		break;
 	}
 		 * find a dimm with two instances of the same uuid.
 		 */
 		dev_err(&nd_region->dev, "%s missing label for %pUb\n",
-				nvdimm_name(nvdimm), nd_label->uuid);
+			nvdimm_name(nvdimm), nsl_uuid_raw(ndd, nd_label));
 		rc = -EINVAL;
 		goto err;
 	}
 	 * the dimm being enabled (i.e. nd_label_reserve_dpa()
 	 * succeeded).
 	 */
-	rc = select_pmem_id(nd_region, nd_label->uuid);
+	nsl_get_uuid(ndd, nd_label, &uuid);
+	rc = select_pmem_id(nd_region, &uuid);
 	if (rc)
 		goto err;
 		WARN_ON(nspm->alt_name || nspm->uuid);
 		nspm->alt_name = kmemdup(nsl_ref_name(ndd, label0),
 					 NSLABEL_NAME_LEN, GFP_KERNEL);
-		nspm->uuid = kmemdup((void __force *) label0->uuid,
+		nsl_get_uuid(ndd, label0, &uuid);
-				NSLABEL_UUID_LEN, GFP_KERNEL);
+		nspm->uuid = kmemdup(&uuid, sizeof(uuid_t), GFP_KERNEL);
 		nspm->lbasize = nsl_get_lbasize(ndd, label0);
 		nspm->nsio.common.claim_class =
 			nsl_get_claim_class(ndd, label0);
 	int i;
 	for (i = 0; i < count; i++) {
-		u8 *uuid = namespace_to_uuid(devs[i]);
+		uuid_t *uuid = namespace_to_uuid(devs[i]);
 		struct resource *res;
-		if (IS_ERR_OR_NULL(uuid)) {
+		if (IS_ERR(uuid)) {
 			WARN_ON(1);
 			continue;
 		}
-		if (memcmp(uuid, nd_label->uuid, NSLABEL_UUID_LEN) != 0)
+		if (!nsl_uuid_equal(ndd, nd_label, uuid))
 			continue;
 		if (is_namespace_blk(devs[i])) {
 			res = nsblk_add_resource(nd_region, ndd,
 			nd_dbg_dpa(nd_region, ndd, res, "%d assign\n", count);
 		} else {
 			dev_err(&nd_region->dev,
-					"error: conflicting extents for uuid: %pUb\n",
+				"error: conflicting extents for uuid: %pUb\n",
-					nd_label->uuid);
+				uuid);
 			return -ENXIO;
 		}
 		break;
 	char name[NSLABEL_NAME_LEN];
 	struct device *dev = NULL;
 	struct resource *res;
+	uuid_t uuid;
 	if (!nsl_validate_type_guid(ndd, nd_label, &nd_set->type_guid))
 		return ERR_PTR(-EAGAIN);
 	dev->parent = &nd_region->dev;
 	nsblk->id = -1;
 	nsblk->lbasize = nsl_get_lbasize(ndd, nd_label);
-	nsblk->uuid = kmemdup(nd_label->uuid, NSLABEL_UUID_LEN, GFP_KERNEL);
+	nsl_get_uuid(ndd, nd_label, &uuid);
+	nsblk->uuid = kmemdup(&uuid, sizeof(uuid_t), GFP_KERNEL);
 	nsblk->common.claim_class = nsl_get_claim_class(ndd, nd_label);
 	if (!nsblk->uuid)
 		goto blk_err;

drivers/nvdimm/nd-core.h+3−2

 void nd_synchronize(void);
 void __nd_device_register(struct device *dev);
 struct nd_label_id;
-char *nd_label_gen_id(struct nd_label_id *label_id, u8 *uuid, u32 flags);
+char *nd_label_gen_id(struct nd_label_id *label_id, const uuid_t *uuid,
-bool nd_is_uuid_unique(struct device *dev, u8 *uuid);
+		      u32 flags);
+bool nd_is_uuid_unique(struct device *dev, uuid_t *uuid);
 struct nd_region;
 struct nvdimm_drvdata;
 struct nd_mapping;

drivers/nvdimm/nd.h+156−29

 	int nslabel_size;
 	struct nd_cmd_get_config_size nsarea;
 	void *data;
+	bool cxl;
 	int ns_current, ns_next;
 	struct resource dpa;
 	struct kref kref;
 static inline const u8 *nsl_ref_name(struct nvdimm_drvdata *ndd,
 				     struct nd_namespace_label *nd_label)
 {
-	return nd_label->name;
+	if (ndd->cxl)
+		return nd_label->cxl.name;
+	return nd_label->efi.name;
 }
 static inline u8 *nsl_get_name(struct nvdimm_drvdata *ndd,
 			       struct nd_namespace_label *nd_label, u8 *name)
 {
-	return memcpy(name, nd_label->name, NSLABEL_NAME_LEN);
+	if (ndd->cxl)
+		return memcpy(name, nd_label->cxl.name, NSLABEL_NAME_LEN);
+	return memcpy(name, nd_label->efi.name, NSLABEL_NAME_LEN);
 }
 static inline u8 *nsl_set_name(struct nvdimm_drvdata *ndd,
 {
 	if (!name)
 		return NULL;
-	return memcpy(nd_label->name, name, NSLABEL_NAME_LEN);
+	if (ndd->cxl)
+		return memcpy(nd_label->cxl.name, name, NSLABEL_NAME_LEN);
+	return memcpy(nd_label->efi.name, name, NSLABEL_NAME_LEN);
 }
 static inline u32 nsl_get_slot(struct nvdimm_drvdata *ndd,
 			       struct nd_namespace_label *nd_label)
 {
-	return __le32_to_cpu(nd_label->slot);
+	if (ndd->cxl)
+		return __le32_to_cpu(nd_label->cxl.slot);
+	return __le32_to_cpu(nd_label->efi.slot);
 }
 static inline void nsl_set_slot(struct nvdimm_drvdata *ndd,
 				struct nd_namespace_label *nd_label, u32 slot)
 {
-	nd_label->slot = __cpu_to_le32(slot);
+	if (ndd->cxl)
+		nd_label->cxl.slot = __cpu_to_le32(slot);
+	else
+		nd_label->efi.slot = __cpu_to_le32(slot);
 }
 static inline u64 nsl_get_checksum(struct nvdimm_drvdata *ndd,
 				   struct nd_namespace_label *nd_label)
 {
-	return __le64_to_cpu(nd_label->checksum);
+	if (ndd->cxl)
+		return __le64_to_cpu(nd_label->cxl.checksum);
+	return __le64_to_cpu(nd_label->efi.checksum);
 }
 static inline void nsl_set_checksum(struct nvdimm_drvdata *ndd,
 				    struct nd_namespace_label *nd_label,
 				    u64 checksum)
 {
-	nd_label->checksum = __cpu_to_le64(checksum);
+	if (ndd->cxl)
+		nd_label->cxl.checksum = __cpu_to_le64(checksum);
+	else
+		nd_label->efi.checksum = __cpu_to_le64(checksum);
 }
 static inline u32 nsl_get_flags(struct nvdimm_drvdata *ndd,
 				struct nd_namespace_label *nd_label)
 {
-	return __le32_to_cpu(nd_label->flags);
+	if (ndd->cxl)
+		return __le32_to_cpu(nd_label->cxl.flags);
+	return __le32_to_cpu(nd_label->efi.flags);
 }
 static inline void nsl_set_flags(struct nvdimm_drvdata *ndd,
 				 struct nd_namespace_label *nd_label, u32 flags)
 {
-	nd_label->flags = __cpu_to_le32(flags);
+	if (ndd->cxl)
+		nd_label->cxl.flags = __cpu_to_le32(flags);
+	else
+		nd_label->efi.flags = __cpu_to_le32(flags);
 }
 static inline u64 nsl_get_dpa(struct nvdimm_drvdata *ndd,
 			      struct nd_namespace_label *nd_label)
 {
-	return __le64_to_cpu(nd_label->dpa);
+	if (ndd->cxl)
+		return __le64_to_cpu(nd_label->cxl.dpa);
+	return __le64_to_cpu(nd_label->efi.dpa);
 }
 static inline void nsl_set_dpa(struct nvdimm_drvdata *ndd,
 			       struct nd_namespace_label *nd_label, u64 dpa)
 {
-	nd_label->dpa = __cpu_to_le64(dpa);
+	if (ndd->cxl)
+		nd_label->cxl.dpa = __cpu_to_le64(dpa);
+	else
+		nd_label->efi.dpa = __cpu_to_le64(dpa);
 }
 static inline u64 nsl_get_rawsize(struct nvdimm_drvdata *ndd,
 				  struct nd_namespace_label *nd_label)
 {
-	return __le64_to_cpu(nd_label->rawsize);
+	if (ndd->cxl)
+		return __le64_to_cpu(nd_label->cxl.rawsize);
+	return __le64_to_cpu(nd_label->efi.rawsize);
 }
 static inline void nsl_set_rawsize(struct nvdimm_drvdata *ndd,
 				   struct nd_namespace_label *nd_label,
 				   u64 rawsize)
 {
-	nd_label->rawsize = __cpu_to_le64(rawsize);
+	if (ndd->cxl)
+		nd_label->cxl.rawsize = __cpu_to_le64(rawsize);
+	else
+		nd_label->efi.rawsize = __cpu_to_le64(rawsize);
 }
 static inline u64 nsl_get_isetcookie(struct nvdimm_drvdata *ndd,
 				     struct nd_namespace_label *nd_label)
 {
-	return __le64_to_cpu(nd_label->isetcookie);
+	/* WARN future refactor attempts that break this assumption */
+	if (dev_WARN_ONCE(ndd->dev, ndd->cxl,
+			  "CXL labels do not use the isetcookie concept\n"))
+		return 0;
+	return __le64_to_cpu(nd_label->efi.isetcookie);
 }
 static inline void nsl_set_isetcookie(struct nvdimm_drvdata *ndd,
 				      struct nd_namespace_label *nd_label,
 				      u64 isetcookie)
 {
-	nd_label->isetcookie = __cpu_to_le64(isetcookie);
+	if (!ndd->cxl)
+		nd_label->efi.isetcookie = __cpu_to_le64(isetcookie);
 }
 static inline bool nsl_validate_isetcookie(struct nvdimm_drvdata *ndd,
 					   struct nd_namespace_label *nd_label,
 					   u64 cookie)
 {
-	return cookie == __le64_to_cpu(nd_label->isetcookie);
+	/*
+	 * Let the EFI and CXL validation comingle, where fields that
+	 * don't matter to CXL always validate.
+	 */
+	if (ndd->cxl)
+		return true;
+	return cookie == __le64_to_cpu(nd_label->efi.isetcookie);
 }
 static inline u16 nsl_get_position(struct nvdimm_drvdata *ndd,
 				   struct nd_namespace_label *nd_label)
 {
-	return __le16_to_cpu(nd_label->position);
+	if (ndd->cxl)
+		return __le16_to_cpu(nd_label->cxl.position);
+	return __le16_to_cpu(nd_label->efi.position);
 }
 static inline void nsl_set_position(struct nvdimm_drvdata *ndd,
 				    struct nd_namespace_label *nd_label,
 				    u16 position)
 {
-	nd_label->position = __cpu_to_le16(position);
+	if (ndd->cxl)
+		nd_label->cxl.position = __cpu_to_le16(position);
+	else
+		nd_label->efi.position = __cpu_to_le16(position);
 }
 static inline u16 nsl_get_nlabel(struct nvdimm_drvdata *ndd,
 				 struct nd_namespace_label *nd_label)
 {
-	return __le16_to_cpu(nd_label->nlabel);
+	if (ndd->cxl)
+		return 0;
+	return __le16_to_cpu(nd_label->efi.nlabel);
 }
 static inline void nsl_set_nlabel(struct nvdimm_drvdata *ndd,
 				  struct nd_namespace_label *nd_label,
 				  u16 nlabel)
 {
-	nd_label->nlabel = __cpu_to_le16(nlabel);
+	if (!ndd->cxl)
+		nd_label->efi.nlabel = __cpu_to_le16(nlabel);
+}
+static inline u16 nsl_get_nrange(struct nvdimm_drvdata *ndd,
+				 struct nd_namespace_label *nd_label)
+{
+	if (ndd->cxl)
+		return __le16_to_cpu(nd_label->cxl.nrange);
+	return 1;
+}
+static inline void nsl_set_nrange(struct nvdimm_drvdata *ndd,
+				  struct nd_namespace_label *nd_label,
+				  u16 nrange)
+{
+	if (ndd->cxl)
+		nd_label->cxl.nrange = __cpu_to_le16(nrange);
 }
 static inline u64 nsl_get_lbasize(struct nvdimm_drvdata *ndd,
 				  struct nd_namespace_label *nd_label)
 {
-	return __le64_to_cpu(nd_label->lbasize);
+	/*
+	 * Yes, for some reason the EFI labels convey a massive 64-bit
+	 * lbasize, that got fixed for CXL.
+	 */
+	if (ndd->cxl)
+		return __le16_to_cpu(nd_label->cxl.lbasize);
+	return __le64_to_cpu(nd_label->efi.lbasize);
 }
 static inline void nsl_set_lbasize(struct nvdimm_drvdata *ndd,
 				   struct nd_namespace_label *nd_label,
 				   u64 lbasize)
 {
-	nd_label->lbasize = __cpu_to_le64(lbasize);
+	if (ndd->cxl)
+		nd_label->cxl.lbasize = __cpu_to_le16(lbasize);
+	else
+		nd_label->efi.lbasize = __cpu_to_le64(lbasize);
+}
+static inline const uuid_t *nsl_get_uuid(struct nvdimm_drvdata *ndd,
+					 struct nd_namespace_label *nd_label,
+					 uuid_t *uuid)
+{
+	if (ndd->cxl)
+		import_uuid(uuid, nd_label->cxl.uuid);
+	else
+		import_uuid(uuid, nd_label->efi.uuid);
+	return uuid;
+}
+static inline const uuid_t *nsl_set_uuid(struct nvdimm_drvdata *ndd,
+					 struct nd_namespace_label *nd_label,
+					 const uuid_t *uuid)
+{
+	if (ndd->cxl)
+		export_uuid(nd_label->cxl.uuid, uuid);
+	else
+		export_uuid(nd_label->efi.uuid, uuid);
+	return uuid;
+}
+static inline bool nsl_uuid_equal(struct nvdimm_drvdata *ndd,
+				  struct nd_namespace_label *nd_label,
+				  const uuid_t *uuid)
+{
+	uuid_t tmp;
+	if (ndd->cxl)
+		import_uuid(&tmp, nd_label->cxl.uuid);
+	else
+		import_uuid(&tmp, nd_label->efi.uuid);
+	return uuid_equal(&tmp, uuid);
+}
+static inline const u8 *nsl_uuid_raw(struct nvdimm_drvdata *ndd,
+				     struct nd_namespace_label *nd_label)
+{
+	if (ndd->cxl)
+		return nd_label->cxl.uuid;
+	return nd_label->efi.uuid;
 }
 bool nsl_validate_blk_isetcookie(struct nvdimm_drvdata *ndd,
 unsigned sizeof_namespace_label(struct nvdimm_drvdata *ndd);
-#define namespace_label_has(ndd, field) \
+#define efi_namespace_label_has(ndd, field) \
-	(offsetof(struct nd_namespace_label, field) \
+	(!ndd->cxl && offsetof(struct nvdimm_efi_label, field) \
 		< sizeof_namespace_label(ndd))
 #define nd_dbg_dpa(r, d, res, fmt, arg...) \
 	struct nd_mapping mapping[];
 };
+static inline bool nsl_validate_nlabel(struct nd_region *nd_region,
+				       struct nvdimm_drvdata *ndd,
+				       struct nd_namespace_label *nd_label)
+{
+	if (ndd->cxl)
+		return true;
+	return nsl_get_nlabel(ndd, nd_label) == nd_region->ndr_mappings;
+}
 struct nd_blk_region {
 	int (*enable)(struct nvdimm_bus *nvdimm_bus, struct device *dev);
 	int (*do_io)(struct nd_blk_region *ndbr, resource_size_t dpa,
 	struct btt *btt;
 	unsigned long lbasize;
 	u64 size;
-	u8 *uuid;
+	uuid_t *uuid;
 	int id;
 	int initial_offset;
 	u16 version_major;
 struct nd_pfn {
 	int id;
-	u8 *uuid;
+	uuid_t *uuid;
 	struct device dev;
 	unsigned long align;
 	unsigned long npfns;
 void nd_device_register(struct device *dev);
 void nd_device_unregister(struct device *dev, enum nd_async_mode mode);
 void nd_device_notify(struct device *dev, enum nvdimm_event event);
-int nd_uuid_store(struct device *dev, u8 **uuid_out, const char *buf,
+int nd_uuid_store(struct device *dev, uuid_t **uuid_out, const char *buf,
 		size_t len);
 ssize_t nd_size_select_show(unsigned long current_size,
 		const unsigned long *supported, char *buf);
 	return false;
 }
 resource_size_t nd_namespace_blk_validate(struct nd_namespace_blk *nsblk);
-const u8 *nd_dev_to_uuid(struct device *dev);
+const uuid_t *nd_dev_to_uuid(struct device *dev);
 bool pmem_should_map_pages(struct device *dev);
 #endif /* __ND_H__ */

drivers/nvdimm/pfn_devs.c+1−1

@@ -452,7 +452,7 @@ int nd_pfn_validate(struct nd_pfn nd_pfn, const char sig) · 451 unmodified lines
452	unsigned long align, start_pad;	452	unsigned long align, start_pad;
453	struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;	453	struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;
454	struct nd_namespace_common *ndns = nd_pfn->ndns;	454	struct nd_namespace_common *ndns = nd_pfn->ndns;
455	const u8 *parent_uuid = nd_dev_to_uuid(&ndns->dev);	455	const uuid_t *parent_uuid = nd_dev_to_uuid(&ndns->dev);
456		456
457	if (!pfn_sb \|\| !ndns)	457	if (!pfn_sb \|\| !ndns)
458	return -ENODEV;	458	return -ENODEV;

drivers/pci/pci.c+32−0

@@ -732,6 +732,38 @@ u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap) · 731 unmodified lines
732	}	732	}
733	EXPORT_SYMBOL_GPL(pci_find_vsec_capability);	733	EXPORT_SYMBOL_GPL(pci_find_vsec_capability);
734		734
		735	/**
		736	* pci_find_dvsec_capability - Find DVSEC for vendor
		737	* @dev: PCI device to query
		738	* @vendor: Vendor ID to match for the DVSEC
		739	* @dvsec: Designated Vendor-specific capability ID
		740	*
		741	* If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability
		742	* offset in config space; otherwise return 0.
		743	*/
		744	u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec)
		745	{
		746	int pos;
		747
		748	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC);
		749	if (!pos)
		750	return 0;
		751
		752	while (pos) {
		753	u16 v, id;
		754
		755	pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v);
		756	pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id);
		757	if (vendor == v && dvsec == id)
		758	return pos;
		759
		760	pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC);
		761	}
		762
		763	return 0;
		764	}
		765	EXPORT_SYMBOL_GPL(pci_find_dvsec_capability);
		766
735	/**	767	/**
736	* pci_find_parent_resource - return resource region of parent bus of given	768	* pci_find_parent_resource - return resource region of parent bus of given
737	* region	769	* region

include/linux/nd.h+2−2

@@ -88,7 +88,7 @@ struct nd_namespace_pmem { · 87 unmodified lines
88	struct nd_namespace_io nsio;	88	struct nd_namespace_io nsio;
89	unsigned long lbasize;	89	unsigned long lbasize;
90	char *alt_name;	90	char *alt_name;
91	u8 *uuid;	91	uuid_t *uuid;
92	int id;	92	int id;
93	};	93	};
94		94
@@ -105,7 +105,7 @@ struct nd_namespace_pmem { · 10 unmodified lines
105	struct nd_namespace_blk {	105	struct nd_namespace_blk {
106	struct nd_namespace_common common;	106	struct nd_namespace_common common;
107	char *alt_name;	107	char *alt_name;
108	u8 *uuid;	108	uuid_t *uuid;
109	int id;	109	int id;
110	unsigned long lbasize;	110	unsigned long lbasize;
111	resource_size_t size;	111	resource_size_t size;

include/linux/pci.h+1−0

@@ -1132,6 +1132,7 @@ u16 pci_find_ext_capability(struct pci_dev *dev, int cap); · 1131 unmodified lines
1132	u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 pos, int cap);	1132	u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 pos, int cap);
1133	struct pci_bus pci_find_next_bus(const struct pci_bus from);	1133	struct pci_bus pci_find_next_bus(const struct pci_bus from);
1134	u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap);	1134	u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap);
		1135	u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec);
1135		1136
1136	u64 pci_get_dsn(struct pci_dev *dev);	1137	u64 pci_get_dsn(struct pci_dev *dev);
1137		1138

tools/testing/cxl/Kbuild+38−0

@@ -0,0 +1,38 @@
	1	# SPDX-License-Identifier: GPL-2.0
	2	ldflags-y += --wrap=is_acpi_device_node
	3	ldflags-y += --wrap=acpi_get_table
	4	ldflags-y += --wrap=acpi_put_table
	5	ldflags-y += --wrap=acpi_evaluate_integer
	6	ldflags-y += --wrap=acpi_pci_find_root
	7	ldflags-y += --wrap=pci_walk_bus
	8	ldflags-y += --wrap=nvdimm_bus_register
	9
	10	DRIVERS := ../../../drivers
	11	CXL_SRC := $(DRIVERS)/cxl
	12	CXL_CORE_SRC := $(DRIVERS)/cxl/core
	13	ccflags-y := -I$(srctree)/drivers/cxl/
	14	ccflags-y += -D__mock=__weak
	15
	16	obj-m += cxl_acpi.o
	17
	18	cxl_acpi-y := $(CXL_SRC)/acpi.o
	19	cxl_acpi-y += mock_acpi.o
	20	cxl_acpi-y += config_check.o
	21
	22	obj-m += cxl_pmem.o
	23
	24	cxl_pmem-y := $(CXL_SRC)/pmem.o
	25	cxl_pmem-y += config_check.o
	26
	27	obj-m += cxl_core.o
	28
	29	cxl_core-y := $(CXL_CORE_SRC)/bus.o
	30	cxl_core-y += $(CXL_CORE_SRC)/pmem.o
	31	cxl_core-y += $(CXL_CORE_SRC)/regs.o
	32	cxl_core-y += $(CXL_CORE_SRC)/memdev.o
	33	cxl_core-y += $(CXL_CORE_SRC)/mbox.o
	34	cxl_core-y += config_check.o
	35
	36	cxl_core-y += mock_pmem.o
	37
	38	obj-m += test/

tools/testing/cxl/config_check.c+13−0

@@ -0,0 +1,13 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/bug.h>
	3
	4	void check(void)
	5	{
	6	/*
	7	* These kconfig symbols must be set to "m" for cxl_test to load
	8	* and operate.
	9	*/
	10	BUILD_BUG_ON(!IS_MODULE(CONFIG_CXL_BUS));
	11	BUILD_BUG_ON(!IS_MODULE(CONFIG_CXL_ACPI));
	12	BUILD_BUG_ON(!IS_MODULE(CONFIG_CXL_PMEM));
	13	}

tools/testing/cxl/mock_acpi.c+109−0

@@ -0,0 +1,109 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
	3
	4	#include <linux/platform_device.h>
	5	#include <linux/device.h>
	6	#include <linux/acpi.h>
	7	#include <linux/pci.h>
	8	#include <cxl.h>
	9	#include "test/mock.h"
	10
	11	struct acpi_device to_cxl_host_bridge(struct device host, struct device *dev)
	12	{
	13	int index;
	14	struct acpi_device adev, found = NULL;
	15	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	16
	17	if (ops && ops->is_mock_bridge(dev)) {
	18	found = ACPI_COMPANION(dev);
	19	goto out;
	20	}
	21
	22	if (dev->bus == &platform_bus_type)
	23	goto out;
	24
	25	adev = to_acpi_device(dev);
	26	if (!acpi_pci_find_root(adev->handle))
	27	goto out;
	28
	29	if (strcmp(acpi_device_hid(adev), "ACPI0016") == 0) {
	30	found = adev;
	31	dev_dbg(host, "found host bridge %s\n", dev_name(&adev->dev));
	32	}
	33	out:
	34	put_cxl_mock_ops(index);
	35	return found;
	36	}
	37
	38	static int match_add_root_port(struct pci_dev pdev, void data)
	39	{
	40	struct cxl_walk_context *ctx = data;
	41	struct pci_bus *root_bus = ctx->root;
	42	struct cxl_port *port = ctx->port;
	43	int type = pci_pcie_type(pdev);
	44	struct device *dev = ctx->dev;
	45	u32 lnkcap, port_num;
	46	int rc;
	47
	48	if (pdev->bus != root_bus)
	49	return 0;
	50	if (!pci_is_pcie(pdev))
	51	return 0;
	52	if (type != PCI_EXP_TYPE_ROOT_PORT)
	53	return 0;
	54	if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP,
	55	&lnkcap) != PCIBIOS_SUCCESSFUL)
	56	return 0;
	57
	58	/* TODO walk DVSEC to find component register base */
	59	port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
	60	rc = cxl_add_dport(port, &pdev->dev, port_num, CXL_RESOURCE_NONE);
	61	if (rc) {
	62	dev_err(dev, "failed to add dport: %s (%d)\n",
	63	dev_name(&pdev->dev), rc);
	64	ctx->error = rc;
	65	return rc;
	66	}
	67	ctx->count++;
	68
	69	dev_dbg(dev, "add dport%d: %s\n", port_num, dev_name(&pdev->dev));
	70
	71	return 0;
	72	}
	73
	74	static int mock_add_root_port(struct platform_device pdev, void data)
	75	{
	76	struct cxl_walk_context *ctx = data;
	77	struct cxl_port *port = ctx->port;
	78	struct device *dev = ctx->dev;
	79	int rc;
	80
	81	rc = cxl_add_dport(port, &pdev->dev, pdev->id, CXL_RESOURCE_NONE);
	82	if (rc) {
	83	dev_err(dev, "failed to add dport: %s (%d)\n",
	84	dev_name(&pdev->dev), rc);
	85	ctx->error = rc;
	86	return rc;
	87	}
	88	ctx->count++;
	89
	90	dev_dbg(dev, "add dport%d: %s\n", pdev->id, dev_name(&pdev->dev));
	91
	92	return 0;
	93	}
	94
	95	int match_add_root_ports(struct pci_dev dev, void data)
	96	{
	97	int index, rc;
	98	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	99	struct platform_device pdev = (struct platform_device ) dev;
	100
	101	if (ops && ops->is_mock_port(pdev))
	102	rc = mock_add_root_port(pdev, data);
	103	else
	104	rc = match_add_root_port(dev, data);
	105
	106	put_cxl_mock_ops(index);
	107
	108	return rc;
	109	}

tools/testing/cxl/mock_pmem.c+24−0

@@ -0,0 +1,24 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
	3	#include <cxl.h>
	4	#include "test/mock.h"
	5	#include <core/core.h>
	6
	7	int match_nvdimm_bridge(struct device dev, const void data)
	8	{
	9	int index, rc = 0;
	10	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	11	const struct cxl_nvdimm *cxl_nvd = data;
	12
	13	if (ops) {
	14	if (dev->type == &cxl_nvdimm_bridge_type &&
	15	(ops->is_mock_dev(dev->parent->parent) ==
	16	ops->is_mock_dev(cxl_nvd->dev.parent->parent)))
	17	rc = 1;
	18	} else
	19	rc = dev->type == &cxl_nvdimm_bridge_type;
	20
	21	put_cxl_mock_ops(index);
	22
	23	return rc;
	24	}

tools/testing/cxl/test/Kbuild+10−0

@@ -0,0 +1,10 @@
	1	# SPDX-License-Identifier: GPL-2.0
	2	ccflags-y := -I$(srctree)/drivers/cxl/
	3
	4	obj-m += cxl_test.o
	5	obj-m += cxl_mock.o
	6	obj-m += cxl_mock_mem.o
	7
	8	cxl_test-y := cxl.o
	9	cxl_mock-y := mock.o
	10	cxl_mock_mem-y := mem.o

tools/testing/cxl/test/cxl.c+576−0

@@ -0,0 +1,576 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	// Copyright(c) 2021 Intel Corporation. All rights reserved.
	3
	4	#include <linux/platform_device.h>
	5	#include <linux/genalloc.h>
	6	#include <linux/module.h>
	7	#include <linux/mutex.h>
	8	#include <linux/acpi.h>
	9	#include <linux/pci.h>
	10	#include <linux/mm.h>
	11	#include "mock.h"
	12
	13	#define NR_CXL_HOST_BRIDGES 4
	14	#define NR_CXL_ROOT_PORTS 2
	15
	16	static struct platform_device *cxl_acpi;
	17	static struct platform_device *cxl_host_bridge[NR_CXL_HOST_BRIDGES];
	18	static struct platform_device
	19	cxl_root_port[NR_CXL_HOST_BRIDGES NR_CXL_ROOT_PORTS];
	20	struct platform_device cxl_mem[NR_CXL_HOST_BRIDGES NR_CXL_ROOT_PORTS];
	21
	22	static struct acpi_device acpi0017_mock;
	23	static struct acpi_device host_bridge[NR_CXL_HOST_BRIDGES] = {
	24	[0] = {
	25	.handle = &host_bridge[0],
	26	},
	27	[1] = {
	28	.handle = &host_bridge[1],
	29	},
	30	[2] = {
	31	.handle = &host_bridge[2],
	32	},
	33	[3] = {
	34	.handle = &host_bridge[3],
	35	},
	36	};
	37
	38	static bool is_mock_dev(struct device *dev)
	39	{
	40	int i;
	41
	42	for (i = 0; i < ARRAY_SIZE(cxl_mem); i++)
	43	if (dev == &cxl_mem[i]->dev)
	44	return true;
	45	if (dev == &cxl_acpi->dev)
	46	return true;
	47	return false;
	48	}
	49
	50	static bool is_mock_adev(struct acpi_device *adev)
	51	{
	52	int i;
	53
	54	if (adev == &acpi0017_mock)
	55	return true;
	56
	57	for (i = 0; i < ARRAY_SIZE(host_bridge); i++)
	58	if (adev == &host_bridge[i])
	59	return true;
	60
	61	return false;
	62	}
	63
	64	static struct {
	65	struct acpi_table_cedt cedt;
	66	struct acpi_cedt_chbs chbs[NR_CXL_HOST_BRIDGES];
	67	struct {
	68	struct acpi_cedt_cfmws cfmws;
	69	u32 target[1];
	70	} cfmws0;
	71	struct {
	72	struct acpi_cedt_cfmws cfmws;
	73	u32 target[4];
	74	} cfmws1;
	75	struct {
	76	struct acpi_cedt_cfmws cfmws;
	77	u32 target[1];
	78	} cfmws2;
	79	struct {
	80	struct acpi_cedt_cfmws cfmws;
	81	u32 target[4];
	82	} cfmws3;
	83	} __packed mock_cedt = {
	84	.cedt = {
	85	.header = {
	86	.signature = "CEDT",
	87	.length = sizeof(mock_cedt),
	88	.revision = 1,
	89	},
	90	},
	91	.chbs[0] = {
	92	.header = {
	93	.type = ACPI_CEDT_TYPE_CHBS,
	94	.length = sizeof(mock_cedt.chbs[0]),
	95	},
	96	.uid = 0,
	97	.cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
	98	},
	99	.chbs[1] = {
	100	.header = {
	101	.type = ACPI_CEDT_TYPE_CHBS,
	102	.length = sizeof(mock_cedt.chbs[0]),
	103	},
	104	.uid = 1,
	105	.cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
	106	},
	107	.chbs[2] = {
	108	.header = {
	109	.type = ACPI_CEDT_TYPE_CHBS,
	110	.length = sizeof(mock_cedt.chbs[0]),
	111	},
	112	.uid = 2,
	113	.cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
	114	},
	115	.chbs[3] = {
	116	.header = {
	117	.type = ACPI_CEDT_TYPE_CHBS,
	118	.length = sizeof(mock_cedt.chbs[0]),
	119	},
	120	.uid = 3,
	121	.cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
	122	},
	123	.cfmws0 = {
	124	.cfmws = {
	125	.header = {
	126	.type = ACPI_CEDT_TYPE_CFMWS,
	127	.length = sizeof(mock_cedt.cfmws0),
	128	},
	129	.interleave_ways = 0,
	130	.granularity = 4,
	131	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
	132	ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
	133	.qtg_id = 0,
	134	.window_size = SZ_256M,
	135	},
	136	.target = { 0 },
	137	},
	138	.cfmws1 = {
	139	.cfmws = {
	140	.header = {
	141	.type = ACPI_CEDT_TYPE_CFMWS,
	142	.length = sizeof(mock_cedt.cfmws1),
	143	},
	144	.interleave_ways = 2,
	145	.granularity = 4,
	146	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
	147	ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
	148	.qtg_id = 1,
	149	.window_size = SZ_256M * 4,
	150	},
	151	.target = { 0, 1, 2, 3 },
	152	},
	153	.cfmws2 = {
	154	.cfmws = {
	155	.header = {
	156	.type = ACPI_CEDT_TYPE_CFMWS,
	157	.length = sizeof(mock_cedt.cfmws2),
	158	},
	159	.interleave_ways = 0,
	160	.granularity = 4,
	161	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
	162	ACPI_CEDT_CFMWS_RESTRICT_PMEM,
	163	.qtg_id = 2,
	164	.window_size = SZ_256M,
	165	},
	166	.target = { 0 },
	167	},
	168	.cfmws3 = {
	169	.cfmws = {
	170	.header = {
	171	.type = ACPI_CEDT_TYPE_CFMWS,
	172	.length = sizeof(mock_cedt.cfmws3),
	173	},
	174	.interleave_ways = 2,
	175	.granularity = 4,
	176	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 \|
	177	ACPI_CEDT_CFMWS_RESTRICT_PMEM,
	178	.qtg_id = 3,
	179	.window_size = SZ_256M * 4,
	180	},
	181	.target = { 0, 1, 2, 3 },
	182	},
	183	};
	184
	185	struct cxl_mock_res {
	186	struct list_head list;
	187	struct range range;
	188	};
	189
	190	static LIST_HEAD(mock_res);
	191	static DEFINE_MUTEX(mock_res_lock);
	192	static struct gen_pool *cxl_mock_pool;
	193
	194	static void depopulate_all_mock_resources(void)
	195	{
	196	struct cxl_mock_res res, _res;
	197
	198	mutex_lock(&mock_res_lock);
	199	list_for_each_entry_safe(res, _res, &mock_res, list) {
	200	gen_pool_free(cxl_mock_pool, res->range.start,
	201	range_len(&res->range));
	202	list_del(&res->list);
	203	kfree(res);
	204	}
	205	mutex_unlock(&mock_res_lock);
	206	}
	207
	208	static struct cxl_mock_res *alloc_mock_res(resource_size_t size)
	209	{
	210	struct cxl_mock_res res = kzalloc(sizeof(res), GFP_KERNEL);
	211	struct genpool_data_align data = {
	212	.align = SZ_256M,
	213	};
	214	unsigned long phys;
	215
	216	INIT_LIST_HEAD(&res->list);
	217	phys = gen_pool_alloc_algo(cxl_mock_pool, size,
	218	gen_pool_first_fit_align, &data);
	219	if (!phys)
	220	return NULL;
	221
	222	res->range = (struct range) {
	223	.start = phys,
	224	.end = phys + size - 1,
	225	};
	226	mutex_lock(&mock_res_lock);
	227	list_add(&res->list, &mock_res);
	228	mutex_unlock(&mock_res_lock);
	229
	230	return res;
	231	}
	232
	233	static int populate_cedt(void)
	234	{
	235	struct acpi_cedt_cfmws *cfmws[4] = {
	236	[0] = &mock_cedt.cfmws0.cfmws,
	237	[1] = &mock_cedt.cfmws1.cfmws,
	238	[2] = &mock_cedt.cfmws2.cfmws,
	239	[3] = &mock_cedt.cfmws3.cfmws,
	240	};
	241	struct cxl_mock_res *res;
	242	int i;
	243
	244	for (i = 0; i < ARRAY_SIZE(mock_cedt.chbs); i++) {
	245	struct acpi_cedt_chbs *chbs = &mock_cedt.chbs[i];
	246	resource_size_t size;
	247
	248	if (chbs->cxl_version == ACPI_CEDT_CHBS_VERSION_CXL20)
	249	size = ACPI_CEDT_CHBS_LENGTH_CXL20;
	250	else
	251	size = ACPI_CEDT_CHBS_LENGTH_CXL11;
	252
	253	res = alloc_mock_res(size);
	254	if (!res)
	255	return -ENOMEM;
	256	chbs->base = res->range.start;
	257	chbs->length = size;
	258	}
	259
	260	for (i = 0; i < ARRAY_SIZE(cfmws); i++) {
	261	struct acpi_cedt_cfmws *window = cfmws[i];
	262
	263	res = alloc_mock_res(window->window_size);
	264	if (!res)
	265	return -ENOMEM;
	266	window->base_hpa = res->range.start;
	267	}
	268
	269	return 0;
	270	}
	271
	272	static acpi_status mock_acpi_get_table(char *signature, u32 instance,
	273	struct acpi_table_header **out_table)
	274	{
	275	if (instance < U32_MAX \|\| strcmp(signature, ACPI_SIG_CEDT) != 0)
	276	return acpi_get_table(signature, instance, out_table);
	277
	278	out_table = (struct acpi_table_header ) &mock_cedt;
	279	return AE_OK;
	280	}
	281
	282	static void mock_acpi_put_table(struct acpi_table_header *table)
	283	{
	284	if (table == (struct acpi_table_header *) &mock_cedt)
	285	return;
	286	acpi_put_table(table);
	287	}
	288
	289	static bool is_mock_bridge(struct device *dev)
	290	{
	291	int i;
	292
	293	for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++)
	294	if (dev == &cxl_host_bridge[i]->dev)
	295	return true;
	296
	297	return false;
	298	}
	299
	300	static int host_bridge_index(struct acpi_device *adev)
	301	{
	302	return adev - host_bridge;
	303	}
	304
	305	static struct acpi_device *find_host_bridge(acpi_handle handle)
	306	{
	307	int i;
	308
	309	for (i = 0; i < ARRAY_SIZE(host_bridge); i++)
	310	if (handle == host_bridge[i].handle)
	311	return &host_bridge[i];
	312	return NULL;
	313	}
	314
	315	static acpi_status
	316	mock_acpi_evaluate_integer(acpi_handle handle, acpi_string pathname,
	317	struct acpi_object_list *arguments,
	318	unsigned long long *data)
	319	{
	320	struct acpi_device *adev = find_host_bridge(handle);
	321
	322	if (!adev \|\| strcmp(pathname, METHOD_NAME__UID) != 0)
	323	return acpi_evaluate_integer(handle, pathname, arguments, data);
	324
	325	*data = host_bridge_index(adev);
	326	return AE_OK;
	327	}
	328
	329	static struct pci_bus mock_pci_bus[NR_CXL_HOST_BRIDGES];
	330	static struct acpi_pci_root mock_pci_root[NR_CXL_HOST_BRIDGES] = {
	331	[0] = {
	332	.bus = &mock_pci_bus[0],
	333	},
	334	[1] = {
	335	.bus = &mock_pci_bus[1],
	336	},
	337	[2] = {
	338	.bus = &mock_pci_bus[2],
	339	},
	340	[3] = {
	341	.bus = &mock_pci_bus[3],
	342	},
	343	};
	344
	345	static struct platform_device mock_cxl_root_port(struct pci_bus bus, int index)
	346	{
	347	int i;
	348
	349	for (i = 0; i < ARRAY_SIZE(mock_pci_bus); i++)
	350	if (bus == &mock_pci_bus[i])
	351	return cxl_root_port[index + i * NR_CXL_ROOT_PORTS];
	352	return NULL;
	353	}
	354
	355	static bool is_mock_port(struct platform_device *pdev)
	356	{
	357	int i;
	358
	359	for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++)
	360	if (pdev == cxl_root_port[i])
	361	return true;
	362	return false;
	363	}
	364
	365	static bool is_mock_bus(struct pci_bus *bus)
	366	{
	367	int i;
	368
	369	for (i = 0; i < ARRAY_SIZE(mock_pci_bus); i++)
	370	if (bus == &mock_pci_bus[i])
	371	return true;
	372	return false;
	373	}
	374
	375	static struct acpi_pci_root *mock_acpi_pci_find_root(acpi_handle handle)
	376	{
	377	struct acpi_device *adev = find_host_bridge(handle);
	378
	379	if (!adev)
	380	return acpi_pci_find_root(handle);
	381	return &mock_pci_root[host_bridge_index(adev)];
	382	}
	383
	384	static struct cxl_mock_ops cxl_mock_ops = {
	385	.is_mock_adev = is_mock_adev,
	386	.is_mock_bridge = is_mock_bridge,
	387	.is_mock_bus = is_mock_bus,
	388	.is_mock_port = is_mock_port,
	389	.is_mock_dev = is_mock_dev,
	390	.mock_port = mock_cxl_root_port,
	391	.acpi_get_table = mock_acpi_get_table,
	392	.acpi_put_table = mock_acpi_put_table,
	393	.acpi_evaluate_integer = mock_acpi_evaluate_integer,
	394	.acpi_pci_find_root = mock_acpi_pci_find_root,
	395	.list = LIST_HEAD_INIT(cxl_mock_ops.list),
	396	};
	397
	398	static void mock_companion(struct acpi_device adev, struct device dev)
	399	{
	400	device_initialize(&adev->dev);
	401	fwnode_init(&adev->fwnode, NULL);
	402	dev->fwnode = &adev->fwnode;
	403	adev->fwnode.dev = dev;
	404	}
	405
	406	#ifndef SZ_64G
	407	#define SZ_64G (SZ_32G * 2)
	408	#endif
	409
	410	#ifndef SZ_512G
	411	#define SZ_512G (SZ_64G * 8)
	412	#endif
	413
	414	static struct platform_device *alloc_memdev(int id)
	415	{
	416	struct resource res[] = {
	417	[0] = {
	418	.flags = IORESOURCE_MEM,
	419	},
	420	[1] = {
	421	.flags = IORESOURCE_MEM,
	422	.desc = IORES_DESC_PERSISTENT_MEMORY,
	423	},
	424	};
	425	struct platform_device *pdev;
	426	int i, rc;
	427
	428	for (i = 0; i < ARRAY_SIZE(res); i++) {
	429	struct cxl_mock_res *r = alloc_mock_res(SZ_256M);
	430
	431	if (!r)
	432	return NULL;
	433	res[i].start = r->range.start;
	434	res[i].end = r->range.end;
	435	}
	436
	437	pdev = platform_device_alloc("cxl_mem", id);
	438	if (!pdev)
	439	return NULL;
	440
	441	rc = platform_device_add_resources(pdev, res, ARRAY_SIZE(res));
	442	if (rc)
	443	goto err;
	444
	445	return pdev;
	446
	447	err:
	448	platform_device_put(pdev);
	449	return NULL;
	450	}
	451
	452	static __init int cxl_test_init(void)
	453	{
	454	int rc, i;
	455
	456	register_cxl_mock_ops(&cxl_mock_ops);
	457
	458	cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE);
	459	if (!cxl_mock_pool) {
	460	rc = -ENOMEM;
	461	goto err_gen_pool_create;
	462	}
	463
	464	rc = gen_pool_add(cxl_mock_pool, SZ_512G, SZ_64G, NUMA_NO_NODE);
	465	if (rc)
	466	goto err_gen_pool_add;
	467
	468	rc = populate_cedt();
	469	if (rc)
	470	goto err_populate;
	471
	472	for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) {
	473	struct acpi_device *adev = &host_bridge[i];
	474	struct platform_device *pdev;
	475
	476	pdev = platform_device_alloc("cxl_host_bridge", i);
	477	if (!pdev)
	478	goto err_bridge;
	479
	480	mock_companion(adev, &pdev->dev);
	481	rc = platform_device_add(pdev);
	482	if (rc) {
	483	platform_device_put(pdev);
	484	goto err_bridge;
	485	}
	486	cxl_host_bridge[i] = pdev;
	487	}
	488
	489	for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++) {
	490	struct platform_device *bridge =
	491	cxl_host_bridge[i / NR_CXL_ROOT_PORTS];
	492	struct platform_device *pdev;
	493
	494	pdev = platform_device_alloc("cxl_root_port", i);
	495	if (!pdev)
	496	goto err_port;
	497	pdev->dev.parent = &bridge->dev;
	498
	499	rc = platform_device_add(pdev);
	500	if (rc) {
	501	platform_device_put(pdev);
	502	goto err_port;
	503	}
	504	cxl_root_port[i] = pdev;
	505	}
	506
	507	BUILD_BUG_ON(ARRAY_SIZE(cxl_mem) != ARRAY_SIZE(cxl_root_port));
	508	for (i = 0; i < ARRAY_SIZE(cxl_mem); i++) {
	509	struct platform_device *port = cxl_root_port[i];
	510	struct platform_device *pdev;
	511
	512	pdev = alloc_memdev(i);
	513	if (!pdev)
	514	goto err_mem;
	515	pdev->dev.parent = &port->dev;
	516
	517	rc = platform_device_add(pdev);
	518	if (rc) {
	519	platform_device_put(pdev);
	520	goto err_mem;
	521	}
	522	cxl_mem[i] = pdev;
	523	}
	524
	525	cxl_acpi = platform_device_alloc("cxl_acpi", 0);
	526	if (!cxl_acpi)
	527	goto err_mem;
	528
	529	mock_companion(&acpi0017_mock, &cxl_acpi->dev);
	530	acpi0017_mock.dev.bus = &platform_bus_type;
	531
	532	rc = platform_device_add(cxl_acpi);
	533	if (rc)
	534	goto err_add;
	535
	536	return 0;
	537
	538	err_add:
	539	platform_device_put(cxl_acpi);
	540	err_mem:
	541	for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--)
	542	platform_device_unregister(cxl_mem[i]);
	543	err_port:
	544	for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
	545	platform_device_unregister(cxl_root_port[i]);
	546	err_bridge:
	547	for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--)
	548	platform_device_unregister(cxl_host_bridge[i]);
	549	err_populate:
	550	depopulate_all_mock_resources();
	551	err_gen_pool_add:
	552	gen_pool_destroy(cxl_mock_pool);
	553	err_gen_pool_create:
	554	unregister_cxl_mock_ops(&cxl_mock_ops);
	555	return rc;
	556	}
	557
	558	static __exit void cxl_test_exit(void)
	559	{
	560	int i;
	561
	562	platform_device_unregister(cxl_acpi);
	563	for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--)
	564	platform_device_unregister(cxl_mem[i]);
	565	for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
	566	platform_device_unregister(cxl_root_port[i]);
	567	for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--)
	568	platform_device_unregister(cxl_host_bridge[i]);
	569	depopulate_all_mock_resources();
	570	gen_pool_destroy(cxl_mock_pool);
	571	unregister_cxl_mock_ops(&cxl_mock_ops);
	572	}
	573
	574	module_init(cxl_test_init);
	575	module_exit(cxl_test_exit);
	576	MODULE_LICENSE("GPL v2");

tools/testing/cxl/test/mem.c+256−0

@@ -0,0 +1,256 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	// Copyright(c) 2021 Intel Corporation. All rights reserved.
	3
	4	#include <linux/platform_device.h>
	5	#include <linux/mod_devicetable.h>
	6	#include <linux/module.h>
	7	#include <linux/sizes.h>
	8	#include <linux/bits.h>
	9	#include <cxlmem.h>
	10
	11	#define LSA_SIZE SZ_128K
	12	#define EFFECT(x) (1U << x)
	13
	14	static struct cxl_cel_entry mock_cel[] = {
	15	{
	16	.opcode = cpu_to_le16(CXL_MBOX_OP_GET_SUPPORTED_LOGS),
	17	.effect = cpu_to_le16(0),
	18	},
	19	{
	20	.opcode = cpu_to_le16(CXL_MBOX_OP_IDENTIFY),
	21	.effect = cpu_to_le16(0),
	22	},
	23	{
	24	.opcode = cpu_to_le16(CXL_MBOX_OP_GET_LSA),
	25	.effect = cpu_to_le16(0),
	26	},
	27	{
	28	.opcode = cpu_to_le16(CXL_MBOX_OP_SET_LSA),
	29	.effect = cpu_to_le16(EFFECT(1) \| EFFECT(2)),
	30	},
	31	};
	32
	33	static struct {
	34	struct cxl_mbox_get_supported_logs gsl;
	35	struct cxl_gsl_entry entry;
	36	} mock_gsl_payload = {
	37	.gsl = {
	38	.entries = cpu_to_le16(1),
	39	},
	40	.entry = {
	41	.uuid = DEFINE_CXL_CEL_UUID,
	42	.size = cpu_to_le32(sizeof(mock_cel)),
	43	},
	44	};
	45
	46	static int mock_gsl(struct cxl_mbox_cmd *cmd)
	47	{
	48	if (cmd->size_out < sizeof(mock_gsl_payload))
	49	return -EINVAL;
	50
	51	memcpy(cmd->payload_out, &mock_gsl_payload, sizeof(mock_gsl_payload));
	52	cmd->size_out = sizeof(mock_gsl_payload);
	53
	54	return 0;
	55	}
	56
	57	static int mock_get_log(struct cxl_mem cxlm, struct cxl_mbox_cmd cmd)
	58	{
	59	struct cxl_mbox_get_log *gl = cmd->payload_in;
	60	u32 offset = le32_to_cpu(gl->offset);
	61	u32 length = le32_to_cpu(gl->length);
	62	uuid_t uuid = DEFINE_CXL_CEL_UUID;
	63	void *data = &mock_cel;
	64
	65	if (cmd->size_in < sizeof(*gl))
	66	return -EINVAL;
	67	if (length > cxlm->payload_size)
	68	return -EINVAL;
	69	if (offset + length > sizeof(mock_cel))
	70	return -EINVAL;
	71	if (!uuid_equal(&gl->uuid, &uuid))
	72	return -EINVAL;
	73	if (length > cmd->size_out)
	74	return -EINVAL;
	75
	76	memcpy(cmd->payload_out, data + offset, length);
	77
	78	return 0;
	79	}
	80
	81	static int mock_id(struct cxl_mem cxlm, struct cxl_mbox_cmd cmd)
	82	{
	83	struct platform_device *pdev = to_platform_device(cxlm->dev);
	84	struct cxl_mbox_identify id = {
	85	.fw_revision = { "mock fw v1 " },
	86	.lsa_size = cpu_to_le32(LSA_SIZE),
	87	/* FIXME: Add partition support */
	88	.partition_align = cpu_to_le64(0),
	89	};
	90	u64 capacity = 0;
	91	int i;
	92
	93	if (cmd->size_out < sizeof(id))
	94	return -EINVAL;
	95
	96	for (i = 0; i < 2; i++) {
	97	struct resource *res;
	98
	99	res = platform_get_resource(pdev, IORESOURCE_MEM, i);
	100	if (!res)
	101	break;
	102
	103	capacity += resource_size(res) / CXL_CAPACITY_MULTIPLIER;
	104
	105	if (le64_to_cpu(id.partition_align))
	106	continue;
	107
	108	if (res->desc == IORES_DESC_PERSISTENT_MEMORY)
	109	id.persistent_capacity = cpu_to_le64(
	110	resource_size(res) / CXL_CAPACITY_MULTIPLIER);
	111	else
	112	id.volatile_capacity = cpu_to_le64(
	113	resource_size(res) / CXL_CAPACITY_MULTIPLIER);
	114	}
	115
	116	id.total_capacity = cpu_to_le64(capacity);
	117
	118	memcpy(cmd->payload_out, &id, sizeof(id));
	119
	120	return 0;
	121	}
	122
	123	static int mock_get_lsa(struct cxl_mem cxlm, struct cxl_mbox_cmd cmd)
	124	{
	125	struct cxl_mbox_get_lsa *get_lsa = cmd->payload_in;
	126	void *lsa = dev_get_drvdata(cxlm->dev);
	127	u32 offset, length;
	128
	129	if (sizeof(*get_lsa) > cmd->size_in)
	130	return -EINVAL;
	131	offset = le32_to_cpu(get_lsa->offset);
	132	length = le32_to_cpu(get_lsa->length);
	133	if (offset + length > LSA_SIZE)
	134	return -EINVAL;
	135	if (length > cmd->size_out)
	136	return -EINVAL;
	137
	138	memcpy(cmd->payload_out, lsa + offset, length);
	139	return 0;
	140	}
	141
	142	static int mock_set_lsa(struct cxl_mem cxlm, struct cxl_mbox_cmd cmd)
	143	{
	144	struct cxl_mbox_set_lsa *set_lsa = cmd->payload_in;
	145	void *lsa = dev_get_drvdata(cxlm->dev);
	146	u32 offset, length;
	147
	148	if (sizeof(*set_lsa) > cmd->size_in)
	149	return -EINVAL;
	150	offset = le32_to_cpu(set_lsa->offset);
	151	length = cmd->size_in - sizeof(*set_lsa);
	152	if (offset + length > LSA_SIZE)
	153	return -EINVAL;
	154
	155	memcpy(lsa + offset, &set_lsa->data[0], length);
	156	return 0;
	157	}
	158
	159	static int cxl_mock_mbox_send(struct cxl_mem cxlm, struct cxl_mbox_cmd cmd)
	160	{
	161	struct device *dev = cxlm->dev;
	162	int rc = -EIO;
	163
	164	switch (cmd->opcode) {
	165	case CXL_MBOX_OP_GET_SUPPORTED_LOGS:
	166	rc = mock_gsl(cmd);
	167	break;
	168	case CXL_MBOX_OP_GET_LOG:
	169	rc = mock_get_log(cxlm, cmd);
	170	break;
	171	case CXL_MBOX_OP_IDENTIFY:
	172	rc = mock_id(cxlm, cmd);
	173	break;
	174	case CXL_MBOX_OP_GET_LSA:
	175	rc = mock_get_lsa(cxlm, cmd);
	176	break;
	177	case CXL_MBOX_OP_SET_LSA:
	178	rc = mock_set_lsa(cxlm, cmd);
	179	break;
	180	default:
	181	break;
	182	}
	183
	184	dev_dbg(dev, "opcode: %#x sz_in: %zd sz_out: %zd rc: %d\n", cmd->opcode,
	185	cmd->size_in, cmd->size_out, rc);
	186
	187	return rc;
	188	}
	189
	190	static void label_area_release(void *lsa)
	191	{
	192	vfree(lsa);
	193	}
	194
	195	static int cxl_mock_mem_probe(struct platform_device *pdev)
	196	{
	197	struct device *dev = &pdev->dev;
	198	struct cxl_memdev *cxlmd;
	199	struct cxl_mem *cxlm;
	200	void *lsa;
	201	int rc;
	202
	203	lsa = vmalloc(LSA_SIZE);
	204	if (!lsa)
	205	return -ENOMEM;
	206	rc = devm_add_action_or_reset(dev, label_area_release, lsa);
	207	if (rc)
	208	return rc;
	209	dev_set_drvdata(dev, lsa);
	210
	211	cxlm = cxl_mem_create(dev);
	212	if (IS_ERR(cxlm))
	213	return PTR_ERR(cxlm);
	214
	215	cxlm->mbox_send = cxl_mock_mbox_send;
	216	cxlm->payload_size = SZ_4K;
	217
	218	rc = cxl_mem_enumerate_cmds(cxlm);
	219	if (rc)
	220	return rc;
	221
	222	rc = cxl_mem_identify(cxlm);
	223	if (rc)
	224	return rc;
	225
	226	rc = cxl_mem_create_range_info(cxlm);
	227	if (rc)
	228	return rc;
	229
	230	cxlmd = devm_cxl_add_memdev(cxlm);
	231	if (IS_ERR(cxlmd))
	232	return PTR_ERR(cxlmd);
	233
	234	if (range_len(&cxlm->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))
	235	rc = devm_cxl_add_nvdimm(dev, cxlmd);
	236
	237	return 0;
	238	}
	239
	240	static const struct platform_device_id cxl_mock_mem_ids[] = {
	241	{ .name = "cxl_mem", },
	242	{ },
	243	};
	244	MODULE_DEVICE_TABLE(platform, cxl_mock_mem_ids);
	245
	246	static struct platform_driver cxl_mock_mem_driver = {
	247	.probe = cxl_mock_mem_probe,
	248	.id_table = cxl_mock_mem_ids,
	249	.driver = {
	250	.name = KBUILD_MODNAME,
	251	},
	252	};
	253
	254	module_platform_driver(cxl_mock_mem_driver);
	255	MODULE_LICENSE("GPL v2");
	256	MODULE_IMPORT_NS(CXL);

tools/testing/cxl/test/mock.c+171−0

@@ -0,0 +1,171 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	//Copyright(c) 2021 Intel Corporation. All rights reserved.
	3
	4	#include <linux/libnvdimm.h>
	5	#include <linux/rculist.h>
	6	#include <linux/device.h>
	7	#include <linux/export.h>
	8	#include <linux/acpi.h>
	9	#include <linux/pci.h>
	10	#include "mock.h"
	11
	12	static LIST_HEAD(mock);
	13
	14	void register_cxl_mock_ops(struct cxl_mock_ops *ops)
	15	{
	16	list_add_rcu(&ops->list, &mock);
	17	}
	18	EXPORT_SYMBOL_GPL(register_cxl_mock_ops);
	19
	20	static DEFINE_SRCU(cxl_mock_srcu);
	21
	22	void unregister_cxl_mock_ops(struct cxl_mock_ops *ops)
	23	{
	24	list_del_rcu(&ops->list);
	25	synchronize_srcu(&cxl_mock_srcu);
	26	}
	27	EXPORT_SYMBOL_GPL(unregister_cxl_mock_ops);
	28
	29	struct cxl_mock_ops get_cxl_mock_ops(int index)
	30	{
	31	*index = srcu_read_lock(&cxl_mock_srcu);
	32	return list_first_or_null_rcu(&mock, struct cxl_mock_ops, list);
	33	}
	34	EXPORT_SYMBOL_GPL(get_cxl_mock_ops);
	35
	36	void put_cxl_mock_ops(int index)
	37	{
	38	srcu_read_unlock(&cxl_mock_srcu, index);
	39	}
	40	EXPORT_SYMBOL_GPL(put_cxl_mock_ops);
	41
	42	bool __wrap_is_acpi_device_node(const struct fwnode_handle *fwnode)
	43	{
	44	struct acpi_device *adev =
	45	container_of(fwnode, struct acpi_device, fwnode);
	46	int index;
	47	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	48	bool retval = false;
	49
	50	if (ops)
	51	retval = ops->is_mock_adev(adev);
	52
	53	if (!retval)
	54	retval = is_acpi_device_node(fwnode);
	55
	56	put_cxl_mock_ops(index);
	57	return retval;
	58	}
	59	EXPORT_SYMBOL(__wrap_is_acpi_device_node);
	60
	61	acpi_status __wrap_acpi_get_table(char *signature, u32 instance,
	62	struct acpi_table_header **out_table)
	63	{
	64	int index;
	65	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	66	acpi_status status;
	67
	68	if (ops)
	69	status = ops->acpi_get_table(signature, instance, out_table);
	70	else
	71	status = acpi_get_table(signature, instance, out_table);
	72
	73	put_cxl_mock_ops(index);
	74
	75	return status;
	76	}
	77	EXPORT_SYMBOL(__wrap_acpi_get_table);
	78
	79	void __wrap_acpi_put_table(struct acpi_table_header *table)
	80	{
	81	int index;
	82	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	83
	84	if (ops)
	85	ops->acpi_put_table(table);
	86	else
	87	acpi_put_table(table);
	88	put_cxl_mock_ops(index);
	89	}
	90	EXPORT_SYMBOL(__wrap_acpi_put_table);
	91
	92	acpi_status __wrap_acpi_evaluate_integer(acpi_handle handle,
	93	acpi_string pathname,
	94	struct acpi_object_list *arguments,
	95	unsigned long long *data)
	96	{
	97	int index;
	98	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	99	acpi_status status;
	100
	101	if (ops)
	102	status = ops->acpi_evaluate_integer(handle, pathname, arguments,
	103	data);
	104	else
	105	status = acpi_evaluate_integer(handle, pathname, arguments,
	106	data);
	107	put_cxl_mock_ops(index);
	108
	109	return status;
	110	}
	111	EXPORT_SYMBOL(__wrap_acpi_evaluate_integer);
	112
	113	struct acpi_pci_root *__wrap_acpi_pci_find_root(acpi_handle handle)
	114	{
	115	int index;
	116	struct acpi_pci_root *root;
	117	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	118
	119	if (ops)
	120	root = ops->acpi_pci_find_root(handle);
	121	else
	122	root = acpi_pci_find_root(handle);
	123
	124	put_cxl_mock_ops(index);
	125
	126	return root;
	127	}
	128	EXPORT_SYMBOL_GPL(__wrap_acpi_pci_find_root);
	129
	130	void __wrap_pci_walk_bus(struct pci_bus *bus,
	131	int (cb)(struct pci_dev , void ), void userdata)
	132	{
	133	int index;
	134	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	135
	136	if (ops && ops->is_mock_bus(bus)) {
	137	int rc, i;
	138
	139	/*
	140	* Simulate 2 root ports per host-bridge and no
	141	* depth recursion.
	142	*/
	143	for (i = 0; i < 2; i++) {
	144	rc = cb((struct pci_dev *) ops->mock_port(bus, i),
	145	userdata);
	146	if (rc)
	147	break;
	148	}
	149	} else
	150	pci_walk_bus(bus, cb, userdata);
	151
	152	put_cxl_mock_ops(index);
	153	}
	154	EXPORT_SYMBOL_GPL(__wrap_pci_walk_bus);
	155
	156	struct nvdimm_bus *
	157	__wrap_nvdimm_bus_register(struct device *dev,
	158	struct nvdimm_bus_descriptor *nd_desc)
	159	{
	160	int index;
	161	struct cxl_mock_ops *ops = get_cxl_mock_ops(&index);
	162
	163	if (ops && ops->is_mock_dev(dev->parent->parent))
	164	nd_desc->provider_name = "cxl_test";
	165	put_cxl_mock_ops(index);
	166
	167	return nvdimm_bus_register(dev, nd_desc);
	168	}
	169	EXPORT_SYMBOL_GPL(__wrap_nvdimm_bus_register);
	170
	171	MODULE_LICENSE("GPL v2");

tools/testing/cxl/test/mock.h+27−0

@@ -0,0 +1,27 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	#include <linux/list.h>
	4	#include <linux/acpi.h>
	5
	6	struct cxl_mock_ops {
	7	struct list_head list;
	8	bool (is_mock_adev)(struct acpi_device dev);
	9	acpi_status (acpi_get_table)(char signature, u32 instance,
	10	struct acpi_table_header **out_table);
	11	void (acpi_put_table)(struct acpi_table_header table);
	12	bool (is_mock_bridge)(struct device dev);
	13	acpi_status (*acpi_evaluate_integer)(acpi_handle handle,
	14	acpi_string pathname,
	15	struct acpi_object_list *arguments,
	16	unsigned long long *data);
	17	struct acpi_pci_root (acpi_pci_find_root)(acpi_handle handle);
	18	struct platform_device (mock_port)(struct pci_bus *bus, int index);
	19	bool (is_mock_bus)(struct pci_bus bus);
	20	bool (is_mock_port)(struct platform_device pdev);
	21	bool (is_mock_dev)(struct device dev);
	22	};
	23
	24	void register_cxl_mock_ops(struct cxl_mock_ops *ops);
	25	void unregister_cxl_mock_ops(struct cxl_mock_ops *ops);
	26	struct cxl_mock_ops get_cxl_mock_ops(int index);
	27	void put_cxl_mock_ops(int index);