// SPDX-License-Identifier: GPL-2.0-only/* Copyright(c) 2020 Intel Corporation. */ #include <linux/io-64-nonatomic-lo-hi.h>#include <linux/firmware.h>#include <linux/device.h>#include <linux/slab.h>#include <linux/idr.h>#include <linux/pci.h>#include <cxlmem.h>#include "trace.h"#include "core.h" static DECLARE_RWSEM(cxl_memdev_rwsem); /* * An entire PCI topology full of devices should be enough for any * config */#define CXL_MEM_MAX_DEVS 65536 static int cxl_mem_major;static DEFINE_IDA(cxl_memdev_ida); static void cxl_memdev_release(struct device *dev){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct device *parent = dev->parent; ida_free(&cxl_memdev_ida, cxlmd->id); kfree(cxlmd); put_device(parent);} static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, kgid_t *gid){ return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));} static ssize_t firmware_version_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%.16s\n", mds->firmware_version);}static DEVICE_ATTR_RO(firmware_version); static ssize_t payload_max_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);}static DEVICE_ATTR_RO(payload_max); static ssize_t label_storage_size_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%zu\n", mds->lsa_size);}static DEVICE_ATTR_RO(label_storage_size); static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds){ /* Static RAM is only expected at partition 0. */ if (cxlds->part[0].mode != CXL_PARTMODE_RAM) return 0; return resource_size(&cxlds->part[0].res);} static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; unsigned long long len = cxl_ram_size(cxlds); return sysfs_emit(buf, "%#llx\n", len);} static struct device_attribute dev_attr_ram_size = __ATTR(size, 0444, ram_size_show, NULL); static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; unsigned long long len = cxl_pmem_size(cxlds); return sysfs_emit(buf, "%#llx\n", len);} static struct device_attribute dev_attr_pmem_size = __ATTR(size, 0444, pmem_size_show, NULL); static ssize_t serial_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; return sysfs_emit(buf, "%#llx\n", cxlds->serial);}static DEVICE_ATTR_RO(serial); static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, char *buf){ return sysfs_emit(buf, "%d\n", dev_to_node(dev));}static DEVICE_ATTR_RO(numa_node); static ssize_t security_state_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); unsigned long state = mds->security.state; int rc = 0; /* sync with latest submission state */ mutex_lock(&cxl_mbox->mbox_mutex); if (mds->security.sanitize_active) rc = sysfs_emit(buf, "sanitize\n"); mutex_unlock(&cxl_mbox->mbox_mutex); if (rc) return rc; if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) return sysfs_emit(buf, "disabled\n"); if (state & CXL_PMEM_SEC_STATE_FROZEN || state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || state & CXL_PMEM_SEC_STATE_USER_PLIMIT) return sysfs_emit(buf, "frozen\n"); if (state & CXL_PMEM_SEC_STATE_LOCKED) return sysfs_emit(buf, "locked\n"); return sysfs_emit(buf, "unlocked\n");}static struct device_attribute dev_attr_security_state = __ATTR(state, 0444, security_state_show, NULL); static ssize_t security_sanitize_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); bool sanitize; ssize_t rc; if (kstrtobool(buf, &sanitize) || !sanitize) return -EINVAL; rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); if (rc) return rc; return len;}static struct device_attribute dev_attr_security_sanitize = __ATTR(sanitize, 0200, NULL, security_sanitize_store); static ssize_t security_erase_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); ssize_t rc; bool erase; if (kstrtobool(buf, &erase) || !erase) return -EINVAL; rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); if (rc) return rc; return len;}static struct device_attribute dev_attr_security_erase = __ATTR(erase, 0200, NULL, security_erase_store); bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd, enum poison_cmd_enabled_bits cmd){ struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); if (!mds) return 0; return test_bit(cmd, mds->poison.enabled_cmds);} static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd){ struct cxl_dev_state *cxlds = cxlmd->cxlds; u64 offset, length; int rc = 0; /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ for (int i = 0; i < cxlds->nr_partitions; i++) { const struct resource *res = &cxlds->part[i].res; offset = res->start; length = resource_size(res); rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); /* * Invalid Physical Address is not an error for * volatile addresses. Device support is optional. */ if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) rc = 0; } return rc;} int cxl_trigger_poison_list(struct cxl_memdev *cxlmd){ struct cxl_port *port; int rc; port = cxlmd->endpoint; if (!port || !is_cxl_endpoint(port)) return -EINVAL; ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc; ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc; if (cxl_num_decoders_committed(port) == 0) { /* No regions mapped to this memdev */ rc = cxl_get_poison_by_memdev(cxlmd); } else { /* Regions mapped, collect poison by endpoint */ rc = cxl_get_poison_by_endpoint(port); } return rc;}EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL"); static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa){ struct cxl_dev_state *cxlds = cxlmd->cxlds; if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0; if (!resource_size(&cxlds->dpa_res)) { dev_dbg(cxlds->dev, "device has no dpa resource\n"); return -EINVAL; } if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) { dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", dpa, &cxlds->dpa_res); return -EINVAL; } if (!IS_ALIGNED(dpa, 64)) { dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); return -EINVAL; } return 0;} int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa){ struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_inject_poison inject; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; struct cxl_region *cxlr; int rc; if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0; lockdep_assert_held(&cxl_rwsem.dpa); lockdep_assert_held(&cxl_rwsem.region); rc = cxl_validate_poison_dpa(cxlmd, dpa); if (rc) return rc; inject.address = cpu_to_le64(dpa); mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_INJECT_POISON, .size_in = sizeof(inject), .payload_in = &inject, }; rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) return rc; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) dev_warn_once(cxl_mbox->host, "poison inject dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev)); record = (struct cxl_poison_record) { .address = cpu_to_le64(dpa), .length = cpu_to_le32(1), }; trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); return 0;} int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa){ int rc; ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc; ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc; return cxl_inject_poison_locked(cxlmd, dpa);}EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL"); int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa){ struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_clear_poison clear; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; struct cxl_region *cxlr; int rc; if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0; lockdep_assert_held(&cxl_rwsem.dpa); lockdep_assert_held(&cxl_rwsem.region); rc = cxl_validate_poison_dpa(cxlmd, dpa); if (rc) return rc; /* * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command * is defined to accept 64 bytes of write-data, along with the * address to clear. This driver uses zeroes as write-data. */ clear = (struct cxl_mbox_clear_poison) { .address = cpu_to_le64(dpa) }; mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_CLEAR_POISON, .size_in = sizeof(clear), .payload_in = &clear, }; rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) return rc; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) dev_warn_once(cxl_mbox->host, "poison clear dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev)); record = (struct cxl_poison_record) { .address = cpu_to_le64(dpa), .length = cpu_to_le32(1), }; trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); return 0;} int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa){ int rc; ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc; ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc; return cxl_clear_poison_locked(cxlmd, dpa);}EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL"); static struct attribute *cxl_memdev_attributes[] = { &dev_attr_serial.attr, &dev_attr_firmware_version.attr, &dev_attr_payload_max.attr, &dev_attr_label_storage_size.attr, &dev_attr_numa_node.attr, NULL,}; static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds){ for (int i = 0; i < cxlds->nr_partitions; i++) if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) return &cxlds->part[i].perf; return NULL;} static ssize_t pmem_qos_class_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class);} static struct device_attribute dev_attr_pmem_qos_class = __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); static struct attribute *cxl_memdev_pmem_attributes[] = { &dev_attr_pmem_size.attr, &dev_attr_pmem_qos_class.attr, NULL,}; static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds){ if (cxlds->part[0].mode != CXL_PARTMODE_RAM) return NULL; return &cxlds->part[0].perf;} static ssize_t ram_qos_class_show(struct device *dev, struct device_attribute *attr, char *buf){ struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class);} static struct device_attribute dev_attr_ram_qos_class = __ATTR(qos_class, 0444, ram_qos_class_show, NULL); static struct attribute *cxl_memdev_ram_attributes[] = { &dev_attr_ram_size.attr, &dev_attr_ram_qos_class.attr, NULL,}; static struct attribute *cxl_memdev_security_attributes[] = { &dev_attr_security_state.attr, &dev_attr_security_sanitize.attr, &dev_attr_security_erase.attr, NULL,}; static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, int n){ if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) return 0; return a->mode;} static struct attribute_group cxl_memdev_attribute_group = { .attrs = cxl_memdev_attributes, .is_visible = cxl_memdev_visible,}; static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)Lines 1–500 of 1317. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2020 Intel Corporation. */ #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/firmware.h> #include <linux/device.h> #include <linux/slab.h> #include <linux/idr.h> #include <linux/pci.h> #include <cxlmem.h> #include "trace.h" #include "core.h" static DECLARE_RWSEM(cxl_memdev_rwsem); /* * An entire PCI topology full of devices should be enough for any * config */ #define CXL_MEM_MAX_DEVS 65536 static int cxl_mem_major; static DEFINE_IDA(cxl_memdev_ida); static void cxl_memdev_release(struct device *dev) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct device *parent = dev->parent; ida_free(&cxl_memdev_ida, cxlmd->id); kfree(cxlmd); put_device(parent); } static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, kgid_t *gid) { return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); } static ssize_t firmware_version_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%.16s\n", mds->firmware_version); } static DEVICE_ATTR_RO(firmware_version); static ssize_t payload_max_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size); } static DEVICE_ATTR_RO(payload_max); static ssize_t label_storage_size_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); if (!mds) return sysfs_emit(buf, "\n"); return sysfs_emit(buf, "%zu\n", mds->lsa_size); } static DEVICE_ATTR_RO(label_storage_size); static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds) { /* Static RAM is only expected at partition 0. */ if (cxlds->part[0].mode != CXL_PARTMODE_RAM) return 0; return resource_size(&cxlds->part[0].res); } static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; unsigned long long len = cxl_ram_size(cxlds); return sysfs_emit(buf, "%#llx\n", len); } static struct device_attribute dev_attr_ram_size = __ATTR(size, 0444, ram_size_show, NULL); static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; unsigned long long len = cxl_pmem_size(cxlds); return sysfs_emit(buf, "%#llx\n", len); } static struct device_attribute dev_attr_pmem_size = __ATTR(size, 0444, pmem_size_show, NULL); static ssize_t serial_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; return sysfs_emit(buf, "%#llx\n", cxlds->serial); } static DEVICE_ATTR_RO(serial); static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, char *buf) { return sysfs_emit(buf, "%d\n", dev_to_node(dev)); } static DEVICE_ATTR_RO(numa_node); static ssize_t security_state_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); unsigned long state = mds->security.state; int rc = 0; /* sync with latest submission state */ mutex_lock(&cxl_mbox->mbox_mutex); if (mds->security.sanitize_active) rc = sysfs_emit(buf, "sanitize\n"); mutex_unlock(&cxl_mbox->mbox_mutex); if (rc) return rc; if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) return sysfs_emit(buf, "disabled\n"); if (state & CXL_PMEM_SEC_STATE_FROZEN || state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || state & CXL_PMEM_SEC_STATE_USER_PLIMIT) return sysfs_emit(buf, "frozen\n"); if (state & CXL_PMEM_SEC_STATE_LOCKED) return sysfs_emit(buf, "locked\n"); return sysfs_emit(buf, "unlocked\n"); } static struct device_attribute dev_attr_security_state = __ATTR(state, 0444, security_state_show, NULL); static ssize_t security_sanitize_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); bool sanitize; ssize_t rc; if (kstrtobool(buf, &sanitize) || !sanitize) return -EINVAL; rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); if (rc) return rc; return len; } static struct device_attribute dev_attr_security_sanitize = __ATTR(sanitize, 0200, NULL, security_sanitize_store); static ssize_t security_erase_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); ssize_t rc; bool erase; if (kstrtobool(buf, &erase) || !erase) return -EINVAL; rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); if (rc) return rc; return len; } static struct device_attribute dev_attr_security_erase = __ATTR(erase, 0200, NULL, security_erase_store); bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd, enum poison_cmd_enabled_bits cmd) { struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); if (!mds) return 0; return test_bit(cmd, mds->poison.enabled_cmds); } static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) { struct cxl_dev_state *cxlds = cxlmd->cxlds; u64 offset, length; int rc = 0; /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ for (int i = 0; i < cxlds->nr_partitions; i++) { const struct resource *res = &cxlds->part[i].res; offset = res->start; length = resource_size(res); rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); /* * Invalid Physical Address is not an error for * volatile addresses. Device support is optional. */ if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) rc = 0; } return rc; } int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) { struct cxl_port *port; int rc; port = cxlmd->endpoint; if (!port || !is_cxl_endpoint(port)) return -EINVAL; ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc; ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc; if (cxl_num_decoders_committed(port) == 0) { /* No regions mapped to this memdev */ rc = cxl_get_poison_by_memdev(cxlmd); } else { /* Regions mapped, collect poison by endpoint */ rc = cxl_get_poison_by_endpoint(port); } return rc; } EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL"); static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) { struct cxl_dev_state *cxlds = cxlmd->cxlds; if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0; if (!resource_size(&cxlds->dpa_res)) { dev_dbg(cxlds->dev, "device has no dpa resource\n"); return -EINVAL; } if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) { dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", dpa, &cxlds->dpa_res); return -EINVAL; } if (!IS_ALIGNED(dpa, 64)) { dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); return -EINVAL; } return 0; } int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa) { struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_inject_poison inject; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; struct cxl_region *cxlr; int rc; if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0; lockdep_assert_held(&cxl_rwsem.dpa); lockdep_assert_held(&cxl_rwsem.region); rc = cxl_validate_poison_dpa(cxlmd, dpa); if (rc) return rc; inject.address = cpu_to_le64(dpa); mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_INJECT_POISON, .size_in = sizeof(inject), .payload_in = &inject, }; rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) return rc; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) dev_warn_once(cxl_mbox->host, "poison inject dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev)); record = (struct cxl_poison_record) { .address = cpu_to_le64(dpa), .length = cpu_to_le32(1), }; trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); return 0; } int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) { int rc; ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc; ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc; return cxl_inject_poison_locked(cxlmd, dpa); } EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL"); int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa) { struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_clear_poison clear; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; struct cxl_region *cxlr; int rc; if (!IS_ENABLED(CONFIG_DEBUG_FS)) return 0; lockdep_assert_held(&cxl_rwsem.dpa); lockdep_assert_held(&cxl_rwsem.region); rc = cxl_validate_poison_dpa(cxlmd, dpa); if (rc) return rc; /* * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command * is defined to accept 64 bytes of write-data, along with the * address to clear. This driver uses zeroes as write-data. */ clear = (struct cxl_mbox_clear_poison) { .address = cpu_to_le64(dpa) }; mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_CLEAR_POISON, .size_in = sizeof(clear), .payload_in = &clear, }; rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) return rc; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) dev_warn_once(cxl_mbox->host, "poison clear dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev)); record = (struct cxl_poison_record) { .address = cpu_to_le64(dpa), .length = cpu_to_le32(1), }; trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); return 0; } int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) { int rc; ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region); if ((rc = ACQUIRE_ERR(rwsem_read_intr, ®ion_rwsem))) return rc; ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa); if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem))) return rc; return cxl_clear_poison_locked(cxlmd, dpa); } EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL"); static struct attribute *cxl_memdev_attributes[] = { &dev_attr_serial.attr, &dev_attr_firmware_version.attr, &dev_attr_payload_max.attr, &dev_attr_label_storage_size.attr, &dev_attr_numa_node.attr, NULL, }; static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds) { for (int i = 0; i < cxlds->nr_partitions; i++) if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) return &cxlds->part[i].perf; return NULL; } static ssize_t pmem_qos_class_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class); } static struct device_attribute dev_attr_pmem_qos_class = __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); static struct attribute *cxl_memdev_pmem_attributes[] = { &dev_attr_pmem_size.attr, &dev_attr_pmem_qos_class.attr, NULL, }; static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds) { if (cxlds->part[0].mode != CXL_PARTMODE_RAM) return NULL; return &cxlds->part[0].perf; } static ssize_t ram_qos_class_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class); } static struct device_attribute dev_attr_ram_qos_class = __ATTR(qos_class, 0444, ram_qos_class_show, NULL); static struct attribute *cxl_memdev_ram_attributes[] = { &dev_attr_ram_size.attr, &dev_attr_ram_qos_class.attr, NULL, }; static struct attribute *cxl_memdev_security_attributes[] = { &dev_attr_security_state.attr, &dev_attr_security_sanitize.attr, &dev_attr_security_erase.attr, NULL, }; static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, int n) { if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) return 0; return a->mode; } static struct attribute_group cxl_memdev_attribute_group = { .attrs = cxl_memdev_attributes, .is_visible = cxl_memdev_visible, }; static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)