History

qedr: Add RoCE driver framework

Adds a skeletal implementation of the qed* RoCE driver - basically the ability to communicate with the qede driver and receive notifications from it regarding various init/exit events.

Ram Amrani committed 10 years ago

Commit 2e0cbc41 parent e9f8f488 files · +331 −11Browse files

Unified · Split

8 files
  1. drivers/infiniband/Kconfig +2 −0
  2. drivers/infiniband/hw/Makefile +1 −0
  3. drivers/infiniband/hw/qedr/Kconfig +7 −0
  4. drivers/infiniband/hw/qedr/Makefile +3 −0
  5. drivers/infiniband/hw/qedr/main.c +254 −0
  6. drivers/infiniband/hw/qedr/qedr.h +61 −0
  7. drivers/net/ethernet/qlogic/Kconfig +0 −11
  8. include/uapi/linux/pci_regs.h +3 −0
drivers/infiniband/Kconfig+2−0
@@ -89,4 +89,6 @@ source "drivers/infiniband/sw/rxe/Kconfig" · 88 unmodified lines
8989
9090 source "drivers/infiniband/hw/hfi1/Kconfig"
9191
92+source "drivers/infiniband/hw/qedr/Kconfig"
93+
9294 endif # INFINIBAND
drivers/infiniband/hw/Makefile+1−0
@@ -10,3 +10,4 @@ obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/ · 9 unmodified lines
1010 obj-$(CONFIG_INFINIBAND_USNIC) += usnic/
1111 obj-$(CONFIG_INFINIBAND_HFI1) += hfi1/
1212 obj-$(CONFIG_INFINIBAND_HNS) += hns/
13+obj-$(CONFIG_INFINIBAND_QEDR) += qedr/
drivers/infiniband/hw/qedr/Kconfig+7−0
@@ -0,0 +1,7 @@
1+config INFINIBAND_QEDR
2+ tristate "QLogic RoCE driver"
3+ depends on 64BIT && QEDE
4+ select QED_LL2
5+ ---help---
6+ This driver provides low-level InfiniBand over Ethernet
7+ support for QLogic QED host channel adapters (HCAs).
drivers/infiniband/hw/qedr/Makefile+3−0
@@ -0,0 +1,3 @@
1+obj-$(CONFIG_INFINIBAND_QEDR) := qedr.o
2+
3+qedr-y := main.o
drivers/infiniband/hw/qedr/main.c+254−0
@@ -0,0 +1,254 @@
1+/* QLogic qedr NIC Driver
2+ * Copyright (c) 2015-2016 QLogic Corporation
3+ *
4+ * This software is available to you under a choice of one of two
5+ * licenses. You may choose to be licensed under the terms of the GNU
6+ * General Public License (GPL) Version 2, available from the file
7+ * COPYING in the main directory of this source tree, or the
8+ * OpenIB.org BSD license below:
9+ *
10+ * Redistribution and use in source and binary forms, with or
11+ * without modification, are permitted provided that the following
12+ * conditions are met:
13+ *
14+ * - Redistributions of source code must retain the above
15+ * copyright notice, this list of conditions and the following
16+ * disclaimer.
17+ *
18+ * - Redistributions in binary form must reproduce the above
19+ * copyright notice, this list of conditions and the following
20+ * disclaimer in the documentation and /or other materials
21+ * provided with the distribution.
22+ *
23+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30+ * SOFTWARE.
31+ */
32+#include <linux/module.h>
33+#include <rdma/ib_verbs.h>
34+#include <rdma/ib_addr.h>
35+#include <linux/netdevice.h>
36+#include <linux/iommu.h>
37+#include <net/addrconf.h>
38+#include <linux/qed/qede_roce.h>
39+#include "qedr.h"
40+
41+MODULE_DESCRIPTION("QLogic 40G/100G ROCE Driver");
42+MODULE_AUTHOR("QLogic Corporation");
43+MODULE_LICENSE("Dual BSD/GPL");
44+MODULE_VERSION(QEDR_MODULE_VERSION);
45+
46+void qedr_ib_dispatch_event(struct qedr_dev *dev, u8 port_num,
47+ enum ib_event_type type)
48+{
49+ struct ib_event ibev;
50+
51+ ibev.device = &dev->ibdev;
52+ ibev.element.port_num = port_num;
53+ ibev.event = type;
54+
55+ ib_dispatch_event(&ibev);
56+}
57+
58+static enum rdma_link_layer qedr_link_layer(struct ib_device *device,
59+ u8 port_num)
60+{
61+ return IB_LINK_LAYER_ETHERNET;
62+}
63+
64+static int qedr_register_device(struct qedr_dev *dev)
65+{
66+ strlcpy(dev->ibdev.name, "qedr%d", IB_DEVICE_NAME_MAX);
67+
68+ memcpy(dev->ibdev.node_desc, QEDR_NODE_DESC, sizeof(QEDR_NODE_DESC));
69+ dev->ibdev.owner = THIS_MODULE;
70+
71+ dev->ibdev.get_link_layer = qedr_link_layer;
72+
73+ return 0;
74+}
75+
76+/* QEDR sysfs interface */
77+static ssize_t show_rev(struct device *device, struct device_attribute *attr,
78+ char *buf)
79+{
80+ struct qedr_dev *dev = dev_get_drvdata(device);
81+
82+ return scnprintf(buf, PAGE_SIZE, "0x%x\n", dev->pdev->vendor);
83+}
84+
85+static ssize_t show_hca_type(struct device *device,
86+ struct device_attribute *attr, char *buf)
87+{
88+ return scnprintf(buf, PAGE_SIZE, "%s\n", "HCA_TYPE_TO_SET");
89+}
90+
91+static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
92+static DEVICE_ATTR(hca_type, S_IRUGO, show_hca_type, NULL);
93+
94+static struct device_attribute *qedr_attributes[] = {
95+ &dev_attr_hw_rev,
96+ &dev_attr_hca_type
97+};
98+
99+static void qedr_remove_sysfiles(struct qedr_dev *dev)
100+{
101+ int i;
102+
103+ for (i = 0; i < ARRAY_SIZE(qedr_attributes); i++)
104+ device_remove_file(&dev->ibdev.dev, qedr_attributes[i]);
105+}
106+
107+static void qedr_pci_set_atomic(struct qedr_dev *dev, struct pci_dev *pdev)
108+{
109+ struct pci_dev *bridge;
110+ u32 val;
111+
112+ dev->atomic_cap = IB_ATOMIC_NONE;
113+
114+ bridge = pdev->bus->self;
115+ if (!bridge)
116+ return;
117+
118+ /* Check whether we are connected directly or via a switch */
119+ while (bridge && bridge->bus->parent) {
120+ DP_DEBUG(dev, QEDR_MSG_INIT,
121+ "Device is not connected directly to root. bridge->bus->number=%d primary=%d\n",
122+ bridge->bus->number, bridge->bus->primary);
123+ /* Need to check Atomic Op Routing Supported all the way to
124+ * root complex.
125+ */
126+ pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &val);
127+ if (!(val & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) {
128+ pcie_capability_clear_word(pdev,
129+ PCI_EXP_DEVCTL2,
130+ PCI_EXP_DEVCTL2_ATOMIC_REQ);
131+ return;
132+ }
133+ bridge = bridge->bus->parent->self;
134+ }
135+ bridge = pdev->bus->self;
136+
137+ /* according to bridge capability */
138+ pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &val);
139+ if (val & PCI_EXP_DEVCAP2_ATOMIC_COMP64) {
140+ pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
141+ PCI_EXP_DEVCTL2_ATOMIC_REQ);
142+ dev->atomic_cap = IB_ATOMIC_GLOB;
143+ } else {
144+ pcie_capability_clear_word(pdev, PCI_EXP_DEVCTL2,
145+ PCI_EXP_DEVCTL2_ATOMIC_REQ);
146+ }
147+}
148+
149+static struct qedr_dev *qedr_add(struct qed_dev *cdev, struct pci_dev *pdev,
150+ struct net_device *ndev)
151+{
152+ struct qedr_dev *dev;
153+ int rc = 0, i;
154+
155+ dev = (struct qedr_dev *)ib_alloc_device(sizeof(*dev));
156+ if (!dev) {
157+ pr_err("Unable to allocate ib device\n");
158+ return NULL;
159+ }
160+
161+ DP_DEBUG(dev, QEDR_MSG_INIT, "qedr add device called\n");
162+
163+ dev->pdev = pdev;
164+ dev->ndev = ndev;
165+ dev->cdev = cdev;
166+
167+ qedr_pci_set_atomic(dev, pdev);
168+
169+ rc = qedr_register_device(dev);
170+ if (rc) {
171+ DP_ERR(dev, "Unable to allocate register device\n");
172+ goto init_err;
173+ }
174+
175+ for (i = 0; i < ARRAY_SIZE(qedr_attributes); i++)
176+ if (device_create_file(&dev->ibdev.dev, qedr_attributes[i]))
177+ goto init_err;
178+
179+ DP_DEBUG(dev, QEDR_MSG_INIT, "qedr driver loaded successfully\n");
180+ return dev;
181+
182+init_err:
183+ ib_dealloc_device(&dev->ibdev);
184+ DP_ERR(dev, "qedr driver load failed rc=%d\n", rc);
185+
186+ return NULL;
187+}
188+
189+static void qedr_remove(struct qedr_dev *dev)
190+{
191+ /* First unregister with stack to stop all the active traffic
192+ * of the registered clients.
193+ */
194+ qedr_remove_sysfiles(dev);
195+
196+ ib_dealloc_device(&dev->ibdev);
197+}
198+
199+static int qedr_close(struct qedr_dev *dev)
200+{
201+ qedr_ib_dispatch_event(dev, 1, IB_EVENT_PORT_ERR);
202+
203+ return 0;
204+}
205+
206+static void qedr_shutdown(struct qedr_dev *dev)
207+{
208+ qedr_close(dev);
209+ qedr_remove(dev);
210+}
211+
212+/* event handling via NIC driver ensures that all the NIC specific
213+ * initialization done before RoCE driver notifies
214+ * event to stack.
215+ */
216+static void qedr_notify(struct qedr_dev *dev, enum qede_roce_event event)
217+{
218+ switch (event) {
219+ case QEDE_UP:
220+ qedr_ib_dispatch_event(dev, 1, IB_EVENT_PORT_ACTIVE);
221+ break;
222+ case QEDE_DOWN:
223+ qedr_close(dev);
224+ break;
225+ case QEDE_CLOSE:
226+ qedr_shutdown(dev);
227+ break;
228+ case QEDE_CHANGE_ADDR:
229+ qedr_ib_dispatch_event(dev, 1, IB_EVENT_GID_CHANGE);
230+ break;
231+ default:
232+ pr_err("Event not supported\n");
233+ }
234+}
235+
236+static struct qedr_driver qedr_drv = {
237+ .name = "qedr_driver",
238+ .add = qedr_add,
239+ .remove = qedr_remove,
240+ .notify = qedr_notify,
241+};
242+
243+static int __init qedr_init_module(void)
244+{
245+ return qede_roce_register_driver(&qedr_drv);
246+}
247+
248+static void __exit qedr_exit_module(void)
249+{
250+ qede_roce_unregister_driver(&qedr_drv);
251+}
252+
253+module_init(qedr_init_module);
254+module_exit(qedr_exit_module);
drivers/infiniband/hw/qedr/qedr.h+61−0
@@ -0,0 +1,61 @@
1+/* QLogic qedr NIC Driver
2+ * Copyright (c) 2015-2016 QLogic Corporation
3+ *
4+ * This software is available to you under a choice of one of two
5+ * licenses. You may choose to be licensed under the terms of the GNU
6+ * General Public License (GPL) Version 2, available from the file
7+ * COPYING in the main directory of this source tree, or the
8+ * OpenIB.org BSD license below:
9+ *
10+ * Redistribution and use in source and binary forms, with or
11+ * without modification, are permitted provided that the following
12+ * conditions are met:
13+ *
14+ * - Redistributions of source code must retain the above
15+ * copyright notice, this list of conditions and the following
16+ * disclaimer.
17+ *
18+ * - Redistributions in binary form must reproduce the above
19+ * copyright notice, this list of conditions and the following
20+ * disclaimer in the documentation and /or other materials
21+ * provided with the distribution.
22+ *
23+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30+ * SOFTWARE.
31+ */
32+#ifndef __QEDR_H__
33+#define __QEDR_H__
34+
35+#include <linux/pci.h>
36+#include <rdma/ib_addr.h>
37+#include <linux/qed/qed_if.h>
38+#include <linux/qed/qede_roce.h>
39+
40+#define QEDR_MODULE_VERSION "8.10.10.0"
41+#define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
42+#define DP_NAME(dev) ((dev)->ibdev.name)
43+
44+#define DP_DEBUG(dev, module, fmt, ...) \
45+ pr_debug("(%s) " module ": " fmt, \
46+ DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
47+
48+#define QEDR_MSG_INIT "INIT"
49+
50+struct qedr_dev {
51+ struct ib_device ibdev;
52+ struct qed_dev *cdev;
53+ struct pci_dev *pdev;
54+ struct net_device *ndev;
55+
56+ enum ib_atomic_cap atomic_cap;
57+
58+ u32 dp_module;
59+ u8 dp_level;
60+};
61+#endif
drivers/net/ethernet/qlogic/Kconfig+0−11
@@ -107,15 +107,4 @@ config QEDE · 106 unmodified lines
107107 ---help---
108108 This enables the support for ...
109109
110−config INFINIBAND_QEDR
111− tristate "QLogic qede RoCE sources [debug]"
112− depends on QEDE && 64BIT
113− select QED_LL2
114− default n
115− ---help---
116− This provides a temporary node that allows the compilation
117− and logical testing of the InfiniBand over Ethernet support
118− for QLogic QED. This would be replaced by the 'real' option
119− once the QEDR driver is added [+relocated].
120−
121110 endif # NET_VENDOR_QLOGIC
include/uapi/linux/pci_regs.h+3−0
@@ -612,6 +612,8 @@ · 611 unmodified lines
612612 */
613613 #define PCI_EXP_DEVCAP2 36 /* Device Capabilities 2 */
614614 #define PCI_EXP_DEVCAP2_ARI 0x00000020 /* Alternative Routing-ID */
615+#define PCI_EXP_DEVCAP2_ATOMIC_ROUTE 0x00000040 /* Atomic Op routing */
616+#define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* Atomic 64-bit compare */
615617 #define PCI_EXP_DEVCAP2_LTR 0x00000800 /* Latency tolerance reporting */
616618 #define PCI_EXP_DEVCAP2_OBFF_MASK 0x000c0000 /* OBFF support mechanism */
617619 #define PCI_EXP_DEVCAP2_OBFF_MSG 0x00040000 /* New message signaling */
@@ -619,6 +621,7 @@ · 1 unmodified line
619621 #define PCI_EXP_DEVCTL2 40 /* Device Control 2 */
620622 #define PCI_EXP_DEVCTL2_COMP_TIMEOUT 0x000f /* Completion Timeout Value */
621623 #define PCI_EXP_DEVCTL2_ARI 0x0020 /* Alternative Routing-ID */
624+#define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 /* Set Atomic requests */
622625 #define PCI_EXP_DEVCTL2_IDO_REQ_EN 0x0100 /* Allow IDO for requests */
623626 #define PCI_EXP_DEVCTL2_IDO_CMP_EN 0x0200 /* Allow IDO for completions */
624627 #define PCI_EXP_DEVCTL2_LTR_EN 0x0400 /* Enable LTR mechanism */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t