History

cifs: change iface_list from array to sorted linked list

A server's published interface list can change over time, and needs to be updated. We've storing iface_list as a simple array, which makes it difficult to manipulate an existing list. With this change, iface_list is modified into a linked list of interfaces, which is kept sorted by speed. Also added a reference counter for an iface entry, so that each channel can maintain a backpointer to the iface and drop it easily when needed.

Shyam Prasad N committed 4 years ago

Commit aa45dad1 parent 9de74996 files · +201 −129Browse files

Unified · Split

6 files
  1. fs/cifs/cifs_debug.c +7 −5
  2. fs/cifs/cifsglob.h +53 −1
  3. fs/cifs/connect.c +4 −2
  4. fs/cifs/misc.c +8 −1
  5. fs/cifs/sess.c +40 −38
  6. fs/cifs/smb2ops.c +89 −82
fs/cifs/cifs_debug.c+7−5
@@ -162,6 +162,8 @@ cifs_dump_iface(struct seq_file *m, struct cifs_server_iface *iface) · 161 unmodified lines
162162 seq_printf(m, "\t\tIPv4: %pI4\n", &ipv4->sin_addr);
163163 else if (iface->sockaddr.ss_family == AF_INET6)
164164 seq_printf(m, "\t\tIPv6: %pI6\n", &ipv6->sin6_addr);
165+ if (!iface->is_active)
166+ seq_puts(m, "\t\t[for-cleanup]\n");
165167 }
166168
167169 static int cifs_debug_files_proc_show(struct seq_file *m, void *v)
@@ -221,6 +223,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) · 53 unmodified lines
221223 struct TCP_Server_Info *server;
222224 struct cifs_ses *ses;
223225 struct cifs_tcon *tcon;
226+ struct cifs_server_iface *iface;
224227 int c, i, j;
225228
226229 seq_puts(m,
@@ -456,11 +459,10 @@ skip_rdma: · 229 unmodified lines
456459 if (ses->iface_count)
457460 seq_printf(m, "\n\n\tServer interfaces: %zu",
458461 ses->iface_count);
459− for (j = 0; j < ses->iface_count; j++) {
460− struct cifs_server_iface *iface;
461−
462− iface = &ses->iface_list[j];
463− seq_printf(m, "\n\t%d)", j+1);
462+ j = 0;
463+ list_for_each_entry(iface, &ses->iface_list,
464+ iface_head) {
465+ seq_printf(m, "\n\t%d)", ++j);
464466 cifs_dump_iface(m, iface);
465467 if (is_ses_using_iface(ses, iface))
466468 seq_puts(m, "\t\t[CONNECTED]\n");
fs/cifs/cifsglob.h+53−1
@@ -933,15 +933,67 @@ static inline void cifs_set_net_ns(struct TCP_Server_Info *srv, struct net *net) · 932 unmodified lines
933933 #endif
934934
935935 struct cifs_server_iface {
936+ struct list_head iface_head;
937+ struct kref refcount;
936938 size_t speed;
937939 unsigned int rdma_capable : 1;
938940 unsigned int rss_capable : 1;
941+ unsigned int is_active : 1; /* unset if non existent */
939942 struct sockaddr_storage sockaddr;
940943 };
941944
945+/* release iface when last ref is dropped */
946+static inline void
947+release_iface(struct kref *ref)
948+{
949+ struct cifs_server_iface *iface = container_of(ref,
950+ struct cifs_server_iface,
951+ refcount);
952+ list_del_init(&iface->iface_head);
953+ kfree(iface);
954+}
955+
956+/*
957+ * compare two interfaces a and b
958+ * return 0 if everything matches.
959+ * return 1 if a has higher link speed, or rdma capable, or rss capable
960+ * return -1 otherwise.
961+ */
962+static inline int
963+iface_cmp(struct cifs_server_iface *a, struct cifs_server_iface *b)
964+{
965+ int cmp_ret = 0;
966+
967+ WARN_ON(!a || !b);
968+ if (a->speed == b->speed) {
969+ if (a->rdma_capable == b->rdma_capable) {
970+ if (a->rss_capable == b->rss_capable) {
971+ cmp_ret = memcmp(&a->sockaddr, &b->sockaddr,
972+ sizeof(a->sockaddr));
973+ if (!cmp_ret)
974+ return 0;
975+ else if (cmp_ret > 0)
976+ return 1;
977+ else
978+ return -1;
979+ } else if (a->rss_capable > b->rss_capable)
980+ return 1;
981+ else
982+ return -1;
983+ } else if (a->rdma_capable > b->rdma_capable)
984+ return 1;
985+ else
986+ return -1;
987+ } else if (a->speed > b->speed)
988+ return 1;
989+ else
990+ return -1;
991+}
992+
942993 struct cifs_chan {
943994 unsigned int in_reconnect : 1; /* if session setup in progress for this channel */
944995 struct TCP_Server_Info *server;
996+ struct cifs_server_iface *iface; /* interface in use */
945997 __u8 signkey[SMB3_SIGN_KEY_SIZE];
946998 };
947999
@@ -993,7 +1045,7 @@ struct cifs_ses { · 45 unmodified lines
9931045 */
9941046 spinlock_t iface_lock;
9951047 /* ========= begin: protected by iface_lock ======== */
996− struct cifs_server_iface *iface_list;
1048+ struct list_head iface_list;
9971049 size_t iface_count;
9981050 unsigned long iface_last_update; /* jiffies */
9991051 /* ========= end: protected by iface_lock ======== */
fs/cifs/connect.c+4−2
@@ -1894,9 +1894,11 @@ void cifs_put_smb_ses(struct cifs_ses *ses) · 1893 unmodified lines
18941894 int i;
18951895
18961896 for (i = 1; i < chan_count; i++) {
1897− spin_unlock(&ses->chan_lock);
1897+ if (ses->chans[i].iface) {
1898+ kref_put(&ses->chans[i].iface->refcount, release_iface);
1899+ ses->chans[i].iface = NULL;
1900+ }
18981901 cifs_put_tcp_session(ses->chans[i].server, 0);
1899− spin_lock(&ses->chan_lock);
19001902 ses->chans[i].server = NULL;
19011903 }
19021904 }
fs/cifs/misc.c+8−1
@@ -75,6 +75,7 @@ sesInfoAlloc(void) · 74 unmodified lines
7575 INIT_LIST_HEAD(&ret_buf->tcon_list);
7676 mutex_init(&ret_buf->session_mutex);
7777 spin_lock_init(&ret_buf->iface_lock);
78+ INIT_LIST_HEAD(&ret_buf->iface_list);
7879 spin_lock_init(&ret_buf->chan_lock);
7980 }
8081 return ret_buf;
@@ -83,6 +84,8 @@ sesInfoAlloc(void) · 2 unmodified lines
8384 void
8485 sesInfoFree(struct cifs_ses *buf_to_free)
8586 {
87+ struct cifs_server_iface *iface = NULL, *niface = NULL;
88+
8689 if (buf_to_free == NULL) {
8790 cifs_dbg(FYI, "Null buffer passed to sesInfoFree\n");
8891 return;
@@ -96,7 +99,11 @@ sesInfoFree(struct cifs_ses *buf_to_free) · 7 unmodified lines
9699 kfree(buf_to_free->user_name);
97100 kfree(buf_to_free->domainName);
98101 kfree_sensitive(buf_to_free->auth_key.response);
99− kfree(buf_to_free->iface_list);
102+ spin_lock(&buf_to_free->iface_lock);
103+ list_for_each_entry_safe(iface, niface, &buf_to_free->iface_list,
104+ iface_head)
105+ kref_put(&iface->refcount, release_iface);
106+ spin_unlock(&buf_to_free->iface_lock);
100107 kfree_sensitive(buf_to_free);
101108 }
102109
fs/cifs/sess.c+40−38
@@ -58,7 +58,7 @@ bool is_ses_using_iface(struct cifs_ses *ses, struct cifs_server_iface *iface) · 57 unmodified lines
5858
5959 spin_lock(&ses->chan_lock);
6060 for (i = 0; i < ses->chan_count; i++) {
61− if (is_server_using_iface(ses->chans[i].server, iface)) {
61+ if (ses->chans[i].iface == iface) {
6262 spin_unlock(&ses->chan_lock);
6363 return true;
6464 }
@@ -151,11 +151,9 @@ int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses) · 86 unmodified lines
151151 {
152152 int old_chan_count, new_chan_count;
153153 int left;
154− int i = 0;
155154 int rc = 0;
156155 int tries = 0;
157− struct cifs_server_iface *ifaces = NULL;
158− size_t iface_count;
156+ struct cifs_server_iface *iface = NULL, *niface = NULL;
159157
160158 spin_lock(&ses->chan_lock);
161159
@@ -184,33 +182,17 @@ int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses) · 22 unmodified lines
184182 }
185183 spin_unlock(&ses->chan_lock);
186184
187− /*
188− * Make a copy of the iface list at the time and use that
189− * instead so as to not hold the iface spinlock for opening
190− * channels
191− */
192− spin_lock(&ses->iface_lock);
193− iface_count = ses->iface_count;
194− if (iface_count <= 0) {
195− spin_unlock(&ses->iface_lock);
196− cifs_dbg(VFS, "no iface list available to open channels\n");
197− return 0;
198− }
199− ifaces = kmemdup(ses->iface_list, iface_count*sizeof(*ifaces),
200− GFP_ATOMIC);
201− if (!ifaces) {
202− spin_unlock(&ses->iface_lock);
203− return 0;
204− }
205− spin_unlock(&ses->iface_lock);
206−
207185 /*
208186 * Keep connecting to same, fastest, iface for all channels as
209187 * long as its RSS. Try next fastest one if not RSS or channel
210188 * creation fails.
211189 */
190+ spin_lock(&ses->iface_lock);
191+ iface = list_first_entry(&ses->iface_list, struct cifs_server_iface,
192+ iface_head);
193+ spin_unlock(&ses->iface_lock);
194+
212195 while (left > 0) {
213− struct cifs_server_iface *iface;
214196
215197 tries++;
216198 if (tries > 3*ses->chan_max) {
@@ -219,27 +201,46 @@ int cifs_try_adding_channels(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses) · 2 unmodified lines
219201 break;
220202 }
221203
222− iface = &ifaces[i];
223− if (is_ses_using_iface(ses, iface) && !iface->rss_capable) {
224− i = (i+1) % iface_count;
225− continue;
204+ spin_lock(&ses->iface_lock);
205+ if (!ses->iface_count) {
206+ spin_unlock(&ses->iface_lock);
207+ break;
226208 }
227209
228− rc = cifs_ses_add_channel(cifs_sb, ses, iface);
229− if (rc) {
230− cifs_dbg(FYI, "failed to open extra channel on iface#%d rc=%d\n",
231− i, rc);
232− i = (i+1) % iface_count;
233− continue;
210+ list_for_each_entry_safe_from(iface, niface, &ses->iface_list,
211+ iface_head) {
212+ /* skip ifaces that are unusable */
213+ if (!iface->is_active ||
214+ (is_ses_using_iface(ses, iface) &&
215+ !iface->rss_capable)) {
216+ continue;
217+ }
218+
219+ /* take ref before unlock */
220+ kref_get(&iface->refcount);
221+
222+ spin_unlock(&ses->iface_lock);
223+ rc = cifs_ses_add_channel(cifs_sb, ses, iface);
224+ spin_lock(&ses->iface_lock);
225+
226+ if (rc) {
227+ cifs_dbg(VFS, "failed to open extra channel on iface:%pIS rc=%d\n",
228+ &iface->sockaddr,
229+ rc);
230+ kref_put(&iface->refcount, release_iface);
231+ continue;
232+ }
233+
234+ cifs_dbg(FYI, "successfully opened new channel on iface:%pIS\n",
235+ &iface->sockaddr);
236+ break;
234237 }
238+ spin_unlock(&ses->iface_lock);
235239
236− cifs_dbg(FYI, "successfully opened new channel on iface#%d\n",
237− i);
238240 left--;
239241 new_chan_count++;
240242 }
241243
242− kfree(ifaces);
243244 return new_chan_count - old_chan_count;
244245 }
245246
@@ -355,6 +356,7 @@ cifs_ses_add_channel(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses, · 109 unmodified lines
355356 spin_unlock(&ses->chan_lock);
356357 goto out;
357358 }
359+ chan->iface = iface;
358360 ses->chan_count++;
359361 atomic_set(&ses->chan_seq, 0);
360362
fs/cifs/smb2ops.c+89−82
@@ -512,73 +512,41 @@ smb3_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) · 511 unmodified lines
512512 static int
513513 parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf,
514514 size_t buf_len,
515− struct cifs_server_iface **iface_list,
516− size_t *iface_count)
515+ struct cifs_ses *ses)
517516 {
518517 struct network_interface_info_ioctl_rsp *p;
519518 struct sockaddr_in *addr4;
520519 struct sockaddr_in6 *addr6;
521520 struct iface_info_ipv4 *p4;
522521 struct iface_info_ipv6 *p6;
523− struct cifs_server_iface *info;
522+ struct cifs_server_iface *info = NULL, *iface = NULL, *niface = NULL;
523+ struct cifs_server_iface tmp_iface;
524524 ssize_t bytes_left;
525525 size_t next = 0;
526526 int nb_iface = 0;
527− int rc = 0;
528−
529− *iface_list = NULL;
530− *iface_count = 0;
531−
532− /*
533− * Fist pass: count and sanity check
534− */
527+ int rc = 0, ret = 0;
535528
536529 bytes_left = buf_len;
537530 p = buf;
538− while (bytes_left >= sizeof(*p)) {
539− nb_iface++;
540− next = le32_to_cpu(p->Next);
541− if (!next) {
542− bytes_left -= sizeof(*p);
543− break;
544− }
545− p = (struct network_interface_info_ioctl_rsp *)((u8 *)p+next);
546− bytes_left -= next;
547− }
548−
549− if (!nb_iface) {
550− cifs_dbg(VFS, "%s: malformed interface info\n", __func__);
551− rc = -EINVAL;
552− goto out;
553− }
554−
555− /* Azure rounds the buffer size up 8, to a 16 byte boundary */
556− if ((bytes_left > 8) || p->Next)
557− cifs_dbg(VFS, "%s: incomplete interface info\n", __func__);
558−
559531
532+ spin_lock(&ses->iface_lock);
560533 /*
561− * Second pass: extract info to internal structure
534+ * Go through iface_list and do kref_put to remove
535+ * any unused ifaces. ifaces in use will be removed
536+ * when the last user calls a kref_put on it
562537 */
563−
564− *iface_list = kcalloc(nb_iface, sizeof(**iface_list), GFP_KERNEL);
565− if (!*iface_list) {
566− rc = -ENOMEM;
567− goto out;
538+ list_for_each_entry_safe(iface, niface, &ses->iface_list,
539+ iface_head) {
540+ iface->is_active = 0;
541+ kref_put(&iface->refcount, release_iface);
568542 }
543+ spin_unlock(&ses->iface_lock);
569544
570− info = *iface_list;
571− bytes_left = buf_len;
572− p = buf;
573545 while (bytes_left >= sizeof(*p)) {
574− info->speed = le64_to_cpu(p->LinkSpeed);
575− info->rdma_capable = le32_to_cpu(p->Capability & RDMA_CAPABLE) ? 1 : 0;
576− info->rss_capable = le32_to_cpu(p->Capability & RSS_CAPABLE) ? 1 : 0;
577−
578− cifs_dbg(FYI, "%s: adding iface %zu\n", __func__, *iface_count);
579− cifs_dbg(FYI, "%s: speed %zu bps\n", __func__, info->speed);
580− cifs_dbg(FYI, "%s: capabilities 0x%08x\n", __func__,
581− le32_to_cpu(p->Capability));
546+ memset(&tmp_iface, 0, sizeof(tmp_iface));
547+ tmp_iface.speed = le64_to_cpu(p->LinkSpeed);
548+ tmp_iface.rdma_capable = le32_to_cpu(p->Capability & RDMA_CAPABLE) ? 1 : 0;
549+ tmp_iface.rss_capable = le32_to_cpu(p->Capability & RSS_CAPABLE) ? 1 : 0;
582550
583551 switch (p->Family) {
584552 /*
@@ -587,7 +555,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, · 2 unmodified lines
587555 * conversion explicit in case either one changes.
588556 */
589557 case INTERNETWORK:
590− addr4 = (struct sockaddr_in *)&info->sockaddr;
558+ addr4 = (struct sockaddr_in *)&tmp_iface.sockaddr;
591559 p4 = (struct iface_info_ipv4 *)p->Buffer;
592560 addr4->sin_family = AF_INET;
593561 memcpy(&addr4->sin_addr, &p4->IPv4Address, 4);
@@ -599,7 +567,7 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, · 5 unmodified lines
599567 &addr4->sin_addr);
600568 break;
601569 case INTERNETWORKV6:
602− addr6 = (struct sockaddr_in6 *)&info->sockaddr;
570+ addr6 = (struct sockaddr_in6 *)&tmp_iface.sockaddr;
603571 p6 = (struct iface_info_ipv6 *)p->Buffer;
604572 addr6->sin6_family = AF_INET6;
605573 memcpy(&addr6->sin6_addr, &p6->IPv6Address, 16);
@@ -619,36 +587,88 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, · 13 unmodified lines
619587 goto next_iface;
620588 }
621589
622− (*iface_count)++;
623− info++;
590+ /*
591+ * The iface_list is assumed to be sorted by speed.
592+ * Check if the new interface exists in that list.
593+ * NEVER change iface. it could be in use.
594+ * Add a new one instead
595+ */
596+ spin_lock(&ses->iface_lock);
597+ iface = niface = NULL;
598+ list_for_each_entry_safe(iface, niface, &ses->iface_list,
599+ iface_head) {
600+ ret = iface_cmp(iface, &tmp_iface);
601+ if (!ret) {
602+ /* just get a ref so that it doesn't get picked/freed */
603+ iface->is_active = 1;
604+ kref_get(&iface->refcount);
605+ spin_unlock(&ses->iface_lock);
606+ goto next_iface;
607+ } else if (ret < 0) {
608+ /* all remaining ifaces are slower */
609+ kref_get(&iface->refcount);
610+ break;
611+ }
612+ }
613+ spin_unlock(&ses->iface_lock);
614+
615+ /* no match. insert the entry in the list */
616+ info = kmalloc(sizeof(struct cifs_server_iface),
617+ GFP_KERNEL);
618+ if (!info) {
619+ rc = -ENOMEM;
620+ goto out;
621+ }
622+ memcpy(info, &tmp_iface, sizeof(tmp_iface));
623+
624+ /* add this new entry to the list */
625+ kref_init(&info->refcount);
626+ info->is_active = 1;
627+
628+ cifs_dbg(FYI, "%s: adding iface %zu\n", __func__, ses->iface_count);
629+ cifs_dbg(FYI, "%s: speed %zu bps\n", __func__, info->speed);
630+ cifs_dbg(FYI, "%s: capabilities 0x%08x\n", __func__,
631+ le32_to_cpu(p->Capability));
632+
633+ spin_lock(&ses->iface_lock);
634+ if (!list_entry_is_head(iface, &ses->iface_list, iface_head)) {
635+ list_add_tail(&info->iface_head, &iface->iface_head);
636+ kref_put(&iface->refcount, release_iface);
637+ } else
638+ list_add_tail(&info->iface_head, &ses->iface_list);
639+ spin_unlock(&ses->iface_lock);
640+
641+ ses->iface_count++;
642+ ses->iface_last_update = jiffies;
624643 next_iface:
644+ nb_iface++;
625645 next = le32_to_cpu(p->Next);
626− if (!next)
646+ if (!next) {
647+ bytes_left -= sizeof(*p);
627648 break;
649+ }
628650 p = (struct network_interface_info_ioctl_rsp *)((u8 *)p+next);
629651 bytes_left -= next;
630652 }
631653
632− if (!*iface_count) {
654+ if (!nb_iface) {
655+ cifs_dbg(VFS, "%s: malformed interface info\n", __func__);
633656 rc = -EINVAL;
634657 goto out;
635658 }
636659
637−out:
638− if (rc) {
639− kfree(*iface_list);
640− *iface_count = 0;
641− *iface_list = NULL;
642− }
643− return rc;
644−}
660+ /* Azure rounds the buffer size up 8, to a 16 byte boundary */
661+ if ((bytes_left > 8) || p->Next)
662+ cifs_dbg(VFS, "%s: incomplete interface info\n", __func__);
645663
646−static int compare_iface(const void *ia, const void *ib)
647−{
648− const struct cifs_server_iface *a = (struct cifs_server_iface *)ia;
649− const struct cifs_server_iface *b = (struct cifs_server_iface *)ib;
650664
651− return a->speed == b->speed ? 0 : (a->speed > b->speed ? -1 : 1);
665+ if (!ses->iface_count) {
666+ rc = -EINVAL;
667+ goto out;
668+ }
669+
670+out:
671+ return rc;
652672 }
653673
654674 static int
@@ -657,8 +677,6 @@ SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon) · 2 unmodified lines
657677 int rc;
658678 unsigned int ret_data_len = 0;
659679 struct network_interface_info_ioctl_rsp *out_buf = NULL;
660− struct cifs_server_iface *iface_list;
661− size_t iface_count;
662680 struct cifs_ses *ses = tcon->ses;
663681
664682 rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID,
@@ -674,21 +692,10 @@ SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon) · 9 unmodified lines
674692 goto out;
675693 }
676694
677− rc = parse_server_interfaces(out_buf, ret_data_len,
678− &iface_list, &iface_count);
695+ rc = parse_server_interfaces(out_buf, ret_data_len, ses);
679696 if (rc)
680697 goto out;
681698
682− /* sort interfaces from fastest to slowest */
683− sort(iface_list, iface_count, sizeof(*iface_list), compare_iface, NULL);
684−
685− spin_lock(&ses->iface_lock);
686− kfree(ses->iface_list);
687− ses->iface_list = iface_list;
688− ses->iface_count = iface_count;
689− ses->iface_last_update = jiffies;
690− spin_unlock(&ses->iface_lock);
691−
692699 out:
693700 kfree(out_buf);
694701 return rc;