| @@ -242,6 +242,7 @@ static void smb_direct_disconnect_rdma_work(struct work_struct *work) · 241 unmodified lines |
| 242 | 242 | * disable[_delayed]_work_sync() |
| 243 | 243 | */ |
| 244 | 244 | disable_work(&sc->disconnect_work); |
| 245 | + disable_work(&sc->connect.work); |
| 245 | 246 | disable_work(&sc->recv_io.posted.refill_work); |
| 246 | 247 | disable_delayed_work(&sc->idle.timer_work); |
| 247 | 248 | disable_work(&sc->idle.immediate_work); |
| @@ -297,6 +298,7 @@ smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc) · 49 unmodified lines |
| 297 | 298 | * not queued again but here we don't block and avoid |
| 298 | 299 | * disable[_delayed]_work_sync() |
| 299 | 300 | */ |
| 301 | + disable_work(&sc->connect.work); |
| 300 | 302 | disable_work(&sc->recv_io.posted.refill_work); |
| 301 | 303 | disable_work(&sc->idle.immediate_work); |
| 302 | 304 | disable_delayed_work(&sc->idle.timer_work); |
| @@ -467,6 +469,7 @@ static void free_transport(struct smb_direct_transport *t) · 164 unmodified lines |
| 467 | 469 | */ |
| 468 | 470 | smb_direct_disconnect_wake_up_all(sc); |
| 469 | 471 | |
| 472 | + disable_work_sync(&sc->connect.work); |
| 470 | 473 | disable_work_sync(&sc->recv_io.posted.refill_work); |
| 471 | 474 | disable_delayed_work_sync(&sc->idle.timer_work); |
| 472 | 475 | disable_work_sync(&sc->idle.immediate_work); |
| @@ -635,28 +638,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) · 162 unmodified lines |
| 635 | 638 | |
| 636 | 639 | switch (sc->recv_io.expected) { |
| 637 | 640 | case SMBDIRECT_EXPECT_NEGOTIATE_REQ: |
| 638 | | − if (wc->byte_len < sizeof(struct smbdirect_negotiate_req)) { |
| 639 | | − put_recvmsg(sc, recvmsg); |
| 640 | | − smb_direct_disconnect_rdma_connection(sc); |
| 641 | | − return; |
| 642 | | − } |
| 643 | | − sc->recv_io.reassembly.full_packet_received = true; |
| 644 | | − /* |
| 645 | | − * Some drivers (at least mlx5_ib) might post a |
| 646 | | − * recv completion before RDMA_CM_EVENT_ESTABLISHED, |
| 647 | | − * we need to adjust our expectation in that case. |
| 648 | | − */ |
| 649 | | − if (!sc->first_error && sc->status == SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING) |
| 650 | | − sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; |
| 651 | | − if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) { |
| 652 | | − put_recvmsg(sc, recvmsg); |
| 653 | | − smb_direct_disconnect_rdma_connection(sc); |
| 654 | | − return; |
| 655 | | − } |
| 656 | | − sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; |
| 657 | | − enqueue_reassembly(sc, recvmsg, 0); |
| 658 | | − wake_up(&sc->status_wait); |
| 659 | | − return; |
| 641 | + /* see smb_direct_negotiate_recv_done */ |
| 642 | + break; |
| 660 | 643 | case SMBDIRECT_EXPECT_DATA_TRANSFER: { |
| 661 | 644 | struct smbdirect_data_transfer *data_transfer = |
| 662 | 645 | (struct smbdirect_data_transfer *)recvmsg->packet; |
| @@ -742,6 +725,126 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) · 79 unmodified lines |
| 742 | 725 | smb_direct_disconnect_rdma_connection(sc); |
| 743 | 726 | } |
| 744 | 727 | |
| 728 | +static void smb_direct_negotiate_recv_work(struct work_struct *work); |
| 729 | + |
| 730 | +static void smb_direct_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) |
| 731 | +{ |
| 732 | + struct smbdirect_recv_io *recv_io = |
| 733 | + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); |
| 734 | + struct smbdirect_socket *sc = recv_io->socket; |
| 735 | + unsigned long flags; |
| 736 | + |
| 737 | + /* |
| 738 | + * reset the common recv_done for later reuse. |
| 739 | + */ |
| 740 | + recv_io->cqe.done = recv_done; |
| 741 | + |
| 742 | + if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { |
| 743 | + put_recvmsg(sc, recv_io); |
| 744 | + if (wc->status != IB_WC_WR_FLUSH_ERR) { |
| 745 | + pr_err("Negotiate Recv error. status='%s (%d)' opcode=%d\n", |
| 746 | + ib_wc_status_msg(wc->status), wc->status, |
| 747 | + wc->opcode); |
| 748 | + smb_direct_disconnect_rdma_connection(sc); |
| 749 | + } |
| 750 | + return; |
| 751 | + } |
| 752 | + |
| 753 | + ksmbd_debug(RDMA, "Negotiate Recv completed. status='%s (%d)', opcode=%d\n", |
| 754 | + ib_wc_status_msg(wc->status), wc->status, |
| 755 | + wc->opcode); |
| 756 | + |
| 757 | + ib_dma_sync_single_for_cpu(sc->ib.dev, |
| 758 | + recv_io->sge.addr, |
| 759 | + recv_io->sge.length, |
| 760 | + DMA_FROM_DEVICE); |
| 761 | + |
| 762 | + /* |
| 763 | + * This is an internal error! |
| 764 | + */ |
| 765 | + if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ)) { |
| 766 | + put_recvmsg(sc, recv_io); |
| 767 | + smb_direct_disconnect_rdma_connection(sc); |
| 768 | + return; |
| 769 | + } |
| 770 | + |
| 771 | + /* |
| 772 | + * Don't reset timer to the keepalive interval in |
| 773 | + * this will be done in smb_direct_negotiate_recv_work. |
| 774 | + */ |
| 775 | + |
| 776 | + /* |
| 777 | + * Only remember the recv_io if it has enough bytes, |
| 778 | + * this gives smb_direct_negotiate_recv_work enough |
| 779 | + * information in order to disconnect if it was not |
| 780 | + * valid. |
| 781 | + */ |
| 782 | + sc->recv_io.reassembly.full_packet_received = true; |
| 783 | + if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req)) |
| 784 | + enqueue_reassembly(sc, recv_io, 0); |
| 785 | + else |
| 786 | + put_recvmsg(sc, recv_io); |
| 787 | + |
| 788 | + /* |
| 789 | + * Some drivers (at least mlx5_ib and irdma in roce mode) |
| 790 | + * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED, |
| 791 | + * we need to adjust our expectation in that case. |
| 792 | + * |
| 793 | + * So we defer further processing of the negotiation |
| 794 | + * to smb_direct_negotiate_recv_work(). |
| 795 | + * |
| 796 | + * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED |
| 797 | + * we queue the work directly otherwise |
| 798 | + * smb_direct_cm_handler() will do it, when |
| 799 | + * RDMA_CM_EVENT_ESTABLISHED arrived. |
| 800 | + */ |
| 801 | + spin_lock_irqsave(&sc->connect.lock, flags); |
| 802 | + if (!sc->first_error) { |
| 803 | + INIT_WORK(&sc->connect.work, smb_direct_negotiate_recv_work); |
| 804 | + if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) |
| 805 | + queue_work(sc->workqueue, &sc->connect.work); |
| 806 | + } |
| 807 | + spin_unlock_irqrestore(&sc->connect.lock, flags); |
| 808 | +} |
| 809 | + |
| 810 | +static void smb_direct_negotiate_recv_work(struct work_struct *work) |
| 811 | +{ |
| 812 | + struct smbdirect_socket *sc = |
| 813 | + container_of(work, struct smbdirect_socket, connect.work); |
| 814 | + const struct smbdirect_socket_parameters *sp = &sc->parameters; |
| 815 | + struct smbdirect_recv_io *recv_io; |
| 816 | + |
| 817 | + if (sc->first_error) |
| 818 | + return; |
| 819 | + |
| 820 | + ksmbd_debug(RDMA, "Negotiate Recv Work running\n"); |
| 821 | + |
| 822 | + /* |
| 823 | + * Reset timer to the keepalive interval in |
| 824 | + * order to trigger our next keepalive message. |
| 825 | + */ |
| 826 | + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; |
| 827 | + mod_delayed_work(sc->workqueue, &sc->idle.timer_work, |
| 828 | + msecs_to_jiffies(sp->keepalive_interval_msec)); |
| 829 | + |
| 830 | + /* |
| 831 | + * If smb_direct_negotiate_recv_done() detected an |
| 832 | + * invalid request we want to disconnect. |
| 833 | + */ |
| 834 | + recv_io = get_first_reassembly(sc); |
| 835 | + if (!recv_io) { |
| 836 | + smb_direct_disconnect_rdma_connection(sc); |
| 837 | + return; |
| 838 | + } |
| 839 | + |
| 840 | + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) { |
| 841 | + smb_direct_disconnect_rdma_connection(sc); |
| 842 | + return; |
| 843 | + } |
| 844 | + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; |
| 845 | + wake_up(&sc->status_wait); |
| 846 | +} |
| 847 | + |
| 745 | 848 | static int smb_direct_post_recv(struct smbdirect_socket *sc, |
| 746 | 849 | struct smbdirect_recv_io *recvmsg) |
| 747 | 850 | { |
| @@ -758,7 +861,6 @@ static int smb_direct_post_recv(struct smbdirect_socket *sc, · 10 unmodified lines |
| 758 | 861 | return ret; |
| 759 | 862 | recvmsg->sge.length = sp->max_recv_size; |
| 760 | 863 | recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey; |
| 761 | | − recvmsg->cqe.done = recv_done; |
| 762 | 864 | |
| 763 | 865 | wr.wr_cqe = &recvmsg->cqe; |
| 764 | 866 | wr.next = NULL; |
| @@ -1732,6 +1834,7 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, · 967 unmodified lines |
| 1732 | 1834 | struct rdma_cm_event *event) |
| 1733 | 1835 | { |
| 1734 | 1836 | struct smbdirect_socket *sc = cm_id->context; |
| 1837 | + unsigned long flags; |
| 1735 | 1838 | |
| 1736 | 1839 | ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n", |
| 1737 | 1840 | cm_id, rdma_event_msg(event->event), event->event); |
| @@ -1739,18 +1842,27 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, · 1 unmodified line |
| 1739 | 1842 | switch (event->event) { |
| 1740 | 1843 | case RDMA_CM_EVENT_ESTABLISHED: { |
| 1741 | 1844 | /* |
| 1742 | | − * Some drivers (at least mlx5_ib) might post a |
| 1743 | | − * recv completion before RDMA_CM_EVENT_ESTABLISHED, |
| 1845 | + * Some drivers (at least mlx5_ib and irdma in roce mode) |
| 1846 | + * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED, |
| 1744 | 1847 | * we need to adjust our expectation in that case. |
| 1745 | 1848 | * |
| 1746 | | − * As we already started the negotiation, we just |
| 1747 | | − * ignore RDMA_CM_EVENT_ESTABLISHED here. |
| 1849 | + * If smb_direct_negotiate_recv_done was called first |
| 1850 | + * it initialized sc->connect.work only for us to |
| 1851 | + * start, so that we turned into |
| 1852 | + * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before |
| 1853 | + * smb_direct_negotiate_recv_work() runs. |
| 1854 | + * |
| 1855 | + * If smb_direct_negotiate_recv_done didn't happen |
| 1856 | + * yet. sc->connect.work is still be disabled and |
| 1857 | + * queue_work() is a no-op. |
| 1748 | 1858 | */ |
| 1749 | | − if (!sc->first_error && sc->status > SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING) |
| 1750 | | − break; |
| 1751 | 1859 | if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) |
| 1752 | 1860 | break; |
| 1753 | 1861 | sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; |
| 1862 | + spin_lock_irqsave(&sc->connect.lock, flags); |
| 1863 | + if (!sc->first_error) |
| 1864 | + queue_work(sc->workqueue, &sc->connect.work); |
| 1865 | + spin_unlock_irqrestore(&sc->connect.lock, flags); |
| 1754 | 1866 | wake_up(&sc->status_wait); |
| 1755 | 1867 | break; |
| 1756 | 1868 | } |
| @@ -1921,6 +2033,7 @@ static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc) · 164 unmodified lines |
| 1921 | 2033 | recvmsg = get_free_recvmsg(sc); |
| 1922 | 2034 | if (!recvmsg) |
| 1923 | 2035 | return -ENOMEM; |
| 2036 | + recvmsg->cqe.done = smb_direct_negotiate_recv_done; |
| 1924 | 2037 | |
| 1925 | 2038 | ret = smb_direct_post_recv(sc, recvmsg); |
| 1926 | 2039 | if (ret) { |
| @@ -2339,6 +2452,7 @@ respond: · 412 unmodified lines |
| 2339 | 2452 | |
| 2340 | 2453 | static int smb_direct_connect(struct smbdirect_socket *sc) |
| 2341 | 2454 | { |
| 2455 | + struct smbdirect_recv_io *recv_io; |
| 2342 | 2456 | int ret; |
| 2343 | 2457 | |
| 2344 | 2458 | ret = smb_direct_init_params(sc); |
| @@ -2353,6 +2467,9 @@ static int smb_direct_connect(struct smbdirect_socket *sc) · 8 unmodified lines |
| 2353 | 2467 | return ret; |
| 2354 | 2468 | } |
| 2355 | 2469 | |
| 2470 | + list_for_each_entry(recv_io, &sc->recv_io.free.list, list) |
| 2471 | + recv_io->cqe.done = recv_done; |
| 2472 | + |
| 2356 | 2473 | ret = smb_direct_create_qpair(sc); |
| 2357 | 2474 | if (ret) { |
| 2358 | 2475 | pr_err("Can't accept RDMA client: %d\n", ret); |