| @@ -242,6 +242,7 @@ static void smb_direct_disconnect_rdma_work(struct work_struct *work) · 241 unmodified lines |
| 242 | * disable[_delayed]_work_sync() | 242 | * disable[_delayed]_work_sync() |
| 243 | */ | 243 | */ |
| 244 | disable_work(&sc->disconnect_work); | 244 | disable_work(&sc->disconnect_work); |
| | 245 | disable_work(&sc->connect.work); |
| 245 | disable_work(&sc->recv_io.posted.refill_work); | 246 | disable_work(&sc->recv_io.posted.refill_work); |
| 246 | disable_delayed_work(&sc->idle.timer_work); | 247 | disable_delayed_work(&sc->idle.timer_work); |
| 247 | disable_work(&sc->idle.immediate_work); | 248 | disable_work(&sc->idle.immediate_work); |
| @@ -297,6 +298,7 @@ smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc) · 49 unmodified lines |
| 297 | * not queued again but here we don't block and avoid | 298 | * not queued again but here we don't block and avoid |
| 298 | * disable[_delayed]_work_sync() | 299 | * disable[_delayed]_work_sync() |
| 299 | */ | 300 | */ |
| | 301 | disable_work(&sc->connect.work); |
| 300 | disable_work(&sc->recv_io.posted.refill_work); | 302 | disable_work(&sc->recv_io.posted.refill_work); |
| 301 | disable_work(&sc->idle.immediate_work); | 303 | disable_work(&sc->idle.immediate_work); |
| 302 | disable_delayed_work(&sc->idle.timer_work); | 304 | disable_delayed_work(&sc->idle.timer_work); |
| @@ -467,6 +469,7 @@ static void free_transport(struct smb_direct_transport *t) · 164 unmodified lines |
| 467 | */ | 469 | */ |
| 468 | smb_direct_disconnect_wake_up_all(sc); | 470 | smb_direct_disconnect_wake_up_all(sc); |
| 469 | | 471 | |
| | 472 | disable_work_sync(&sc->connect.work); |
| 470 | disable_work_sync(&sc->recv_io.posted.refill_work); | 473 | disable_work_sync(&sc->recv_io.posted.refill_work); |
| 471 | disable_delayed_work_sync(&sc->idle.timer_work); | 474 | disable_delayed_work_sync(&sc->idle.timer_work); |
| 472 | disable_work_sync(&sc->idle.immediate_work); | 475 | disable_work_sync(&sc->idle.immediate_work); |
| @@ -635,28 +638,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) · 162 unmodified lines |
| 635 | | 638 | |
| 636 | switch (sc->recv_io.expected) { | 639 | switch (sc->recv_io.expected) { |
| 637 | case SMBDIRECT_EXPECT_NEGOTIATE_REQ: | 640 | case SMBDIRECT_EXPECT_NEGOTIATE_REQ: |
| 638 | if (wc->byte_len < sizeof(struct smbdirect_negotiate_req)) { | 641 | /* see smb_direct_negotiate_recv_done */ |
| 639 | put_recvmsg(sc, recvmsg); | 642 | break; |
| 640 | smb_direct_disconnect_rdma_connection(sc); | | |
| 641 | return; | | |
| 642 | } | | |
| 643 | sc->recv_io.reassembly.full_packet_received = true; | | |
| 644 | /* | | |
| 645 | * Some drivers (at least mlx5_ib) might post a | | |
| 646 | * recv completion before RDMA_CM_EVENT_ESTABLISHED, | | |
| 647 | * we need to adjust our expectation in that case. | | |
| 648 | */ | | |
| 649 | if (!sc->first_error && sc->status == SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING) | | |
| 650 | sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; | | |
| 651 | if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) { | | |
| 652 | put_recvmsg(sc, recvmsg); | | |
| 653 | smb_direct_disconnect_rdma_connection(sc); | | |
| 654 | return; | | |
| 655 | } | | |
| 656 | sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; | | |
| 657 | enqueue_reassembly(sc, recvmsg, 0); | | |
| 658 | wake_up(&sc->status_wait); | | |
| 659 | return; | | |
| 660 | case SMBDIRECT_EXPECT_DATA_TRANSFER: { | 643 | case SMBDIRECT_EXPECT_DATA_TRANSFER: { |
| 661 | struct smbdirect_data_transfer *data_transfer = | 644 | struct smbdirect_data_transfer *data_transfer = |
| 662 | (struct smbdirect_data_transfer *)recvmsg->packet; | 645 | (struct smbdirect_data_transfer *)recvmsg->packet; |
| @@ -742,6 +725,126 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) · 79 unmodified lines |
| 742 | smb_direct_disconnect_rdma_connection(sc); | 725 | smb_direct_disconnect_rdma_connection(sc); |
| 743 | } | 726 | } |
| 744 | | 727 | |
| | 728 | static void smb_direct_negotiate_recv_work(struct work_struct *work); |
| | 729 | |
| | 730 | static void smb_direct_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) |
| | 731 | { |
| | 732 | struct smbdirect_recv_io *recv_io = |
| | 733 | container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); |
| | 734 | struct smbdirect_socket *sc = recv_io->socket; |
| | 735 | unsigned long flags; |
| | 736 | |
| | 737 | /* |
| | 738 | * reset the common recv_done for later reuse. |
| | 739 | */ |
| | 740 | recv_io->cqe.done = recv_done; |
| | 741 | |
| | 742 | if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { |
| | 743 | put_recvmsg(sc, recv_io); |
| | 744 | if (wc->status != IB_WC_WR_FLUSH_ERR) { |
| | 745 | pr_err("Negotiate Recv error. status='%s (%d)' opcode=%d\n", |
| | 746 | ib_wc_status_msg(wc->status), wc->status, |
| | 747 | wc->opcode); |
| | 748 | smb_direct_disconnect_rdma_connection(sc); |
| | 749 | } |
| | 750 | return; |
| | 751 | } |
| | 752 | |
| | 753 | ksmbd_debug(RDMA, "Negotiate Recv completed. status='%s (%d)', opcode=%d\n", |
| | 754 | ib_wc_status_msg(wc->status), wc->status, |
| | 755 | wc->opcode); |
| | 756 | |
| | 757 | ib_dma_sync_single_for_cpu(sc->ib.dev, |
| | 758 | recv_io->sge.addr, |
| | 759 | recv_io->sge.length, |
| | 760 | DMA_FROM_DEVICE); |
| | 761 | |
| | 762 | /* |
| | 763 | * This is an internal error! |
| | 764 | */ |
| | 765 | if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ)) { |
| | 766 | put_recvmsg(sc, recv_io); |
| | 767 | smb_direct_disconnect_rdma_connection(sc); |
| | 768 | return; |
| | 769 | } |
| | 770 | |
| | 771 | /* |
| | 772 | * Don't reset timer to the keepalive interval in |
| | 773 | * this will be done in smb_direct_negotiate_recv_work. |
| | 774 | */ |
| | 775 | |
| | 776 | /* |
| | 777 | * Only remember the recv_io if it has enough bytes, |
| | 778 | * this gives smb_direct_negotiate_recv_work enough |
| | 779 | * information in order to disconnect if it was not |
| | 780 | * valid. |
| | 781 | */ |
| | 782 | sc->recv_io.reassembly.full_packet_received = true; |
| | 783 | if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req)) |
| | 784 | enqueue_reassembly(sc, recv_io, 0); |
| | 785 | else |
| | 786 | put_recvmsg(sc, recv_io); |
| | 787 | |
| | 788 | /* |
| | 789 | * Some drivers (at least mlx5_ib and irdma in roce mode) |
| | 790 | * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED, |
| | 791 | * we need to adjust our expectation in that case. |
| | 792 | * |
| | 793 | * So we defer further processing of the negotiation |
| | 794 | * to smb_direct_negotiate_recv_work(). |
| | 795 | * |
| | 796 | * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED |
| | 797 | * we queue the work directly otherwise |
| | 798 | * smb_direct_cm_handler() will do it, when |
| | 799 | * RDMA_CM_EVENT_ESTABLISHED arrived. |
| | 800 | */ |
| | 801 | spin_lock_irqsave(&sc->connect.lock, flags); |
| | 802 | if (!sc->first_error) { |
| | 803 | INIT_WORK(&sc->connect.work, smb_direct_negotiate_recv_work); |
| | 804 | if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) |
| | 805 | queue_work(sc->workqueue, &sc->connect.work); |
| | 806 | } |
| | 807 | spin_unlock_irqrestore(&sc->connect.lock, flags); |
| | 808 | } |
| | 809 | |
| | 810 | static void smb_direct_negotiate_recv_work(struct work_struct *work) |
| | 811 | { |
| | 812 | struct smbdirect_socket *sc = |
| | 813 | container_of(work, struct smbdirect_socket, connect.work); |
| | 814 | const struct smbdirect_socket_parameters *sp = &sc->parameters; |
| | 815 | struct smbdirect_recv_io *recv_io; |
| | 816 | |
| | 817 | if (sc->first_error) |
| | 818 | return; |
| | 819 | |
| | 820 | ksmbd_debug(RDMA, "Negotiate Recv Work running\n"); |
| | 821 | |
| | 822 | /* |
| | 823 | * Reset timer to the keepalive interval in |
| | 824 | * order to trigger our next keepalive message. |
| | 825 | */ |
| | 826 | sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; |
| | 827 | mod_delayed_work(sc->workqueue, &sc->idle.timer_work, |
| | 828 | msecs_to_jiffies(sp->keepalive_interval_msec)); |
| | 829 | |
| | 830 | /* |
| | 831 | * If smb_direct_negotiate_recv_done() detected an |
| | 832 | * invalid request we want to disconnect. |
| | 833 | */ |
| | 834 | recv_io = get_first_reassembly(sc); |
| | 835 | if (!recv_io) { |
| | 836 | smb_direct_disconnect_rdma_connection(sc); |
| | 837 | return; |
| | 838 | } |
| | 839 | |
| | 840 | if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) { |
| | 841 | smb_direct_disconnect_rdma_connection(sc); |
| | 842 | return; |
| | 843 | } |
| | 844 | sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; |
| | 845 | wake_up(&sc->status_wait); |
| | 846 | } |
| | 847 | |
| 745 | static int smb_direct_post_recv(struct smbdirect_socket *sc, | 848 | static int smb_direct_post_recv(struct smbdirect_socket *sc, |
| 746 | struct smbdirect_recv_io *recvmsg) | 849 | struct smbdirect_recv_io *recvmsg) |
| 747 | { | 850 | { |
| @@ -758,7 +861,6 @@ static int smb_direct_post_recv(struct smbdirect_socket *sc, · 10 unmodified lines |
| 758 | return ret; | 861 | return ret; |
| 759 | recvmsg->sge.length = sp->max_recv_size; | 862 | recvmsg->sge.length = sp->max_recv_size; |
| 760 | recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey; | 863 | recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey; |
| 761 | recvmsg->cqe.done = recv_done; | | |
| 762 | | 864 | |
| 763 | wr.wr_cqe = &recvmsg->cqe; | 865 | wr.wr_cqe = &recvmsg->cqe; |
| 764 | wr.next = NULL; | 866 | wr.next = NULL; |
| @@ -1732,6 +1834,7 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, · 967 unmodified lines |
| 1732 | struct rdma_cm_event *event) | 1834 | struct rdma_cm_event *event) |
| 1733 | { | 1835 | { |
| 1734 | struct smbdirect_socket *sc = cm_id->context; | 1836 | struct smbdirect_socket *sc = cm_id->context; |
| | 1837 | unsigned long flags; |
| 1735 | | 1838 | |
| 1736 | ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n", | 1839 | ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n", |
| 1737 | cm_id, rdma_event_msg(event->event), event->event); | 1840 | cm_id, rdma_event_msg(event->event), event->event); |
| @@ -1739,18 +1842,27 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, · 1 unmodified line |
| 1739 | switch (event->event) { | 1842 | switch (event->event) { |
| 1740 | case RDMA_CM_EVENT_ESTABLISHED: { | 1843 | case RDMA_CM_EVENT_ESTABLISHED: { |
| 1741 | /* | 1844 | /* |
| 1742 | * Some drivers (at least mlx5_ib) might post a | 1845 | * Some drivers (at least mlx5_ib and irdma in roce mode) |
| 1743 | * recv completion before RDMA_CM_EVENT_ESTABLISHED, | 1846 | * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED, |
| 1744 | * we need to adjust our expectation in that case. | 1847 | * we need to adjust our expectation in that case. |
| 1745 | * | 1848 | * |
| 1746 | * As we already started the negotiation, we just | 1849 | * If smb_direct_negotiate_recv_done was called first |
| 1747 | * ignore RDMA_CM_EVENT_ESTABLISHED here. | 1850 | * it initialized sc->connect.work only for us to |
| | 1851 | * start, so that we turned into |
| | 1852 | * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before |
| | 1853 | * smb_direct_negotiate_recv_work() runs. |
| | 1854 | * |
| | 1855 | * If smb_direct_negotiate_recv_done didn't happen |
| | 1856 | * yet. sc->connect.work is still be disabled and |
| | 1857 | * queue_work() is a no-op. |
| 1748 | */ | 1858 | */ |
| 1749 | if (!sc->first_error && sc->status > SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING) | | |
| 1750 | break; | | |
| 1751 | if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) | 1859 | if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) |
| 1752 | break; | 1860 | break; |
| 1753 | sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; | 1861 | sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; |
| | 1862 | spin_lock_irqsave(&sc->connect.lock, flags); |
| | 1863 | if (!sc->first_error) |
| | 1864 | queue_work(sc->workqueue, &sc->connect.work); |
| | 1865 | spin_unlock_irqrestore(&sc->connect.lock, flags); |
| 1754 | wake_up(&sc->status_wait); | 1866 | wake_up(&sc->status_wait); |
| 1755 | break; | 1867 | break; |
| 1756 | } | 1868 | } |
| @@ -1921,6 +2033,7 @@ static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc) · 164 unmodified lines |
| 1921 | recvmsg = get_free_recvmsg(sc); | 2033 | recvmsg = get_free_recvmsg(sc); |
| 1922 | if (!recvmsg) | 2034 | if (!recvmsg) |
| 1923 | return -ENOMEM; | 2035 | return -ENOMEM; |
| | 2036 | recvmsg->cqe.done = smb_direct_negotiate_recv_done; |
| 1924 | | 2037 | |
| 1925 | ret = smb_direct_post_recv(sc, recvmsg); | 2038 | ret = smb_direct_post_recv(sc, recvmsg); |
| 1926 | if (ret) { | 2039 | if (ret) { |
| @@ -2339,6 +2452,7 @@ respond: · 412 unmodified lines |
| 2339 | | 2452 | |
| 2340 | static int smb_direct_connect(struct smbdirect_socket *sc) | 2453 | static int smb_direct_connect(struct smbdirect_socket *sc) |
| 2341 | { | 2454 | { |
| | 2455 | struct smbdirect_recv_io *recv_io; |
| 2342 | int ret; | 2456 | int ret; |
| 2343 | | 2457 | |
| 2344 | ret = smb_direct_init_params(sc); | 2458 | ret = smb_direct_init_params(sc); |
| @@ -2353,6 +2467,9 @@ static int smb_direct_connect(struct smbdirect_socket *sc) · 8 unmodified lines |
| 2353 | return ret; | 2467 | return ret; |
| 2354 | } | 2468 | } |
| 2355 | | 2469 | |
| | 2470 | list_for_each_entry(recv_io, &sc->recv_io.free.list, list) |
| | 2471 | recv_io->cqe.done = recv_done; |
| | 2472 | |
| 2356 | ret = smb_direct_create_qpair(sc); | 2473 | ret = smb_direct_create_qpair(sc); |
| 2357 | if (ret) { | 2474 | if (ret) { |
| 2358 | pr_err("Can't accept RDMA client: %d\n", ret); | 2475 | pr_err("Can't accept RDMA client: %d\n", ret); |