When a FUSE daemon panics and failover, we aim to minimize the impact on applications by reusing the existing FUSE connection. During this process, another daemon is employed to preserve the FUSE connection's file descriptor. The new started FUSE Daemon will takeover the fd and continue to provide service. However, it is possible for some inflight requests to be lost and never returned. As a result, applications awaiting replies would become stuck forever. To address this, we can resend these pending requests to the new started FUSE daemon. This patch introduces a new notification type "FUSE_NOTIFY_RESEND", which can trigger resending of the pending requests, ensuring they are properly processed again.
Unified · Split
| @@ -1775,6 +1775,59 @@ copy_finish: · 1774 unmodified lines | |||
| 1775 | return err; | 1775 | return err; |
| 1776 | } | 1776 | } |
| 1777 | 1777 | ||
| 1778 | /* | ||
| 1779 | * Resending all processing queue requests. | ||
| 1780 | * | ||
| 1781 | * During a FUSE daemon panics and failover, it is possible for some inflight | ||
| 1782 | * requests to be lost and never returned. As a result, applications awaiting | ||
| 1783 | * replies would become stuck forever. To address this, we can use notification | ||
| 1784 | * to trigger resending of these pending requests to the FUSE daemon, ensuring | ||
| 1785 | * they are properly processed again. | ||
| 1786 | * | ||
| 1787 | * Please note that this strategy is applicable only to idempotent requests or | ||
| 1788 | * if the FUSE daemon takes careful measures to avoid processing duplicated | ||
| 1789 | * non-idempotent requests. | ||
| 1790 | */ | ||
| 1791 | static void fuse_resend(struct fuse_conn *fc) | ||
| 1792 | { | ||
| 1793 | struct fuse_dev *fud; | ||
| 1794 | struct fuse_req *req, *next; | ||
| 1795 | struct fuse_iqueue *fiq = &fc->iq; | ||
| 1796 | LIST_HEAD(to_queue); | ||
| 1797 | unsigned int i; | ||
| 1798 | |||
| 1799 | spin_lock(&fc->lock); | ||
| 1800 | if (!fc->connected) { | ||
| 1801 | spin_unlock(&fc->lock); | ||
| 1802 | return; | ||
| 1803 | } | ||
| 1804 | |||
| 1805 | list_for_each_entry(fud, &fc->devices, entry) { | ||
| 1806 | struct fuse_pqueue *fpq = &fud->pq; | ||
| 1807 | |||
| 1808 | spin_lock(&fpq->lock); | ||
| 1809 | for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) | ||
| 1810 | list_splice_tail_init(&fpq->processing[i], &to_queue); | ||
| 1811 | spin_unlock(&fpq->lock); | ||
| 1812 | } | ||
| 1813 | spin_unlock(&fc->lock); | ||
| 1814 | |||
| 1815 | list_for_each_entry_safe(req, next, &to_queue, list) { | ||
| 1816 | __set_bit(FR_PENDING, &req->flags); | ||
| 1817 | } | ||
| 1818 | |||
| 1819 | spin_lock(&fiq->lock); | ||
| 1820 | /* iq and pq requests are both oldest to newest */ | ||
| 1821 | list_splice(&to_queue, &fiq->pending); | ||
| 1822 | fiq->ops->wake_pending_and_unlock(fiq); | ||
| 1823 | } | ||
| 1824 | |||
| 1825 | static int fuse_notify_resend(struct fuse_conn *fc) | ||
| 1826 | { | ||
| 1827 | fuse_resend(fc); | ||
| 1828 | return 0; | ||
| 1829 | } | ||
| 1830 | |||
| 1778 | static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, | 1831 | static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, |
| 1779 | unsigned int size, struct fuse_copy_state *cs) | 1832 | unsigned int size, struct fuse_copy_state *cs) |
| 1780 | { | 1833 | { |
| @@ -1800,6 +1853,9 @@ static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, · 19 unmodified lines | |||
| 1800 | case FUSE_NOTIFY_DELETE: | 1853 | case FUSE_NOTIFY_DELETE: |
| 1801 | return fuse_notify_delete(fc, size, cs); | 1854 | return fuse_notify_delete(fc, size, cs); |
| 1802 | 1855 | ||
| 1856 | case FUSE_NOTIFY_RESEND: | ||
| 1857 | return fuse_notify_resend(fc); | ||
| 1858 | |||
| 1803 | default: | 1859 | default: |
| 1804 | fuse_copy_finish(cs); | 1860 | fuse_copy_finish(cs); |
| 1805 | return -EINVAL; | 1861 | return -EINVAL; |
| @@ -216,6 +216,7 @@ · 215 unmodified lines | |||
| 216 | * - add max_stack_depth to fuse_init_out, add FUSE_PASSTHROUGH init flag | 216 | * - add max_stack_depth to fuse_init_out, add FUSE_PASSTHROUGH init flag |
| 217 | * - add backing_id to fuse_open_out, add FOPEN_PASSTHROUGH open flag | 217 | * - add backing_id to fuse_open_out, add FOPEN_PASSTHROUGH open flag |
| 218 | * - add FUSE_NO_EXPORT_SUPPORT init flag | 218 | * - add FUSE_NO_EXPORT_SUPPORT init flag |
| 219 | * - add FUSE_NOTIFY_RESEND | ||
| 219 | */ | 220 | */ |
| 220 | 221 | ||
| 221 | #ifndef _LINUX_FUSE_H | 222 | #ifndef _LINUX_FUSE_H |
| @@ -645,6 +646,7 @@ enum fuse_notify_code { · 423 unmodified lines | |||
| 645 | FUSE_NOTIFY_STORE = 4, | 646 | FUSE_NOTIFY_STORE = 4, |
| 646 | FUSE_NOTIFY_RETRIEVE = 5, | 647 | FUSE_NOTIFY_RETRIEVE = 5, |
| 647 | FUSE_NOTIFY_DELETE = 6, | 648 | FUSE_NOTIFY_DELETE = 6, |
| 649 | FUSE_NOTIFY_RESEND = 7, | ||
| 648 | FUSE_NOTIFY_CODE_MAX, | 650 | FUSE_NOTIFY_CODE_MAX, |
| 649 | }; | 651 | }; |
| 650 | 652 | ||