We have never been able to track down and address the underlying cause of the performance issues with workqueue-based service support. svo_enqueue_xprt is called multiple times per RPC, so it adds instruction path length, but always ends up at the same function: svc_xprt_do_enqueue(). We do not anticipate needing this flexibility for dynamic nfsd thread management support. As a micro-optimization, remove .svo_enqueue_xprt because Spectre/Meltdown makes virtual function calls more costly. This change essentially reverts commit b9e13cdfac70 ("nfsd/sunrpc: turn enqueueing a svc_xprt into a svc_serv operation").
Unified · Split
| @@ -353,7 +353,6 @@ static struct notifier_block lockd_inet6addr_notifier = { · 352 unmodified lines | |||
| 353 | static const struct svc_serv_ops lockd_sv_ops = { | 353 | static const struct svc_serv_ops lockd_sv_ops = { |
| 354 | .svo_shutdown = svc_rpcb_cleanup, | 354 | .svo_shutdown = svc_rpcb_cleanup, |
| 355 | .svo_function = lockd, | 355 | .svo_function = lockd, |
| 356 | .svo_enqueue_xprt = svc_xprt_do_enqueue, | ||
| 357 | .svo_module = THIS_MODULE, | 356 | .svo_module = THIS_MODULE, |
| 358 | }; | 357 | }; |
| 359 | 358 | ||
| @@ -234,13 +234,11 @@ err_bind: · 233 unmodified lines | |||
| 234 | 234 | ||
| 235 | static const struct svc_serv_ops nfs40_cb_sv_ops = { | 235 | static const struct svc_serv_ops nfs40_cb_sv_ops = { |
| 236 | .svo_function = nfs4_callback_svc, | 236 | .svo_function = nfs4_callback_svc, |
| 237 | .svo_enqueue_xprt = svc_xprt_do_enqueue, | ||
| 238 | .svo_module = THIS_MODULE, | 237 | .svo_module = THIS_MODULE, |
| 239 | }; | 238 | }; |
| 240 | #if defined(CONFIG_NFS_V4_1) | 239 | #if defined(CONFIG_NFS_V4_1) |
| 241 | static const struct svc_serv_ops nfs41_cb_sv_ops = { | 240 | static const struct svc_serv_ops nfs41_cb_sv_ops = { |
| 242 | .svo_function = nfs41_callback_svc, | 241 | .svo_function = nfs41_callback_svc, |
| 243 | .svo_enqueue_xprt = svc_xprt_do_enqueue, | ||
| 244 | .svo_module = THIS_MODULE, | 242 | .svo_module = THIS_MODULE, |
| 245 | }; | 243 | }; |
| 246 | 244 | ||
| @@ -615,7 +615,6 @@ static int nfsd_get_default_max_blksize(void) · 614 unmodified lines | |||
| 615 | static const struct svc_serv_ops nfsd_thread_sv_ops = { | 615 | static const struct svc_serv_ops nfsd_thread_sv_ops = { |
| 616 | .svo_shutdown = nfsd_last_thread, | 616 | .svo_shutdown = nfsd_last_thread, |
| 617 | .svo_function = nfsd, | 617 | .svo_function = nfsd, |
| 618 | .svo_enqueue_xprt = svc_xprt_do_enqueue, | ||
| 619 | .svo_module = THIS_MODULE, | 618 | .svo_module = THIS_MODULE, |
| 620 | }; | 619 | }; |
| 621 | 620 | ||
| @@ -61,9 +61,6 @@ struct svc_serv_ops { · 60 unmodified lines | |||
| 61 | /* function for service threads to run */ | 61 | /* function for service threads to run */ |
| 62 | int (*svo_function)(void *); | 62 | int (*svo_function)(void *); |
| 63 | 63 | ||
| 64 | /* queue up a transport for servicing */ | ||
| 65 | void (*svo_enqueue_xprt)(struct svc_xprt *); | ||
| 66 | |||
| 67 | /* optional module to count when adding threads. | 64 | /* optional module to count when adding threads. |
| 68 | * Thread function must call module_put_and_kthread_exit() to exit. | 65 | * Thread function must call module_put_and_kthread_exit() to exit. |
| 69 | */ | 66 | */ |
| @@ -131,7 +131,6 @@ int svc_create_xprt(struct svc_serv *, const char *, struct net *, · 130 unmodified lines | |||
| 131 | const int, const unsigned short, int, | 131 | const int, const unsigned short, int, |
| 132 | const struct cred *); | 132 | const struct cred *); |
| 133 | void svc_xprt_received(struct svc_xprt *xprt); | 133 | void svc_xprt_received(struct svc_xprt *xprt); |
| 134 | void svc_xprt_do_enqueue(struct svc_xprt *xprt); | ||
| 135 | void svc_xprt_enqueue(struct svc_xprt *xprt); | 134 | void svc_xprt_enqueue(struct svc_xprt *xprt); |
| 136 | void svc_xprt_put(struct svc_xprt *xprt); | 135 | void svc_xprt_put(struct svc_xprt *xprt); |
| 137 | void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt); | 136 | void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt); |
| @@ -32,6 +32,7 @@ static int svc_deferred_recv(struct svc_rqst *rqstp); · 31 unmodified lines | |||
| 32 | static struct cache_deferred_req *svc_defer(struct cache_req *req); | 32 | static struct cache_deferred_req *svc_defer(struct cache_req *req); |
| 33 | static void svc_age_temp_xprts(struct timer_list *t); | 33 | static void svc_age_temp_xprts(struct timer_list *t); |
| 34 | static void svc_delete_xprt(struct svc_xprt *xprt); | 34 | static void svc_delete_xprt(struct svc_xprt *xprt); |
| 35 | static void svc_xprt_do_enqueue(struct svc_xprt *xprt); | ||
| 35 | 36 | ||
| 36 | /* apparently the "standard" is that clients close | 37 | /* apparently the "standard" is that clients close |
| 37 | * idle connections after 5 minutes, servers after | 38 | * idle connections after 5 minutes, servers after |
| @@ -266,12 +267,12 @@ void svc_xprt_received(struct svc_xprt *xprt) · 228 unmodified lines | |||
| 266 | } | 267 | } |
| 267 | 268 | ||
| 268 | /* As soon as we clear busy, the xprt could be closed and | 269 | /* As soon as we clear busy, the xprt could be closed and |
| 269 | * 'put', so we need a reference to call svc_enqueue_xprt with: | 270 | * 'put', so we need a reference to call svc_xprt_do_enqueue with: |
| 270 | */ | 271 | */ |
| 271 | svc_xprt_get(xprt); | 272 | svc_xprt_get(xprt); |
| 272 | smp_mb__before_atomic(); | 273 | smp_mb__before_atomic(); |
| 273 | clear_bit(XPT_BUSY, &xprt->xpt_flags); | 274 | clear_bit(XPT_BUSY, &xprt->xpt_flags); |
| 274 | xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt); | 275 | svc_xprt_do_enqueue(xprt); |
| 275 | svc_xprt_put(xprt); | 276 | svc_xprt_put(xprt); |
| 276 | } | 277 | } |
| 277 | EXPORT_SYMBOL_GPL(svc_xprt_received); | 278 | EXPORT_SYMBOL_GPL(svc_xprt_received); |
| @@ -423,7 +424,7 @@ static bool svc_xprt_ready(struct svc_xprt *xprt) · 145 unmodified lines | |||
| 423 | return false; | 424 | return false; |
| 424 | } | 425 | } |
| 425 | 426 | ||
| 426 | void svc_xprt_do_enqueue(struct svc_xprt *xprt) | 427 | static void svc_xprt_do_enqueue(struct svc_xprt *xprt) |
| 427 | { | 428 | { |
| 428 | struct svc_pool *pool; | 429 | struct svc_pool *pool; |
| 429 | struct svc_rqst *rqstp = NULL; | 430 | struct svc_rqst *rqstp = NULL; |
| @@ -467,7 +468,6 @@ out_unlock: · 37 unmodified lines | |||
| 467 | put_cpu(); | 468 | put_cpu(); |
| 468 | trace_svc_xprt_enqueue(xprt, rqstp); | 469 | trace_svc_xprt_enqueue(xprt, rqstp); |
| 469 | } | 470 | } |
| 470 | EXPORT_SYMBOL_GPL(svc_xprt_do_enqueue); | ||
| 471 | 471 | ||
| 472 | /* | 472 | /* |
| 473 | * Queue up a transport with data pending. If there are idle nfsd | 473 | * Queue up a transport with data pending. If there are idle nfsd |
| @@ -478,7 +478,7 @@ void svc_xprt_enqueue(struct svc_xprt *xprt) · 4 unmodified lines | |||
| 478 | { | 478 | { |
| 479 | if (test_bit(XPT_BUSY, &xprt->xpt_flags)) | 479 | if (test_bit(XPT_BUSY, &xprt->xpt_flags)) |
| 480 | return; | 480 | return; |
| 481 | xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt); | 481 | svc_xprt_do_enqueue(xprt); |
| 482 | } | 482 | } |
| 483 | EXPORT_SYMBOL_GPL(svc_xprt_enqueue); | 483 | EXPORT_SYMBOL_GPL(svc_xprt_enqueue); |
| 484 | 484 | ||