We have never been able to track down and address the underlying cause of the performance issues with workqueue-based service support. svo_enqueue_xprt is called multiple times per RPC, so it adds instruction path length, but always ends up at the same function: svc_xprt_do_enqueue(). We do not anticipate needing this flexibility for dynamic nfsd thread management support. As a micro-optimization, remove .svo_enqueue_xprt because Spectre/Meltdown makes virtual function calls more costly. This change essentially reverts commit b9e13cdfac70 ("nfsd/sunrpc: turn enqueueing a svc_xprt into a svc_serv operation").
Unified · Split
| @@ -353,7 +353,6 @@ static struct notifier_block lockd_inet6addr_notifier = { · 352 unmodified lines | ||
| 353 | 353 | static const struct svc_serv_ops lockd_sv_ops = { |
| 354 | 354 | .svo_shutdown = svc_rpcb_cleanup, |
| 355 | 355 | .svo_function = lockd, |
| 356 | − .svo_enqueue_xprt = svc_xprt_do_enqueue, | |
| 357 | 356 | .svo_module = THIS_MODULE, |
| 358 | 357 | }; |
| 359 | 358 | |
| @@ -234,13 +234,11 @@ err_bind: · 233 unmodified lines | ||
| 234 | 234 | |
| 235 | 235 | static const struct svc_serv_ops nfs40_cb_sv_ops = { |
| 236 | 236 | .svo_function = nfs4_callback_svc, |
| 237 | − .svo_enqueue_xprt = svc_xprt_do_enqueue, | |
| 238 | 237 | .svo_module = THIS_MODULE, |
| 239 | 238 | }; |
| 240 | 239 | #if defined(CONFIG_NFS_V4_1) |
| 241 | 240 | static const struct svc_serv_ops nfs41_cb_sv_ops = { |
| 242 | 241 | .svo_function = nfs41_callback_svc, |
| 243 | − .svo_enqueue_xprt = svc_xprt_do_enqueue, | |
| 244 | 242 | .svo_module = THIS_MODULE, |
| 245 | 243 | }; |
| 246 | 244 | |
| @@ -615,7 +615,6 @@ static int nfsd_get_default_max_blksize(void) · 614 unmodified lines | ||
| 615 | 615 | static const struct svc_serv_ops nfsd_thread_sv_ops = { |
| 616 | 616 | .svo_shutdown = nfsd_last_thread, |
| 617 | 617 | .svo_function = nfsd, |
| 618 | − .svo_enqueue_xprt = svc_xprt_do_enqueue, | |
| 619 | 618 | .svo_module = THIS_MODULE, |
| 620 | 619 | }; |
| 621 | 620 | |
| @@ -61,9 +61,6 @@ struct svc_serv_ops { · 60 unmodified lines | ||
| 61 | 61 | /* function for service threads to run */ |
| 62 | 62 | int (*svo_function)(void *); |
| 63 | 63 | |
| 64 | − /* queue up a transport for servicing */ | |
| 65 | − void (*svo_enqueue_xprt)(struct svc_xprt *); | |
| 66 | − | |
| 67 | 64 | /* optional module to count when adding threads. |
| 68 | 65 | * Thread function must call module_put_and_kthread_exit() to exit. |
| 69 | 66 | */ |
| @@ -131,7 +131,6 @@ int svc_create_xprt(struct svc_serv *, const char *, struct net *, · 130 unmodified lines | ||
| 131 | 131 | const int, const unsigned short, int, |
| 132 | 132 | const struct cred *); |
| 133 | 133 | void svc_xprt_received(struct svc_xprt *xprt); |
| 134 | −void svc_xprt_do_enqueue(struct svc_xprt *xprt); | |
| 135 | 134 | void svc_xprt_enqueue(struct svc_xprt *xprt); |
| 136 | 135 | void svc_xprt_put(struct svc_xprt *xprt); |
| 137 | 136 | void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt); |
| @@ -32,6 +32,7 @@ static int svc_deferred_recv(struct svc_rqst *rqstp); · 31 unmodified lines | ||
| 32 | 32 | static struct cache_deferred_req *svc_defer(struct cache_req *req); |
| 33 | 33 | static void svc_age_temp_xprts(struct timer_list *t); |
| 34 | 34 | static void svc_delete_xprt(struct svc_xprt *xprt); |
| 35 | +static void svc_xprt_do_enqueue(struct svc_xprt *xprt); | |
| 35 | 36 | |
| 36 | 37 | /* apparently the "standard" is that clients close |
| 37 | 38 | * idle connections after 5 minutes, servers after |
| @@ -266,12 +267,12 @@ void svc_xprt_received(struct svc_xprt *xprt) · 228 unmodified lines | ||
| 266 | 267 | } |
| 267 | 268 | |
| 268 | 269 | /* As soon as we clear busy, the xprt could be closed and |
| 269 | − * 'put', so we need a reference to call svc_enqueue_xprt with: | |
| 270 | + * 'put', so we need a reference to call svc_xprt_do_enqueue with: | |
| 270 | 271 | */ |
| 271 | 272 | svc_xprt_get(xprt); |
| 272 | 273 | smp_mb__before_atomic(); |
| 273 | 274 | clear_bit(XPT_BUSY, &xprt->xpt_flags); |
| 274 | − xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt); | |
| 275 | + svc_xprt_do_enqueue(xprt); | |
| 275 | 276 | svc_xprt_put(xprt); |
| 276 | 277 | } |
| 277 | 278 | EXPORT_SYMBOL_GPL(svc_xprt_received); |
| @@ -423,7 +424,7 @@ static bool svc_xprt_ready(struct svc_xprt *xprt) · 145 unmodified lines | ||
| 423 | 424 | return false; |
| 424 | 425 | } |
| 425 | 426 | |
| 426 | −void svc_xprt_do_enqueue(struct svc_xprt *xprt) | |
| 427 | +static void svc_xprt_do_enqueue(struct svc_xprt *xprt) | |
| 427 | 428 | { |
| 428 | 429 | struct svc_pool *pool; |
| 429 | 430 | struct svc_rqst *rqstp = NULL; |
| @@ -467,7 +468,6 @@ out_unlock: · 37 unmodified lines | ||
| 467 | 468 | put_cpu(); |
| 468 | 469 | trace_svc_xprt_enqueue(xprt, rqstp); |
| 469 | 470 | } |
| 470 | −EXPORT_SYMBOL_GPL(svc_xprt_do_enqueue); | |
| 471 | 471 | |
| 472 | 472 | /* |
| 473 | 473 | * Queue up a transport with data pending. If there are idle nfsd |
| @@ -478,7 +478,7 @@ void svc_xprt_enqueue(struct svc_xprt *xprt) · 4 unmodified lines | ||
| 478 | 478 | { |
| 479 | 479 | if (test_bit(XPT_BUSY, &xprt->xpt_flags)) |
| 480 | 480 | return; |
| 481 | − xprt->xpt_server->sv_ops->svo_enqueue_xprt(xprt); | |
| 481 | + svc_xprt_do_enqueue(xprt); | |
| 482 | 482 | } |
| 483 | 483 | EXPORT_SYMBOL_GPL(svc_xprt_enqueue); |
| 484 | 484 | |