Improve the efficiency of buffered reads in a number of ways: (1) Overhaul the algorithm in general so that it's a lot more compact and split the read submission code between buffered and unbuffered versions. The unbuffered version can be vastly simplified. (2) Read-result collection is handed off to a work queue rather than being done in the I/O thread. Multiple subrequests can be processes simultaneously. (3) When a subrequest is collected, any folios it fully spans are collected and "spare" data on either side is donated to either the previous or the next subrequest in the sequence. Notes: (*) Readahead expansion is massively slows down fio, presumably because it causes a load of extra allocations, both folio and xarray, up front before RPC requests can be transmitted. (*) RDMA with cifs does appear to work, both with SIW and RXE. (*) PG_private_2-based reading and copy-to-cache is split out into its own file and altered to use folio_queue. Note that the copy to the cache now creates a new write transaction against the cache and adds the folios to be copied into it. This allows it to use part of the writeback I/O code.
Showing the changes to include/trace (1 of 28 files). Show all
Unified · Split
| @@ -20,6 +20,7 @@ · 19 unmodified lines | ||
| 20 | 20 | EM(netfs_read_trace_expanded, "EXPANDED ") \ |
| 21 | 21 | EM(netfs_read_trace_readahead, "READAHEAD") \ |
| 22 | 22 | EM(netfs_read_trace_readpage, "READPAGE ") \ |
| 23 | + EM(netfs_read_trace_read_gaps, "READ-GAPS") \ | |
| 23 | 24 | EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \ |
| 24 | 25 | E_(netfs_read_trace_write_begin, "WRITEBEGN") |
| 25 | 26 | |
| @@ -33,12 +34,14 @@ · 7 unmodified lines | ||
| 33 | 34 | #define netfs_rreq_origins \ |
| 34 | 35 | EM(NETFS_READAHEAD, "RA") \ |
| 35 | 36 | EM(NETFS_READPAGE, "RP") \ |
| 37 | + EM(NETFS_READ_GAPS, "RG") \ | |
| 36 | 38 | EM(NETFS_READ_FOR_WRITE, "RW") \ |
| 37 | 39 | EM(NETFS_DIO_READ, "DR") \ |
| 38 | 40 | EM(NETFS_WRITEBACK, "WB") \ |
| 39 | 41 | EM(NETFS_WRITETHROUGH, "WT") \ |
| 40 | 42 | EM(NETFS_UNBUFFERED_WRITE, "UW") \ |
| 41 | − E_(NETFS_DIO_WRITE, "DW") | |
| 43 | + EM(NETFS_DIO_WRITE, "DW") \ | |
| 44 | + E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C") | |
| 42 | 45 | |
| 43 | 46 | #define netfs_rreq_traces \ |
| 44 | 47 | EM(netfs_rreq_trace_assess, "ASSESS ") \ |
| @@ -69,15 +72,25 @@ · 24 unmodified lines | ||
| 69 | 72 | E_(NETFS_INVALID_WRITE, "INVL") |
| 70 | 73 | |
| 71 | 74 | #define netfs_sreq_traces \ |
| 75 | + EM(netfs_sreq_trace_add_donations, "+DON ") \ | |
| 76 | + EM(netfs_sreq_trace_added, "ADD ") \ | |
| 77 | + EM(netfs_sreq_trace_clear, "CLEAR") \ | |
| 72 | 78 | EM(netfs_sreq_trace_discard, "DSCRD") \ |
| 79 | + EM(netfs_sreq_trace_donate_to_prev, "DON-P") \ | |
| 80 | + EM(netfs_sreq_trace_donate_to_next, "DON-N") \ | |
| 73 | 81 | EM(netfs_sreq_trace_download_instead, "RDOWN") \ |
| 74 | 82 | EM(netfs_sreq_trace_fail, "FAIL ") \ |
| 75 | 83 | EM(netfs_sreq_trace_free, "FREE ") \ |
| 84 | + EM(netfs_sreq_trace_hit_eof, "EOF ") \ | |
| 85 | + EM(netfs_sreq_trace_io_progress, "IO ") \ | |
| 76 | 86 | EM(netfs_sreq_trace_limited, "LIMIT") \ |
| 77 | 87 | EM(netfs_sreq_trace_prepare, "PREP ") \ |
| 78 | 88 | EM(netfs_sreq_trace_prep_failed, "PRPFL") \ |
| 79 | − EM(netfs_sreq_trace_resubmit_short, "SHORT") \ | |
| 89 | + EM(netfs_sreq_trace_progress, "PRGRS") \ | |
| 90 | + EM(netfs_sreq_trace_reprep_failed, "REPFL") \ | |
| 80 | 91 | EM(netfs_sreq_trace_retry, "RETRY") \ |
| 92 | + EM(netfs_sreq_trace_short, "SHORT") \ | |
| 93 | + EM(netfs_sreq_trace_split, "SPLIT") \ | |
| 81 | 94 | EM(netfs_sreq_trace_submit, "SUBMT") \ |
| 82 | 95 | EM(netfs_sreq_trace_terminated, "TERM ") \ |
| 83 | 96 | EM(netfs_sreq_trace_write, "WRITE") \ |
| @@ -118,7 +131,7 @@ · 34 unmodified lines | ||
| 118 | 131 | EM(netfs_sreq_trace_new, "NEW ") \ |
| 119 | 132 | EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \ |
| 120 | 133 | EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \ |
| 121 | − EM(netfs_sreq_trace_put_discard, "PUT DISCARD") \ | |
| 134 | + EM(netfs_sreq_trace_put_consumed, "PUT CONSUME") \ | |
| 122 | 135 | EM(netfs_sreq_trace_put_done, "PUT DONE ") \ |
| 123 | 136 | EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \ |
| 124 | 137 | EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \ |
| @@ -138,6 +151,7 @@ · 13 unmodified lines | ||
| 138 | 151 | EM(netfs_flush_content, "flush") \ |
| 139 | 152 | EM(netfs_streaming_filled_page, "mod-streamw-f") \ |
| 140 | 153 | EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \ |
| 154 | + EM(netfs_folio_trace_abandon, "abandon") \ | |
| 141 | 155 | EM(netfs_folio_trace_cancel_copy, "cancel-copy") \ |
| 142 | 156 | EM(netfs_folio_trace_clear, "clear") \ |
| 143 | 157 | EM(netfs_folio_trace_clear_cc, "clear-cc") \ |
| @@ -154,7 +168,11 @@ · 10 unmodified lines | ||
| 154 | 168 | EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \ |
| 155 | 169 | EM(netfs_folio_trace_not_under_wback, "!wback") \ |
| 156 | 170 | EM(netfs_folio_trace_put, "put") \ |
| 171 | + EM(netfs_folio_trace_read, "read") \ | |
| 172 | + EM(netfs_folio_trace_read_done, "read-done") \ | |
| 157 | 173 | EM(netfs_folio_trace_read_gaps, "read-gaps") \ |
| 174 | + EM(netfs_folio_trace_read_put, "read-put") \ | |
| 175 | + EM(netfs_folio_trace_read_unlock, "read-unlock") \ | |
| 158 | 176 | EM(netfs_folio_trace_redirtied, "redirtied") \ |
| 159 | 177 | EM(netfs_folio_trace_store, "store") \ |
| 160 | 178 | EM(netfs_folio_trace_store_copy, "store-copy") \ |
| @@ -167,6 +185,12 @@ · 6 unmodified lines | ||
| 167 | 185 | EM(netfs_contig_trace_jump, "-->JUMP-->") \ |
| 168 | 186 | E_(netfs_contig_trace_unlock, "Unlock") |
| 169 | 187 | |
| 188 | +#define netfs_donate_traces \ | |
| 189 | + EM(netfs_trace_donate_tail_to_prev, "tail-to-prev") \ | |
| 190 | + EM(netfs_trace_donate_to_prev, "to-prev") \ | |
| 191 | + EM(netfs_trace_donate_to_next, "to-next") \ | |
| 192 | + E_(netfs_trace_donate_to_deferred_next, "defer-next") | |
| 193 | + | |
| 170 | 194 | #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY |
| 171 | 195 | #define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY |
| 172 | 196 | |
| @@ -184,6 +208,7 @@ enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); · 11 unmodified lines | ||
| 184 | 208 | enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte); |
| 185 | 209 | enum netfs_folio_trace { netfs_folio_traces } __mode(byte); |
| 186 | 210 | enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte); |
| 211 | +enum netfs_donate_trace { netfs_donate_traces } __mode(byte); | |
| 187 | 212 | |
| 188 | 213 | #endif |
| 189 | 214 | |
| @@ -206,6 +231,7 @@ netfs_rreq_ref_traces; · 16 unmodified lines | ||
| 206 | 231 | netfs_sreq_ref_traces; |
| 207 | 232 | netfs_folio_traces; |
| 208 | 233 | netfs_collect_contig_traces; |
| 234 | +netfs_donate_traces; | |
| 209 | 235 | |
| 210 | 236 | /* |
| 211 | 237 | * Now redefine the EM() and E_() macros to map the enums to the strings that |
| @@ -226,6 +252,7 @@ TRACE_EVENT(netfs_read, · 14 unmodified lines | ||
| 226 | 252 | TP_STRUCT__entry( |
| 227 | 253 | __field(unsigned int, rreq ) |
| 228 | 254 | __field(unsigned int, cookie ) |
| 255 | + __field(loff_t, i_size ) | |
| 229 | 256 | __field(loff_t, start ) |
| 230 | 257 | __field(size_t, len ) |
| 231 | 258 | __field(enum netfs_read_trace, what ) |
| @@ -235,18 +262,19 @@ TRACE_EVENT(netfs_read, · 3 unmodified lines | ||
| 235 | 262 | TP_fast_assign( |
| 236 | 263 | __entry->rreq = rreq->debug_id; |
| 237 | 264 | __entry->cookie = rreq->cache_resources.debug_id; |
| 265 | + __entry->i_size = rreq->i_size; | |
| 238 | 266 | __entry->start = start; |
| 239 | 267 | __entry->len = len; |
| 240 | 268 | __entry->what = what; |
| 241 | 269 | __entry->netfs_inode = rreq->inode->i_ino; |
| 242 | 270 | ), |
| 243 | 271 | |
| 244 | − TP_printk("R=%08x %s c=%08x ni=%x s=%llx %zx", | |
| 272 | + TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx", | |
| 245 | 273 | __entry->rreq, |
| 246 | 274 | __print_symbolic(__entry->what, netfs_read_traces), |
| 247 | 275 | __entry->cookie, |
| 248 | 276 | __entry->netfs_inode, |
| 249 | − __entry->start, __entry->len) | |
| 277 | + __entry->start, __entry->len, __entry->i_size) | |
| 250 | 278 | ); |
| 251 | 279 | |
| 252 | 280 | TRACE_EVENT(netfs_rreq, |
| @@ -651,6 +679,71 @@ TRACE_EVENT(netfs_collect_stream, · 398 unmodified lines | ||
| 651 | 679 | __entry->collected_to, __entry->front) |
| 652 | 680 | ); |
| 653 | 681 | |
| 682 | +TRACE_EVENT(netfs_progress, | |
| 683 | + TP_PROTO(const struct netfs_io_subrequest *subreq, | |
| 684 | + unsigned long long start, size_t avail, size_t part), | |
| 685 | + | |
| 686 | + TP_ARGS(subreq, start, avail, part), | |
| 687 | + | |
| 688 | + TP_STRUCT__entry( | |
| 689 | + __field(unsigned int, rreq) | |
| 690 | + __field(unsigned int, subreq) | |
| 691 | + __field(unsigned int, consumed) | |
| 692 | + __field(unsigned int, transferred) | |
| 693 | + __field(unsigned long long, f_start) | |
| 694 | + __field(unsigned int, f_avail) | |
| 695 | + __field(unsigned int, f_part) | |
| 696 | + __field(unsigned char, slot) | |
| 697 | + ), | |
| 698 | + | |
| 699 | + TP_fast_assign( | |
| 700 | + __entry->rreq = subreq->rreq->debug_id; | |
| 701 | + __entry->subreq = subreq->debug_index; | |
| 702 | + __entry->consumed = subreq->consumed; | |
| 703 | + __entry->transferred = subreq->transferred; | |
| 704 | + __entry->f_start = start; | |
| 705 | + __entry->f_avail = avail; | |
| 706 | + __entry->f_part = part; | |
| 707 | + __entry->slot = subreq->curr_folioq_slot; | |
| 708 | + ), | |
| 709 | + | |
| 710 | + TP_printk("R=%08x[%02x] s=%llx ct=%x/%x pa=%x/%x sl=%x", | |
| 711 | + __entry->rreq, __entry->subreq, __entry->f_start, | |
| 712 | + __entry->consumed, __entry->transferred, | |
| 713 | + __entry->f_part, __entry->f_avail, __entry->slot) | |
| 714 | + ); | |
| 715 | + | |
| 716 | +TRACE_EVENT(netfs_donate, | |
| 717 | + TP_PROTO(const struct netfs_io_request *rreq, | |
| 718 | + const struct netfs_io_subrequest *from, | |
| 719 | + const struct netfs_io_subrequest *to, | |
| 720 | + size_t amount, | |
| 721 | + enum netfs_donate_trace trace), | |
| 722 | + | |
| 723 | + TP_ARGS(rreq, from, to, amount, trace), | |
| 724 | + | |
| 725 | + TP_STRUCT__entry( | |
| 726 | + __field(unsigned int, rreq) | |
| 727 | + __field(unsigned int, from) | |
| 728 | + __field(unsigned int, to) | |
| 729 | + __field(unsigned int, amount) | |
| 730 | + __field(enum netfs_donate_trace, trace) | |
| 731 | + ), | |
| 732 | + | |
| 733 | + TP_fast_assign( | |
| 734 | + __entry->rreq = rreq->debug_id; | |
| 735 | + __entry->from = from->debug_index; | |
| 736 | + __entry->to = to ? to->debug_index : -1; | |
| 737 | + __entry->amount = amount; | |
| 738 | + __entry->trace = trace; | |
| 739 | + ), | |
| 740 | + | |
| 741 | + TP_printk("R=%08x[%02x] -> [%02x] %s am=%x", | |
| 742 | + __entry->rreq, __entry->from, __entry->to, | |
| 743 | + __print_symbolic(__entry->trace, netfs_donate_traces), | |
| 744 | + __entry->amount) | |
| 745 | + ); | |
| 746 | + | |
| 654 | 747 | #undef EM |
| 655 | 748 | #undef E_ |
| 656 | 749 | #endif /* _TRACE_NETFS_H */ |