| @@ -10,103 +10,239 @@ struct blk_mq_tags; · 9 unmodified lines |
| 10 | struct blk_flush_queue; | 10 | struct blk_flush_queue; |
| 11 | | 11 | |
| 12 | /** | 12 | /** |
| 13 | * struct blk_mq_hw_ctx - State for a hardware queue facing the hardware block device | 13 | * struct blk_mq_hw_ctx - State for a hardware queue facing the hardware |
| | 14 | * block device |
| 14 | */ | 15 | */ |
| 15 | struct blk_mq_hw_ctx { | 16 | struct blk_mq_hw_ctx { |
| 16 | struct { | 17 | struct { |
| | 18 | /** @lock: Protects the dispatch list. */ |
| 17 | spinlock_t lock; | 19 | spinlock_t lock; |
| | 20 | /** |
| | 21 | * @dispatch: Used for requests that are ready to be |
| | 22 | * dispatched to the hardware but for some reason (e.g. lack of |
| | 23 | * resources) could not be sent to the hardware. As soon as the |
| | 24 | * driver can send new requests, requests at this list will |
| | 25 | * be sent first for a fairer dispatch. |
| | 26 | */ |
| 18 | struct list_head dispatch; | 27 | struct list_head dispatch; |
| 19 | unsigned long state; /* BLK_MQ_S_* flags */ | 28 | /** |
| | 29 | * @state: BLK_MQ_S_* flags. Defines the state of the hw |
| | 30 | * queue (active, scheduled to restart, stopped). |
| | 31 | */ |
| | 32 | unsigned long state; |
| 20 | } ____cacheline_aligned_in_smp; | 33 | } ____cacheline_aligned_in_smp; |
| 21 | | 34 | |
| | 35 | /** |
| | 36 | * @run_work: Used for scheduling a hardware queue run at a later time. |
| | 37 | */ |
| 22 | struct delayed_work run_work; | 38 | struct delayed_work run_work; |
| | 39 | /** @cpumask: Map of available CPUs where this hctx can run. */ |
| 23 | cpumask_var_t cpumask; | 40 | cpumask_var_t cpumask; |
| | 41 | /** |
| | 42 | * @next_cpu: Used by blk_mq_hctx_next_cpu() for round-robin CPU |
| | 43 | * selection from @cpumask. |
| | 44 | */ |
| 24 | int next_cpu; | 45 | int next_cpu; |
| | 46 | /** |
| | 47 | * @next_cpu_batch: Counter of how many works left in the batch before |
| | 48 | * changing to the next CPU. |
| | 49 | */ |
| 25 | int next_cpu_batch; | 50 | int next_cpu_batch; |
| 26 | | 51 | |
| 27 | unsigned long flags; /* BLK_MQ_F_* flags */ | 52 | /** @flags: BLK_MQ_F_* flags. Defines the behaviour of the queue. */ |
| | 53 | unsigned long flags; |
| 28 | | 54 | |
| | 55 | /** |
| | 56 | * @sched_data: Pointer owned by the IO scheduler attached to a request |
| | 57 | * queue. It's up to the IO scheduler how to use this pointer. |
| | 58 | */ |
| 29 | void *sched_data; | 59 | void *sched_data; |
| | 60 | /** |
| | 61 | * @queue: Pointer to the request queue that owns this hardware context. |
| | 62 | */ |
| 30 | struct request_queue *queue; | 63 | struct request_queue *queue; |
| | 64 | /** @fq: Queue of requests that need to perform a flush operation. */ |
| 31 | struct blk_flush_queue *fq; | 65 | struct blk_flush_queue *fq; |
| 32 | | 66 | |
| | 67 | /** |
| | 68 | * @driver_data: Pointer to data owned by the block driver that created |
| | 69 | * this hctx |
| | 70 | */ |
| 33 | void *driver_data; | 71 | void *driver_data; |
| 34 | | 72 | |
| | 73 | /** |
| | 74 | * @ctx_map: Bitmap for each software queue. If bit is on, there is a |
| | 75 | * pending request in that software queue. |
| | 76 | */ |
| 35 | struct sbitmap ctx_map; | 77 | struct sbitmap ctx_map; |
| 36 | | 78 | |
| | 79 | /** |
| | 80 | * @dispatch_from: Software queue to be used when no scheduler was |
| | 81 | * selected. |
| | 82 | */ |
| 37 | struct blk_mq_ctx *dispatch_from; | 83 | struct blk_mq_ctx *dispatch_from; |
| | 84 | /** |
| | 85 | * @dispatch_busy: Number used by blk_mq_update_dispatch_busy() to |
| | 86 | * decide if the hw_queue is busy using Exponential Weighted Moving |
| | 87 | * Average algorithm. |
| | 88 | */ |
| 38 | unsigned int dispatch_busy; | 89 | unsigned int dispatch_busy; |
| 39 | | 90 | |
| | 91 | /** @type: HCTX_TYPE_* flags. Type of hardware queue. */ |
| 40 | unsigned short type; | 92 | unsigned short type; |
| | 93 | /** @nr_ctx: Number of software queues. */ |
| 41 | unsigned short nr_ctx; | 94 | unsigned short nr_ctx; |
| | 95 | /** @ctxs: Array of software queues. */ |
| 42 | struct blk_mq_ctx **ctxs; | 96 | struct blk_mq_ctx **ctxs; |
| 43 | | 97 | |
| | 98 | /** @dispatch_wait_lock: Lock for dispatch_wait queue. */ |
| 44 | spinlock_t dispatch_wait_lock; | 99 | spinlock_t dispatch_wait_lock; |
| | 100 | /** |
| | 101 | * @dispatch_wait: Waitqueue to put requests when there is no tag |
| | 102 | * available at the moment, to wait for another try in the future. |
| | 103 | */ |
| 45 | wait_queue_entry_t dispatch_wait; | 104 | wait_queue_entry_t dispatch_wait; |
| | 105 | |
| | 106 | /** |
| | 107 | * @wait_index: Index of next available dispatch_wait queue to insert |
| | 108 | * requests. |
| | 109 | */ |
| 46 | atomic_t wait_index; | 110 | atomic_t wait_index; |
| 47 | | 111 | |
| | 112 | /** |
| | 113 | * @tags: Tags owned by the block driver. A tag at this set is only |
| | 114 | * assigned when a request is dispatched from a hardware queue. |
| | 115 | */ |
| 48 | struct blk_mq_tags *tags; | 116 | struct blk_mq_tags *tags; |
| | 117 | /** |
| | 118 | * @sched_tags: Tags owned by I/O scheduler. If there is an I/O |
| | 119 | * scheduler associated with a request queue, a tag is assigned when |
| | 120 | * that request is allocated. Else, this member is not used. |
| | 121 | */ |
| 49 | struct blk_mq_tags *sched_tags; | 122 | struct blk_mq_tags *sched_tags; |
| 50 | | 123 | |
| | 124 | /** @queued: Number of queued requests. */ |
| 51 | unsigned long queued; | 125 | unsigned long queued; |
| | 126 | /** @run: Number of dispatched requests. */ |
| 52 | unsigned long run; | 127 | unsigned long run; |
| 53 | #define BLK_MQ_MAX_DISPATCH_ORDER 7 | 128 | #define BLK_MQ_MAX_DISPATCH_ORDER 7 |
| | 129 | /** @dispatched: Number of dispatch requests by queue. */ |
| 54 | unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; | 130 | unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; |
| 55 | | 131 | |
| | 132 | /** @numa_node: NUMA node the storage adapter has been connected to. */ |
| 56 | unsigned int numa_node; | 133 | unsigned int numa_node; |
| | 134 | /** @queue_num: Index of this hardware queue. */ |
| 57 | unsigned int queue_num; | 135 | unsigned int queue_num; |
| 58 | | 136 | |
| | 137 | /** |
| | 138 | * @nr_active: Number of active requests. Only used when a tag set is |
| | 139 | * shared across request queues. |
| | 140 | */ |
| 59 | atomic_t nr_active; | 141 | atomic_t nr_active; |
| 60 | | 142 | |
| | 143 | /** @cpuhp_dead: List to store request if some CPU die. */ |
| 61 | struct hlist_node cpuhp_dead; | 144 | struct hlist_node cpuhp_dead; |
| | 145 | /** @kobj: Kernel object for sysfs. */ |
| 62 | struct kobject kobj; | 146 | struct kobject kobj; |
| 63 | | 147 | |
| | 148 | /** @poll_considered: Count times blk_poll() was called. */ |
| 64 | unsigned long poll_considered; | 149 | unsigned long poll_considered; |
| | 150 | /** @poll_invoked: Count how many requests blk_poll() polled. */ |
| 65 | unsigned long poll_invoked; | 151 | unsigned long poll_invoked; |
| | 152 | /** @poll_success: Count how many polled requests were completed. */ |
| 66 | unsigned long poll_success; | 153 | unsigned long poll_success; |
| 67 | | 154 | |
| 68 | #ifdef CONFIG_BLK_DEBUG_FS | 155 | #ifdef CONFIG_BLK_DEBUG_FS |
| | 156 | /** |
| | 157 | * @debugfs_dir: debugfs directory for this hardware queue. Named |
| | 158 | * as cpu<cpu_number>. |
| | 159 | */ |
| 69 | struct dentry *debugfs_dir; | 160 | struct dentry *debugfs_dir; |
| | 161 | /** @sched_debugfs_dir: debugfs directory for the scheduler. */ |
| 70 | struct dentry *sched_debugfs_dir; | 162 | struct dentry *sched_debugfs_dir; |
| 71 | #endif | 163 | #endif |
| 72 | | 164 | |
| | 165 | /** @hctx_list: List of all hardware queues. */ |
| 73 | struct list_head hctx_list; | 166 | struct list_head hctx_list; |
| 74 | | 167 | |
| 75 | /* Must be the last member - see also blk_mq_hw_ctx_size(). */ | 168 | /** |
| | 169 | * @srcu: Sleepable RCU. Use as lock when type of the hardware queue is |
| | 170 | * blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also |
| | 171 | * blk_mq_hw_ctx_size(). |
| | 172 | */ |
| 76 | struct srcu_struct srcu[0]; | 173 | struct srcu_struct srcu[0]; |
| 77 | }; | 174 | }; |
| 78 | | 175 | |
| | 176 | /** |
| | 177 | * struct blk_mq_queue_map - Map software queues to hardware queues |
| | 178 | * @mq_map: CPU ID to hardware queue index map. This is an array |
| | 179 | * with nr_cpu_ids elements. Each element has a value in the range |
| | 180 | * [@queue_offset, @queue_offset + @nr_queues). |
| | 181 | * @nr_queues: Number of hardware queues to map CPU IDs onto. |
| | 182 | * @queue_offset: First hardware queue to map onto. Used by the PCIe NVMe |
| | 183 | * driver to map each hardware queue type (enum hctx_type) onto a distinct |
| | 184 | * set of hardware queues. |
| | 185 | */ |
| 79 | struct blk_mq_queue_map { | 186 | struct blk_mq_queue_map { |
| 80 | unsigned int *mq_map; | 187 | unsigned int *mq_map; |
| 81 | unsigned int nr_queues; | 188 | unsigned int nr_queues; |
| 82 | unsigned int queue_offset; | 189 | unsigned int queue_offset; |
| 83 | }; | 190 | }; |
| 84 | | 191 | |
| | 192 | /** |
| | 193 | * enum hctx_type - Type of hardware queue |
| | 194 | * @HCTX_TYPE_DEFAULT: All I/O not otherwise accounted for. |
| | 195 | * @HCTX_TYPE_READ: Just for READ I/O. |
| | 196 | * @HCTX_TYPE_POLL: Polled I/O of any kind. |
| | 197 | * @HCTX_MAX_TYPES: Number of types of hctx. |
| | 198 | */ |
| 85 | enum hctx_type { | 199 | enum hctx_type { |
| 86 | HCTX_TYPE_DEFAULT, /* all I/O not otherwise accounted for */ | 200 | HCTX_TYPE_DEFAULT, |
| 87 | HCTX_TYPE_READ, /* just for READ I/O */ | 201 | HCTX_TYPE_READ, |
| 88 | HCTX_TYPE_POLL, /* polled I/O of any kind */ | 202 | HCTX_TYPE_POLL, |
| 89 | | 203 | |
| 90 | HCTX_MAX_TYPES, | 204 | HCTX_MAX_TYPES, |
| 91 | }; | 205 | }; |
| 92 | | 206 | |
| | 207 | /** |
| | 208 | * struct blk_mq_tag_set - tag set that can be shared between request queues |
| | 209 | * @map: One or more ctx -> hctx mappings. One map exists for each |
| | 210 | * hardware queue type (enum hctx_type) that the driver wishes |
| | 211 | * to support. There are no restrictions on maps being of the |
| | 212 | * same size, and it's perfectly legal to share maps between |
| | 213 | * types. |
| | 214 | * @nr_maps: Number of elements in the @map array. A number in the range |
| | 215 | * [1, HCTX_MAX_TYPES]. |
| | 216 | * @ops: Pointers to functions that implement block driver behavior. |
| | 217 | * @nr_hw_queues: Number of hardware queues supported by the block driver that |
| | 218 | * owns this data structure. |
| | 219 | * @queue_depth: Number of tags per hardware queue, reserved tags included. |
| | 220 | * @reserved_tags: Number of tags to set aside for BLK_MQ_REQ_RESERVED tag |
| | 221 | * allocations. |
| | 222 | * @cmd_size: Number of additional bytes to allocate per request. The block |
| | 223 | * driver owns these additional bytes. |
| | 224 | * @numa_node: NUMA node the storage adapter has been connected to. |
| | 225 | * @timeout: Request processing timeout in jiffies. |
| | 226 | * @flags: Zero or more BLK_MQ_F_* flags. |
| | 227 | * @driver_data: Pointer to data owned by the block driver that created this |
| | 228 | * tag set. |
| | 229 | * @tags: Tag sets. One tag set per hardware queue. Has @nr_hw_queues |
| | 230 | * elements. |
| | 231 | * @tag_list_lock: Serializes tag_list accesses. |
| | 232 | * @tag_list: List of the request queues that use this tag set. See also |
| | 233 | * request_queue.tag_set_list. |
| | 234 | */ |
| 93 | struct blk_mq_tag_set { | 235 | struct blk_mq_tag_set { |
| 94 | /* | | |
| 95 | * map[] holds ctx -> hctx mappings, one map exists for each type | | |
| 96 | * that the driver wishes to support. There are no restrictions | | |
| 97 | * on maps being of the same size, and it's perfectly legal to | | |
| 98 | * share maps between types. | | |
| 99 | */ | | |
| 100 | struct blk_mq_queue_map map[HCTX_MAX_TYPES]; | 236 | struct blk_mq_queue_map map[HCTX_MAX_TYPES]; |
| 101 | unsigned int nr_maps; /* nr entries in map[] */ | 237 | unsigned int nr_maps; |
| 102 | const struct blk_mq_ops *ops; | 238 | const struct blk_mq_ops *ops; |
| 103 | unsigned int nr_hw_queues; /* nr hw queues across maps */ | 239 | unsigned int nr_hw_queues; |
| 104 | unsigned int queue_depth; /* max hw supported */ | 240 | unsigned int queue_depth; |
| 105 | unsigned int reserved_tags; | 241 | unsigned int reserved_tags; |
| 106 | unsigned int cmd_size; /* per-request extra data */ | 242 | unsigned int cmd_size; |
| 107 | int numa_node; | 243 | int numa_node; |
| 108 | unsigned int timeout; | 244 | unsigned int timeout; |
| 109 | unsigned int flags; /* BLK_MQ_F_* */ | 245 | unsigned int flags; |
| 110 | void *driver_data; | 246 | void *driver_data; |
| 111 | | 247 | |
| 112 | struct blk_mq_tags **tags; | 248 | struct blk_mq_tags **tags; |
| @@ -115,6 +251,12 @@ struct blk_mq_tag_set { · 2 unmodified lines |
| 115 | struct list_head tag_list; | 251 | struct list_head tag_list; |
| 116 | }; | 252 | }; |
| 117 | | 253 | |
| | 254 | /** |
| | 255 | * struct blk_mq_queue_data - Data about a request inserted in a queue |
| | 256 | * |
| | 257 | * @rq: Request pointer. |
| | 258 | * @last: If it is the last request in the queue. |
| | 259 | */ |
| 118 | struct blk_mq_queue_data { | 260 | struct blk_mq_queue_data { |
| 119 | struct request *rq; | 261 | struct request *rq; |
| 120 | bool last; | 262 | bool last; |
| @@ -142,81 +284,101 @@ typedef bool (busy_fn)(struct request_queue *); · 21 unmodified lines |
| 142 | typedef void (complete_fn)(struct request *); | 284 | typedef void (complete_fn)(struct request *); |
| 143 | typedef void (cleanup_rq_fn)(struct request *); | 285 | typedef void (cleanup_rq_fn)(struct request *); |
| 144 | | 286 | |
| 145 | | 287 | /** |
| | 288 | * struct blk_mq_ops - Callback functions that implements block driver |
| | 289 | * behaviour. |
| | 290 | */ |
| 146 | struct blk_mq_ops { | 291 | struct blk_mq_ops { |
| 147 | /* | 292 | /** |
| 148 | * Queue request | 293 | * @queue_rq: Queue a new request from block IO. |
| 149 | */ | 294 | */ |
| 150 | queue_rq_fn *queue_rq; | 295 | queue_rq_fn *queue_rq; |
| 151 | | 296 | |
| 152 | /* | 297 | /** |
| 153 | * If a driver uses bd->last to judge when to submit requests to | 298 | * @commit_rqs: If a driver uses bd->last to judge when to submit |
| 154 | * hardware, it must define this function. In case of errors that | 299 | * requests to hardware, it must define this function. In case of errors |
| 155 | * make us stop issuing further requests, this hook serves the | 300 | * that make us stop issuing further requests, this hook serves the |
| 156 | * purpose of kicking the hardware (which the last request otherwise | 301 | * purpose of kicking the hardware (which the last request otherwise |
| 157 | * would have done). | 302 | * would have done). |
| 158 | */ | 303 | */ |
| 159 | commit_rqs_fn *commit_rqs; | 304 | commit_rqs_fn *commit_rqs; |
| 160 | | 305 | |
| 161 | /* | 306 | /** |
| 162 | * Reserve budget before queue request, once .queue_rq is | 307 | * @get_budget: Reserve budget before queue request, once .queue_rq is |
| 163 | * run, it is driver's responsibility to release the | 308 | * run, it is driver's responsibility to release the |
| 164 | * reserved budget. Also we have to handle failure case | 309 | * reserved budget. Also we have to handle failure case |
| 165 | * of .get_budget for avoiding I/O deadlock. | 310 | * of .get_budget for avoiding I/O deadlock. |
| 166 | */ | 311 | */ |
| 167 | get_budget_fn *get_budget; | 312 | get_budget_fn *get_budget; |
| | 313 | /** |
| | 314 | * @put_budget: Release the reserved budget. |
| | 315 | */ |
| 168 | put_budget_fn *put_budget; | 316 | put_budget_fn *put_budget; |
| 169 | | 317 | |
| 170 | /* | 318 | /** |
| 171 | * Called on request timeout | 319 | * @timeout: Called on request timeout. |
| 172 | */ | 320 | */ |
| 173 | timeout_fn *timeout; | 321 | timeout_fn *timeout; |
| 174 | | 322 | |
| 175 | /* | 323 | /** |
| 176 | * Called to poll for completion of a specific tag. | 324 | * @poll: Called to poll for completion of a specific tag. |
| 177 | */ | 325 | */ |
| 178 | poll_fn *poll; | 326 | poll_fn *poll; |
| 179 | | 327 | |
| | 328 | /** |
| | 329 | * @complete: Mark the request as complete. |
| | 330 | */ |
| 180 | complete_fn *complete; | 331 | complete_fn *complete; |
| 181 | | 332 | |
| 182 | /* | 333 | /** |
| 183 | * Called when the block layer side of a hardware queue has been | 334 | * @init_hctx: Called when the block layer side of a hardware queue has |
| 184 | * set up, allowing the driver to allocate/init matching structures. | 335 | * been set up, allowing the driver to allocate/init matching |
| 185 | * Ditto for exit/teardown. | 336 | * structures. |
| 186 | */ | 337 | */ |
| 187 | init_hctx_fn *init_hctx; | 338 | init_hctx_fn *init_hctx; |
| | 339 | /** |
| | 340 | * @exit_hctx: Ditto for exit/teardown. |
| | 341 | */ |
| 188 | exit_hctx_fn *exit_hctx; | 342 | exit_hctx_fn *exit_hctx; |
| 189 | | 343 | |
| 190 | /* | 344 | /** |
| 191 | * Called for every command allocated by the block layer to allow | 345 | * @init_request: Called for every command allocated by the block layer |
| 192 | * the driver to set up driver specific data. | 346 | * to allow the driver to set up driver specific data. |
| 193 | * | 347 | * |
| 194 | * Tag greater than or equal to queue_depth is for setting up | 348 | * Tag greater than or equal to queue_depth is for setting up |
| 195 | * flush request. | 349 | * flush request. |
| 196 | * | | |
| 197 | * Ditto for exit/teardown. | | |
| 198 | */ | 350 | */ |
| 199 | init_request_fn *init_request; | 351 | init_request_fn *init_request; |
| | 352 | /** |
| | 353 | * @exit_request: Ditto for exit/teardown. |
| | 354 | */ |
| 200 | exit_request_fn *exit_request; | 355 | exit_request_fn *exit_request; |
| 201 | /* Called from inside blk_get_request() */ | 356 | |
| | 357 | /** |
| | 358 | * @initialize_rq_fn: Called from inside blk_get_request(). |
| | 359 | */ |
| 202 | void (*initialize_rq_fn)(struct request *rq); | 360 | void (*initialize_rq_fn)(struct request *rq); |
| 203 | | 361 | |
| 204 | /* | 362 | /** |
| 205 | * Called before freeing one request which isn't completed yet, | 363 | * @cleanup_rq: Called before freeing one request which isn't completed |
| 206 | * and usually for freeing the driver private data | 364 | * yet, and usually for freeing the driver private data. |
| 207 | */ | 365 | */ |
| 208 | cleanup_rq_fn *cleanup_rq; | 366 | cleanup_rq_fn *cleanup_rq; |
| 209 | | 367 | |
| 210 | /* | 368 | /** |
| 211 | * If set, returns whether or not this queue currently is busy | 369 | * @busy: If set, returns whether or not this queue currently is busy. |
| 212 | */ | 370 | */ |
| 213 | busy_fn *busy; | 371 | busy_fn *busy; |
| 214 | | 372 | |
| | 373 | /** |
| | 374 | * @map_queues: This allows drivers specify their own queue mapping by |
| | 375 | * overriding the setup-time function that builds the mq_map. |
| | 376 | */ |
| 215 | map_queues_fn *map_queues; | 377 | map_queues_fn *map_queues; |
| 216 | | 378 | |
| 217 | #ifdef CONFIG_BLK_DEBUG_FS | 379 | #ifdef CONFIG_BLK_DEBUG_FS |
| 218 | /* | 380 | /** |
| 219 | * Used by the debugfs implementation to show driver-specific | 381 | * @show_rq: Used by the debugfs implementation to show driver-specific |
| 220 | * information about a request. | 382 | * information about a request. |
| 221 | */ | 383 | */ |
| 222 | void (*show_rq)(struct seq_file *m, struct request *rq); | 384 | void (*show_rq)(struct seq_file *m, struct request *rq); |
| @@ -262,7 +424,6 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set); · 39 unmodified lines |
| 262 | void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); | 424 | void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); |
| 263 | | 425 | |
| 264 | void blk_mq_free_request(struct request *rq); | 426 | void blk_mq_free_request(struct request *rq); |
| 265 | bool blk_mq_can_queue(struct blk_mq_hw_ctx *); | | |
| 266 | | 427 | |
| 267 | bool blk_mq_queue_inflight(struct request_queue *q); | 428 | bool blk_mq_queue_inflight(struct request_queue *q); |
| 268 | | 429 | |
| @@ -301,9 +462,25 @@ static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) · 32 unmodified lines |
| 301 | return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; | 462 | return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; |
| 302 | } | 463 | } |
| 303 | | 464 | |
| | 465 | /** |
| | 466 | * blk_mq_rq_state() - read the current MQ_RQ_* state of a request |
| | 467 | * @rq: target request. |
| | 468 | */ |
| | 469 | static inline enum mq_rq_state blk_mq_rq_state(struct request *rq) |
| | 470 | { |
| | 471 | return READ_ONCE(rq->state); |
| | 472 | } |
| | 473 | |
| | 474 | static inline int blk_mq_request_started(struct request *rq) |
| | 475 | { |
| | 476 | return blk_mq_rq_state(rq) != MQ_RQ_IDLE; |
| | 477 | } |
| | 478 | |
| | 479 | static inline int blk_mq_request_completed(struct request *rq) |
| | 480 | { |
| | 481 | return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE; |
| | 482 | } |
| 304 | | 483 | |
| 305 | int blk_mq_request_started(struct request *rq); | | |
| 306 | int blk_mq_request_completed(struct request *rq); | | |
| 307 | void blk_mq_start_request(struct request *rq); | 484 | void blk_mq_start_request(struct request *rq); |
| 308 | void blk_mq_end_request(struct request *rq, blk_status_t error); | 485 | void blk_mq_end_request(struct request *rq, blk_status_t error); |
| 309 | void __blk_mq_end_request(struct request *rq, blk_status_t error); | 486 | void __blk_mq_end_request(struct request *rq, blk_status_t error); |
| @@ -324,7 +501,7 @@ void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); · 14 unmodified lines |
| 324 | void blk_mq_quiesce_queue(struct request_queue *q); | 501 | void blk_mq_quiesce_queue(struct request_queue *q); |
| 325 | void blk_mq_unquiesce_queue(struct request_queue *q); | 502 | void blk_mq_unquiesce_queue(struct request_queue *q); |
| 326 | void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); | 503 | void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); |
| 327 | bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); | 504 | void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); |
| 328 | void blk_mq_run_hw_queues(struct request_queue *q, bool async); | 505 | void blk_mq_run_hw_queues(struct request_queue *q, bool async); |
| 329 | void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, | 506 | void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, |
| 330 | busy_tag_iter_fn *fn, void *priv); | 507 | busy_tag_iter_fn *fn, void *priv); |
| @@ -343,14 +520,29 @@ void blk_mq_quiesce_queue_nowait(struct request_queue *q); · 12 unmodified lines |
| 343 | | 520 | |
| 344 | unsigned int blk_mq_rq_cpu(struct request *rq); | 521 | unsigned int blk_mq_rq_cpu(struct request *rq); |
| 345 | | 522 | |
| 346 | /* | 523 | /** |
| | 524 | * blk_mq_rq_from_pdu - cast a PDU to a request |
| | 525 | * @pdu: the PDU (Protocol Data Unit) to be casted |
| | 526 | * |
| | 527 | * Return: request |
| | 528 | * |
| 347 | * Driver command data is immediately after the request. So subtract request | 529 | * Driver command data is immediately after the request. So subtract request |
| 348 | * size to get back to the original request, add request size to get the PDU. | 530 | * size to get back to the original request. |
| 349 | */ | 531 | */ |
| 350 | static inline struct request *blk_mq_rq_from_pdu(void *pdu) | 532 | static inline struct request *blk_mq_rq_from_pdu(void *pdu) |
| 351 | { | 533 | { |
| 352 | return pdu - sizeof(struct request); | 534 | return pdu - sizeof(struct request); |
| 353 | } | 535 | } |
| | 536 | |
| | 537 | /** |
| | 538 | * blk_mq_rq_to_pdu - cast a request to a PDU |
| | 539 | * @rq: the request to be casted |
| | 540 | * |
| | 541 | * Return: pointer to the PDU |
| | 542 | * |
| | 543 | * Driver command data is immediately after the request. So add request to get |
| | 544 | * the PDU. |
| | 545 | */ |
| 354 | static inline void *blk_mq_rq_to_pdu(struct request *rq) | 546 | static inline void *blk_mq_rq_to_pdu(struct request *rq) |
| 355 | { | 547 | { |
| 356 | return rq + 1; | 548 | return rq + 1; |