Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : /*
3 : * Portions Copyright (C) 1992 Drew Eckhardt
4 : */
5 : #ifndef _LINUX_BLKDEV_H
6 : #define _LINUX_BLKDEV_H
7 :
8 : #include <linux/types.h>
9 : #include <linux/blk_types.h>
10 : #include <linux/device.h>
11 : #include <linux/list.h>
12 : #include <linux/llist.h>
13 : #include <linux/minmax.h>
14 : #include <linux/timer.h>
15 : #include <linux/workqueue.h>
16 : #include <linux/wait.h>
17 : #include <linux/bio.h>
18 : #include <linux/gfp.h>
19 : #include <linux/kdev_t.h>
20 : #include <linux/rcupdate.h>
21 : #include <linux/percpu-refcount.h>
22 : #include <linux/blkzoned.h>
23 : #include <linux/sched.h>
24 : #include <linux/sbitmap.h>
25 : #include <linux/uuid.h>
26 : #include <linux/xarray.h>
27 :
28 : struct module;
29 : struct request_queue;
30 : struct elevator_queue;
31 : struct blk_trace;
32 : struct request;
33 : struct sg_io_hdr;
34 : struct blkcg_gq;
35 : struct blk_flush_queue;
36 : struct kiocb;
37 : struct pr_ops;
38 : struct rq_qos;
39 : struct blk_queue_stats;
40 : struct blk_stat_callback;
41 : struct blk_crypto_profile;
42 :
43 : extern const struct device_type disk_type;
44 : extern const struct device_type part_type;
45 : extern struct class block_class;
46 :
47 : /*
48 : * Maximum number of blkcg policies allowed to be registered concurrently.
49 : * Defined here to simplify include dependency.
50 : */
51 : #define BLKCG_MAX_POLS 6
52 :
53 : #define DISK_MAX_PARTS 256
54 : #define DISK_NAME_LEN 32
55 :
56 : #define PARTITION_META_INFO_VOLNAMELTH 64
57 : /*
58 : * Enough for the string representation of any kind of UUID plus NULL.
59 : * EFI UUID is 36 characters. MSDOS UUID is 11 characters.
60 : */
61 : #define PARTITION_META_INFO_UUIDLTH (UUID_STRING_LEN + 1)
62 :
63 : struct partition_meta_info {
64 : char uuid[PARTITION_META_INFO_UUIDLTH];
65 : u8 volname[PARTITION_META_INFO_VOLNAMELTH];
66 : };
67 :
68 : /**
69 : * DOC: genhd capability flags
70 : *
71 : * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to
72 : * removable media. When set, the device remains present even when media is not
73 : * inserted. Shall not be set for devices which are removed entirely when the
74 : * media is removed.
75 : *
76 : * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events,
77 : * doesn't appear in sysfs, and can't be opened from userspace or using
78 : * blkdev_get*. Used for the underlying components of multipath devices.
79 : *
80 : * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not
81 : * scan for partitions from add_disk, and users can't add partitions manually.
82 : *
83 : */
84 : enum {
85 : GENHD_FL_REMOVABLE = 1 << 0,
86 : GENHD_FL_HIDDEN = 1 << 1,
87 : GENHD_FL_NO_PART = 1 << 2,
88 : };
89 :
90 : enum {
91 : DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */
92 : DISK_EVENT_EJECT_REQUEST = 1 << 1, /* eject requested */
93 : };
94 :
95 : enum {
96 : /* Poll even if events_poll_msecs is unset */
97 : DISK_EVENT_FLAG_POLL = 1 << 0,
98 : /* Forward events to udev */
99 : DISK_EVENT_FLAG_UEVENT = 1 << 1,
100 : /* Block event polling when open for exclusive write */
101 : DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2,
102 : };
103 :
104 : struct disk_events;
105 : struct badblocks;
106 :
107 : struct blk_integrity {
108 : const struct blk_integrity_profile *profile;
109 : unsigned char flags;
110 : unsigned char tuple_size;
111 : unsigned char interval_exp;
112 : unsigned char tag_size;
113 : };
114 :
115 : typedef unsigned int __bitwise blk_mode_t;
116 :
117 : /* open for reading */
118 : #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0))
119 : /* open for writing */
120 : #define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1))
121 : /* open exclusively (vs other exclusive openers */
122 : #define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2))
123 : /* opened with O_NDELAY */
124 : #define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3))
125 : /* open for "writes" only for ioctls (specialy hack for floppy.c) */
126 : #define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4))
127 :
128 : struct gendisk {
129 : /*
130 : * major/first_minor/minors should not be set by any new driver, the
131 : * block core will take care of allocating them automatically.
132 : */
133 : int major;
134 : int first_minor;
135 : int minors;
136 :
137 : char disk_name[DISK_NAME_LEN]; /* name of major driver */
138 :
139 : unsigned short events; /* supported events */
140 : unsigned short event_flags; /* flags related to event processing */
141 :
142 : struct xarray part_tbl;
143 : struct block_device *part0;
144 :
145 : const struct block_device_operations *fops;
146 : struct request_queue *queue;
147 : void *private_data;
148 :
149 : struct bio_set bio_split;
150 :
151 : int flags;
152 : unsigned long state;
153 : #define GD_NEED_PART_SCAN 0
154 : #define GD_READ_ONLY 1
155 : #define GD_DEAD 2
156 : #define GD_NATIVE_CAPACITY 3
157 : #define GD_ADDED 4
158 : #define GD_SUPPRESS_PART_SCAN 5
159 : #define GD_OWNS_QUEUE 6
160 :
161 : struct mutex open_mutex; /* open/close mutex */
162 : unsigned open_partitions; /* number of open partitions */
163 :
164 : struct backing_dev_info *bdi;
165 : struct kobject queue_kobj; /* the queue/ directory */
166 : struct kobject *slave_dir;
167 : #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
168 : struct list_head slave_bdevs;
169 : #endif
170 : struct timer_rand_state *random;
171 : atomic_t sync_io; /* RAID */
172 : struct disk_events *ev;
173 :
174 : #ifdef CONFIG_BLK_DEV_ZONED
175 : /*
176 : * Zoned block device information for request dispatch control.
177 : * nr_zones is the total number of zones of the device. This is always
178 : * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
179 : * bits which indicates if a zone is conventional (bit set) or
180 : * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
181 : * bits which indicates if a zone is write locked, that is, if a write
182 : * request targeting the zone was dispatched.
183 : *
184 : * Reads of this information must be protected with blk_queue_enter() /
185 : * blk_queue_exit(). Modifying this information is only allowed while
186 : * no requests are being processed. See also blk_mq_freeze_queue() and
187 : * blk_mq_unfreeze_queue().
188 : */
189 : unsigned int nr_zones;
190 : unsigned int max_open_zones;
191 : unsigned int max_active_zones;
192 : unsigned long *conv_zones_bitmap;
193 : unsigned long *seq_zones_wlock;
194 : #endif /* CONFIG_BLK_DEV_ZONED */
195 :
196 : #if IS_ENABLED(CONFIG_CDROM)
197 : struct cdrom_device_info *cdi;
198 : #endif
199 : int node_id;
200 : struct badblocks *bb;
201 : struct lockdep_map lockdep_map;
202 : u64 diskseq;
203 : blk_mode_t open_mode;
204 :
205 : /*
206 : * Independent sector access ranges. This is always NULL for
207 : * devices that do not have multiple independent access ranges.
208 : */
209 : struct blk_independent_access_ranges *ia_ranges;
210 : };
211 :
212 : static inline bool disk_live(struct gendisk *disk)
213 : {
214 : return !inode_unhashed(disk->part0->bd_inode);
215 : }
216 :
217 : /**
218 : * disk_openers - returns how many openers are there for a disk
219 : * @disk: disk to check
220 : *
221 : * This returns the number of openers for a disk. Note that this value is only
222 : * stable if disk->open_mutex is held.
223 : *
224 : * Note: Due to a quirk in the block layer open code, each open partition is
225 : * only counted once even if there are multiple openers.
226 : */
227 : static inline unsigned int disk_openers(struct gendisk *disk)
228 : {
229 : return atomic_read(&disk->part0->bd_openers);
230 : }
231 :
232 : /*
233 : * The gendisk is refcounted by the part0 block_device, and the bd_device
234 : * therein is also used for device model presentation in sysfs.
235 : */
236 : #define dev_to_disk(device) \
237 : (dev_to_bdev(device)->bd_disk)
238 : #define disk_to_dev(disk) \
239 : (&((disk)->part0->bd_device))
240 :
241 : #if IS_REACHABLE(CONFIG_CDROM)
242 : #define disk_to_cdi(disk) ((disk)->cdi)
243 : #else
244 : #define disk_to_cdi(disk) NULL
245 : #endif
246 :
247 : static inline dev_t disk_devt(struct gendisk *disk)
248 : {
249 : return MKDEV(disk->major, disk->first_minor);
250 : }
251 :
252 : static inline int blk_validate_block_size(unsigned long bsize)
253 : {
254 : if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize))
255 : return -EINVAL;
256 :
257 : return 0;
258 : }
259 :
260 : static inline bool blk_op_is_passthrough(blk_opf_t op)
261 : {
262 : op &= REQ_OP_MASK;
263 : return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
264 : }
265 :
266 : /*
267 : * Zoned block device models (zoned limit).
268 : *
269 : * Note: This needs to be ordered from the least to the most severe
270 : * restrictions for the inheritance in blk_stack_limits() to work.
271 : */
272 : enum blk_zoned_model {
273 : BLK_ZONED_NONE = 0, /* Regular block device */
274 : BLK_ZONED_HA, /* Host-aware zoned block device */
275 : BLK_ZONED_HM, /* Host-managed zoned block device */
276 : };
277 :
278 : /*
279 : * BLK_BOUNCE_NONE: never bounce (default)
280 : * BLK_BOUNCE_HIGH: bounce all highmem pages
281 : */
282 : enum blk_bounce {
283 : BLK_BOUNCE_NONE,
284 : BLK_BOUNCE_HIGH,
285 : };
286 :
287 : struct queue_limits {
288 : enum blk_bounce bounce;
289 : unsigned long seg_boundary_mask;
290 : unsigned long virt_boundary_mask;
291 :
292 : unsigned int max_hw_sectors;
293 : unsigned int max_dev_sectors;
294 : unsigned int chunk_sectors;
295 : unsigned int max_sectors;
296 : unsigned int max_user_sectors;
297 : unsigned int max_segment_size;
298 : unsigned int physical_block_size;
299 : unsigned int logical_block_size;
300 : unsigned int alignment_offset;
301 : unsigned int io_min;
302 : unsigned int io_opt;
303 : unsigned int max_discard_sectors;
304 : unsigned int max_hw_discard_sectors;
305 : unsigned int max_secure_erase_sectors;
306 : unsigned int max_write_zeroes_sectors;
307 : unsigned int max_zone_append_sectors;
308 : unsigned int discard_granularity;
309 : unsigned int discard_alignment;
310 : unsigned int zone_write_granularity;
311 :
312 : unsigned short max_segments;
313 : unsigned short max_integrity_segments;
314 : unsigned short max_discard_segments;
315 :
316 : unsigned char misaligned;
317 : unsigned char discard_misaligned;
318 : unsigned char raid_partial_stripes_expensive;
319 : enum blk_zoned_model zoned;
320 :
321 : /*
322 : * Drivers that set dma_alignment to less than 511 must be prepared to
323 : * handle individual bvec's that are not a multiple of a SECTOR_SIZE
324 : * due to possible offsets.
325 : */
326 : unsigned int dma_alignment;
327 : };
328 :
329 : typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
330 : void *data);
331 :
332 : void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
333 :
334 : #ifdef CONFIG_BLK_DEV_ZONED
335 : #define BLK_ALL_ZONES ((unsigned int)-1)
336 : int blkdev_report_zones(struct block_device *bdev, sector_t sector,
337 : unsigned int nr_zones, report_zones_cb cb, void *data);
338 : unsigned int bdev_nr_zones(struct block_device *bdev);
339 : extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
340 : sector_t sectors, sector_t nr_sectors,
341 : gfp_t gfp_mask);
342 : int blk_revalidate_disk_zones(struct gendisk *disk,
343 : void (*update_driver_data)(struct gendisk *disk));
344 : #else /* CONFIG_BLK_DEV_ZONED */
345 : static inline unsigned int bdev_nr_zones(struct block_device *bdev)
346 : {
347 : return 0;
348 : }
349 : #endif /* CONFIG_BLK_DEV_ZONED */
350 :
351 : /*
352 : * Independent access ranges: struct blk_independent_access_range describes
353 : * a range of contiguous sectors that can be accessed using device command
354 : * execution resources that are independent from the resources used for
355 : * other access ranges. This is typically found with single-LUN multi-actuator
356 : * HDDs where each access range is served by a different set of heads.
357 : * The set of independent ranges supported by the device is defined using
358 : * struct blk_independent_access_ranges. The independent ranges must not overlap
359 : * and must include all sectors within the disk capacity (no sector holes
360 : * allowed).
361 : * For a device with multiple ranges, requests targeting sectors in different
362 : * ranges can be executed in parallel. A request can straddle an access range
363 : * boundary.
364 : */
365 : struct blk_independent_access_range {
366 : struct kobject kobj;
367 : sector_t sector;
368 : sector_t nr_sectors;
369 : };
370 :
371 : struct blk_independent_access_ranges {
372 : struct kobject kobj;
373 : bool sysfs_registered;
374 : unsigned int nr_ia_ranges;
375 : struct blk_independent_access_range ia_range[];
376 : };
377 :
378 : struct request_queue {
379 : struct request *last_merge;
380 : struct elevator_queue *elevator;
381 :
382 : struct percpu_ref q_usage_counter;
383 :
384 : struct blk_queue_stats *stats;
385 : struct rq_qos *rq_qos;
386 : struct mutex rq_qos_mutex;
387 :
388 : const struct blk_mq_ops *mq_ops;
389 :
390 : /* sw queues */
391 : struct blk_mq_ctx __percpu *queue_ctx;
392 :
393 : unsigned int queue_depth;
394 :
395 : /* hw dispatch queues */
396 : struct xarray hctx_table;
397 : unsigned int nr_hw_queues;
398 :
399 : /*
400 : * The queue owner gets to use this for whatever they like.
401 : * ll_rw_blk doesn't touch it.
402 : */
403 : void *queuedata;
404 :
405 : /*
406 : * various queue flags, see QUEUE_* below
407 : */
408 : unsigned long queue_flags;
409 : /*
410 : * Number of contexts that have called blk_set_pm_only(). If this
411 : * counter is above zero then only RQF_PM requests are processed.
412 : */
413 : atomic_t pm_only;
414 :
415 : /*
416 : * ida allocated id for this queue. Used to index queues from
417 : * ioctx.
418 : */
419 : int id;
420 :
421 : spinlock_t queue_lock;
422 :
423 : struct gendisk *disk;
424 :
425 : refcount_t refs;
426 :
427 : /*
428 : * mq queue kobject
429 : */
430 : struct kobject *mq_kobj;
431 :
432 : #ifdef CONFIG_BLK_DEV_INTEGRITY
433 : struct blk_integrity integrity;
434 : #endif /* CONFIG_BLK_DEV_INTEGRITY */
435 :
436 : #ifdef CONFIG_PM
437 : struct device *dev;
438 : enum rpm_status rpm_status;
439 : #endif
440 :
441 : /*
442 : * queue settings
443 : */
444 : unsigned long nr_requests; /* Max # of requests */
445 :
446 : unsigned int dma_pad_mask;
447 :
448 : #ifdef CONFIG_BLK_INLINE_ENCRYPTION
449 : struct blk_crypto_profile *crypto_profile;
450 : struct kobject *crypto_kobject;
451 : #endif
452 :
453 : unsigned int rq_timeout;
454 :
455 : struct timer_list timeout;
456 : struct work_struct timeout_work;
457 :
458 : atomic_t nr_active_requests_shared_tags;
459 :
460 : struct blk_mq_tags *sched_shared_tags;
461 :
462 : struct list_head icq_list;
463 : #ifdef CONFIG_BLK_CGROUP
464 : DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
465 : struct blkcg_gq *root_blkg;
466 : struct list_head blkg_list;
467 : struct mutex blkcg_mutex;
468 : #endif
469 :
470 : struct queue_limits limits;
471 :
472 : unsigned int required_elevator_features;
473 :
474 : int node;
475 : #ifdef CONFIG_BLK_DEV_IO_TRACE
476 : struct blk_trace __rcu *blk_trace;
477 : #endif
478 : /*
479 : * for flush operations
480 : */
481 : struct blk_flush_queue *fq;
482 : struct list_head flush_list;
483 :
484 : struct list_head requeue_list;
485 : spinlock_t requeue_lock;
486 : struct delayed_work requeue_work;
487 :
488 : struct mutex sysfs_lock;
489 : struct mutex sysfs_dir_lock;
490 :
491 : /*
492 : * for reusing dead hctx instance in case of updating
493 : * nr_hw_queues
494 : */
495 : struct list_head unused_hctx_list;
496 : spinlock_t unused_hctx_lock;
497 :
498 : int mq_freeze_depth;
499 :
500 : #ifdef CONFIG_BLK_DEV_THROTTLING
501 : /* Throttle data */
502 : struct throtl_data *td;
503 : #endif
504 : struct rcu_head rcu_head;
505 : wait_queue_head_t mq_freeze_wq;
506 : /*
507 : * Protect concurrent access to q_usage_counter by
508 : * percpu_ref_kill() and percpu_ref_reinit().
509 : */
510 : struct mutex mq_freeze_lock;
511 :
512 : int quiesce_depth;
513 :
514 : struct blk_mq_tag_set *tag_set;
515 : struct list_head tag_set_list;
516 :
517 : struct dentry *debugfs_dir;
518 : struct dentry *sched_debugfs_dir;
519 : struct dentry *rqos_debugfs_dir;
520 : /*
521 : * Serializes all debugfs metadata operations using the above dentries.
522 : */
523 : struct mutex debugfs_mutex;
524 :
525 : bool mq_sysfs_init_done;
526 : };
527 :
528 : /* Keep blk_queue_flag_name[] in sync with the definitions below */
529 : #define QUEUE_FLAG_STOPPED 0 /* queue is stopped */
530 : #define QUEUE_FLAG_DYING 1 /* queue being torn down */
531 : #define QUEUE_FLAG_NOMERGES 3 /* disable merge attempts */
532 : #define QUEUE_FLAG_SAME_COMP 4 /* complete on same CPU-group */
533 : #define QUEUE_FLAG_FAIL_IO 5 /* fake timeout */
534 : #define QUEUE_FLAG_NONROT 6 /* non-rotational device (SSD) */
535 : #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
536 : #define QUEUE_FLAG_IO_STAT 7 /* do disk/partitions IO accounting */
537 : #define QUEUE_FLAG_NOXMERGES 9 /* No extended merges */
538 : #define QUEUE_FLAG_ADD_RANDOM 10 /* Contributes to random pool */
539 : #define QUEUE_FLAG_SYNCHRONOUS 11 /* always completes in submit context */
540 : #define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */
541 : #define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */
542 : #define QUEUE_FLAG_STABLE_WRITES 15 /* don't modify blks until WB is done */
543 : #define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */
544 : #define QUEUE_FLAG_WC 17 /* Write back caching */
545 : #define QUEUE_FLAG_FUA 18 /* device supports FUA writes */
546 : #define QUEUE_FLAG_DAX 19 /* device supports DAX */
547 : #define QUEUE_FLAG_STATS 20 /* track IO start and completion times */
548 : #define QUEUE_FLAG_REGISTERED 22 /* queue has been registered to a disk */
549 : #define QUEUE_FLAG_QUIESCED 24 /* queue has been quiesced */
550 : #define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */
551 : #define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */
552 : #define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */
553 : #define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */
554 : #define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */
555 : #define QUEUE_FLAG_SQ_SCHED 30 /* single queue style io dispatch */
556 : #define QUEUE_FLAG_SKIP_TAGSET_QUIESCE 31 /* quiesce_tagset skip the queue*/
557 :
558 : #define QUEUE_FLAG_MQ_DEFAULT ((1UL << QUEUE_FLAG_IO_STAT) | \
559 : (1UL << QUEUE_FLAG_SAME_COMP) | \
560 : (1UL << QUEUE_FLAG_NOWAIT))
561 :
562 : void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
563 : void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
564 : bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
565 :
566 : #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
567 : #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
568 : #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
569 : #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
570 : #define blk_queue_noxmerges(q) \
571 : test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
572 : #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
573 : #define blk_queue_stable_writes(q) \
574 : test_bit(QUEUE_FLAG_STABLE_WRITES, &(q)->queue_flags)
575 : #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
576 : #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
577 : #define blk_queue_zone_resetall(q) \
578 : test_bit(QUEUE_FLAG_ZONE_RESETALL, &(q)->queue_flags)
579 : #define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
580 : #define blk_queue_pci_p2pdma(q) \
581 : test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
582 : #ifdef CONFIG_BLK_RQ_ALLOC_TIME
583 : #define blk_queue_rq_alloc_time(q) \
584 : test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
585 : #else
586 : #define blk_queue_rq_alloc_time(q) false
587 : #endif
588 :
589 : #define blk_noretry_request(rq) \
590 : ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
591 : REQ_FAILFAST_DRIVER))
592 : #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
593 : #define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
594 : #define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
595 : #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
596 : #define blk_queue_skip_tagset_quiesce(q) \
597 : test_bit(QUEUE_FLAG_SKIP_TAGSET_QUIESCE, &(q)->queue_flags)
598 :
599 : extern void blk_set_pm_only(struct request_queue *q);
600 : extern void blk_clear_pm_only(struct request_queue *q);
601 :
602 : #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
603 :
604 : #define dma_map_bvec(dev, bv, dir, attrs) \
605 : dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
606 : (dir), (attrs))
607 :
608 : static inline bool queue_is_mq(struct request_queue *q)
609 : {
610 : return q->mq_ops;
611 : }
612 :
613 : #ifdef CONFIG_PM
614 : static inline enum rpm_status queue_rpm_status(struct request_queue *q)
615 : {
616 : return q->rpm_status;
617 : }
618 : #else
619 : static inline enum rpm_status queue_rpm_status(struct request_queue *q)
620 : {
621 : return RPM_ACTIVE;
622 : }
623 : #endif
624 :
625 : static inline enum blk_zoned_model
626 : blk_queue_zoned_model(struct request_queue *q)
627 : {
628 1713718 : if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
629 1713718 : return q->limits.zoned;
630 : return BLK_ZONED_NONE;
631 : }
632 :
633 : static inline bool blk_queue_is_zoned(struct request_queue *q)
634 : {
635 1710502 : switch (blk_queue_zoned_model(q)) {
636 : case BLK_ZONED_HA:
637 : case BLK_ZONED_HM:
638 : return true;
639 : default:
640 1710502 : return false;
641 : }
642 : }
643 :
644 : #ifdef CONFIG_BLK_DEV_ZONED
645 : static inline unsigned int disk_nr_zones(struct gendisk *disk)
646 : {
647 : return blk_queue_is_zoned(disk->queue) ? disk->nr_zones : 0;
648 : }
649 :
650 : static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
651 : {
652 : if (!blk_queue_is_zoned(disk->queue))
653 : return 0;
654 : return sector >> ilog2(disk->queue->limits.chunk_sectors);
655 : }
656 :
657 : static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector)
658 : {
659 : if (!blk_queue_is_zoned(disk->queue))
660 : return false;
661 : if (!disk->conv_zones_bitmap)
662 : return true;
663 : return !test_bit(disk_zone_no(disk, sector), disk->conv_zones_bitmap);
664 : }
665 :
666 : static inline void disk_set_max_open_zones(struct gendisk *disk,
667 : unsigned int max_open_zones)
668 : {
669 : disk->max_open_zones = max_open_zones;
670 : }
671 :
672 : static inline void disk_set_max_active_zones(struct gendisk *disk,
673 : unsigned int max_active_zones)
674 : {
675 : disk->max_active_zones = max_active_zones;
676 : }
677 :
678 : static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
679 : {
680 : return bdev->bd_disk->max_open_zones;
681 : }
682 :
683 : static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
684 : {
685 0 : return bdev->bd_disk->max_active_zones;
686 : }
687 :
688 : #else /* CONFIG_BLK_DEV_ZONED */
689 : static inline unsigned int disk_nr_zones(struct gendisk *disk)
690 : {
691 : return 0;
692 : }
693 : static inline bool disk_zone_is_seq(struct gendisk *disk, sector_t sector)
694 : {
695 : return false;
696 : }
697 : static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
698 : {
699 : return 0;
700 : }
701 : static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
702 : {
703 : return 0;
704 : }
705 :
706 : static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
707 : {
708 : return 0;
709 : }
710 : #endif /* CONFIG_BLK_DEV_ZONED */
711 :
712 : static inline unsigned int blk_queue_depth(struct request_queue *q)
713 : {
714 : if (q->queue_depth)
715 : return q->queue_depth;
716 :
717 : return q->nr_requests;
718 : }
719 :
720 : /*
721 : * default timeout for SG_IO if none specified
722 : */
723 : #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
724 : #define BLK_MIN_SG_TIMEOUT (7 * HZ)
725 :
726 : /* This should not be used directly - use rq_for_each_segment */
727 : #define for_each_bio(_bio) \
728 : for (; _bio; _bio = _bio->bi_next)
729 :
730 : int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
731 : const struct attribute_group **groups);
732 : static inline int __must_check add_disk(struct gendisk *disk)
733 : {
734 : return device_add_disk(NULL, disk, NULL);
735 : }
736 : void del_gendisk(struct gendisk *gp);
737 : void invalidate_disk(struct gendisk *disk);
738 : void set_disk_ro(struct gendisk *disk, bool read_only);
739 : void disk_uevent(struct gendisk *disk, enum kobject_action action);
740 :
741 503503 : static inline int get_disk_ro(struct gendisk *disk)
742 : {
743 1007006 : return disk->part0->bd_read_only ||
744 503503 : test_bit(GD_READ_ONLY, &disk->state);
745 : }
746 :
747 503574 : static inline int bdev_read_only(struct block_device *bdev)
748 : {
749 503574 : return bdev->bd_read_only || get_disk_ro(bdev->bd_disk);
750 : }
751 :
752 : bool set_capacity_and_notify(struct gendisk *disk, sector_t size);
753 : bool disk_force_media_change(struct gendisk *disk, unsigned int events);
754 :
755 : void add_disk_randomness(struct gendisk *disk) __latent_entropy;
756 : void rand_initialize_disk(struct gendisk *disk);
757 :
758 : static inline sector_t get_start_sect(struct block_device *bdev)
759 : {
760 : return bdev->bd_start_sect;
761 : }
762 :
763 : static inline sector_t bdev_nr_sectors(struct block_device *bdev)
764 : {
765 1058842 : return bdev->bd_nr_sectors;
766 : }
767 :
768 : static inline loff_t bdev_nr_bytes(struct block_device *bdev)
769 : {
770 1056317 : return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT;
771 : }
772 :
773 : static inline sector_t get_capacity(struct gendisk *disk)
774 : {
775 : return bdev_nr_sectors(disk->part0);
776 : }
777 :
778 2525 : static inline u64 sb_bdev_nr_blocks(struct super_block *sb)
779 : {
780 0 : return bdev_nr_sectors(sb->s_bdev) >>
781 2525 : (sb->s_blocksize_bits - SECTOR_SHIFT);
782 : }
783 :
784 : int bdev_disk_changed(struct gendisk *disk, bool invalidate);
785 :
786 : void put_disk(struct gendisk *disk);
787 : struct gendisk *__blk_alloc_disk(int node, struct lock_class_key *lkclass);
788 :
789 : /**
790 : * blk_alloc_disk - allocate a gendisk structure
791 : * @node_id: numa node to allocate on
792 : *
793 : * Allocate and pre-initialize a gendisk structure for use with BIO based
794 : * drivers.
795 : *
796 : * Context: can sleep
797 : */
798 : #define blk_alloc_disk(node_id) \
799 : ({ \
800 : static struct lock_class_key __key; \
801 : \
802 : __blk_alloc_disk(node_id, &__key); \
803 : })
804 :
805 : int __register_blkdev(unsigned int major, const char *name,
806 : void (*probe)(dev_t devt));
807 : #define register_blkdev(major, name) \
808 : __register_blkdev(major, name, NULL)
809 : void unregister_blkdev(unsigned int major, const char *name);
810 :
811 : bool disk_check_media_change(struct gendisk *disk);
812 : int __invalidate_device(struct block_device *bdev, bool kill_dirty);
813 : void set_capacity(struct gendisk *disk, sector_t size);
814 :
815 : #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
816 : int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk);
817 : void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk);
818 : #else
819 : static inline int bd_link_disk_holder(struct block_device *bdev,
820 : struct gendisk *disk)
821 : {
822 : return 0;
823 : }
824 : static inline void bd_unlink_disk_holder(struct block_device *bdev,
825 : struct gendisk *disk)
826 : {
827 : }
828 : #endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */
829 :
830 : dev_t part_devt(struct gendisk *disk, u8 partno);
831 : void inc_diskseq(struct gendisk *disk);
832 : void blk_request_module(dev_t devt);
833 :
834 : extern int blk_register_queue(struct gendisk *disk);
835 : extern void blk_unregister_queue(struct gendisk *disk);
836 : void submit_bio_noacct(struct bio *bio);
837 : struct bio *bio_split_to_limits(struct bio *bio);
838 :
839 : extern int blk_lld_busy(struct request_queue *q);
840 : extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
841 : extern void blk_queue_exit(struct request_queue *q);
842 : extern void blk_sync_queue(struct request_queue *q);
843 :
844 : /* Helper to convert REQ_OP_XXX to its string format XXX */
845 : extern const char *blk_op_str(enum req_op op);
846 :
847 : int blk_status_to_errno(blk_status_t status);
848 : blk_status_t errno_to_blk_status(int errno);
849 :
850 : /* only poll the hardware once, don't continue until a completion was found */
851 : #define BLK_POLL_ONESHOT (1 << 0)
852 : int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
853 : int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
854 : unsigned int flags);
855 :
856 : static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
857 : {
858 65165756 : return bdev->bd_queue; /* this is never NULL */
859 : }
860 :
861 : /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */
862 : const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
863 :
864 : static inline unsigned int bio_zone_no(struct bio *bio)
865 : {
866 : return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
867 : }
868 :
869 : static inline unsigned int bio_zone_is_seq(struct bio *bio)
870 : {
871 : return disk_zone_is_seq(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
872 : }
873 :
874 : /*
875 : * Return how much of the chunk is left to be used for I/O at a given offset.
876 : */
877 : static inline unsigned int blk_chunk_sectors_left(sector_t offset,
878 : unsigned int chunk_sectors)
879 : {
880 : if (unlikely(!is_power_of_2(chunk_sectors)))
881 : return chunk_sectors - sector_div(offset, chunk_sectors);
882 : return chunk_sectors - (offset & (chunk_sectors - 1));
883 : }
884 :
885 : /*
886 : * Access functions for manipulating queue properties
887 : */
888 : void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce limit);
889 : extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
890 : extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
891 : extern void blk_queue_max_segments(struct request_queue *, unsigned short);
892 : extern void blk_queue_max_discard_segments(struct request_queue *,
893 : unsigned short);
894 : void blk_queue_max_secure_erase_sectors(struct request_queue *q,
895 : unsigned int max_sectors);
896 : extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
897 : extern void blk_queue_max_discard_sectors(struct request_queue *q,
898 : unsigned int max_discard_sectors);
899 : extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
900 : unsigned int max_write_same_sectors);
901 : extern void blk_queue_logical_block_size(struct request_queue *, unsigned int);
902 : extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
903 : unsigned int max_zone_append_sectors);
904 : extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
905 : void blk_queue_zone_write_granularity(struct request_queue *q,
906 : unsigned int size);
907 : extern void blk_queue_alignment_offset(struct request_queue *q,
908 : unsigned int alignment);
909 : void disk_update_readahead(struct gendisk *disk);
910 : extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
911 : extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
912 : extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
913 : extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
914 : extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
915 : extern void blk_set_stacking_limits(struct queue_limits *lim);
916 : extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
917 : sector_t offset);
918 : extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
919 : sector_t offset);
920 : extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
921 : extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
922 : extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
923 : extern void blk_queue_dma_alignment(struct request_queue *, int);
924 : extern void blk_queue_update_dma_alignment(struct request_queue *, int);
925 : extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
926 : extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
927 :
928 : struct blk_independent_access_ranges *
929 : disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges);
930 : void disk_set_independent_access_ranges(struct gendisk *disk,
931 : struct blk_independent_access_ranges *iars);
932 :
933 : /*
934 : * Elevator features for blk_queue_required_elevator_features:
935 : */
936 : /* Supports zoned block devices sequential write constraint */
937 : #define ELEVATOR_F_ZBD_SEQ_WRITE (1U << 0)
938 :
939 : extern void blk_queue_required_elevator_features(struct request_queue *q,
940 : unsigned int features);
941 : extern bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
942 : struct device *dev);
943 :
944 : bool __must_check blk_get_queue(struct request_queue *);
945 : extern void blk_put_queue(struct request_queue *);
946 :
947 : void blk_mark_disk_dead(struct gendisk *disk);
948 :
949 : #ifdef CONFIG_BLOCK
950 : /*
951 : * blk_plug permits building a queue of related requests by holding the I/O
952 : * fragments for a short period. This allows merging of sequential requests
953 : * into single larger request. As the requests are moved from a per-task list to
954 : * the device's request_queue in a batch, this results in improved scalability
955 : * as the lock contention for request_queue lock is reduced.
956 : *
957 : * It is ok not to disable preemption when adding the request to the plug list
958 : * or when attempting a merge. For details, please see schedule() where
959 : * blk_flush_plug() is called.
960 : */
961 : struct blk_plug {
962 : struct request *mq_list; /* blk-mq requests */
963 :
964 : /* if ios_left is > 1, we can batch tag/rq allocations */
965 : struct request *cached_rq;
966 : unsigned short nr_ios;
967 :
968 : unsigned short rq_count;
969 :
970 : bool multiple_queues;
971 : bool has_elevator;
972 : bool nowait;
973 :
974 : struct list_head cb_list; /* md requires an unplug callback */
975 : };
976 :
977 : struct blk_plug_cb;
978 : typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
979 : struct blk_plug_cb {
980 : struct list_head list;
981 : blk_plug_cb_fn callback;
982 : void *data;
983 : };
984 : extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
985 : void *data, int size);
986 : extern void blk_start_plug(struct blk_plug *);
987 : extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short);
988 : extern void blk_finish_plug(struct blk_plug *);
989 :
990 : void __blk_flush_plug(struct blk_plug *plug, bool from_schedule);
991 : static inline void blk_flush_plug(struct blk_plug *plug, bool async)
992 : {
993 1347041 : if (plug)
994 1983 : __blk_flush_plug(plug, async);
995 : }
996 :
997 : int blkdev_issue_flush(struct block_device *bdev);
998 : long nr_blockdev_pages(void);
999 : #else /* CONFIG_BLOCK */
1000 : struct blk_plug {
1001 : };
1002 :
1003 : static inline void blk_start_plug_nr_ios(struct blk_plug *plug,
1004 : unsigned short nr_ios)
1005 : {
1006 : }
1007 :
1008 : static inline void blk_start_plug(struct blk_plug *plug)
1009 : {
1010 : }
1011 :
1012 : static inline void blk_finish_plug(struct blk_plug *plug)
1013 : {
1014 : }
1015 :
1016 : static inline void blk_flush_plug(struct blk_plug *plug, bool async)
1017 : {
1018 : }
1019 :
1020 : static inline int blkdev_issue_flush(struct block_device *bdev)
1021 : {
1022 : return 0;
1023 : }
1024 :
1025 : static inline long nr_blockdev_pages(void)
1026 : {
1027 : return 0;
1028 : }
1029 : #endif /* CONFIG_BLOCK */
1030 :
1031 : extern void blk_io_schedule(void);
1032 :
1033 : int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1034 : sector_t nr_sects, gfp_t gfp_mask);
1035 : int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1036 : sector_t nr_sects, gfp_t gfp_mask, struct bio **biop);
1037 : int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
1038 : sector_t nr_sects, gfp_t gfp);
1039 :
1040 : #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */
1041 : #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */
1042 :
1043 : extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1044 : sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1045 : unsigned flags);
1046 : extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1047 : sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1048 :
1049 11975 : static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1050 : sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1051 : {
1052 47900 : return blkdev_issue_discard(sb->s_bdev,
1053 11975 : block << (sb->s_blocksize_bits -
1054 : SECTOR_SHIFT),
1055 11975 : nr_blocks << (sb->s_blocksize_bits -
1056 : SECTOR_SHIFT),
1057 : gfp_mask);
1058 : }
1059 8501 : static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1060 : sector_t nr_blocks, gfp_t gfp_mask)
1061 : {
1062 34005 : return blkdev_issue_zeroout(sb->s_bdev,
1063 8501 : block << (sb->s_blocksize_bits -
1064 : SECTOR_SHIFT),
1065 8501 : nr_blocks << (sb->s_blocksize_bits -
1066 : SECTOR_SHIFT),
1067 : gfp_mask, 0);
1068 : }
1069 :
1070 : static inline bool bdev_is_partition(struct block_device *bdev)
1071 : {
1072 0 : return bdev->bd_partno;
1073 : }
1074 :
1075 : enum blk_default_limits {
1076 : BLK_MAX_SEGMENTS = 128,
1077 : BLK_SAFE_MAX_SECTORS = 255,
1078 : BLK_MAX_SEGMENT_SIZE = 65536,
1079 : BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1080 : };
1081 :
1082 : #define BLK_DEF_MAX_SECTORS 2560u
1083 :
1084 : static inline unsigned long queue_segment_boundary(const struct request_queue *q)
1085 : {
1086 : return q->limits.seg_boundary_mask;
1087 : }
1088 :
1089 : static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1090 : {
1091 : return q->limits.virt_boundary_mask;
1092 : }
1093 :
1094 : static inline unsigned int queue_max_sectors(const struct request_queue *q)
1095 : {
1096 : return q->limits.max_sectors;
1097 : }
1098 :
1099 : static inline unsigned int queue_max_bytes(struct request_queue *q)
1100 : {
1101 : return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9;
1102 : }
1103 :
1104 : static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1105 : {
1106 : return q->limits.max_hw_sectors;
1107 : }
1108 :
1109 : static inline unsigned short queue_max_segments(const struct request_queue *q)
1110 : {
1111 : return q->limits.max_segments;
1112 : }
1113 :
1114 : static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1115 : {
1116 : return q->limits.max_discard_segments;
1117 : }
1118 :
1119 : static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1120 : {
1121 : return q->limits.max_segment_size;
1122 : }
1123 :
1124 : static inline unsigned int queue_max_zone_append_sectors(const struct request_queue *q)
1125 : {
1126 :
1127 : const struct queue_limits *l = &q->limits;
1128 :
1129 : return min(l->max_zone_append_sectors, l->max_sectors);
1130 : }
1131 :
1132 : static inline unsigned int
1133 : bdev_max_zone_append_sectors(struct block_device *bdev)
1134 : {
1135 : return queue_max_zone_append_sectors(bdev_get_queue(bdev));
1136 : }
1137 :
1138 : static inline unsigned int bdev_max_segments(struct block_device *bdev)
1139 : {
1140 : return queue_max_segments(bdev_get_queue(bdev));
1141 : }
1142 :
1143 : static inline unsigned queue_logical_block_size(const struct request_queue *q)
1144 : {
1145 62840259 : int retval = 512;
1146 :
1147 62840259 : if (q && q->limits.logical_block_size)
1148 62840288 : retval = q->limits.logical_block_size;
1149 :
1150 32174958 : return retval;
1151 : }
1152 :
1153 : static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
1154 : {
1155 32174958 : return queue_logical_block_size(bdev_get_queue(bdev));
1156 : }
1157 :
1158 : static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1159 : {
1160 : return q->limits.physical_block_size;
1161 : }
1162 :
1163 : static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1164 : {
1165 : return queue_physical_block_size(bdev_get_queue(bdev));
1166 : }
1167 :
1168 : static inline unsigned int queue_io_min(const struct request_queue *q)
1169 : {
1170 : return q->limits.io_min;
1171 : }
1172 :
1173 : static inline int bdev_io_min(struct block_device *bdev)
1174 : {
1175 : return queue_io_min(bdev_get_queue(bdev));
1176 : }
1177 :
1178 : static inline unsigned int queue_io_opt(const struct request_queue *q)
1179 : {
1180 : return q->limits.io_opt;
1181 : }
1182 :
1183 : static inline int bdev_io_opt(struct block_device *bdev)
1184 : {
1185 : return queue_io_opt(bdev_get_queue(bdev));
1186 : }
1187 :
1188 : static inline unsigned int
1189 : queue_zone_write_granularity(const struct request_queue *q)
1190 : {
1191 : return q->limits.zone_write_granularity;
1192 : }
1193 :
1194 : static inline unsigned int
1195 : bdev_zone_write_granularity(struct block_device *bdev)
1196 : {
1197 : return queue_zone_write_granularity(bdev_get_queue(bdev));
1198 : }
1199 :
1200 : int bdev_alignment_offset(struct block_device *bdev);
1201 : unsigned int bdev_discard_alignment(struct block_device *bdev);
1202 :
1203 : static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev)
1204 : {
1205 75621 : return bdev_get_queue(bdev)->limits.max_discard_sectors;
1206 : }
1207 :
1208 : static inline unsigned int bdev_discard_granularity(struct block_device *bdev)
1209 : {
1210 26025 : return bdev_get_queue(bdev)->limits.discard_granularity;
1211 : }
1212 :
1213 : static inline unsigned int
1214 : bdev_max_secure_erase_sectors(struct block_device *bdev)
1215 : {
1216 : return bdev_get_queue(bdev)->limits.max_secure_erase_sectors;
1217 : }
1218 :
1219 : static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1220 : {
1221 : struct request_queue *q = bdev_get_queue(bdev);
1222 :
1223 : if (q)
1224 : return q->limits.max_write_zeroes_sectors;
1225 :
1226 : return 0;
1227 : }
1228 :
1229 : static inline bool bdev_nonrot(struct block_device *bdev)
1230 : {
1231 5759 : return blk_queue_nonrot(bdev_get_queue(bdev));
1232 : }
1233 :
1234 : static inline bool bdev_synchronous(struct block_device *bdev)
1235 : {
1236 : return test_bit(QUEUE_FLAG_SYNCHRONOUS,
1237 : &bdev_get_queue(bdev)->queue_flags);
1238 : }
1239 :
1240 : static inline bool bdev_stable_writes(struct block_device *bdev)
1241 : {
1242 62769 : return test_bit(QUEUE_FLAG_STABLE_WRITES,
1243 : &bdev_get_queue(bdev)->queue_flags);
1244 : }
1245 :
1246 : static inline bool bdev_write_cache(struct block_device *bdev)
1247 : {
1248 442594 : return test_bit(QUEUE_FLAG_WC, &bdev_get_queue(bdev)->queue_flags);
1249 : }
1250 :
1251 : static inline bool bdev_fua(struct block_device *bdev)
1252 : {
1253 0 : return test_bit(QUEUE_FLAG_FUA, &bdev_get_queue(bdev)->queue_flags);
1254 : }
1255 :
1256 : static inline bool bdev_nowait(struct block_device *bdev)
1257 : {
1258 : return test_bit(QUEUE_FLAG_NOWAIT, &bdev_get_queue(bdev)->queue_flags);
1259 : }
1260 :
1261 : static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
1262 : {
1263 3216 : return blk_queue_zoned_model(bdev_get_queue(bdev));
1264 : }
1265 :
1266 : static inline bool bdev_is_zoned(struct block_device *bdev)
1267 : {
1268 1710502 : return blk_queue_is_zoned(bdev_get_queue(bdev));
1269 : }
1270 :
1271 : static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec)
1272 : {
1273 : return disk_zone_no(bdev->bd_disk, sec);
1274 : }
1275 :
1276 : /* Whether write serialization is required for @op on zoned devices. */
1277 : static inline bool op_needs_zoned_write_locking(enum req_op op)
1278 : {
1279 : return op == REQ_OP_WRITE || op == REQ_OP_WRITE_ZEROES;
1280 : }
1281 :
1282 : static inline bool bdev_op_is_zoned_write(struct block_device *bdev,
1283 : enum req_op op)
1284 : {
1285 : return bdev_is_zoned(bdev) && op_needs_zoned_write_locking(op);
1286 : }
1287 :
1288 : static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1289 : {
1290 0 : struct request_queue *q = bdev_get_queue(bdev);
1291 :
1292 0 : if (!blk_queue_is_zoned(q))
1293 : return 0;
1294 0 : return q->limits.chunk_sectors;
1295 : }
1296 :
1297 : static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev,
1298 : sector_t sector)
1299 : {
1300 : return sector & (bdev_zone_sectors(bdev) - 1);
1301 : }
1302 :
1303 : static inline bool bdev_is_zone_start(struct block_device *bdev,
1304 : sector_t sector)
1305 : {
1306 : return bdev_offset_from_zone_start(bdev, sector) == 0;
1307 : }
1308 :
1309 : static inline int queue_dma_alignment(const struct request_queue *q)
1310 : {
1311 30665301 : return q ? q->limits.dma_alignment : 511;
1312 : }
1313 :
1314 : static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
1315 : {
1316 61330602 : return queue_dma_alignment(bdev_get_queue(bdev));
1317 : }
1318 :
1319 30665301 : static inline bool bdev_iter_is_aligned(struct block_device *bdev,
1320 : struct iov_iter *iter)
1321 : {
1322 91995903 : return iov_iter_is_aligned(iter, bdev_dma_alignment(bdev),
1323 : bdev_logical_block_size(bdev) - 1);
1324 : }
1325 :
1326 : static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1327 : unsigned int len)
1328 : {
1329 : unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1330 : return !(addr & alignment) && !(len & alignment);
1331 : }
1332 :
1333 : /* assumes size > 256 */
1334 3216 : static inline unsigned int blksize_bits(unsigned int size)
1335 : {
1336 1043039 : return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT;
1337 : }
1338 :
1339 122884 : static inline unsigned int block_size(struct block_device *bdev)
1340 : {
1341 122884 : return 1 << bdev->bd_inode->i_blkbits;
1342 : }
1343 :
1344 : int kblockd_schedule_work(struct work_struct *work);
1345 : int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1346 :
1347 : #define MODULE_ALIAS_BLOCKDEV(major,minor) \
1348 : MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1349 : #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1350 : MODULE_ALIAS("block-major-" __stringify(major) "-*")
1351 :
1352 : #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1353 :
1354 : bool blk_crypto_register(struct blk_crypto_profile *profile,
1355 : struct request_queue *q);
1356 :
1357 : #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1358 :
1359 : static inline bool blk_crypto_register(struct blk_crypto_profile *profile,
1360 : struct request_queue *q)
1361 : {
1362 : return true;
1363 : }
1364 :
1365 : #endif /* CONFIG_BLK_INLINE_ENCRYPTION */
1366 :
1367 : enum blk_unique_id {
1368 : /* these match the Designator Types specified in SPC */
1369 : BLK_UID_T10 = 1,
1370 : BLK_UID_EUI64 = 2,
1371 : BLK_UID_NAA = 3,
1372 : };
1373 :
1374 : struct block_device_operations {
1375 : void (*submit_bio)(struct bio *bio);
1376 : int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob,
1377 : unsigned int flags);
1378 : int (*open)(struct gendisk *disk, blk_mode_t mode);
1379 : void (*release)(struct gendisk *disk);
1380 : int (*ioctl)(struct block_device *bdev, blk_mode_t mode,
1381 : unsigned cmd, unsigned long arg);
1382 : int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode,
1383 : unsigned cmd, unsigned long arg);
1384 : unsigned int (*check_events) (struct gendisk *disk,
1385 : unsigned int clearing);
1386 : void (*unlock_native_capacity) (struct gendisk *);
1387 : int (*getgeo)(struct block_device *, struct hd_geometry *);
1388 : int (*set_read_only)(struct block_device *bdev, bool ro);
1389 : void (*free_disk)(struct gendisk *disk);
1390 : /* this callback is with swap_lock and sometimes page table lock held */
1391 : void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1392 : int (*report_zones)(struct gendisk *, sector_t sector,
1393 : unsigned int nr_zones, report_zones_cb cb, void *data);
1394 : char *(*devnode)(struct gendisk *disk, umode_t *mode);
1395 : /* returns the length of the identifier or a negative errno: */
1396 : int (*get_unique_id)(struct gendisk *disk, u8 id[16],
1397 : enum blk_unique_id id_type);
1398 : struct module *owner;
1399 : const struct pr_ops *pr_ops;
1400 :
1401 : /*
1402 : * Special callback for probing GPT entry at a given sector.
1403 : * Needed by Android devices, used by GPT scanner and MMC blk
1404 : * driver.
1405 : */
1406 : int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector);
1407 : };
1408 :
1409 : #ifdef CONFIG_COMPAT
1410 : extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t,
1411 : unsigned int, unsigned long);
1412 : #else
1413 : #define blkdev_compat_ptr_ioctl NULL
1414 : #endif
1415 :
1416 16853836 : static inline void blk_wake_io_task(struct task_struct *waiter)
1417 : {
1418 : /*
1419 : * If we're polling, the task itself is doing the completions. For
1420 : * that case, we don't need to signal a wakeup, it's enough to just
1421 : * mark us as RUNNING.
1422 : */
1423 16853836 : if (waiter == current)
1424 11 : __set_current_state(TASK_RUNNING);
1425 : else
1426 16853825 : wake_up_process(waiter);
1427 16853835 : }
1428 :
1429 : unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
1430 : unsigned long start_time);
1431 : void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
1432 : unsigned int sectors, unsigned long start_time);
1433 :
1434 : unsigned long bio_start_io_acct(struct bio *bio);
1435 : void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1436 : struct block_device *orig_bdev);
1437 :
1438 : /**
1439 : * bio_end_io_acct - end I/O accounting for bio based drivers
1440 : * @bio: bio to end account for
1441 : * @start_time: start time returned by bio_start_io_acct()
1442 : */
1443 : static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
1444 : {
1445 : return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
1446 : }
1447 :
1448 : int bdev_read_only(struct block_device *bdev);
1449 : int set_blocksize(struct block_device *bdev, int size);
1450 :
1451 : int lookup_bdev(const char *pathname, dev_t *dev);
1452 :
1453 : void blkdev_show(struct seq_file *seqf, off_t offset);
1454 :
1455 : #define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */
1456 : #define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */
1457 : #ifdef CONFIG_BLOCK
1458 : #define BLKDEV_MAJOR_MAX 512
1459 : #else
1460 : #define BLKDEV_MAJOR_MAX 0
1461 : #endif
1462 :
1463 : struct blk_holder_ops {
1464 : void (*mark_dead)(struct block_device *bdev);
1465 : };
1466 :
1467 : /*
1468 : * Return the correct open flags for blkdev_get_by_* for super block flags
1469 : * as stored in sb->s_flags.
1470 : */
1471 : #define sb_open_mode(flags) \
1472 : (BLK_OPEN_READ | (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
1473 :
1474 : struct block_device *blkdev_get_by_dev(dev_t dev, blk_mode_t mode, void *holder,
1475 : const struct blk_holder_ops *hops);
1476 : struct block_device *blkdev_get_by_path(const char *path, blk_mode_t mode,
1477 : void *holder, const struct blk_holder_ops *hops);
1478 : int bd_prepare_to_claim(struct block_device *bdev, void *holder,
1479 : const struct blk_holder_ops *hops);
1480 : void bd_abort_claiming(struct block_device *bdev, void *holder);
1481 : void blkdev_put(struct block_device *bdev, void *holder);
1482 :
1483 : /* just for blk-cgroup, don't use elsewhere */
1484 : struct block_device *blkdev_get_no_open(dev_t dev);
1485 : void blkdev_put_no_open(struct block_device *bdev);
1486 :
1487 : struct block_device *I_BDEV(struct inode *inode);
1488 :
1489 : #ifdef CONFIG_BLOCK
1490 : void invalidate_bdev(struct block_device *bdev);
1491 : int sync_blockdev(struct block_device *bdev);
1492 : int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
1493 : int sync_blockdev_nowait(struct block_device *bdev);
1494 : void sync_bdevs(bool wait);
1495 : void bdev_statx_dioalign(struct inode *inode, struct kstat *stat);
1496 : void printk_all_partitions(void);
1497 : int __init early_lookup_bdev(const char *pathname, dev_t *dev);
1498 : #else
1499 : static inline void invalidate_bdev(struct block_device *bdev)
1500 : {
1501 : }
1502 : static inline int sync_blockdev(struct block_device *bdev)
1503 : {
1504 : return 0;
1505 : }
1506 : static inline int sync_blockdev_nowait(struct block_device *bdev)
1507 : {
1508 : return 0;
1509 : }
1510 : static inline void sync_bdevs(bool wait)
1511 : {
1512 : }
1513 : static inline void bdev_statx_dioalign(struct inode *inode, struct kstat *stat)
1514 : {
1515 : }
1516 : static inline void printk_all_partitions(void)
1517 : {
1518 : }
1519 : static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
1520 : {
1521 : return -EINVAL;
1522 : }
1523 : #endif /* CONFIG_BLOCK */
1524 :
1525 : int fsync_bdev(struct block_device *bdev);
1526 :
1527 : int freeze_bdev(struct block_device *bdev);
1528 : int thaw_bdev(struct block_device *bdev);
1529 :
1530 : struct io_comp_batch {
1531 : struct request *req_list;
1532 : bool need_ts;
1533 : void (*complete)(struct io_comp_batch *);
1534 : };
1535 :
1536 : #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { }
1537 :
1538 : #endif /* _LINUX_BLKDEV_H */
|
