kybaaz/linux-xiaomi-chiron
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

block: switch polling to be bio based

Browse source 
Replace the blk_poll interface that requires the caller to keep a queue
and cookie from the submissions with polling based on the bio.

Polling for the bio itself leads to a few advantages:

 - the cookie construction can made entirely private in blk-mq.c
 - the caller does not need to remember the request_queue and cookie
   separately and thus sidesteps their lifetime issues
 - keeping the device and the cookie inside the bio allows to trivially
   support polling BIOs remapping by stacking drivers
 - a lot of code to propagate the cookie back up the submission path can
   be removed entirely.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Tested-by: Mark Wunderlich <mark.wunderlich@intel.com>
Link: https://lore.kernel.org/r/20211012111226.760968-15-hch@lst.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Christoph Hellwig 2021-10-12 13:12:24 +02:00 • committed by Jens Axboe
parent 19416123ab
commit 3e08773c38
39 changed files with 232 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/linux/bio.h
View file

@ -349,7 +349,7 @@ static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned short nr_iovecs)
return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
}
extern blk_qc_t submit_bio(struct bio *);
void submit_bio(struct bio *bio);
extern void bio_endio(struct bio *);

15
include/linux/blk-mq.h
View file

@ -359,9 +359,9 @@ struct blk_mq_hw_ctx {
/** @kobj: Kernel object for sysfs. */
struct kobject kobj;
/** @poll_considered: Count times blk_poll() was called. */
/** @poll_considered: Count times blk_mq_poll() was called. */
unsigned long poll_considered;
/** @poll_invoked: Count how many requests blk_poll() polled. */
/** @poll_invoked: Count how many requests blk_mq_poll() polled. */
unsigned long poll_invoked;
/** @poll_success: Count how many polled requests were completed. */
unsigned long poll_success;
@ -815,16 +815,6 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)
for ((i) = 0; (i) < (hctx)->nr_ctx && \
({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++)
static inline blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx,
struct request *rq)
{
if (rq->tag != -1)
return rq->tag | (hctx->queue_num << BLK_QC_T_SHIFT);
return rq->internal_tag | (hctx->queue_num << BLK_QC_T_SHIFT) |
BLK_QC_T_INTERNAL;
}
static inline void blk_mq_cleanup_rq(struct request *rq)
{
if (rq->q->mq_ops->cleanup_rq)
@ -843,7 +833,6 @@ static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,
rq->rq_disk = bio->bi_bdev->bd_disk;
}
blk_qc_t blk_mq_submit_bio(struct bio *bio);
void blk_mq_hctx_set_fq_lock_class(struct blk_mq_hw_ctx *hctx,
struct lock_class_key *key);

12
include/linux/blk_types.h
View file

@ -208,6 +208,9 @@ static inline void bio_issue_init(struct bio_issue *issue,
((u64)size << BIO_ISSUE_SIZE_SHIFT));
}
typedef unsigned int blk_qc_t;
#define BLK_QC_T_NONE -1U
/*
* main unit of I/O for the block layer and lower layers (ie drivers and
* stacking drivers)
@ -227,8 +230,8 @@ struct bio {
struct bvec_iter bi_iter;
blk_qc_t bi_cookie;
bio_end_io_t *bi_end_io;
void *bi_private;
#ifdef CONFIG_BLK_CGROUP
/*
@ -384,7 +387,7 @@ enum req_flag_bits {
/* command specific flags for REQ_OP_WRITE_ZEROES: */
__REQ_NOUNMAP, /* do not free blocks when zeroing */
__REQ_POLLED, /* caller polls for completion using blk_poll */
__REQ_POLLED, /* caller polls for completion using bio_poll */
/* for driver use */
__REQ_DRV,
@ -495,11 +498,6 @@ static inline int op_stat_group(unsigned int op)
return op_is_write(op);
}
typedef unsigned int blk_qc_t;
#define BLK_QC_T_NONE -1U
#define BLK_QC_T_SHIFT 16
#define BLK_QC_T_INTERNAL (1U << 31)
struct blk_rq_stat {
u64 mean;
u64 min;

8
include/linux/blkdev.h
View file

@ -25,6 +25,7 @@ struct request;
struct sg_io_hdr;
struct blkcg_gq;
struct blk_flush_queue;
struct kiocb;
struct pr_ops;
struct rq_qos;
struct blk_queue_stats;
@ -550,7 +551,7 @@ static inline unsigned int blk_queue_depth(struct request_queue *q)
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
blk_qc_t submit_bio_noacct(struct bio *bio);
void submit_bio_noacct(struct bio *bio);
extern int blk_lld_busy(struct request_queue *q);
extern void blk_queue_split(struct bio **);
@ -568,7 +569,8 @@ blk_status_t errno_to_blk_status(int errno);
#define BLK_POLL_ONESHOT (1 << 0)
/* do not sleep to wait for the expected completion time */
#define BLK_POLL_NOSLEEP (1 << 1)
int blk_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags);
int bio_poll(struct bio *bio, unsigned int flags);
int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags);
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
{
@ -1176,7 +1178,7 @@ static inline void blk_ksm_unregister(struct request_queue *q) { }
struct block_device_operations {
blk_qc_t (*submit_bio) (struct bio *bio);
void (*submit_bio)(struct bio *bio);
int (*open) (struct block_device *, fmode_t);
void (*release) (struct gendisk *, fmode_t);
int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int);

6
include/linux/fs.h
View file

@ -334,11 +334,7 @@ struct kiocb {
int ki_flags;
u16 ki_hint;
u16 ki_ioprio; /* See linux/ioprio.h */
union {
unsigned int ki_cookie; /* for ->iopoll */
struct wait_page_queue *ki_waitq; /* for async buffered IO */
};
struct wait_page_queue *ki_waitq; /* for async buffered IO */
randomized_struct_fields_end
};

5
include/linux/iomap.h
View file

@ -313,8 +313,8 @@ int iomap_writepages(struct address_space *mapping,
struct iomap_dio_ops {
int (*end_io)(struct kiocb *iocb, ssize_t size, int error,
unsigned flags);
blk_qc_t (*submit_io)(const struct iomap_iter *iter, struct bio *bio,
loff_t file_offset);
void (*submit_io)(const struct iomap_iter *iter, struct bio *bio,
loff_t file_offset);
};
/*
@ -337,7 +337,6 @@ struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
unsigned int dio_flags);
ssize_t iomap_dio_complete(struct iomap_dio *dio);
int iomap_dio_iopoll(struct kiocb *kiocb, unsigned int flags);
#ifdef CONFIG_SWAP
struct file;