4bb659b156
blk-mq currently uses percpu_ida for tag allocation. But that only works well if the ratio between tag space and number of CPUs is sufficiently high. For most devices and systems, that is not the case. The end result if that we either only utilize the tag space partially, or we end up attempting to fully exhaust it and run into lots of lock contention with stealing between CPUs. This is not optimal. This new tagging scheme is a hybrid bitmap allocator. It uses two tricks to both be SMP friendly and allow full exhaustion of the space: 1) We cache the last allocated (or freed) tag on a per blk-mq software context basis. This allows us to limit the space we have to search. The key element here is not caching it in the shared tag structure, otherwise we end up dirtying more shared cache lines on each allocate/free operation. 2) The tag space is split into cache line sized groups, and each context will start off randomly in that space. Even up to full utilization of the space, this divides the tag users efficiently into cache line groups, avoiding dirtying the same one both between allocators and between allocator and freeer. This scheme shows drastically better behaviour, both on small tag spaces but on large ones as well. It has been tested extensively to show better performance for all the cases blk-mq cares about. Signed-off-by: Jens Axboe <axboe@fb.com>
56 lines
1.5 KiB
C
56 lines
1.5 KiB
C
#ifndef INT_BLK_MQ_H
|
|
#define INT_BLK_MQ_H
|
|
|
|
struct blk_mq_tag_set;
|
|
|
|
struct blk_mq_ctx {
|
|
struct {
|
|
spinlock_t lock;
|
|
struct list_head rq_list;
|
|
} ____cacheline_aligned_in_smp;
|
|
|
|
unsigned int cpu;
|
|
unsigned int index_hw;
|
|
|
|
unsigned int last_tag ____cacheline_aligned_in_smp;
|
|
|
|
/* incremented at dispatch time */
|
|
unsigned long rq_dispatched[2];
|
|
unsigned long rq_merged;
|
|
|
|
/* incremented at completion time */
|
|
unsigned long ____cacheline_aligned_in_smp rq_completed[2];
|
|
|
|
struct request_queue *queue;
|
|
struct kobject kobj;
|
|
} ____cacheline_aligned_in_smp;
|
|
|
|
void __blk_mq_complete_request(struct request *rq);
|
|
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
|
|
void blk_mq_init_flush(struct request_queue *q);
|
|
void blk_mq_drain_queue(struct request_queue *q);
|
|
void blk_mq_free_queue(struct request_queue *q);
|
|
void blk_mq_clone_flush_request(struct request *flush_rq,
|
|
struct request *orig_rq);
|
|
|
|
/*
|
|
* CPU hotplug helpers
|
|
*/
|
|
struct blk_mq_cpu_notifier;
|
|
void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
|
|
void (*fn)(void *, unsigned long, unsigned int),
|
|
void *data);
|
|
void blk_mq_register_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
|
|
void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
|
|
void blk_mq_cpu_init(void);
|
|
void blk_mq_enable_hotplug(void);
|
|
void blk_mq_disable_hotplug(void);
|
|
|
|
/*
|
|
* CPU -> queue mappings
|
|
*/
|
|
extern unsigned int *blk_mq_make_queue_map(struct blk_mq_tag_set *set);
|
|
extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues);
|
|
|
|
#endif
|