5 #include "blk-mq-tag.h"
7 int blk_mq_sched_init_hctx_data(struct request_queue
*q
, size_t size
,
8 int (*init
)(struct blk_mq_hw_ctx
*),
9 void (*exit
)(struct blk_mq_hw_ctx
*));
11 void blk_mq_sched_free_hctx_data(struct request_queue
*q
,
12 void (*exit
)(struct blk_mq_hw_ctx
*));
14 struct request
*blk_mq_sched_get_request(struct request_queue
*q
, struct bio
*bio
, unsigned int op
, struct blk_mq_alloc_data
*data
);
15 void blk_mq_sched_put_request(struct request
*rq
);
17 void blk_mq_sched_request_inserted(struct request
*rq
);
18 bool blk_mq_sched_try_merge(struct request_queue
*q
, struct bio
*bio
,
19 struct request
**merged_request
);
20 bool __blk_mq_sched_bio_merge(struct request_queue
*q
, struct bio
*bio
);
21 bool blk_mq_sched_try_insert_merge(struct request_queue
*q
, struct request
*rq
);
22 void blk_mq_sched_restart(struct blk_mq_hw_ctx
*hctx
);
24 void blk_mq_sched_insert_request(struct request
*rq
, bool at_head
,
25 bool run_queue
, bool async
, bool can_block
);
26 void blk_mq_sched_insert_requests(struct request_queue
*q
,
27 struct blk_mq_ctx
*ctx
,
28 struct list_head
*list
, bool run_queue_async
);
30 void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx
*hctx
);
31 void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx
*hctx
,
32 struct list_head
*rq_list
,
33 struct request
*(*get_rq
)(struct blk_mq_hw_ctx
*));
35 int blk_mq_init_sched(struct request_queue
*q
, struct elevator_type
*e
);
36 void blk_mq_exit_sched(struct request_queue
*q
, struct elevator_queue
*e
);
38 int blk_mq_sched_init_hctx(struct request_queue
*q
, struct blk_mq_hw_ctx
*hctx
,
39 unsigned int hctx_idx
);
40 void blk_mq_sched_exit_hctx(struct request_queue
*q
, struct blk_mq_hw_ctx
*hctx
,
41 unsigned int hctx_idx
);
43 int blk_mq_sched_init(struct request_queue
*q
);
46 blk_mq_sched_bio_merge(struct request_queue
*q
, struct bio
*bio
)
48 struct elevator_queue
*e
= q
->elevator
;
50 if (!e
|| blk_queue_nomerges(q
) || !bio_mergeable(bio
))
53 return __blk_mq_sched_bio_merge(q
, bio
);
56 static inline int blk_mq_sched_get_rq_priv(struct request_queue
*q
,
60 struct elevator_queue
*e
= q
->elevator
;
62 if (e
&& e
->type
->ops
.mq
.get_rq_priv
)
63 return e
->type
->ops
.mq
.get_rq_priv(q
, rq
, bio
);
68 static inline void blk_mq_sched_put_rq_priv(struct request_queue
*q
,
71 struct elevator_queue
*e
= q
->elevator
;
73 if (e
&& e
->type
->ops
.mq
.put_rq_priv
)
74 e
->type
->ops
.mq
.put_rq_priv(q
, rq
);
78 blk_mq_sched_allow_merge(struct request_queue
*q
, struct request
*rq
,
81 struct elevator_queue
*e
= q
->elevator
;
83 if (e
&& e
->type
->ops
.mq
.allow_merge
)
84 return e
->type
->ops
.mq
.allow_merge(q
, rq
, bio
);
90 blk_mq_sched_completed_request(struct blk_mq_hw_ctx
*hctx
, struct request
*rq
)
92 struct elevator_queue
*e
= hctx
->queue
->elevator
;
94 if (e
&& e
->type
->ops
.mq
.completed_request
)
95 e
->type
->ops
.mq
.completed_request(hctx
, rq
);
97 BUG_ON(rq
->internal_tag
== -1);
99 blk_mq_put_tag(hctx
, hctx
->sched_tags
, rq
->mq_ctx
, rq
->internal_tag
);
102 static inline void blk_mq_sched_started_request(struct request
*rq
)
104 struct request_queue
*q
= rq
->q
;
105 struct elevator_queue
*e
= q
->elevator
;
107 if (e
&& e
->type
->ops
.mq
.started_request
)
108 e
->type
->ops
.mq
.started_request(rq
);
111 static inline void blk_mq_sched_requeue_request(struct request
*rq
)
113 struct request_queue
*q
= rq
->q
;
114 struct elevator_queue
*e
= q
->elevator
;
116 if (e
&& e
->type
->ops
.mq
.requeue_request
)
117 e
->type
->ops
.mq
.requeue_request(rq
);
120 static inline bool blk_mq_sched_has_work(struct blk_mq_hw_ctx
*hctx
)
122 struct elevator_queue
*e
= hctx
->queue
->elevator
;
124 if (e
&& e
->type
->ops
.mq
.has_work
)
125 return e
->type
->ops
.mq
.has_work(hctx
);
131 * Mark a hardware queue as needing a restart.
133 static inline void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx
*hctx
)
135 if (!test_bit(BLK_MQ_S_SCHED_RESTART
, &hctx
->state
))
136 set_bit(BLK_MQ_S_SCHED_RESTART
, &hctx
->state
);
139 static inline bool blk_mq_sched_needs_restart(struct blk_mq_hw_ctx
*hctx
)
141 return test_bit(BLK_MQ_S_SCHED_RESTART
, &hctx
->state
);