diff options
| author | Ming Lei <tom.leiming@gmail.com> | 2014-06-01 00:43:37 +0800 |
|---|---|---|
| committer | Jens Axboe <axboe@fb.com> | 2014-06-03 21:04:39 -0600 |
| commit | cb96a42cc1f50ba1c7b1e9b2343bec80b926107f (patch) | |
| tree | c69f953fcd7d04288903abe73a0f92f155882bfd /block/blk-mq-tag.c | |
| parent | 1aecfe4887713838c79bc52f774609a57db4f988 (diff) | |
blk-mq: fix schedule from atomic context
blk_mq_put_ctx() has to be called before io_schedule() in
bt_get().
This patch fixes the problem by taking similar approach from
percpu_ida allocation for the situation.
Signed-off-by: Ming Lei <tom.leiming@gmail.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'block/blk-mq-tag.c')
| -rw-r--r-- | block/blk-mq-tag.c | 48 |
1 files changed, 31 insertions, 17 deletions
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index d90c4aeb7dd3..1aab39f71d95 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -221,8 +221,10 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt, return bs; } -static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx, - unsigned int *last_tag, gfp_t gfp) +static int bt_get(struct blk_mq_alloc_data *data, + struct blk_mq_bitmap_tags *bt, + struct blk_mq_hw_ctx *hctx, + unsigned int *last_tag) { struct bt_wait_state *bs; DEFINE_WAIT(wait); @@ -232,7 +234,7 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx, if (tag != -1) return tag; - if (!(gfp & __GFP_WAIT)) + if (!(data->gfp & __GFP_WAIT)) return -1; bs = bt_wait_ptr(bt, hctx); @@ -249,50 +251,62 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx, if (was_empty) atomic_set(&bs->wait_cnt, bt->wake_cnt); + blk_mq_put_ctx(data->ctx); + io_schedule(); + + data->ctx = blk_mq_get_ctx(data->q); + data->hctx = data->q->mq_ops->map_queue(data->q, + data->ctx->cpu); + if (data->reserved) { + bt = &data->hctx->tags->breserved_tags; + } else { + last_tag = &data->ctx->last_tag; + hctx = data->hctx; + bt = &hctx->tags->bitmap_tags; + } + finish_wait(&bs->wait, &wait); + bs = bt_wait_ptr(bt, hctx); } while (1); finish_wait(&bs->wait, &wait); return tag; } -static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags, - struct blk_mq_hw_ctx *hctx, - unsigned int *last_tag, gfp_t gfp) +static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data) { int tag; - tag = bt_get(&tags->bitmap_tags, hctx, last_tag, gfp); + tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx, + &data->ctx->last_tag); if (tag >= 0) - return tag + tags->nr_reserved_tags; + return tag + data->hctx->tags->nr_reserved_tags; return BLK_MQ_TAG_FAIL; } -static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags, - gfp_t gfp) +static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data) { int tag, zero = 0; - if (unlikely(!tags->nr_reserved_tags)) { + if (unlikely(!data->hctx->tags->nr_reserved_tags)) { WARN_ON_ONCE(1); return BLK_MQ_TAG_FAIL; } - tag = bt_get(&tags->breserved_tags, NULL, &zero, gfp); + tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero); if (tag < 0) return BLK_MQ_TAG_FAIL; return tag; } -unsigned int blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, unsigned int *last_tag, - gfp_t gfp, bool reserved) +unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) { - if (!reserved) - return __blk_mq_get_tag(hctx->tags, hctx, last_tag, gfp); + if (!data->reserved) + return __blk_mq_get_tag(data); - return __blk_mq_get_reserved_tag(hctx->tags, gfp); + return __blk_mq_get_reserved_tag(data); } static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt) |