/** * __get_request - get a free request * @rl: request list to allocate from * @op: REQ_OP_READ/REQ_OP_WRITE * @op_flags: rq_flag_bits * @bio: bio to allocate request for (can be %NULL) * @gfp_mask: allocation mask * * Get a free request from @q. This function may fail under memory * pressure or if @q is dead. * * Must be called with @q->queue_lock held and, * Returns ERR_PTR on failure, with @q->queue_lock held. * Returns request pointer on success, with @q->queue_lock *not held*. */ static struct request *__get_request(struct request_list *rl, int op, int op_flags, struct bio *bio, gfp_t gfp_mask) { struct request_queue *q = rl->q; struct request *rq; struct elevator_type *et = q->elevator->type; struct io_context *ioc = rq_ioc(bio); struct io_cq *icq = NULL; const bool is_sync = rw_is_sync(op, op_flags) != 0; int may_queue; if (unlikely(blk_queue_dying(q))) return ERR_PTR(-ENODEV); may_queue = elv_may_queue(q, op, op_flags); if (may_queue == ELV_MQUEUE_NO) goto rq_starved; if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) { if (rl->count[is_sync]+1 >= q->nr_requests) { if (!blk_rl_full(rl, is_sync)) { ioc_set_batching(q, ioc); blk_set_rl_full(rl, is_sync); } else { if (may_queue != ELV_MQUEUE_MUST && !ioc_batching(q, ioc)) { return ERR_PTR(-ENOMEM); } } } blk_set_congested(rl, is_sync); } if (rl->count[is_sync] >= (3 * q->nr_requests / 2)) return ERR_PTR(-ENOMEM); q->nr_rqs[is_sync]++; rl->count[is_sync]++; rl->starved[is_sync] = 0; if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) { op_flags |= REQ_ELVPRIV; q->nr_rqs_elvpriv++; if (et->icq_cache && ioc) icq = ioc_lookup_icq(ioc, q); } if (blk_queue_io_stat(q)) op_flags |= REQ_IO_STAT; spin_unlock_irq(q->queue_lock); /* allocate and init request */ rq = mempool_alloc(rl->rq_pool, gfp_mask); if (!rq) goto fail_alloc; blk_rq_init(q, rq); blk_rq_set_rl(rq, rl); req_set_op_attrs(rq, op, op_flags | REQ_ALLOCED); /* init elvpriv */ if (op_flags & REQ_ELVPRIV) { if (unlikely(et->icq_cache && !icq)) { if (ioc) icq = ioc_create_icq(ioc, q, gfp_mask); if (!icq) goto fail_elvpriv; } rq->elv.icq = icq; if (unlikely(elv_set_request(q, rq, bio, gfp_mask))) goto fail_elvpriv; /* @rq->elv.icq holds io_context until @rq is freed */ if (icq) get_io_context(icq->ioc); }
out: if (ioc_batching(q, ioc)) ioc->nr_batch_requests--; trace_block_getrq(q, bio, op); return rq; fail_elvpriv: printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n", __func__, dev_name(q->backing_dev_info.dev)); rq->cmd_flags &= ~REQ_ELVPRIV; rq->elv.icq = NULL; spin_lock_irq(q->queue_lock); q->nr_rqs_elvpriv--; spin_unlock_irq(q->queue_lock); goto out; fail_alloc: spin_lock_irq(q->queue_lock); freed_request(rl, op, op_flags); rq_starved: if (unlikely(rl->count[is_sync] == 0)) rl->starved[is_sync] = 1; return ERR_PTR(-ENOMEM); }
static struct io_context *rq_ioc(struct bio *bio) { #ifdef CONFIG_BLK_CGROUP if (bio && bio->bi_ioc) return bio->bi_ioc; #endif return current->io_context; }
bio_associate_current:blk_throtl_bio<-- blkcg_bio_issue_check<-- generic_make_request_checks
bio_disassociate_task: __bio_free < bio_free, bio_reset>