Linuxカーネルのファイルアクセスの処理を追いかける (16) mq-deadline: dispatch_request

// 787:
static bool dd_has_work(struct blk_mq_hw_ctx *hctx)
{
    struct deadline_data *dd = hctx->queue->elevator->elevator_data;
    enum dd_prio prio;

    for (prio = 0; prio <= DD_PRIO_MAX; prio++)
        if (dd_has_work_for_prio(&dd->per_prio[prio]))
            return true;

    return false;
}

// 780:
static bool dd_has_work_for_prio(struct dd_per_prio *per_prio)
{
    return !list_empty_careful(&per_prio->dispatch) ||
        !list_empty_careful(&per_prio->fifo_list[DD_READ]) ||
        !list_empty_careful(&per_prio->fifo_list[DD_WRITE]);
}

// 477:
static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
{
    struct deadline_data *dd = hctx->queue->elevator->elevator_data;
    struct request *rq;
    enum dd_prio prio;

    spin_lock(&dd->lock);
    for (prio = 0; prio <= DD_PRIO_MAX; prio++) {
        rq = __dd_dispatch_request(dd, &dd->per_prio[prio]);
        if (rq)
            break;
    }
    spin_unlock(&dd->lock);

    return rq;
}

// 362:
static struct request *__dd_dispatch_request(struct deadline_data *dd,
                         struct dd_per_prio *per_prio)
{
    struct request *rq, *next_rq;
    enum dd_data_dir data_dir;
    enum dd_prio prio;
    u8 ioprio_class;

    lockdep_assert_held(&dd->lock);

    if (!list_empty(&per_prio->dispatch)) {
        rq = list_first_entry(&per_prio->dispatch, struct request,
                      queuelist);
        list_del_init(&rq->queuelist);
        goto done;
    }

    /*
    * batches are currently reads XOR writes
    */
    rq = deadline_next_request(dd, per_prio, dd->last_dir);
    if (rq && dd->batching < dd->fifo_batch)
        /* we have a next request are still entitled to batch */
        goto dispatch_request;

    /*
    * at this point we are not running a batch. select the appropriate
    * data direction (read / write)
    */

    if (!list_empty(&per_prio->fifo_list[DD_READ])) {
        BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_READ]));

        if (deadline_fifo_request(dd, per_prio, DD_WRITE) &&
            (dd->starved++ >= dd->writes_starved))
            goto dispatch_writes;

        data_dir = DD_READ;

        goto dispatch_find_request;
    }

    /*
    * there are either no reads or writes have been starved
    */

    if (!list_empty(&per_prio->fifo_list[DD_WRITE])) {
dispatch_writes:
        BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_WRITE]));

        dd->starved = 0;

        data_dir = DD_WRITE;

        goto dispatch_find_request;
    }

    return NULL;

dispatch_find_request:
    /*
    * we are not running a batch, find best request for selected data_dir
    */
    next_rq = deadline_next_request(dd, per_prio, data_dir);
    if (deadline_check_fifo(per_prio, data_dir) || !next_rq) {
        /*
        * A deadline has expired, the last request was in the other
        * direction, or we have run out of higher-sectored requests.
        * Start again from the request with the earliest expiry time.
        */
        rq = deadline_fifo_request(dd, per_prio, data_dir);
    } else {
        /*
        * The last req was the same dir and we have a next request in
        * sort order. No expired requests so continue on from here.
        */
        rq = next_rq;
    }

    /*
    * For a zoned block device, if we only have writes queued and none of
    * them can be dispatched, rq will be NULL.
    */
    if (!rq)
        return NULL;

    dd->last_dir = data_dir;
    dd->batching = 0;

dispatch_request:
    /*
    * rq is the selected appropriate request.
    */
    dd->batching++;
    deadline_move_request(dd, per_prio, rq);
done:
    ioprio_class = dd_rq_ioclass(rq);
    prio = ioprio_class_to_prio[ioprio_class];
    dd_count(dd, dispatched, prio);
    /*
    * If the request needs its target zone locked, do it.
    */
    blk_req_zone_write_lock(rq);
    rq->rq_flags |= RQF_STARTED;
    return rq;
}

// 277:
static inline int deadline_check_fifo(struct dd_per_prio *per_prio,
                      enum dd_data_dir data_dir)
{
    struct request *rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);

    /*
    * rq is expired!
    */
    if (time_after_eq(jiffies, (unsigned long)rq->fifo_time))
        return 1;

    return 0;
}

// 259:
static void
deadline_move_request(struct deadline_data *dd, struct dd_per_prio *per_prio,
              struct request *rq)
{
    const enum dd_data_dir data_dir = rq_data_dir(rq);

    per_prio->next_rq[data_dir] = deadline_latter_request(rq);

    /*
    * take it off the sort and fifo list
    */
    deadline_remove_request(rq->q, per_prio, rq);
}

// 192:
static void deadline_remove_request(struct request_queue *q,
                    struct dd_per_prio *per_prio,
                    struct request *rq)
{
    list_del_init(&rq->queuelist);

    /*
    * We might not be on the rbtree, if we are doing an insert merge
    */
    if (!RB_EMPTY_NODE(&rq->rb_node))
        deadline_del_rq_rb(per_prio, rq);

    elv_rqhash_del(q, rq);
    if (q->last_merge == rq)
        q->last_merge = NULL;
}