blk-core.c 54.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5
/*
 * Copyright (C) 1991, 1992 Linus Torvalds
 * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
 * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6 7
 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
 *	-  July2000
Linus Torvalds's avatar
Linus Torvalds committed
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 */

/*
 * This handles all read/write requests to block devices
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
28
#include <linux/task_io_accounting_ops.h>
29 30
#include <linux/interrupt.h>
#include <linux/cpu.h>
31
#include <linux/blktrace_api.h>
32
#include <linux/fault-inject.h>
Linus Torvalds's avatar
Linus Torvalds committed
33

34 35
#include "blk.h"

36
static int __make_request(struct request_queue *q, struct bio *bio);
Linus Torvalds's avatar
Linus Torvalds committed
37 38 39 40

/*
 * For the allocated request tables
 */
41
static struct kmem_cache *request_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
42 43 44 45

/*
 * For queue allocation
 */
46
struct kmem_cache *blk_requestq_cachep;
Linus Torvalds's avatar
Linus Torvalds committed
47 48 49 50

/*
 * Controlling structure to kblockd
 */
51
static struct workqueue_struct *kblockd_workqueue;
Linus Torvalds's avatar
Linus Torvalds committed
52

53 54
static DEFINE_PER_CPU(struct list_head, blk_cpu_done);

55 56
static void drive_stat_acct(struct request *rq, int new_io)
{
57
	struct hd_struct *part;
58
	int rw = rq_data_dir(rq);
Tejun Heo's avatar
Tejun Heo committed
59
	int cpu;
60 61 62 63

	if (!blk_fs_request(rq) || !rq->rq_disk)
		return;

64
	cpu = part_stat_lock();
65
	part = disk_map_sector_rcu(rq->rq_disk, rq->sector);
Tejun Heo's avatar
Tejun Heo committed
66

67
	if (!new_io)
68
		part_stat_inc(cpu, part, merges[rw]);
69
	else {
70 71
		part_round_stats(cpu, part);
		part_inc_in_flight(part);
72
	}
73

74
	part_stat_unlock();
75 76
}

77
void blk_queue_congestion_threshold(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
{
	int nr;

	nr = q->nr_requests - (q->nr_requests / 8) + 1;
	if (nr > q->nr_requests)
		nr = q->nr_requests;
	q->nr_congestion_on = nr;

	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
	if (nr < 1)
		nr = 1;
	q->nr_congestion_off = nr;
}

/**
 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
 * @bdev:	device
 *
 * Locates the passed device's request queue and returns the address of its
 * backing_dev_info
 *
 * Will return NULL if the request queue cannot be located.
 */
struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
{
	struct backing_dev_info *ret = NULL;
104
	struct request_queue *q = bdev_get_queue(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
105 106 107 108 109 110 111

	if (q)
		ret = &q->backing_dev_info;
	return ret;
}
EXPORT_SYMBOL(blk_get_backing_dev_info);

112
void blk_rq_init(struct request_queue *q, struct request *rq)
Linus Torvalds's avatar
Linus Torvalds committed
113
{
114 115
	memset(rq, 0, sizeof(*rq));

Linus Torvalds's avatar
Linus Torvalds committed
116
	INIT_LIST_HEAD(&rq->queuelist);
117
	INIT_LIST_HEAD(&rq->donelist);
Jens Axboe's avatar
Jens Axboe committed
118 119
	rq->q = q;
	rq->sector = rq->hard_sector = (sector_t) -1;
120 121
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
122
	rq->cmd = rq->__cmd;
Jens Axboe's avatar
Jens Axboe committed
123
	rq->tag = -1;
Linus Torvalds's avatar
Linus Torvalds committed
124 125
	rq->ref_count = 1;
}
126
EXPORT_SYMBOL(blk_rq_init);
Linus Torvalds's avatar
Linus Torvalds committed
127

128 129
static void req_bio_endio(struct request *rq, struct bio *bio,
			  unsigned int nbytes, int error)
Linus Torvalds's avatar
Linus Torvalds committed
130
{
131
	struct request_queue *q = rq->q;
132

133 134 135 136 137
	if (&q->bar_rq != rq) {
		if (error)
			clear_bit(BIO_UPTODATE, &bio->bi_flags);
		else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
			error = -EIO;
138

139
		if (unlikely(nbytes > bio->bi_size)) {
140
			printk(KERN_ERR "%s: want %u bytes done, %u left\n",
141
			       __func__, nbytes, bio->bi_size);
142 143
			nbytes = bio->bi_size;
		}
144

145 146
		bio->bi_size -= nbytes;
		bio->bi_sector += (nbytes >> 9);
147 148 149 150

		if (bio_integrity(bio))
			bio_integrity_advance(bio, nbytes);

151
		if (bio->bi_size == 0)
152
			bio_endio(bio, error);
153 154 155 156 157 158 159 160 161
	} else {

		/*
		 * Okay, this is the barrier request in progress, just
		 * record the error;
		 */
		if (error && !q->orderr)
			q->orderr = error;
	}
Linus Torvalds's avatar
Linus Torvalds committed
162 163 164 165 166 167
}

void blk_dump_rq_flags(struct request *rq, char *msg)
{
	int bit;

168
	printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
169 170
		rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
		rq->cmd_flags);
Linus Torvalds's avatar
Linus Torvalds committed
171

172 173 174 175 176 177 178 179
	printk(KERN_INFO "  sector %llu, nr/cnr %lu/%u\n",
						(unsigned long long)rq->sector,
						rq->nr_sectors,
						rq->current_nr_sectors);
	printk(KERN_INFO "  bio %p, biotail %p, buffer %p, data %p, len %u\n",
						rq->bio, rq->biotail,
						rq->buffer, rq->data,
						rq->data_len);
Linus Torvalds's avatar
Linus Torvalds committed
180

181
	if (blk_pc_request(rq)) {
182
		printk(KERN_INFO "  cdb: ");
183
		for (bit = 0; bit < BLK_MAX_CDB; bit++)
Linus Torvalds's avatar
Linus Torvalds committed
184 185 186 187 188 189 190 191 192 193 194 195 196 197
			printk("%02x ", rq->cmd[bit]);
		printk("\n");
	}
}
EXPORT_SYMBOL(blk_dump_rq_flags);

/*
 * "plug" the device if there are no outstanding requests: this will
 * force the transfer to start only after we have put all the requests
 * on the list.
 *
 * This is called with interrupts off and no requests on the queue and
 * with the queue lock held.
 */
198
void blk_plug_device(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
199 200 201 202 203 204 205
{
	WARN_ON(!irqs_disabled());

	/*
	 * don't plug a stopped queue, it must be paired with blk_start_queue()
	 * which will restart the queueing
	 */
206
	if (blk_queue_stopped(q))
Linus Torvalds's avatar
Linus Torvalds committed
207 208
		return;

Jens Axboe's avatar
Jens Axboe committed
209
	if (!queue_flag_test_and_set(QUEUE_FLAG_PLUGGED, q)) {
Linus Torvalds's avatar
Linus Torvalds committed
210
		mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
211 212
		blk_add_trace_generic(q, NULL, 0, BLK_TA_PLUG);
	}
Linus Torvalds's avatar
Linus Torvalds committed
213 214 215
}
EXPORT_SYMBOL(blk_plug_device);

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
/**
 * blk_plug_device_unlocked - plug a device without queue lock held
 * @q:    The &struct request_queue to plug
 *
 * Description:
 *   Like @blk_plug_device(), but grabs the queue lock and disables
 *   interrupts.
 **/
void blk_plug_device_unlocked(struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	blk_plug_device(q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_plug_device_unlocked);

Linus Torvalds's avatar
Linus Torvalds committed
234 235 236 237
/*
 * remove the queue from the plugged list, if present. called with
 * queue lock held and interrupts disabled.
 */
238
int blk_remove_plug(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
239 240 241
{
	WARN_ON(!irqs_disabled());

Jens Axboe's avatar
Jens Axboe committed
242
	if (!queue_flag_test_and_clear(QUEUE_FLAG_PLUGGED, q))
Linus Torvalds's avatar
Linus Torvalds committed
243 244 245 246 247 248 249 250 251 252
		return 0;

	del_timer(&q->unplug_timer);
	return 1;
}
EXPORT_SYMBOL(blk_remove_plug);

/*
 * remove the plug and let it rip..
 */
253
void __generic_unplug_device(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
254
{
255
	if (unlikely(blk_queue_stopped(q)))
Linus Torvalds's avatar
Linus Torvalds committed
256 257 258 259 260
		return;

	if (!blk_remove_plug(q))
		return;

261
	q->request_fn(q);
Linus Torvalds's avatar
Linus Torvalds committed
262 263 264 265 266
}
EXPORT_SYMBOL(__generic_unplug_device);

/**
 * generic_unplug_device - fire a request queue
267
 * @q:    The &struct request_queue in question
Linus Torvalds's avatar
Linus Torvalds committed
268 269 270 271 272 273 274 275
 *
 * Description:
 *   Linux uses plugging to build bigger requests queues before letting
 *   the device have at them. If a queue is plugged, the I/O scheduler
 *   is still adding and merging requests on the queue. Once the queue
 *   gets unplugged, the request_fn defined for the queue is invoked and
 *   transfers started.
 **/
276
void generic_unplug_device(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
277
{
278 279 280 281 282
	if (blk_queue_plugged(q)) {
		spin_lock_irq(q->queue_lock);
		__generic_unplug_device(q);
		spin_unlock_irq(q->queue_lock);
	}
Linus Torvalds's avatar
Linus Torvalds committed
283 284 285 286 287 288
}
EXPORT_SYMBOL(generic_unplug_device);

static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
				   struct page *page)
{
289
	struct request_queue *q = bdi->unplug_io_data;
Linus Torvalds's avatar
Linus Torvalds committed
290

291
	blk_unplug(q);
Linus Torvalds's avatar
Linus Torvalds committed
292 293
}

294
void blk_unplug_work(struct work_struct *work)
Linus Torvalds's avatar
Linus Torvalds committed
295
{
296 297
	struct request_queue *q =
		container_of(work, struct request_queue, unplug_work);
Linus Torvalds's avatar
Linus Torvalds committed
298

299 300 301
	blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
				q->rq.count[READ] + q->rq.count[WRITE]);

Linus Torvalds's avatar
Linus Torvalds committed
302 303 304
	q->unplug_fn(q);
}

305
void blk_unplug_timeout(unsigned long data)
Linus Torvalds's avatar
Linus Torvalds committed
306
{
307
	struct request_queue *q = (struct request_queue *)data;
Linus Torvalds's avatar
Linus Torvalds committed
308

309 310 311
	blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_TIMER, NULL,
				q->rq.count[READ] + q->rq.count[WRITE]);

Linus Torvalds's avatar
Linus Torvalds committed
312 313 314
	kblockd_schedule_work(&q->unplug_work);
}

315 316 317 318 319 320 321 322 323 324 325 326 327 328
void blk_unplug(struct request_queue *q)
{
	/*
	 * devices don't necessarily have an ->unplug_fn defined
	 */
	if (q->unplug_fn) {
		blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
					q->rq.count[READ] + q->rq.count[WRITE]);

		q->unplug_fn(q);
	}
}
EXPORT_SYMBOL(blk_unplug);

Linus Torvalds's avatar
Linus Torvalds committed
329 330
/**
 * blk_start_queue - restart a previously stopped queue
331
 * @q:    The &struct request_queue in question
Linus Torvalds's avatar
Linus Torvalds committed
332 333 334 335 336 337
 *
 * Description:
 *   blk_start_queue() will clear the stop flag on the queue, and call
 *   the request_fn for the queue if it was in a stopped state when
 *   entered. Also see blk_stop_queue(). Queue lock must be held.
 **/
338
void blk_start_queue(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
339
{
340 341
	WARN_ON(!irqs_disabled());

342
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
Linus Torvalds's avatar
Linus Torvalds committed
343 344 345 346 347

	/*
	 * one level of recursion is ok and is much faster than kicking
	 * the unplug handling
	 */
Jens Axboe's avatar
Jens Axboe committed
348
	if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
Linus Torvalds's avatar
Linus Torvalds committed
349
		q->request_fn(q);
350
		queue_flag_clear(QUEUE_FLAG_REENTER, q);
Linus Torvalds's avatar
Linus Torvalds committed
351 352 353 354 355 356 357 358 359
	} else {
		blk_plug_device(q);
		kblockd_schedule_work(&q->unplug_work);
	}
}
EXPORT_SYMBOL(blk_start_queue);

/**
 * blk_stop_queue - stop a queue
360
 * @q:    The &struct request_queue in question
Linus Torvalds's avatar
Linus Torvalds committed
361 362 363 364 365 366 367 368 369 370 371
 *
 * Description:
 *   The Linux block layer assumes that a block driver will consume all
 *   entries on the request queue when the request_fn strategy is called.
 *   Often this will not happen, because of hardware limitations (queue
 *   depth settings). If a device driver gets a 'queue full' response,
 *   or if it simply chooses not to queue more I/O at one point, it can
 *   call this function to prevent the request_fn from being called until
 *   the driver has signalled it's ready to go again. This happens by calling
 *   blk_start_queue() to restart queue operations. Queue lock must be held.
 **/
372
void blk_stop_queue(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
373 374
{
	blk_remove_plug(q);
375
	queue_flag_set(QUEUE_FLAG_STOPPED, q);
Linus Torvalds's avatar
Linus Torvalds committed
376 377 378 379 380 381 382 383 384 385 386 387
}
EXPORT_SYMBOL(blk_stop_queue);

/**
 * blk_sync_queue - cancel any pending callbacks on a queue
 * @q: the queue
 *
 * Description:
 *     The block layer may perform asynchronous callback activity
 *     on a queue, such as calling the unplug function after a timeout.
 *     A block device may call blk_sync_queue to ensure that any
 *     such activity is cancelled, thus allowing it to release resources
388
 *     that the callbacks might use. The caller must already have made sure
Linus Torvalds's avatar
Linus Torvalds committed
389 390 391 392 393 394 395
 *     that its ->make_request_fn will not re-add plugging prior to calling
 *     this function.
 *
 */
void blk_sync_queue(struct request_queue *q)
{
	del_timer_sync(&q->unplug_timer);
396
	kblockd_flush_work(&q->unplug_work);
Linus Torvalds's avatar
Linus Torvalds committed
397 398 399 400 401 402 403
}
EXPORT_SYMBOL(blk_sync_queue);

/**
 * blk_run_queue - run a single device queue
 * @q:	The queue to run
 */
404
void __blk_run_queue(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
405 406
{
	blk_remove_plug(q);
407 408 409 410 411 412

	/*
	 * Only recurse once to avoid overrunning the stack, let the unplug
	 * handling reinvoke the handler shortly if we already got there.
	 */
	if (!elv_queue_empty(q)) {
Jens Axboe's avatar
Jens Axboe committed
413
		if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
414
			q->request_fn(q);
415
			queue_flag_clear(QUEUE_FLAG_REENTER, q);
416 417 418 419 420
		} else {
			blk_plug_device(q);
			kblockd_schedule_work(&q->unplug_work);
		}
	}
421 422
}
EXPORT_SYMBOL(__blk_run_queue);
423

424 425 426 427 428 429 430 431 432 433
/**
 * blk_run_queue - run a single device queue
 * @q: The queue to run
 */
void blk_run_queue(struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	__blk_run_queue(q);
Linus Torvalds's avatar
Linus Torvalds committed
434 435 436 437
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_run_queue);

438
void blk_put_queue(struct request_queue *q)
439 440 441 442
{
	kobject_put(&q->kobj);
}

443
void blk_cleanup_queue(struct request_queue *q)
444 445
{
	mutex_lock(&q->sysfs_lock);
446
	queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
447 448 449 450 451 452 453
	mutex_unlock(&q->sysfs_lock);

	if (q->elevator)
		elevator_exit(q->elevator);

	blk_put_queue(q);
}
Linus Torvalds's avatar
Linus Torvalds committed
454 455
EXPORT_SYMBOL(blk_cleanup_queue);

456
static int blk_init_free_list(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
457 458 459 460 461
{
	struct request_list *rl = &q->rq;

	rl->count[READ] = rl->count[WRITE] = 0;
	rl->starved[READ] = rl->starved[WRITE] = 0;
462
	rl->elvpriv = 0;
Linus Torvalds's avatar
Linus Torvalds committed
463 464 465
	init_waitqueue_head(&rl->wait[READ]);
	init_waitqueue_head(&rl->wait[WRITE]);

466 467
	rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
				mempool_free_slab, request_cachep, q->node);
Linus Torvalds's avatar
Linus Torvalds committed
468 469 470 471 472 473 474

	if (!rl->rq_pool)
		return -ENOMEM;

	return 0;
}

475
struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
476
{
477 478 479
	return blk_alloc_queue_node(gfp_mask, -1);
}
EXPORT_SYMBOL(blk_alloc_queue);
Linus Torvalds's avatar
Linus Torvalds committed
480

481
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
482
{
483
	struct request_queue *q;
Peter Zijlstra's avatar
Peter Zijlstra committed
484
	int err;
485

486
	q = kmem_cache_alloc_node(blk_requestq_cachep,
487
				gfp_mask | __GFP_ZERO, node_id);
Linus Torvalds's avatar
Linus Torvalds committed
488 489 490
	if (!q)
		return NULL;

Peter Zijlstra's avatar
Peter Zijlstra committed
491 492 493 494
	q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
	q->backing_dev_info.unplug_io_data = q;
	err = bdi_init(&q->backing_dev_info);
	if (err) {
495
		kmem_cache_free(blk_requestq_cachep, q);
Peter Zijlstra's avatar
Peter Zijlstra committed
496 497 498
		return NULL;
	}

Linus Torvalds's avatar
Linus Torvalds committed
499
	init_timer(&q->unplug_timer);
500

501
	kobject_init(&q->kobj, &blk_queue_ktype);
Linus Torvalds's avatar
Linus Torvalds committed
502

503
	mutex_init(&q->sysfs_lock);
504
	spin_lock_init(&q->__queue_lock);
505

Linus Torvalds's avatar
Linus Torvalds committed
506 507
	return q;
}
508
EXPORT_SYMBOL(blk_alloc_queue_node);
Linus Torvalds's avatar
Linus Torvalds committed
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531

/**
 * blk_init_queue  - prepare a request queue for use with a block device
 * @rfn:  The function to be called to process requests that have been
 *        placed on the queue.
 * @lock: Request queue spin lock
 *
 * Description:
 *    If a block device wishes to use the standard request handling procedures,
 *    which sorts requests and coalesces adjacent requests, then it must
 *    call blk_init_queue().  The function @rfn will be called when there
 *    are requests on the queue that need to be processed.  If the device
 *    supports plugging, then @rfn may not be called immediately when requests
 *    are available on the queue, but may be called at some time later instead.
 *    Plugged queues are generally unplugged when a buffer belonging to one
 *    of the requests on the queue is needed, or due to memory pressure.
 *
 *    @rfn is not required, or even expected, to remove all requests off the
 *    queue, but only as many as it can handle at a time.  If it does leave
 *    requests on the queue, it is responsible for arranging that the requests
 *    get dealt with eventually.
 *
 *    The queue spin lock must be held while manipulating the requests on the
532 533
 *    request queue; this lock will be taken also from interrupt context, so irq
 *    disabling is needed for it.
Linus Torvalds's avatar
Linus Torvalds committed
534
 *
535
 *    Function returns a pointer to the initialized request queue, or %NULL if
Linus Torvalds's avatar
Linus Torvalds committed
536 537 538 539 540 541
 *    it didn't succeed.
 *
 * Note:
 *    blk_init_queue() must be paired with a blk_cleanup_queue() call
 *    when the block device is deactivated (such as at module unload).
 **/
542

543
struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
Linus Torvalds's avatar
Linus Torvalds committed
544
{
545 546 547 548
	return blk_init_queue_node(rfn, lock, -1);
}
EXPORT_SYMBOL(blk_init_queue);

549
struct request_queue *
550 551
blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
{
552
	struct request_queue *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
Linus Torvalds's avatar
Linus Torvalds committed
553 554 555 556

	if (!q)
		return NULL;

557
	q->node = node_id;
558
	if (blk_init_free_list(q)) {
559
		kmem_cache_free(blk_requestq_cachep, q);
560 561
		return NULL;
	}
Linus Torvalds's avatar
Linus Torvalds committed
562

's avatar
committed
563 564 565 566
	/*
	 * if caller didn't supply a lock, they get per-queue locking with
	 * our embedded lock
	 */
567
	if (!lock)
's avatar
committed
568 569
		lock = &q->__queue_lock;

Linus Torvalds's avatar
Linus Torvalds committed
570 571 572 573 574 575 576 577 578 579 580 581 582 583
	q->request_fn		= rfn;
	q->prep_rq_fn		= NULL;
	q->unplug_fn		= generic_unplug_device;
	q->queue_flags		= (1 << QUEUE_FLAG_CLUSTER);
	q->queue_lock		= lock;

	blk_queue_segment_boundary(q, 0xffffffff);

	blk_queue_make_request(q, __make_request);
	blk_queue_max_segment_size(q, MAX_SEGMENT_SIZE);

	blk_queue_max_hw_segments(q, MAX_HW_SEGMENTS);
	blk_queue_max_phys_segments(q, MAX_PHYS_SEGMENTS);

584 585
	q->sg_reserved_size = INT_MAX;

586 587
	blk_set_cmd_filter_defaults(&q->cmd_filter);

Linus Torvalds's avatar
Linus Torvalds committed
588 589 590 591 592 593 594 595
	/*
	 * all done
	 */
	if (!elevator_init(q, NULL)) {
		blk_queue_congestion_threshold(q);
		return q;
	}

596
	blk_put_queue(q);
Linus Torvalds's avatar
Linus Torvalds committed
597 598
	return NULL;
}
599
EXPORT_SYMBOL(blk_init_queue_node);
Linus Torvalds's avatar
Linus Torvalds committed
600

601
int blk_get_queue(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
602
{
Nick Piggin's avatar
Nick Piggin committed
603
	if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
604
		kobject_get(&q->kobj);
Linus Torvalds's avatar
Linus Torvalds committed
605 606 607 608 609 610
		return 0;
	}

	return 1;
}

611
static inline void blk_free_request(struct request_queue *q, struct request *rq)
Linus Torvalds's avatar
Linus Torvalds committed
612
{
613
	if (rq->cmd_flags & REQ_ELVPRIV)
614
		elv_put_request(q, rq);
Linus Torvalds's avatar
Linus Torvalds committed
615 616 617
	mempool_free(rq, q->rq.rq_pool);
}

Jens Axboe's avatar
Jens Axboe committed
618
static struct request *
619
blk_alloc_request(struct request_queue *q, int rw, int priv, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
620 621 622 623 624 625
{
	struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);

	if (!rq)
		return NULL;

626
	blk_rq_init(q, rq);
627

628
	rq->cmd_flags = rw | REQ_ALLOCED;
Linus Torvalds's avatar
Linus Torvalds committed
629

630
	if (priv) {
631
		if (unlikely(elv_set_request(q, rq, gfp_mask))) {
632 633 634
			mempool_free(rq, q->rq.rq_pool);
			return NULL;
		}
635
		rq->cmd_flags |= REQ_ELVPRIV;
636
	}
Linus Torvalds's avatar
Linus Torvalds committed
637

638
	return rq;
Linus Torvalds's avatar
Linus Torvalds committed
639 640 641 642 643 644
}

/*
 * ioc_batching returns true if the ioc is a valid batching request and
 * should be given priority access to a request.
 */
645
static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds's avatar
Linus Torvalds committed
646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
{
	if (!ioc)
		return 0;

	/*
	 * Make sure the process is able to allocate at least 1 request
	 * even if the batch times out, otherwise we could theoretically
	 * lose wakeups.
	 */
	return ioc->nr_batch_requests == q->nr_batching ||
		(ioc->nr_batch_requests > 0
		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
}

/*
 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
 * will cause the process to be a "batcher" on all queues in the system. This
 * is the behaviour we want though - once it gets a wakeup it should be given
 * a nice run.
 */
666
static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds's avatar
Linus Torvalds committed
667 668 669 670 671 672 673 674
{
	if (!ioc || ioc_batching(q, ioc))
		return;

	ioc->nr_batch_requests = q->nr_batching;
	ioc->last_waited = jiffies;
}

675
static void __freed_request(struct request_queue *q, int rw)
Linus Torvalds's avatar
Linus Torvalds committed
676 677 678 679
{
	struct request_list *rl = &q->rq;

	if (rl->count[rw] < queue_congestion_off_threshold(q))
680
		blk_clear_queue_congested(q, rw);
Linus Torvalds's avatar
Linus Torvalds committed
681 682 683 684 685 686 687 688 689 690 691 692 693

	if (rl->count[rw] + 1 <= q->nr_requests) {
		if (waitqueue_active(&rl->wait[rw]))
			wake_up(&rl->wait[rw]);

		blk_clear_queue_full(q, rw);
	}
}

/*
 * A request has just been released.  Account for it, update the full and
 * congestion status, wake up any waiters.   Called under q->queue_lock.
 */
694
static void freed_request(struct request_queue *q, int rw, int priv)
Linus Torvalds's avatar
Linus Torvalds committed
695 696 697 698
{
	struct request_list *rl = &q->rq;

	rl->count[rw]--;
699 700
	if (priv)
		rl->elvpriv--;
Linus Torvalds's avatar
Linus Torvalds committed
701 702 703 704 705 706 707 708 709

	__freed_request(q, rw);

	if (unlikely(rl->starved[rw ^ 1]))
		__freed_request(q, rw ^ 1);
}

#define blkdev_free_rq(list) list_entry((list)->next, struct request, queuelist)
/*
Nick Piggin's avatar
Nick Piggin committed
710 711 712
 * Get a free request, queue_lock must be held.
 * Returns NULL on failure, with queue_lock held.
 * Returns !NULL on success, with queue_lock *not held*.
Linus Torvalds's avatar
Linus Torvalds committed
713
 */
714
static struct request *get_request(struct request_queue *q, int rw_flags,
715
				   struct bio *bio, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
716 717 718
{
	struct request *rq = NULL;
	struct request_list *rl = &q->rq;
719
	struct io_context *ioc = NULL;
720
	const int rw = rw_flags & 0x01;
721 722
	int may_queue, priv;

723
	may_queue = elv_may_queue(q, rw_flags);
724 725 726 727 728
	if (may_queue == ELV_MQUEUE_NO)
		goto rq_starved;

	if (rl->count[rw]+1 >= queue_congestion_on_threshold(q)) {
		if (rl->count[rw]+1 >= q->nr_requests) {
729
			ioc = current_io_context(GFP_ATOMIC, q->node);
730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
			/*
			 * The queue will fill after this allocation, so set
			 * it as full, and mark this process as "batching".
			 * This process will be allowed to complete a batch of
			 * requests, others will be blocked.
			 */
			if (!blk_queue_full(q, rw)) {
				ioc_set_batching(q, ioc);
				blk_set_queue_full(q, rw);
			} else {
				if (may_queue != ELV_MQUEUE_MUST
						&& !ioc_batching(q, ioc)) {
					/*
					 * The queue is full and the allocating
					 * process is not a "batcher", and not
					 * exempted by the IO scheduler
					 */
					goto out;
				}
			}
Linus Torvalds's avatar
Linus Torvalds committed
750
		}
751
		blk_set_queue_congested(q, rw);
Linus Torvalds's avatar
Linus Torvalds committed
752 753
	}

754 755 756 757 758
	/*
	 * Only allow batching queuers to allocate up to 50% over the defined
	 * limit of requests, otherwise we could have thousands of requests
	 * allocated with any setting of ->nr_requests
	 */
759
	if (rl->count[rw] >= (3 * q->nr_requests / 2))
760
		goto out;
761

Linus Torvalds's avatar
Linus Torvalds committed
762 763
	rl->count[rw]++;
	rl->starved[rw] = 0;
764

765
	priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
766 767 768
	if (priv)
		rl->elvpriv++;

Linus Torvalds's avatar
Linus Torvalds committed
769 770
	spin_unlock_irq(q->queue_lock);

771
	rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
772
	if (unlikely(!rq)) {
Linus Torvalds's avatar
Linus Torvalds committed
773 774 775 776 777 778 779 780
		/*
		 * Allocation failed presumably due to memory. Undo anything
		 * we might have messed up.
		 *
		 * Allocating task should really be put onto the front of the
		 * wait queue, but this is pretty rare.
		 */
		spin_lock_irq(q->queue_lock);
781
		freed_request(q, rw, priv);
Linus Torvalds's avatar
Linus Torvalds committed
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796

		/*
		 * in the very unlikely event that allocation failed and no
		 * requests for this direction was pending, mark us starved
		 * so that freeing of a request in the other direction will
		 * notice us. another possible fix would be to split the
		 * rq mempool into READ and WRITE
		 */
rq_starved:
		if (unlikely(rl->count[rw] == 0))
			rl->starved[rw] = 1;

		goto out;
	}

797 798 799 800 801 802
	/*
	 * ioc may be NULL here, and ioc_batching will be false. That's
	 * OK, if the queue is under the request limit then requests need
	 * not count toward the nr_batch_requests limit. There will always
	 * be some limit enforced by BLK_BATCH_TIME.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
803 804
	if (ioc_batching(q, ioc))
		ioc->nr_batch_requests--;
805

806
	blk_add_trace_generic(q, bio, rw, BLK_TA_GETRQ);
Linus Torvalds's avatar
Linus Torvalds committed
807 808 809 810 811 812 813
out:
	return rq;
}

/*
 * No available requests for this queue, unplug the device and wait for some
 * requests to become available.
Nick Piggin's avatar
Nick Piggin committed
814 815
 *
 * Called with q->queue_lock held, and returns with it unlocked.
Linus Torvalds's avatar
Linus Torvalds committed
816
 */
817
static struct request *get_request_wait(struct request_queue *q, int rw_flags,
818
					struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
819
{
820
	const int rw = rw_flags & 0x01;
Linus Torvalds's avatar
Linus Torvalds committed
821 822
	struct request *rq;

823
	rq = get_request(q, rw_flags, bio, GFP_NOIO);
824 825
	while (!rq) {
		DEFINE_WAIT(wait);
826
		struct io_context *ioc;
Linus Torvalds's avatar
Linus Torvalds committed
827 828 829 830 831
		struct request_list *rl = &q->rq;

		prepare_to_wait_exclusive(&rl->wait[rw], &wait,
				TASK_UNINTERRUPTIBLE);

832
		blk_add_trace_generic(q, bio, rw, BLK_TA_SLEEPRQ);
Linus Torvalds's avatar
Linus Torvalds committed
833

834 835 836
		__generic_unplug_device(q);
		spin_unlock_irq(q->queue_lock);
		io_schedule();
Linus Torvalds's avatar
Linus Torvalds committed
837

838 839 840 841 842 843 844 845
		/*
		 * After sleeping, we become a "batching" process and
		 * will be able to allocate at least one request, and
		 * up to a big batch of them for a small period time.
		 * See ioc_batching, ioc_set_batching
		 */
		ioc = current_io_context(GFP_NOIO, q->node);
		ioc_set_batching(q, ioc);
Nick Piggin's avatar
Nick Piggin committed
846

847
		spin_lock_irq(q->queue_lock);
Linus Torvalds's avatar
Linus Torvalds committed
848
		finish_wait(&rl->wait[rw], &wait);
849 850 851

		rq = get_request(q, rw_flags, bio, GFP_NOIO);
	};
Linus Torvalds's avatar
Linus Torvalds committed
852 853 854 855

	return rq;
}

856
struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
857 858 859 860 861
{
	struct request *rq;

	BUG_ON(rw != READ && rw != WRITE);

Nick Piggin's avatar
Nick Piggin committed
862 863
	spin_lock_irq(q->queue_lock);
	if (gfp_mask & __GFP_WAIT) {
864
		rq = get_request_wait(q, rw, NULL);
Nick Piggin's avatar
Nick Piggin committed
865
	} else {
866
		rq = get_request(q, rw, NULL, gfp_mask);
Nick Piggin's avatar
Nick Piggin committed
867 868 869 870
		if (!rq)
			spin_unlock_irq(q->queue_lock);
	}
	/* q->queue_lock is unlocked at this point */
Linus Torvalds's avatar
Linus Torvalds committed
871 872 873 874 875

	return rq;
}
EXPORT_SYMBOL(blk_get_request);

876 877 878 879 880 881 882 883 884 885
/**
 * blk_start_queueing - initiate dispatch of requests to device
 * @q:		request queue to kick into gear
 *
 * This is basically a helper to remove the need to know whether a queue
 * is plugged or not if someone just wants to initiate dispatch of requests
 * for this queue.
 *
 * The queue lock must be held with interrupts disabled.
 */
886
void blk_start_queueing(struct request_queue *q)
887 888 889 890 891 892 893 894
{
	if (!blk_queue_plugged(q))
		q->request_fn(q);
	else
		__generic_unplug_device(q);
}
EXPORT_SYMBOL(blk_start_queueing);

Linus Torvalds's avatar
Linus Torvalds committed
895 896 897 898 899 900 901 902 903 904
/**
 * blk_requeue_request - put a request back on queue
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 *
 * Description:
 *    Drivers often keep queueing requests until the hardware cannot accept
 *    more, when that condition happens we need to put the request back
 *    on the queue. Must be called with queue lock held.
 */
905
void blk_requeue_request(struct request_queue *q, struct request *rq)
Linus Torvalds's avatar
Linus Torvalds committed
906
{
907 908
	blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);

Linus Torvalds's avatar
Linus Torvalds committed
909 910 911 912 913 914 915 916
	if (blk_rq_tagged(rq))
		blk_queue_end_tag(q, rq);

	elv_requeue_request(q, rq);
}
EXPORT_SYMBOL(blk_requeue_request);

/**
917
 * blk_insert_request - insert a special request into a request queue
Linus Torvalds's avatar
Linus Torvalds committed
918 919 920 921 922 923 924 925 926
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 * @at_head:	insert request at head or tail of queue
 * @data:	private data
 *
 * Description:
 *    Many block devices need to execute commands asynchronously, so they don't
 *    block the whole kernel from preemption during request execution.  This is
 *    accomplished normally by inserting aritficial requests tagged as
927 928
 *    REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them
 *    be scheduled for actual execution by the request queue.
Linus Torvalds's avatar
Linus Torvalds committed
929 930 931 932 933 934
 *
 *    We have the option of inserting the head or the tail of the queue.
 *    Typically we use the tail for new ioctls and so forth.  We use the head
 *    of the queue for things like a QUEUE_FULL message from a device, or a
 *    host that is unable to accept a particular command.
 */
935
void blk_insert_request(struct request_queue *q, struct request *rq,
936
			int at_head, void *data)
Linus Torvalds's avatar
Linus Torvalds committed
937
{
938
	int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
Linus Torvalds's avatar
Linus Torvalds committed
939 940 941 942 943 944 945
	unsigned long flags;

	/*
	 * tell I/O scheduler that this isn't a regular read/write (ie it
	 * must not attempt merges on this) and that it acts as a soft
	 * barrier
	 */
946 947
	rq->cmd_type = REQ_TYPE_SPECIAL;
	rq->cmd_flags |= REQ_SOFTBARRIER;
Linus Torvalds's avatar
Linus Torvalds committed
948 949 950 951 952 953 954 955

	rq->special = data;

	spin_lock_irqsave(q->queue_lock, flags);

	/*
	 * If command is tagged, release the tag
	 */
956 957
	if (blk_rq_tagged(rq))
		blk_queue_end_tag(q, rq);
Linus Torvalds's avatar
Linus Torvalds committed
958

959
	drive_stat_acct(rq, 1);
960
	__elv_add_request(q, rq, where, 0);
961
	blk_start_queueing(q);
Linus Torvalds's avatar
Linus Torvalds committed
962 963 964 965 966 967 968 969 970
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_insert_request);

/*
 * add-request adds a request to the linked list.
 * queue lock is held and interrupts disabled, as we muck with the
 * request queue list.
 */
971
static inline void add_request(struct request_queue *q, struct request *req)
Linus Torvalds's avatar
Linus Torvalds committed
972
{
973
	drive_stat_acct(req, 1);
Linus Torvalds's avatar
Linus Torvalds committed
974 975 976 977 978 979 980

	/*
	 * elevator indicated where it wants this request to be
	 * inserted at elevator_merge time
	 */
	__elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
}
981

982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
static void part_round_stats_single(int cpu, struct hd_struct *part,
				    unsigned long now)
{
	if (now == part->stamp)
		return;

	if (part->in_flight) {
		__part_stat_add(cpu, part, time_in_queue,
				part->in_flight * (now - part->stamp));
		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
	}
	part->stamp = now;
}

/**
 * part_round_stats()	- Round off the performance stats on a struct
Linus Torvalds's avatar
Linus Torvalds committed
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
 * disk_stats.
 *
 * The average IO queue length and utilisation statistics are maintained
 * by observing the current state of the queue length and the amount of
 * time it has been in this state for.
 *
 * Normally, that accounting is done on IO completion, but that can result
 * in more than a second's worth of IO being accounted for within any one
 * second, leading to >100% utilisation.  To deal with that, we call this
 * function to do a round-off before returning the results when reading
 * /proc/diskstats.  This accounts immediately for all queue usage up to
 * the current jiffies and restarts the counters again.
 */
Tejun Heo's avatar
Tejun Heo committed
1011
void part_round_stats(int cpu, struct hd_struct *part)
1012 1013 1014
{
	unsigned long now = jiffies;

1015 1016 1017
	if (part->partno)
		part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
	part_round_stats_single(cpu, part, now);
1018
}
1019
EXPORT_SYMBOL_GPL(part_round_stats);
1020

Linus Torvalds's avatar
Linus Torvalds committed
1021 1022 1023
/*
 * queue lock must be held
 */
1024
void __blk_put_request(struct request_queue *q, struct request *req)
Linus Torvalds's avatar
Linus Torvalds committed
1025 1026 1027 1028 1029 1030
{
	if (unlikely(!q))
		return;
	if (unlikely(--req->ref_count))
		return;

1031 1032
	elv_completed_request(q, req);

Linus Torvalds's avatar
Linus Torvalds committed
1033 1034 1035 1036
	/*
	 * Request may not have originated from ll_rw_blk. if not,
	 * it didn't come out of our reserved rq pools
	 */
1037
	if (req->cmd_flags & REQ_ALLOCED) {
Linus Torvalds's avatar
Linus Torvalds committed
1038
		int rw = rq_data_dir(req);
1039
		int priv = req->cmd_flags & REQ_ELVPRIV;
Linus Torvalds's avatar
Linus Torvalds committed
1040 1041

		BUG_ON(!list_empty(&req->queuelist));
1042
		BUG_ON(!hlist_unhashed(&req->hash));
Linus Torvalds's avatar
Linus Torvalds committed
1043 1044

		blk_free_request(q, req);
1045
		freed_request(q, rw, priv);
Linus Torvalds's avatar
Linus Torvalds committed
1046 1047
	}
}
1048 1049
EXPORT_SYMBOL_GPL(__blk_put_request);

Linus Torvalds's avatar
Linus Torvalds committed
1050 1051
void blk_put_request(struct request *req)
{
1052
	unsigned long flags;
1053
	struct request_queue *q = req->q;
1054

1055 1056 1057
	spin_lock_irqsave(q->queue_lock, flags);
	__blk_put_request(q, req);
	spin_unlock_irqrestore(q->queue_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
1058 1059 1060
}
EXPORT_SYMBOL(blk_put_request);

1061
void init_request_from_bio(struct request *req, struct bio *bio)
1062
{
1063
	req->cmd_type = REQ_TYPE_FS;
1064 1065 1066 1067 1068

	/*
	 * inherit FAILFAST from bio (for read-ahead, and explicit FAILFAST)
	 */
	if (bio_rw_ahead(bio) || bio_failfast(bio))
1069
		req->cmd_flags |= REQ_FAILFAST;
1070 1071 1072 1073

	/*
	 * REQ_BARRIER implies no merging, but lets make it explicit
	 */
1074
	if (unlikely(bio_discard(bio))) {
1075 1076 1077
		req->cmd_flags |= REQ_DISCARD;
		if (bio_barrier(bio))
			req->cmd_flags |= REQ_SOFTBARRIER;
1078
		req->q->prepare_discard_fn(req->q, req);
1079 1080
	} else if (unlikely(bio_barrier(bio)))
		req->cmd_flags |= (REQ_HARDBARRIER | REQ_NOMERGE);
1081

Jens Axboe's avatar
Jens Axboe committed
1082
	if (bio_sync(bio))
1083
		req->cmd_flags |= REQ_RW_SYNC;
1084 1085
	if (bio_rw_meta(bio))
		req->cmd_flags |= REQ_RW_META;
Jens Axboe's avatar
Jens Axboe committed
1086

1087 1088 1089 1090
	req->errors = 0;
	req->hard_sector = req->sector = bio->bi_sector;
	req->ioprio = bio_prio(bio);
	req->start_time = jiffies;
1091
	blk_rq_bio_prep(req->q, req, bio);
1092 1093
}

1094
static int __make_request(struct request_queue *q, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
1095
{
1096
	struct request *req;
1097
	int el_ret, nr_sectors, barrier, discard, err;
1098 1099
	const unsigned short prio = bio_prio(bio);
	const int sync = bio_sync(bio);
1100
	int rw_flags;
Linus Torvalds's avatar
Linus Torvalds committed
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111

	nr_sectors = bio_sectors(bio);

	/*
	 * low level driver can indicate that it wants pages above a
	 * certain limit bounced to low memory (ie for highmem, or even
	 * ISA dma in theory)
	 */
	blk_queue_bounce(q, &bio);

	barrier = bio_barrier(bio);
1112 1113
	if (unlikely(barrier) && bio_has_data(bio) &&
	    (q->next_ordered == QUEUE_ORDERED_NONE)) {
Linus Torvalds's avatar
Linus Torvalds committed
1114 1115 1116 1117
		err = -EOPNOTSUPP;
		goto end_io;
	}

1118 1119 1120 1121 1122 1123
	discard = bio_discard(bio);
	if (unlikely(discard) && !q->prepare_discard_fn) {
		err = -EOPNOTSUPP;
		goto end_io;
	}

Linus Torvalds's avatar
Linus Torvalds committed
1124 1125
	spin_lock_irq(q->queue_lock);

1126
	if (unlikely(barrier) || elv_queue_empty(q))
Linus Torvalds's avatar
Linus Torvalds committed
1127 1128 1129 1130
		goto get_rq;

	el_ret = elv_merge(q, &req, bio);
	switch (el_ret) {
1131 1132
	case ELEVATOR_BACK_MERGE:
		BUG_ON(!rq_mergeable(req));
Linus Torvalds's avatar
Linus Torvalds committed
1133

1134 1135
		if (!ll_back_merge_fn(q, req, bio))
			break;
Linus Torvalds's avatar
Linus Torvalds committed
1136

1137
		blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);
1138

1139 1140 1141 1142 1143 1144 1145 1146
		req->biotail->bi_next = bio;
		req->biotail = bio;
		req->nr_sectors = req->hard_nr_sectors += nr_sectors;
		req->ioprio = ioprio_best(req->ioprio, prio);
		drive_stat_acct(req, 0);
		if (!attempt_back_merge(q, req))
			elv_merged_request(q, req, el_ret);
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
1147

1148 1149
	case ELEVATOR_FRONT_MERGE:
		BUG_ON(!rq_mergeable(req));
Linus Torvalds's avatar
Linus Torvalds committed
1150

1151 1152
		if (!ll_front_merge_fn(q, req, bio))
			break;
Linus Torvalds's avatar
Linus Torvalds committed
1153