raid5.c 133 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4
/*
 * raid5.c : Multiple Devices driver for Linux
 *	   Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *	   Copyright (C) 1999, 2000 Ingo Molnar
5
 *	   Copyright (C) 2002, 2003 H. Peter Anvin
Linus Torvalds's avatar
Linus Torvalds committed
6
 *
7 8 9
 * RAID-4/5/6 management functions.
 * Thanks to Penguin Computing for making the RAID-6 development possible
 * by donating a test server!
Linus Torvalds's avatar
Linus Torvalds committed
10 11 12 13 14 15 16 17 18 19 20
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
/*
 * BITMAP UNPLUGGING:
 *
 * The sequencing for updating the bitmap reliably is a little
 * subtle (and I got it wrong the first time) so it deserves some
 * explanation.
 *
 * We group bitmap updates into batches.  Each batch has a number.
 * We may write out several batches at once, but that isn't very important.
 * conf->bm_write is the number of the last batch successfully written.
 * conf->bm_flush is the number of the last batch that was closed to
 *    new additions.
 * When we discover that we will need to write to any block in a stripe
 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
 * the number of the batch it will be in. This is bm_flush+1.
 * When we are ready to do a write, if that batch hasn't been written yet,
 *   we plug the array and queue the stripe for later.
 * When an unplug happens, we increment bm_flush, thus closing the current
 *   batch.
 * When we notice that bm_flush > bm_write, we write out all pending updates
 * to the bitmap, and advance bm_write to where bm_flush was.
 * This may occasionally write a bit out twice, but is sure never to
 * miss any bits.
 */
Linus Torvalds's avatar
Linus Torvalds committed
45 46 47 48 49

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/bitops.h>
50
#include <linux/kthread.h>
Linus Torvalds's avatar
Linus Torvalds committed
51
#include <asm/atomic.h>
52
#include "raid6.h"
Linus Torvalds's avatar
Linus Torvalds committed
53

54
#include <linux/raid/bitmap.h>
55
#include <linux/async_tx.h>
56

Linus Torvalds's avatar
Linus Torvalds committed
57 58 59 60 61 62 63 64 65
/*
 * Stripe cache
 */

#define NR_STRIPES		256
#define STRIPE_SIZE		PAGE_SIZE
#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
#define	IO_THRESHOLD		1
66
#define BYPASS_THRESHOLD	1
67
#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
Linus Torvalds's avatar
Linus Torvalds committed
68 69
#define HASH_MASK		(NR_HASH - 1)

70
#define stripe_hash(conf, sect)	(&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK]))
Linus Torvalds's avatar
Linus Torvalds committed
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91

/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 * order without overlap.  There may be several bio's per stripe+device, and
 * a bio could span several devices.
 * When walking this list for a particular stripe+device, we must never proceed
 * beyond a bio that extends past this device, as the next bio might no longer
 * be valid.
 * This macro is used to determine the 'next' bio in the list, given the sector
 * of the current stripe+device
 */
#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
/*
 * The following can be used to debug the driver
 */
#define RAID5_PARANOIA	1
#if RAID5_PARANOIA && defined(CONFIG_SMP)
# define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)
#else
# define CHECK_DEVLOCK()
#endif

92
#ifdef DEBUG
Linus Torvalds's avatar
Linus Torvalds committed
93 94 95 96
#define inline
#define __inline__
#endif

97 98
#define printk_rl(args...) ((void) (printk_ratelimit() && printk(args)))

99 100 101 102 103
#if !RAID6_USE_EMPTY_ZERO_PAGE
/* In .bss so it's zeroed */
const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));
#endif

104
/*
105 106
 * We maintain a biased count of active stripes in the bottom 16 bits of
 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
107 108 109
 */
static inline int raid5_bi_phys_segments(struct bio *bio)
{
110
	return bio->bi_phys_segments & 0xffff;
111 112 113 114
}

static inline int raid5_bi_hw_segments(struct bio *bio)
{
115
	return (bio->bi_phys_segments >> 16) & 0xffff;
116 117 118 119 120 121 122 123 124 125 126 127 128
}

static inline int raid5_dec_bi_phys_segments(struct bio *bio)
{
	--bio->bi_phys_segments;
	return raid5_bi_phys_segments(bio);
}

static inline int raid5_dec_bi_hw_segments(struct bio *bio)
{
	unsigned short val = raid5_bi_hw_segments(bio);

	--val;
129
	bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio);
130 131 132 133 134
	return val;
}

static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt)
{
135
	bio->bi_phys_segments = raid5_bi_phys_segments(bio) || (cnt << 16);
136 137
}

138 139 140 141 142
static inline int raid6_next_disk(int disk, int raid_disks)
{
	disk++;
	return (disk < raid_disks) ? disk : 0;
}
143 144 145 146 147 148 149 150 151

static void return_io(struct bio *return_bi)
{
	struct bio *bi = return_bi;
	while (bi) {

		return_bi = bi->bi_next;
		bi->bi_next = NULL;
		bi->bi_size = 0;
152
		bio_endio(bi, 0);
153 154 155 156
		bi = return_bi;
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
157 158
static void print_raid5_conf (raid5_conf_t *conf);

159 160 161 162 163 164 165
static int stripe_operations_active(struct stripe_head *sh)
{
	return sh->check_state || sh->reconstruct_state ||
	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);
}

166
static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh)
Linus Torvalds's avatar
Linus Torvalds committed
167 168
{
	if (atomic_dec_and_test(&sh->count)) {
169 170
		BUG_ON(!list_empty(&sh->lru));
		BUG_ON(atomic_read(&conf->active_stripes)==0);
Linus Torvalds's avatar
Linus Torvalds committed
171
		if (test_bit(STRIPE_HANDLE, &sh->state)) {
172
			if (test_bit(STRIPE_DELAYED, &sh->state)) {
Linus Torvalds's avatar
Linus Torvalds committed
173
				list_add_tail(&sh->lru, &conf->delayed_list);
174 175
				blk_plug_device(conf->mddev->queue);
			} else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
176
				   sh->bm_seq - conf->seq_write > 0) {
177
				list_add_tail(&sh->lru, &conf->bitmap_list);
178 179
				blk_plug_device(conf->mddev->queue);
			} else {
180
				clear_bit(STRIPE_BIT_DELAY, &sh->state);
Linus Torvalds's avatar
Linus Torvalds committed
181
				list_add_tail(&sh->lru, &conf->handle_list);
182
			}
Linus Torvalds's avatar
Linus Torvalds committed
183 184
			md_wakeup_thread(conf->mddev->thread);
		} else {
185
			BUG_ON(stripe_operations_active(sh));
Linus Torvalds's avatar
Linus Torvalds committed
186 187 188 189 190 191
			if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
				atomic_dec(&conf->preread_active_stripes);
				if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
					md_wakeup_thread(conf->mddev->thread);
			}
			atomic_dec(&conf->active_stripes);
192 193
			if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
				list_add_tail(&sh->lru, &conf->inactive_list);
Linus Torvalds's avatar
Linus Torvalds committed
194
				wake_up(&conf->wait_for_stripe);
195 196
				if (conf->retry_read_aligned)
					md_wakeup_thread(conf->mddev->thread);
197
			}
Linus Torvalds's avatar
Linus Torvalds committed
198 199 200 201 202 203 204
		}
	}
}
static void release_stripe(struct stripe_head *sh)
{
	raid5_conf_t *conf = sh->raid_conf;
	unsigned long flags;
205

Linus Torvalds's avatar
Linus Torvalds committed
206 207 208 209 210
	spin_lock_irqsave(&conf->device_lock, flags);
	__release_stripe(conf, sh);
	spin_unlock_irqrestore(&conf->device_lock, flags);
}

211
static inline void remove_hash(struct stripe_head *sh)
Linus Torvalds's avatar
Linus Torvalds committed
212
{
213 214
	pr_debug("remove_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
Linus Torvalds's avatar
Linus Torvalds committed
215

216
	hlist_del_init(&sh->hash);
Linus Torvalds's avatar
Linus Torvalds committed
217 218
}

219
static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh)
Linus Torvalds's avatar
Linus Torvalds committed
220
{
221
	struct hlist_head *hp = stripe_hash(conf, sh->sector);
Linus Torvalds's avatar
Linus Torvalds committed
222

223 224
	pr_debug("insert_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
Linus Torvalds's avatar
Linus Torvalds committed
225 226

	CHECK_DEVLOCK();
227
	hlist_add_head(&sh->hash, hp);
Linus Torvalds's avatar
Linus Torvalds committed
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
}


/* find an idle stripe, make sure it is unhashed, and return it. */
static struct stripe_head *get_free_stripe(raid5_conf_t *conf)
{
	struct stripe_head *sh = NULL;
	struct list_head *first;

	CHECK_DEVLOCK();
	if (list_empty(&conf->inactive_list))
		goto out;
	first = conf->inactive_list.next;
	sh = list_entry(first, struct stripe_head, lru);
	list_del_init(first);
	remove_hash(sh);
	atomic_inc(&conf->active_stripes);
out:
	return sh;
}

static void shrink_buffers(struct stripe_head *sh, int num)
{
	struct page *p;
	int i;

	for (i=0; i<num ; i++) {
		p = sh->dev[i].page;
		if (!p)
			continue;
		sh->dev[i].page = NULL;
259
		put_page(p);
Linus Torvalds's avatar
Linus Torvalds committed
260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
	}
}

static int grow_buffers(struct stripe_head *sh, int num)
{
	int i;

	for (i=0; i<num; i++) {
		struct page *page;

		if (!(page = alloc_page(GFP_KERNEL))) {
			return 1;
		}
		sh->dev[i].page = page;
	}
	return 0;
}

static void raid5_build_block (struct stripe_head *sh, int i);

280
static void init_stripe(struct stripe_head *sh, sector_t sector, int pd_idx, int disks)
Linus Torvalds's avatar
Linus Torvalds committed
281 282
{
	raid5_conf_t *conf = sh->raid_conf;
283
	int i;
Linus Torvalds's avatar
Linus Torvalds committed
284

285 286
	BUG_ON(atomic_read(&sh->count) != 0);
	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
287
	BUG_ON(stripe_operations_active(sh));
288

Linus Torvalds's avatar
Linus Torvalds committed
289
	CHECK_DEVLOCK();
290
	pr_debug("init_stripe called, stripe %llu\n",
Linus Torvalds's avatar
Linus Torvalds committed
291 292 293
		(unsigned long long)sh->sector);

	remove_hash(sh);
294

Linus Torvalds's avatar
Linus Torvalds committed
295 296 297 298
	sh->sector = sector;
	sh->pd_idx = pd_idx;
	sh->state = 0;

299 300 301
	sh->disks = disks;

	for (i = sh->disks; i--; ) {
Linus Torvalds's avatar
Linus Torvalds committed
302 303
		struct r5dev *dev = &sh->dev[i];

304
		if (dev->toread || dev->read || dev->towrite || dev->written ||
Linus Torvalds's avatar
Linus Torvalds committed
305
		    test_bit(R5_LOCKED, &dev->flags)) {
306
			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
Linus Torvalds's avatar
Linus Torvalds committed
307
			       (unsigned long long)sh->sector, i, dev->toread,
308
			       dev->read, dev->towrite, dev->written,
Linus Torvalds's avatar
Linus Torvalds committed
309 310 311 312 313 314 315 316 317
			       test_bit(R5_LOCKED, &dev->flags));
			BUG();
		}
		dev->flags = 0;
		raid5_build_block(sh, i);
	}
	insert_hash(conf, sh);
}

318
static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector, int disks)
Linus Torvalds's avatar
Linus Torvalds committed
319 320
{
	struct stripe_head *sh;
321
	struct hlist_node *hn;
Linus Torvalds's avatar
Linus Torvalds committed
322 323

	CHECK_DEVLOCK();
324
	pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
325
	hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash)
326
		if (sh->sector == sector && sh->disks == disks)
Linus Torvalds's avatar
Linus Torvalds committed
327
			return sh;
328
	pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
Linus Torvalds's avatar
Linus Torvalds committed
329 330 331 332
	return NULL;
}

static void unplug_slaves(mddev_t *mddev);
333
static void raid5_unplug_device(struct request_queue *q);
Linus Torvalds's avatar
Linus Torvalds committed
334

335 336
static struct stripe_head *get_active_stripe(raid5_conf_t *conf, sector_t sector, int disks,
					     int pd_idx, int noblock)
Linus Torvalds's avatar
Linus Torvalds committed
337 338 339
{
	struct stripe_head *sh;

340
	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
Linus Torvalds's avatar
Linus Torvalds committed
341 342 343 344

	spin_lock_irq(&conf->device_lock);

	do {
345 346 347
		wait_event_lock_irq(conf->wait_for_stripe,
				    conf->quiesce == 0,
				    conf->device_lock, /* nothing */);
348
		sh = __find_stripe(conf, sector, disks);
Linus Torvalds's avatar
Linus Torvalds committed
349 350 351 352 353 354 355 356 357
		if (!sh) {
			if (!conf->inactive_blocked)
				sh = get_free_stripe(conf);
			if (noblock && sh == NULL)
				break;
			if (!sh) {
				conf->inactive_blocked = 1;
				wait_event_lock_irq(conf->wait_for_stripe,
						    !list_empty(&conf->inactive_list) &&
358 359
						    (atomic_read(&conf->active_stripes)
						     < (conf->max_nr_stripes *3/4)
Linus Torvalds's avatar
Linus Torvalds committed
360 361
						     || !conf->inactive_blocked),
						    conf->device_lock,
362
						    raid5_unplug_device(conf->mddev->queue)
Linus Torvalds's avatar
Linus Torvalds committed
363 364 365
					);
				conf->inactive_blocked = 0;
			} else
366
				init_stripe(sh, sector, pd_idx, disks);
Linus Torvalds's avatar
Linus Torvalds committed
367 368
		} else {
			if (atomic_read(&sh->count)) {
369
			  BUG_ON(!list_empty(&sh->lru));
Linus Torvalds's avatar
Linus Torvalds committed
370 371 372
			} else {
				if (!test_bit(STRIPE_HANDLE, &sh->state))
					atomic_inc(&conf->active_stripes);
373 374
				if (list_empty(&sh->lru) &&
				    !test_bit(STRIPE_EXPANDING, &sh->state))
375 376
					BUG();
				list_del_init(&sh->lru);
Linus Torvalds's avatar
Linus Torvalds committed
377 378 379 380 381 382 383 384 385 386 387
			}
		}
	} while (sh == NULL);

	if (sh)
		atomic_inc(&sh->count);

	spin_unlock_irq(&conf->device_lock);
	return sh;
}

388 389 390 391
static void
raid5_end_read_request(struct bio *bi, int error);
static void
raid5_end_write_request(struct bio *bi, int error);
392

393
static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
{
	raid5_conf_t *conf = sh->raid_conf;
	int i, disks = sh->disks;

	might_sleep();

	for (i = disks; i--; ) {
		int rw;
		struct bio *bi;
		mdk_rdev_t *rdev;
		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags))
			rw = WRITE;
		else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
			rw = READ;
		else
			continue;

		bi = &sh->dev[i].req;

		bi->bi_rw = rw;
		if (rw == WRITE)
			bi->bi_end_io = raid5_end_write_request;
		else
			bi->bi_end_io = raid5_end_read_request;

		rcu_read_lock();
		rdev = rcu_dereference(conf->disks[i].rdev);
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
		if (rdev)
			atomic_inc(&rdev->nr_pending);
		rcu_read_unlock();

		if (rdev) {
428
			if (s->syncing || s->expanding || s->expanded)
429 430
				md_sync_acct(rdev->bdev, STRIPE_SECTORS);

Dan Williams's avatar
Dan Williams committed
431 432
			set_bit(STRIPE_IO_STARTED, &sh->state);

433 434
			bi->bi_bdev = rdev->bdev;
			pr_debug("%s: for %llu schedule op %ld on disc %d\n",
435
				__func__, (unsigned long long)sh->sector,
436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
				bi->bi_rw, i);
			atomic_inc(&sh->count);
			bi->bi_sector = sh->sector + rdev->data_offset;
			bi->bi_flags = 1 << BIO_UPTODATE;
			bi->bi_vcnt = 1;
			bi->bi_max_vecs = 1;
			bi->bi_idx = 0;
			bi->bi_io_vec = &sh->dev[i].vec;
			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			bi->bi_io_vec[0].bv_offset = 0;
			bi->bi_size = STRIPE_SIZE;
			bi->bi_next = NULL;
			if (rw == WRITE &&
			    test_bit(R5_ReWrite, &sh->dev[i].flags))
				atomic_add(STRIPE_SECTORS,
					&rdev->corrected_errors);
			generic_make_request(bi);
		} else {
			if (rw == WRITE)
				set_bit(STRIPE_DEGRADED, &sh->state);
			pr_debug("skip op %ld on disc %d for sector %llu\n",
				bi->bi_rw, i, (unsigned long long)sh->sector);
			clear_bit(R5_LOCKED, &sh->dev[i].flags);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
	}
}

static struct dma_async_tx_descriptor *
async_copy_data(int frombio, struct bio *bio, struct page *page,
	sector_t sector, struct dma_async_tx_descriptor *tx)
{
	struct bio_vec *bvl;
	struct page *bio_page;
	int i;
	int page_offset;

	if (bio->bi_sector >= sector)
		page_offset = (signed)(bio->bi_sector - sector) * 512;
	else
		page_offset = (signed)(sector - bio->bi_sector) * -512;
	bio_for_each_segment(bvl, bio, i) {
		int len = bio_iovec_idx(bio, i)->bv_len;
		int clen;
		int b_offset = 0;

		if (page_offset < 0) {
			b_offset = -page_offset;
			page_offset += b_offset;
			len -= b_offset;
		}

		if (len > 0 && page_offset + len > STRIPE_SIZE)
			clen = STRIPE_SIZE - page_offset;
		else
			clen = len;

		if (clen > 0) {
			b_offset += bio_iovec_idx(bio, i)->bv_offset;
			bio_page = bio_iovec_idx(bio, i)->bv_page;
			if (frombio)
				tx = async_memcpy(page, bio_page, page_offset,
					b_offset, clen,
499
					ASYNC_TX_DEP_ACK,
500 501 502 503
					tx, NULL, NULL);
			else
				tx = async_memcpy(bio_page, page, b_offset,
					page_offset, clen,
504
					ASYNC_TX_DEP_ACK,
505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
					tx, NULL, NULL);
		}
		if (clen < len) /* hit end of page */
			break;
		page_offset +=  len;
	}

	return tx;
}

static void ops_complete_biofill(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	struct bio *return_bi = NULL;
	raid5_conf_t *conf = sh->raid_conf;
520
	int i;
521

522
	pr_debug("%s: stripe %llu\n", __func__,
523 524 525
		(unsigned long long)sh->sector);

	/* clear completed biofills */
526
	spin_lock_irq(&conf->device_lock);
527 528 529 530
	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];

		/* acknowledge completion of a biofill operation */
531 532
		/* and check if we need to reply to a read request,
		 * new R5_Wantfill requests are held off until
533
		 * !STRIPE_BIOFILL_RUN
534 535
		 */
		if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
536 537 538 539 540 541 542 543
			struct bio *rbi, *rbi2;

			BUG_ON(!dev->read);
			rbi = dev->read;
			dev->read = NULL;
			while (rbi && rbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
				rbi2 = r5_next_bio(rbi, dev->sector);
544
				if (!raid5_dec_bi_phys_segments(rbi)) {
545 546 547 548 549 550 551
					rbi->bi_next = return_bi;
					return_bi = rbi;
				}
				rbi = rbi2;
			}
		}
	}
552 553
	spin_unlock_irq(&conf->device_lock);
	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
554 555 556

	return_io(return_bi);

557
	set_bit(STRIPE_HANDLE, &sh->state);
558 559 560 561 562 563 564 565 566
	release_stripe(sh);
}

static void ops_run_biofill(struct stripe_head *sh)
{
	struct dma_async_tx_descriptor *tx = NULL;
	raid5_conf_t *conf = sh->raid_conf;
	int i;

567
	pr_debug("%s: stripe %llu\n", __func__,
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
		(unsigned long long)sh->sector);

	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (test_bit(R5_Wantfill, &dev->flags)) {
			struct bio *rbi;
			spin_lock_irq(&conf->device_lock);
			dev->read = rbi = dev->toread;
			dev->toread = NULL;
			spin_unlock_irq(&conf->device_lock);
			while (rbi && rbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
				tx = async_copy_data(0, rbi, dev->page,
					dev->sector, tx);
				rbi = r5_next_bio(rbi, dev->sector);
			}
		}
	}

	atomic_inc(&sh->count);
	async_trigger_callback(ASYNC_TX_DEP_ACK | ASYNC_TX_ACK, tx,
		ops_complete_biofill, sh);
}

static void ops_complete_compute5(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	int target = sh->ops.target;
	struct r5dev *tgt = &sh->dev[target];

598
	pr_debug("%s: stripe %llu\n", __func__,
599 600 601 602 603
		(unsigned long long)sh->sector);

	set_bit(R5_UPTODATE, &tgt->flags);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	clear_bit(R5_Wantcompute, &tgt->flags);
604 605 606
	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
	if (sh->check_state == check_state_compute_run)
		sh->check_state = check_state_compute_result;
607 608 609 610
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

611
static struct dma_async_tx_descriptor *ops_run_compute5(struct stripe_head *sh)
612 613 614 615 616 617 618 619 620 621 622 623
{
	/* kernel stack size limits the total number of disks */
	int disks = sh->disks;
	struct page *xor_srcs[disks];
	int target = sh->ops.target;
	struct r5dev *tgt = &sh->dev[target];
	struct page *xor_dest = tgt->page;
	int count = 0;
	struct dma_async_tx_descriptor *tx;
	int i;

	pr_debug("%s: stripe %llu block: %d\n",
624
		__func__, (unsigned long long)sh->sector, target);
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));

	for (i = disks; i--; )
		if (i != target)
			xor_srcs[count++] = sh->dev[i].page;

	atomic_inc(&sh->count);

	if (unlikely(count == 1))
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE,
			0, NULL, ops_complete_compute5, sh);
	else
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
			ASYNC_TX_XOR_ZERO_DST, NULL,
			ops_complete_compute5, sh);

	return tx;
}

static void ops_complete_prexor(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

648
	pr_debug("%s: stripe %llu\n", __func__,
649 650 651 652 653 654 655 656 657 658 659 660 661 662
		(unsigned long long)sh->sector);
}

static struct dma_async_tx_descriptor *
ops_run_prexor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
{
	/* kernel stack size limits the total number of disks */
	int disks = sh->disks;
	struct page *xor_srcs[disks];
	int count = 0, pd_idx = sh->pd_idx, i;

	/* existing parity data subtracted */
	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;

663
	pr_debug("%s: stripe %llu\n", __func__,
664 665 666 667 668
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		/* Only process blocks that are known to be uptodate */
669
		if (test_bit(R5_Wantdrain, &dev->flags))
670 671 672 673 674 675 676 677 678 679 680
			xor_srcs[count++] = dev->page;
	}

	tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
		ASYNC_TX_DEP_ACK | ASYNC_TX_XOR_DROP_DST, tx,
		ops_complete_prexor, sh);

	return tx;
}

static struct dma_async_tx_descriptor *
681
ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
682 683
{
	int disks = sh->disks;
684
	int i;
685

686
	pr_debug("%s: stripe %llu\n", __func__,
687 688 689 690 691 692
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		struct bio *chosen;

693
		if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
			struct bio *wbi;

			spin_lock(&sh->lock);
			chosen = dev->towrite;
			dev->towrite = NULL;
			BUG_ON(dev->written);
			wbi = dev->written = chosen;
			spin_unlock(&sh->lock);

			while (wbi && wbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
				tx = async_copy_data(1, wbi, dev->page,
					dev->sector, tx);
				wbi = r5_next_bio(wbi, dev->sector);
			}
		}
	}

	return tx;
}

static void ops_complete_postxor(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	int disks = sh->disks, i, pd_idx = sh->pd_idx;

720
	pr_debug("%s: stripe %llu\n", __func__,
721 722 723 724 725 726 727 728
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (dev->written || i == pd_idx)
			set_bit(R5_UPTODATE, &dev->flags);
	}

729 730 731 732 733 734 735 736
	if (sh->reconstruct_state == reconstruct_state_drain_run)
		sh->reconstruct_state = reconstruct_state_drain_result;
	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
		sh->reconstruct_state = reconstruct_state_prexor_drain_result;
	else {
		BUG_ON(sh->reconstruct_state != reconstruct_state_run);
		sh->reconstruct_state = reconstruct_state_result;
	}
737 738 739 740 741 742

	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

static void
743
ops_run_postxor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
744 745 746 747 748 749 750
{
	/* kernel stack size limits the total number of disks */
	int disks = sh->disks;
	struct page *xor_srcs[disks];

	int count = 0, pd_idx = sh->pd_idx, i;
	struct page *xor_dest;
751
	int prexor = 0;
752 753
	unsigned long flags;

754
	pr_debug("%s: stripe %llu\n", __func__,
755 756 757 758 759
		(unsigned long long)sh->sector);

	/* check if prexor is active which means only process blocks
	 * that are part of a read-modify-write (written)
	 */
760 761
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
		prexor = 1;
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789
		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (dev->written)
				xor_srcs[count++] = dev->page;
		}
	} else {
		xor_dest = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i != pd_idx)
				xor_srcs[count++] = dev->page;
		}
	}

	/* 1/ if we prexor'd then the dest is reused as a source
	 * 2/ if we did not prexor then we are redoing the parity
	 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
	 * for the synchronous xor case
	 */
	flags = ASYNC_TX_DEP_ACK | ASYNC_TX_ACK |
		(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);

	atomic_inc(&sh->count);

	if (unlikely(count == 1)) {
		flags &= ~(ASYNC_TX_XOR_DROP_DST | ASYNC_TX_XOR_ZERO_DST);
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE,
790
			flags, tx, ops_complete_postxor, sh);
791 792
	} else
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
793
			flags, tx, ops_complete_postxor, sh);
794 795 796 797 798 799
}

static void ops_complete_check(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

800
	pr_debug("%s: stripe %llu\n", __func__,
801 802
		(unsigned long long)sh->sector);

803
	sh->check_state = check_state_check_result;
804 805 806 807 808 809 810 811 812 813 814 815 816 817
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

static void ops_run_check(struct stripe_head *sh)
{
	/* kernel stack size limits the total number of disks */
	int disks = sh->disks;
	struct page *xor_srcs[disks];
	struct dma_async_tx_descriptor *tx;

	int count = 0, pd_idx = sh->pd_idx, i;
	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;

818
	pr_debug("%s: stripe %llu\n", __func__,
819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (i != pd_idx)
			xor_srcs[count++] = dev->page;
	}

	tx = async_xor_zero_sum(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
		&sh->ops.zero_sum_result, 0, NULL, NULL, NULL);

	atomic_inc(&sh->count);
	tx = async_trigger_callback(ASYNC_TX_DEP_ACK | ASYNC_TX_ACK, tx,
		ops_complete_check, sh);
}

835
static void raid5_run_ops(struct stripe_head *sh, unsigned long ops_request)
836 837 838 839
{
	int overlap_clear = 0, i, disks = sh->disks;
	struct dma_async_tx_descriptor *tx = NULL;

840
	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
841 842 843 844
		ops_run_biofill(sh);
		overlap_clear++;
	}

845 846 847 848 849 850
	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
		tx = ops_run_compute5(sh);
		/* terminate the chain if postxor is not set to be run */
		if (tx && !test_bit(STRIPE_OP_POSTXOR, &ops_request))
			async_tx_ack(tx);
	}
851

852
	if (test_bit(STRIPE_OP_PREXOR, &ops_request))
853 854
		tx = ops_run_prexor(sh, tx);

855
	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
856
		tx = ops_run_biodrain(sh, tx);
857 858 859
		overlap_clear++;
	}

860
	if (test_bit(STRIPE_OP_POSTXOR, &ops_request))
861
		ops_run_postxor(sh, tx);
862

863
	if (test_bit(STRIPE_OP_CHECK, &ops_request))
864 865 866 867 868 869 870 871 872 873
		ops_run_check(sh);

	if (overlap_clear)
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_Overlap, &dev->flags))
				wake_up(&sh->raid_conf->wait_for_overlap);
		}
}

874
static int grow_one_stripe(raid5_conf_t *conf)
Linus Torvalds's avatar
Linus Torvalds committed
875 876
{
	struct stripe_head *sh;
877 878 879 880 881 882 883 884 885 886 887 888
	sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL);
	if (!sh)
		return 0;
	memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev));
	sh->raid_conf = conf;
	spin_lock_init(&sh->lock);

	if (grow_buffers(sh, conf->raid_disks)) {
		shrink_buffers(sh, conf->raid_disks);
		kmem_cache_free(conf->slab_cache, sh);
		return 0;
	}
889
	sh->disks = conf->raid_disks;
890 891 892 893 894 895 896 897 898 899
	/* we just created an active stripe so... */
	atomic_set(&sh->count, 1);
	atomic_inc(&conf->active_stripes);
	INIT_LIST_HEAD(&sh->lru);
	release_stripe(sh);
	return 1;
}

static int grow_stripes(raid5_conf_t *conf, int num)
{
900
	struct kmem_cache *sc;
Linus Torvalds's avatar
Linus Torvalds committed
901 902
	int devs = conf->raid_disks;

903 904
	sprintf(conf->cache_name[0], "raid5-%s", mdname(conf->mddev));
	sprintf(conf->cache_name[1], "raid5-%s-alt", mdname(conf->mddev));
905 906
	conf->active_name = 0;
	sc = kmem_cache_create(conf->cache_name[conf->active_name],
Linus Torvalds's avatar
Linus Torvalds committed
907
			       sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
908
			       0, 0, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
909 910 911
	if (!sc)
		return 1;
	conf->slab_cache = sc;
912
	conf->pool_size = devs;
913
	while (num--)
914
		if (!grow_one_stripe(conf))
Linus Torvalds's avatar
Linus Torvalds committed
915 916 917
			return 1;
	return 0;
}
918 919

#ifdef CONFIG_MD_RAID5_RESHAPE
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947
static int resize_stripes(raid5_conf_t *conf, int newsize)
{
	/* Make all the stripes able to hold 'newsize' devices.
	 * New slots in each stripe get 'page' set to a new page.
	 *
	 * This happens in stages:
	 * 1/ create a new kmem_cache and allocate the required number of
	 *    stripe_heads.
	 * 2/ gather all the old stripe_heads and tranfer the pages across
	 *    to the new stripe_heads.  This will have the side effect of
	 *    freezing the array as once all stripe_heads have been collected,
	 *    no IO will be possible.  Old stripe heads are freed once their
	 *    pages have been transferred over, and the old kmem_cache is
	 *    freed when all stripes are done.
	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
	 *    we simple return a failre status - no need to clean anything up.
	 * 4/ allocate new pages for the new slots in the new stripe_heads.
	 *    If this fails, we don't bother trying the shrink the
	 *    stripe_heads down again, we just leave them as they are.
	 *    As each stripe_head is processed the new one is released into
	 *    active service.
	 *
	 * Once step2 is started, we cannot afford to wait for a write,
	 * so we use GFP_NOIO allocations.
	 */
	struct stripe_head *osh, *nsh;
	LIST_HEAD(newstripes);
	struct disk_info *ndisks;
948
	int err;
949
	struct kmem_cache *sc;
950 951 952 953 954
	int i;

	if (newsize <= conf->pool_size)
		return 0; /* never bother to shrink */

955 956 957
	err = md_allow_write(conf->mddev);
	if (err)
		return err;
958

959 960 961
	/* Step 1 */
	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
			       sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
962
			       0, 0, NULL);
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996
	if (!sc)
		return -ENOMEM;

	for (i = conf->max_nr_stripes; i; i--) {
		nsh = kmem_cache_alloc(sc, GFP_KERNEL);
		if (!nsh)
			break;

		memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev));

		nsh->raid_conf = conf;
		spin_lock_init(&nsh->lock);

		list_add(&nsh->lru, &newstripes);
	}
	if (i) {
		/* didn't get enough, give up */
		while (!list_empty(&newstripes)) {
			nsh = list_entry(newstripes.next, struct stripe_head, lru);
			list_del(&nsh->lru);
			kmem_cache_free(sc, nsh);
		}
		kmem_cache_destroy(sc);
		return -ENOMEM;
	}
	/* Step 2 - Must use GFP_NOIO now.
	 * OK, we have enough stripes, start collecting inactive
	 * stripes and copying them over
	 */
	list_for_each_entry(nsh, &newstripes, lru) {
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    !list_empty(&conf->inactive_list),
				    conf->device_lock,
997
				    unplug_slaves(conf->mddev)
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
			);
		osh = get_free_stripe(conf);
		spin_unlock_irq(&conf->device_lock);
		atomic_set(&nsh->count, 1);
		for(i=0; i<conf->pool_size; i++)
			nsh->dev[i].page = osh->dev[i].page;
		for( ; i<newsize; i++)
			nsh->dev[i].page = NULL;
		kmem_cache_free(conf->slab_cache, osh);
	}
	kmem_cache_destroy(conf->slab_cache);

	/* Step 3.
	 * At this point, we are holding all the stripes so the array
	 * is completely stalled, so now is a good time to resize
	 * conf->disks.
	 */
	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
	if (ndisks) {
		for (i=0; i<conf->raid_disks; i++)
			ndisks[i] = conf->disks[i];
		kfree(conf->disks);
		conf->disks = ndisks;
	} else
		err = -ENOMEM;

	/* Step 4, return new stripes to service */
	while(!list_empty(&newstripes)) {
		nsh = list_entry(newstripes.next, struct stripe_head, lru);
		list_del_init(&nsh->lru);
		for (i=conf->raid_disks; i < newsize; i++)
			if (nsh->dev[i].page == NULL) {
				struct page *p = alloc_page(GFP_NOIO);
				nsh->dev[i].page = p;
				if (!p)
					err = -ENOMEM;
			}
		release_stripe(nsh);
	}
	/* critical section pass, GFP_NOIO no longer needed */

	conf->slab_cache = sc;
	conf->active_name = 1-conf->active_name;
	conf->pool_size = newsize;
	return err;
}
1044
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1045

1046
static int drop_one_stripe(raid5_conf_t *conf)
Linus Torvalds's avatar
Linus Torvalds committed
1047 1048 1049
{
	struct stripe_head *sh;

1050 1051 1052 1053 1054
	spin_lock_irq(&conf->device_lock);
	sh = get_free_stripe(conf);
	spin_unlock_irq(&conf->device_lock);
	if (!sh)
		return 0;
1055
	BUG_ON(atomic_read(&sh->count));
1056
	shrink_buffers(sh, conf->pool_size);
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
	kmem_cache_free(conf->slab_cache, sh);
	atomic_dec(&conf->active_stripes);
	return 1;
}

static void shrink_stripes(raid5_conf_t *conf)
{
	while (drop_one_stripe(conf))
		;

1067 1068
	if (conf->slab_cache)
		kmem_cache_destroy(conf->slab_cache);
Linus Torvalds's avatar
Linus Torvalds committed
1069 1070 1071
	conf->slab_cache = NULL;
}

1072
static void raid5_end_read_request(struct bio * bi, int error)
Linus Torvalds's avatar
Linus Torvalds committed
1073 1074 1075
{
 	struct stripe_head *sh = bi->bi_private;
	raid5_conf_t *conf = sh->raid_conf;
1076
	int disks = sh->disks, i;
Linus Torvalds's avatar
Linus Torvalds committed
1077
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1078 1079
	char b[BDEVNAME_SIZE];
	mdk_rdev_t *rdev;
Linus Torvalds's avatar
Linus Torvalds committed
1080 1081 1082 1083 1084 1085


	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

1086 1087
	pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
Linus Torvalds's avatar
Linus Torvalds committed
1088 1089 1090
		uptodate);
	if (i == disks) {
		BUG();
1091
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1092 1093 1094 1095
	}

	if (uptodate) {
		set_bit(R5_UPTODATE, &sh->dev[i].flags);
1096
		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1097
			rdev = conf->disks[i].rdev;
1098 1099 1100 1101 1102 1103
			printk_rl(KERN_INFO "raid5:%s: read error corrected"
				  " (%lu sectors at %llu on %s)\n",
				  mdname(conf->mddev), STRIPE_SECTORS,
				  (unsigned long long)(sh->sector
						       + rdev->data_offset),
				  bdevname(rdev->bdev, b));
1104 1105 1106
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
		}
1107 1108
		if (atomic_read(&conf->disks[i].rdev->read_errors))
			atomic_set(&conf->disks[i].rdev->read_errors, 0);
Linus Torvalds's avatar
Linus Torvalds committed
1109
	} else {
1110
		const char *bdn = bdevname(conf->disks[i].rdev->bdev, b);
1111
		int retry = 0;
1112 1113
		rdev = conf->disks[i].rdev;

Linus Torvalds's avatar
Linus Torvalds committed
1114
		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
1115
		atomic_inc(&rdev->read_errors);
1116
		if (conf->mddev->degraded)
1117 1118 1119 1120 1121 1122 1123
			printk_rl(KERN_WARNING
				  "raid5:%s: read error not correctable "
				  "(sector %llu on %s).\n",
				  mdname(conf->mddev),
				  (unsigned long long)(sh->sector
						       + rdev->data_offset),
				  bdn);
1124
		else if (test_bit(R5_ReWrite, &sh->dev[i].flags))
1125
			/* Oh, no!!! */
1126 1127 1128 1129 1130 1131 1132
			printk_rl(KERN_WARNING
				  "raid5:%s: read error NOT corrected!! "
				  "(sector %llu on %s).\n",
				  mdname(conf->mddev),
				  (unsigned long long)(sh->sector
						       + rdev->data_offset),
				  bdn);
1133
		else if (atomic_read(&rdev->read_errors)
1134
			 > conf->max_nr_stripes)
1135
			printk(KERN_WARNING
1136 1137
			       "raid5:%s: Too many read errors, failing device %s.\n",
			       mdname(conf->mddev), bdn);
1138 1139 1140 1141 1142
		else
			retry = 1;
		if (retry)
			set_bit(R5_ReadError, &sh->dev[i].flags);
		else {
1143 1144
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
1145
			md_error(conf->mddev, rdev);
1146
		}
Linus Torvalds's avatar
Linus Torvalds committed
1147 1148 1149 1150 1151 1152 1153
	}
	rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1154
static void raid5_end_write_request (struct bio *bi, int error)
Linus Torvalds's avatar
Linus Torvalds committed
1155 1156 1157
{
 	struct stripe_head *sh = bi->bi_private;
	raid5_conf_t *conf = sh->raid_conf;
1158
	int disks = sh->disks, i;
Linus Torvalds's avatar
Linus Torvalds committed
1159 1160 1161 1162 1163 1164
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);

	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

1165
	pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
Linus Torvalds's avatar
Linus Torvalds committed
1166 1167 1168 1169
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
		uptodate);
	if (i == disks) {
		BUG();
1170
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1171 1172 1173 1174 1175 1176 1177 1178 1179
	}

	if (!uptodate)
		md_error(conf->mddev, conf->disks[i].rdev);

	rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
	
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
1180
	release_stripe(sh);
Linus Torvalds's avatar
Linus Torvalds committed
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
}


static sector_t compute_blocknr(struct stripe_head *sh, int i);
	
static void raid5_build_block (struct stripe_head *sh, int i)
{
	struct r5dev *dev = &sh->dev[i];

	bio_init(&dev->req);
	dev->req.bi_io_vec = &dev->vec;
	dev->req.bi_vcnt++;
	dev->req.bi_max_vecs++;
	dev->vec.bv_page = dev->page;
	dev->vec.bv_len = STRIPE_SIZE;
	dev->vec.bv_offset = 0;

	dev->req.bi_sector = sh->sector;
	dev->req.bi_private = sh;

	dev->flags = 0;
1202
	dev->sector = compute_blocknr(sh, i);
Linus Torvalds's avatar
Linus Torvalds committed
1203 1204 1205 1206 1207 1208
}

static void error(mddev_t *mddev, mdk_rdev_t *rdev)
{
	char b[BDEVNAME_SIZE];
	raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
1209
	pr_debug("raid5: error called\n");
Linus Torvalds's avatar
Linus Torvalds committed
1210

1211
	if (!test_bit(Faulty, &rdev->flags)) {
1212
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
1213 1214 1215
		if (test_and_clear_bit(In_sync, &rdev->flags)) {
			unsigned long flags;
			spin_lock_irqsave(&conf->device_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
1216
			mddev->degraded++;
1217
			spin_unlock_irqrestore(&conf->device_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
1218 1219 1220
			/*
			 * if recovery was running, make sure it aborts.
			 */
1221
			set_bit(MD_RECOVERY_INTR, &mddev->recovery);
Linus Torvalds's avatar
Linus Torvalds committed
1222
		}
1223
		set_bit(Faulty, &rdev->flags);
Linus Torvalds's avatar
Linus Torvalds committed
1224
		printk (KERN_ALERT
1225 1226
			"raid5: Disk failure on %s, disabling device.\n"
			"raid5: Operation continuing on %d devices.\n",
1227
			bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded);
Linus Torvalds's avatar
Linus Torvalds committed
1228
	}
1229
}
Linus Torvalds's avatar
Linus Torvalds committed
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266

/*
 * Input: a 'big' sector number,
 * Output: index of the data and parity disk, and the sector # in them.
 */
static sector_t raid5_compute_sector(sector_t r_sector, unsigned int raid_disks,
			unsigned int data_disks, unsigned int * dd_idx,
			unsigned int * pd_idx, raid5_conf_t *conf)
{
	long stripe;
	unsigned long chunk_number;
	unsigned int chunk_offset;
	sector_t new_sector;
	int sectors_per_chunk = conf->chunk_size >> 9;

	/* First compute the information on this sector */

	/*
	 * Compute the chunk number and the sector offset inside the chunk
	 */
	chunk_offset = sector_div(r_sector, sectors_per_chunk);
	chunk_number = r_sector;
	BUG_ON(r_sector != chunk_number);

	/*
	 * Compute the stripe number
	 */
	stripe = chunk_number / data_disks;

	/*
	 * Compute the data disk and parity disk indexes inside the stripe
	 */
	*dd_idx = chunk_number % data_disks;

	/*
	 * Select the parity disk based on the user selected algorithm.
	 */
1267 1268
	switch(conf->level) {
	case 4:
Linus Torvalds's avatar
Linus Torvalds committed
1269
		*pd_idx = data_disks;
1270 1271 1272
		break;
	case 5:
		switch (conf->algorithm) {
Linus Torvalds's avatar
Linus Torvalds committed
1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
		case ALGORITHM_LEFT_ASYMMETRIC:
			*pd_idx = data_disks - stripe % raid_disks;
			if (*dd_idx >= *pd_idx)
				(*dd_idx)++;
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
			*pd_idx = stripe % raid_disks;
			if (*dd_idx >= *pd_idx)
				(*dd_idx)++;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
			*pd_idx = data_disks - stripe % raid_disks;
			*dd_idx = (*pd_idx + 1 + *dd_idx) % raid_disks;
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
			*pd_idx = stripe % raid_disks;
			*dd_idx = (*pd_idx + 1 + *dd_idx) % raid_disks;
			break;
		default: