raid5.h 27.9 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5
#ifndef _RAID5_H
#define _RAID5_H

#include <linux/raid/xor.h>
Dan Williams's avatar
Dan Williams committed
6
#include <linux/dmaengine.h>
Linus Torvalds's avatar
Linus Torvalds committed
7 8 9

/*
 *
10
 * Each stripe contains one buffer per device.  Each buffer can be in
Linus Torvalds's avatar
Linus Torvalds committed
11
 * one of a number of states stored in "flags".  Changes between
12 13 14
 * these states happen *almost* exclusively under the protection of the
 * STRIPE_ACTIVE flag.  Some very specific changes can happen in bi_end_io, and
 * these are not protected by STRIPE_ACTIVE.
Linus Torvalds's avatar
Linus Torvalds committed
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
 *
 * The flag bits that are used to represent these states are:
 *   R5_UPTODATE and R5_LOCKED
 *
 * State Empty == !UPTODATE, !LOCK
 *        We have no data, and there is no active request
 * State Want == !UPTODATE, LOCK
 *        A read request is being submitted for this block
 * State Dirty == UPTODATE, LOCK
 *        Some new data is in this buffer, and it is being written out
 * State Clean == UPTODATE, !LOCK
 *        We have valid data which is the same as on disc
 *
 * The possible state transitions are:
 *
 *  Empty -> Want   - on read or write to get old data for  parity calc
NeilBrown's avatar
NeilBrown committed
31
 *  Empty -> Dirty  - on compute_parity to satisfy write/sync request.
Linus Torvalds's avatar
Linus Torvalds committed
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
 *  Empty -> Clean  - on compute_block when computing a block for failed drive
 *  Want  -> Empty  - on failed read
 *  Want  -> Clean  - on successful completion of read request
 *  Dirty -> Clean  - on successful completion of write request
 *  Dirty -> Clean  - on failed write
 *  Clean -> Dirty  - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW)
 *
 * The Want->Empty, Want->Clean, Dirty->Clean, transitions
 * all happen in b_end_io at interrupt time.
 * Each sets the Uptodate bit before releasing the Lock bit.
 * This leaves one multi-stage transition:
 *    Want->Dirty->Clean
 * This is safe because thinking that a Clean buffer is actually dirty
 * will at worst delay some action, and the stripe will be scheduled
 * for attention after the transition is complete.
 *
 * There is one possibility that is not covered by these states.  That
 * is if one drive has failed and there is a spare being rebuilt.  We
 * can't distinguish between a clean block that has been generated
 * from parity calculations, and a clean block that has been
 * successfully written to the spare ( or to parity when resyncing).
53
 * To distinguish these states we have a stripe bit STRIPE_INSYNC that
Linus Torvalds's avatar
Linus Torvalds committed
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
 * is set whenever a write is scheduled to the spare, or to the parity
 * disc if there is no spare.  A sync request clears this bit, and
 * when we find it set with no buffers locked, we know the sync is
 * complete.
 *
 * Buffers for the md device that arrive via make_request are attached
 * to the appropriate stripe in one of two lists linked on b_reqnext.
 * One list (bh_read) for read requests, one (bh_write) for write.
 * There should never be more than one buffer on the two lists
 * together, but we are not guaranteed of that so we allow for more.
 *
 * If a buffer is on the read list when the associated cache buffer is
 * Uptodate, the data is copied into the read buffer and it's b_end_io
 * routine is called.  This may happen in the end_request routine only
 * if the buffer has just successfully been read.  end_request should
 * remove the buffers from the list and then set the Uptodate bit on
 * the buffer.  Other threads may do this only if they first check
 * that the Uptodate bit is set.  Once they have checked that they may
 * take buffers off the read queue.
 *
 * When a buffer on the write list is committed for write it is copied
 * into the cache buffer, which is then marked dirty, and moved onto a
 * third list, the written list (bh_written).  Once both the parity
 * block and the cached buffer are successfully written, any buffer on
 * a written list can be returned with b_end_io.
 *
80 81 82 83
 * The write list and read list both act as fifos.  The read list,
 * write list and written list are protected by the device_lock.
 * The device_lock is only for list manipulations and will only be
 * held for a very short time.  It can be claimed from interrupts.
Linus Torvalds's avatar
Linus Torvalds committed
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
 *
 *
 * Stripes in the stripe cache can be on one of two lists (or on
 * neither).  The "inactive_list" contains stripes which are not
 * currently being used for any request.  They can freely be reused
 * for another stripe.  The "handle_list" contains stripes that need
 * to be handled in some way.  Both of these are fifo queues.  Each
 * stripe is also (potentially) linked to a hash bucket in the hash
 * table so that it can be found by sector number.  Stripes that are
 * not hashed must be on the inactive_list, and will normally be at
 * the front.  All stripes start life this way.
 *
 * The inactive_list, handle_list and hash bucket lists are all protected by the
 * device_lock.
 *  - stripes have a reference counter. If count==0, they are on a list.
 *  - If a stripe might need handling, STRIPE_HANDLE is set.
 *  - When refcount reaches zero, then if STRIPE_HANDLE it is put on
 *    handle_list else inactive_list
 *
 * This, combined with the fact that STRIPE_HANDLE is only ever
 * cleared while a stripe has a non-zero count means that if the
 * refcount is 0 and STRIPE_HANDLE is set, then it is on the
 * handle_list and if recount is 0 and STRIPE_HANDLE is not set, then
 * the stripe is on inactive_list.
 *
 * The possible transitions are:
 *  activate an unhashed/inactive stripe (get_active_stripe())
 *     lockdev check-hash unlink-stripe cnt++ clean-stripe hash-stripe unlockdev
 *  activate a hashed, possibly active stripe (get_active_stripe())
 *     lockdev check-hash if(!cnt++)unlink-stripe unlockdev
 *  attach a request to an active stripe (add_stripe_bh())
 *     lockdev attach-buffer unlockdev
 *  handle a stripe (handle_stripe())
117
 *     setSTRIPE_ACTIVE,  clrSTRIPE_HANDLE ...
118 119
 *		(lockdev check-buffers unlockdev) ..
 *		change-state ..
120
 *		record io/ops needed clearSTRIPE_ACTIVE schedule io/ops
Linus Torvalds's avatar
Linus Torvalds committed
121 122 123 124 125
 *  release an active stripe (release_stripe())
 *     lockdev if (!--cnt) { if  STRIPE_HANDLE, add to handle_list else add to inactive-list } unlockdev
 *
 * The refcount counts each thread that have activated the stripe,
 * plus raid5d if it is handling it, plus one for each active request
126 127 128
 * on a cached buffer, and plus one if the stripe is undergoing stripe
 * operations.
 *
129
 * The stripe operations are:
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
 * -copying data between the stripe cache and user application buffers
 * -computing blocks to save a disk access, or to recover a missing block
 * -updating the parity on a write operation (reconstruct write and
 *  read-modify-write)
 * -checking parity correctness
 * -running i/o to disk
 * These operations are carried out by raid5_run_ops which uses the async_tx
 * api to (optionally) offload operations to dedicated hardware engines.
 * When requesting an operation handle_stripe sets the pending bit for the
 * operation and increments the count.  raid5_run_ops is then run whenever
 * the count is non-zero.
 * There are some critical dependencies between the operations that prevent some
 * from being requested while another is in flight.
 * 1/ Parity check operations destroy the in cache version of the parity block,
 *    so we prevent parity dependent operations like writes and compute_blocks
 *    from starting while a check is in progress.  Some dma engines can perform
 *    the check without damaging the parity block, in these cases the parity
 *    block is re-marked up to date (assuming the check was successful) and is
 *    not re-read from disk.
 * 2/ When a write operation is requested we immediately lock the affected
 *    blocks, and mark them as not up to date.  This causes new read requests
 *    to be held off, as well as parity checks and compute block operations.
 * 3/ Once a compute block operation has been requested handle_stripe treats
 *    that block as if it is up to date.  raid5_run_ops guaruntees that any
 *    operation that is dependent on the compute block result is initiated after
 *    the compute block completes.
Linus Torvalds's avatar
Linus Torvalds committed
156 157
 */

158
/*
159
 * Operations state - intermediate states that are visible outside of
160
 *   STRIPE_ACTIVE.
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
 * In general _idle indicates nothing is running, _run indicates a data
 * processing operation is active, and _result means the data processing result
 * is stable and can be acted upon.  For simple operations like biofill and
 * compute that only have an _idle and _run state they are indicated with
 * sh->state flags (STRIPE_BIOFILL_RUN and STRIPE_COMPUTE_RUN)
 */
/**
 * enum check_states - handles syncing / repairing a stripe
 * @check_state_idle - check operations are quiesced
 * @check_state_run - check operation is running
 * @check_state_result - set outside lock when check result is valid
 * @check_state_compute_run - check failed and we are repairing
 * @check_state_compute_result - set outside lock when compute result is valid
 */
enum check_states {
	check_state_idle = 0,
177 178 179
	check_state_run, /* xor parity check */
	check_state_run_q, /* q-parity check */
	check_state_run_pq, /* pq dual parity check */
180 181 182 183 184 185 186 187 188 189
	check_state_check_result,
	check_state_compute_run, /* parity repair */
	check_state_compute_result,
};

/**
 * enum reconstruct_states - handles writing or expanding a stripe
 */
enum reconstruct_states {
	reconstruct_state_idle = 0,
190
	reconstruct_state_prexor_drain_run,	/* prexor-write */
191 192
	reconstruct_state_drain_run,		/* write */
	reconstruct_state_run,			/* expand */
193
	reconstruct_state_prexor_drain_result,
194 195 196 197
	reconstruct_state_drain_result,
	reconstruct_state_result,
};

Linus Torvalds's avatar
Linus Torvalds committed
198
struct stripe_head {
199
	struct hlist_node	hash;
200
	struct list_head	lru;	      /* inactive_list or handle_list */
201
	struct llist_node	release_list;
202
	struct r5conf		*raid_conf;
203 204
	short			generation;	/* increments with every
						 * reshape */
205 206 207
	sector_t		sector;		/* sector of this row */
	short			pd_idx;		/* parity disk index */
	short			qd_idx;		/* 'Q' disk index for raid6 */
208
	short			ddf_layout;/* use DDF ordering to calculate Q */
209
	short			hash_lock_index;
210 211
	unsigned long		state;		/* state flags */
	atomic_t		count;	      /* nr of active thread/requests */
212
	int			bm_seq;	/* sequence number for bitmap flushes */
213
	int			disks;		/* disks in stripe */
214 215 216 217
	int			overwrite_disks; /* total overwrite disks in stripe,
						  * this is only checked when stripe
						  * has STRIPE_BATCH_READY
						  */
218
	enum check_states	check_state;
219
	enum reconstruct_states reconstruct_state;
Shaohua Li's avatar
Shaohua Li committed
220
	spinlock_t		stripe_lock;
221
	int			cpu;
222
	struct r5worker_group	*group;
223 224 225 226

	struct stripe_head	*batch_head; /* protected by stripe lock */
	spinlock_t		batch_lock; /* only header's lock is useful */
	struct list_head	batch_list; /* protected by head's batch lock*/
Shaohua Li's avatar
Shaohua Li committed
227

228 229 230 231 232
	union {
		struct r5l_io_unit	*log_io;
		struct ppl_io_unit	*ppl_io;
	};

Shaohua Li's avatar
Shaohua Li committed
233
	struct list_head	log_list;
234 235
	sector_t		log_start; /* first meta block on the journal */
	struct list_head	r5c; /* for r5c_cache->stripe_in_journal */
236 237

	struct page		*ppl_page; /* partial parity of this stripe */
238 239
	/**
	 * struct stripe_operations
240
	 * @target - STRIPE_OP_COMPUTE_BLK target
241 242 243
	 * @target2 - 2nd compute target in the raid6 case
	 * @zero_sum_result - P and Q verification flags
	 * @request - async service request flags for raid_run_ops
244 245
	 */
	struct stripe_operations {
246
		int 		     target, target2;
Dan Williams's avatar
Dan Williams committed
247
		enum sum_check_flags zero_sum_result;
248
	} ops;
Linus Torvalds's avatar
Linus Torvalds committed
249
	struct r5dev {
250 251 252 253 254
		/* rreq and rvec are used for the replacement device when
		 * writing data to both devices.
		 */
		struct bio	req, rreq;
		struct bio_vec	vec, rvec;
255
		struct page	*page, *orig_page;
256
		struct bio	*toread, *read, *towrite, *written;
Linus Torvalds's avatar
Linus Torvalds committed
257 258
		sector_t	sector;			/* sector of this page */
		unsigned long	flags;
Shaohua Li's avatar
Shaohua Li committed
259
		u32		log_checksum;
260
		unsigned short	write_hint;
Linus Torvalds's avatar
Linus Torvalds committed
261 262
	} dev[1]; /* allocated with extra space depending of RAID geometry */
};
263 264

/* stripe_head_state - collects and tracks the dynamic state of a stripe_head
265
 *     for handle_stripe.
266 267
 */
struct stripe_head_state {
268 269 270 271 272 273 274
	/* 'syncing' means that we need to read all devices, either
	 * to check/correct parity, or to reconstruct a missing device.
	 * 'replacing' means we are replacing one or more drives and
	 * the source is valid at this point so we don't need to
	 * read all devices, just the replacement targets.
	 */
	int syncing, expanding, expanded, replacing;
275
	int locked, uptodate, to_read, to_write, failed, written;
276
	int to_fill, compute, req_compute, non_overwrite;
277
	int injournal, just_cached;
278 279
	int failed_num[2];
	int p_failed, q_failed;
280 281 282
	int dec_preread_active;
	unsigned long ops_request;

283
	struct md_rdev *blocked_rdev;
284
	int handle_bad_blocks;
Shaohua Li's avatar
Shaohua Li committed
285
	int log_failed;
286
	int waiting_extra_page;
287 288
};

289 290 291 292
/* Flags for struct r5dev.flags */
enum r5dev_flags {
	R5_UPTODATE,	/* page contains current data */
	R5_LOCKED,	/* IO has been submitted on "req" */
293
	R5_DOUBLE_LOCKED,/* Cannot clear R5_LOCKED until 2 writes complete */
294
	R5_OVERWRITE,	/* towrite covers whole page */
Linus Torvalds's avatar
Linus Torvalds committed
295
/* and some that are internal to handle_stripe */
296 297 298 299 300
	R5_Insync,	/* rdev && rdev->in_sync at start */
	R5_Wantread,	/* want to schedule a read */
	R5_Wantwrite,
	R5_Overlap,	/* There is a pending overlapping request
			 * on this block */
301
	R5_ReadNoMerge, /* prevent bio from merging in block-layer */
302 303
	R5_ReadError,	/* seen a read error here recently */
	R5_ReWrite,	/* have tried to over-write the readerror */
Linus Torvalds's avatar
Linus Torvalds committed
304

305 306 307 308 309 310 311 312 313
	R5_Expanded,	/* This block now has post-expand data */
	R5_Wantcompute,	/* compute_block in progress treat as
			 * uptodate
			 */
	R5_Wantfill,	/* dev->toread contains a bio that needs
			 * filling
			 */
	R5_Wantdrain,	/* dev->towrite needs to be drained */
	R5_WantFUA,	/* Write should be FUA */
Shaohua Li's avatar
Shaohua Li committed
314
	R5_SyncIO,	/* The IO is sync */
315 316 317 318 319
	R5_WriteError,	/* got a write error - need to record it */
	R5_MadeGood,	/* A bad block has been fixed by writing to it */
	R5_ReadRepl,	/* Will/did read from replacement rather than orig */
	R5_MadeGoodRepl,/* A bad block on the replacement device has been
			 * fixed by writing to it */
320 321 322 323 324
	R5_NeedReplace,	/* This device has a replacement which is not
			 * up-to-date at this stripe. */
	R5_WantReplace, /* We need to update the replacement, we have read
			 * data in, and now is a good time to write it out.
			 */
Shaohua Li's avatar
Shaohua Li committed
325
	R5_Discard,	/* Discard the stripe */
326
	R5_SkipCopy,	/* Don't copy data from bio to stripe cache */
327 328 329 330 331
	R5_InJournal,	/* data being written is in the journal device.
			 * if R5_InJournal is set for parity pd_idx, all the
			 * data and parity being written are in the journal
			 * device
			 */
332 333 334 335 336
	R5_OrigPageUPTDODATE,	/* with write back cache, we read old data into
				 * dev->orig_page for prexor. When this flag is
				 * set, orig_page contains latest data in the
				 * raid disk.
				 */
337
};
Linus Torvalds's avatar
Linus Torvalds committed
338 339 340 341

/*
 * Stripe state
 */
342
enum {
343
	STRIPE_ACTIVE,
344 345 346 347
	STRIPE_HANDLE,
	STRIPE_SYNC_REQUESTED,
	STRIPE_SYNCING,
	STRIPE_INSYNC,
348
	STRIPE_REPLACED,
349 350 351 352 353 354 355 356 357 358 359 360
	STRIPE_PREREAD_ACTIVE,
	STRIPE_DELAYED,
	STRIPE_DEGRADED,
	STRIPE_BIT_DELAY,
	STRIPE_EXPANDING,
	STRIPE_EXPAND_SOURCE,
	STRIPE_EXPAND_READY,
	STRIPE_IO_STARTED,	/* do not count towards 'bypass_count' */
	STRIPE_FULL_WRITE,	/* all blocks are set to be overwritten */
	STRIPE_BIOFILL_RUN,
	STRIPE_COMPUTE_RUN,
	STRIPE_OPS_REQ_PENDING,
361
	STRIPE_ON_UNPLUG_LIST,
362
	STRIPE_DISCARD,
363
	STRIPE_ON_RELEASE_LIST,
364
	STRIPE_BATCH_READY,
365
	STRIPE_BATCH_ERR,
366 367 368
	STRIPE_BITMAP_PENDING,	/* Being added to bitmap, don't add
				 * to batch yet.
				 */
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
	STRIPE_LOG_TRAPPED,	/* trapped into log (see raid5-cache.c)
				 * this bit is used in two scenarios:
				 *
				 * 1. write-out phase
				 *  set in first entry of r5l_write_stripe
				 *  clear in second entry of r5l_write_stripe
				 *  used to bypass logic in handle_stripe
				 *
				 * 2. caching phase
				 *  set in r5c_try_caching_write()
				 *  clear when journal write is done
				 *  used to initiate r5c_cache_data()
				 *  also used to bypass logic in handle_stripe
				 */
	STRIPE_R5C_CACHING,	/* the stripe is in caching phase
				 * see more detail in the raid5-cache.c
				 */
386 387 388 389 390 391
	STRIPE_R5C_PARTIAL_STRIPE,	/* in r5c cache (to-be/being handled or
					 * in conf->r5c_partial_stripe_list)
					 */
	STRIPE_R5C_FULL_STRIPE,	/* in r5c cache (to-be/being handled or
				 * in conf->r5c_full_stripe_list)
				 */
Song Liu's avatar
Song Liu committed
392
	STRIPE_R5C_PREFLUSH,	/* need to flush journal device */
393
};
394

395
#define STRIPE_EXPAND_SYNC_FLAGS \
396 397 398 399
	((1 << STRIPE_EXPAND_SOURCE) |\
	(1 << STRIPE_EXPAND_READY) |\
	(1 << STRIPE_EXPANDING) |\
	(1 << STRIPE_SYNC_REQUESTED))
400
/*
401
 * Operation request flags
402
 */
NeilBrown's avatar
NeilBrown committed
403 404 405 406 407 408 409
enum {
	STRIPE_OP_BIOFILL,
	STRIPE_OP_COMPUTE_BLK,
	STRIPE_OP_PREXOR,
	STRIPE_OP_BIODRAIN,
	STRIPE_OP_RECONSTRUCT,
	STRIPE_OP_CHECK,
410
	STRIPE_OP_PARTIAL_PARITY,
NeilBrown's avatar
NeilBrown committed
411
};
412 413 414 415 416 417 418

/*
 * RAID parity calculation preferences
 */
enum {
	PARITY_DISABLE_RMW = 0,
	PARITY_ENABLE_RMW,
419
	PARITY_PREFER_RMW,
420 421 422 423 424 425 426 427 428 429
};

/*
 * Pages requested from set_syndrome_sources()
 */
enum {
	SYNDROME_SRC_ALL,
	SYNDROME_SRC_WANT_DRAIN,
	SYNDROME_SRC_WRITTEN,
};
Linus Torvalds's avatar
Linus Torvalds committed
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
/*
 * Plugging:
 *
 * To improve write throughput, we need to delay the handling of some
 * stripes until there has been a chance that several write requests
 * for the one stripe have all been collected.
 * In particular, any write request that would require pre-reading
 * is put on a "delayed" queue until there are no stripes currently
 * in a pre-read phase.  Further, if the "delayed" queue is empty when
 * a stripe is put on it then we "plug" the queue and do not process it
 * until an unplug call is made. (the unplug_io_fn() is called).
 *
 * When preread is initiated on a stripe, we set PREREAD_ACTIVE and add
 * it to the count of prereading stripes.
 * When write is initiated, or the stripe refcnt == 0 (just in case) we
 * clear the PREREAD_ACTIVE flag and decrement the count
446 447 448
 * Whenever the 'handle' queue is empty and the device is not plugged, we
 * move any strips from delayed to handle and clear the DELAYED flag and set
 * PREREAD_ACTIVE.
Linus Torvalds's avatar
Linus Torvalds committed
449 450
 * In stripe_handle, if we find pre-reading is necessary, we do it if
 * PREREAD_ACTIVE is set, else we set DELAYED which will send it to the delayed queue.
451
 * HANDLE gets cleared if stripe_handle leaves nothing locked.
Linus Torvalds's avatar
Linus Torvalds committed
452
 */
453

454 455 456 457 458 459 460 461 462 463 464 465
/* Note: disk_info.rdev can be set to NULL asynchronously by raid5_remove_disk.
 * There are three safe ways to access disk_info.rdev.
 * 1/ when holding mddev->reconfig_mutex
 * 2/ when resync/recovery/reshape is known to be happening - i.e. in code that
 *    is called as part of performing resync/recovery/reshape.
 * 3/ while holding rcu_read_lock(), use rcu_dereference to get the pointer
 *    and if it is non-NULL, increment rdev->nr_pending before dropping the RCU
 *    lock.
 * When .rdev is set to NULL, the nr_pending count checked again and if
 * it has been incremented, the pointer is put back in .rdev.
 */

Linus Torvalds's avatar
Linus Torvalds committed
466
struct disk_info {
467
	struct md_rdev	*rdev, *replacement;
468
	struct page	*extra_page; /* extra page to use in prexor */
Linus Torvalds's avatar
Linus Torvalds committed
469 470
};

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
/*
 * Stripe cache
 */

#define NR_STRIPES		256
#define STRIPE_SIZE		PAGE_SIZE
#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
#define	IO_THRESHOLD		1
#define BYPASS_THRESHOLD	1
#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
#define HASH_MASK		(NR_HASH - 1)
#define MAX_STRIPE_BATCH	8

/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 * order without overlap.  There may be several bio's per stripe+device, and
 * a bio could span several devices.
 * When walking this list for a particular stripe+device, we must never proceed
 * beyond a bio that extends past this device, as the next bio might no longer
 * be valid.
 * This function is used to determine the 'next' bio in the list, given the
 * sector of the current stripe+device
 */
static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
{
	int sectors = bio_sectors(bio);

	if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
		return bio->bi_next;
	else
		return NULL;
}

504 505 506 507 508 509 510 511
/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
 * This is because we sometimes take all the spinlocks
 * and creating that much locking depth can cause
 * problems.
 */
#define NR_STRIPE_HASH_LOCKS 8
#define STRIPE_HASH_LOCKS_MASK (NR_STRIPE_HASH_LOCKS - 1)

512 513 514
struct r5worker {
	struct work_struct work;
	struct r5worker_group *group;
515
	struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
516
	bool working;
517 518 519 520
};

struct r5worker_group {
	struct list_head handle_list;
521
	struct list_head loprio_list;
522 523
	struct r5conf *conf;
	struct r5worker *workers;
524
	int stripes_cnt;
525 526
};

527 528 529 530 531 532 533 534 535 536
/*
 * r5c journal modes of the array: write-back or write-through.
 * write-through mode has identical behavior as existing log only
 * implementation.
 */
enum r5c_journal_mode {
	R5C_JOURNAL_MODE_WRITE_THROUGH = 0,
	R5C_JOURNAL_MODE_WRITE_BACK = 1,
};

537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
enum r5_cache_state {
	R5_INACTIVE_BLOCKED,	/* release of inactive stripes blocked,
				 * waiting for 25% to be free
				 */
	R5_ALLOC_MORE,		/* It might help to allocate another
				 * stripe.
				 */
	R5_DID_ALLOC,		/* A stripe was allocated, don't allocate
				 * more until at least one has been
				 * released.  This avoids flooding
				 * the cache.
				 */
	R5C_LOG_TIGHT,		/* log device space tight, need to
				 * prioritize stripes at last_checkpoint
				 */
	R5C_LOG_CRITICAL,	/* log device is running out of space,
				 * only process stripes that are already
				 * occupying the log
				 */
556 557 558
	R5C_EXTRA_PAGE_IN_USE,	/* a stripe is using disk_info.extra_page
				 * for prexor
				 */
559 560
};

Shaohua Li's avatar
Shaohua Li committed
561 562 563 564 565 566 567 568
#define PENDING_IO_MAX 512
#define PENDING_IO_ONE_FLUSH 128
struct r5pending_data {
	struct list_head sibling;
	sector_t sector; /* stripe sector */
	struct bio_list bios;
};

569
struct r5conf {
570
	struct hlist_head	*stripe_hashtbl;
571 572
	/* only protect corresponding hash list and inactive_list */
	spinlock_t		hash_locks[NR_STRIPE_HASH_LOCKS];
573
	struct mddev		*mddev;
574
	int			chunk_sectors;
575
	int			level, algorithm, rmw_level;
576
	int			max_degraded;
577
	int			raid_disks;
Linus Torvalds's avatar
Linus Torvalds committed
578
	int			max_nr_stripes;
579
	int			min_nr_stripes;
Linus Torvalds's avatar
Linus Torvalds committed
580

581 582 583 584 585 586 587 588 589 590
	/* reshape_progress is the leading edge of a 'reshape'
	 * It has value MaxSector when no reshape is happening
	 * If delta_disks < 0, it is the last sector we started work on,
	 * else is it the next sector to work on.
	 */
	sector_t		reshape_progress;
	/* reshape_safe is the trailing edge of a reshape.  We know that
	 * before (or after) this address, all reshape has completed.
	 */
	sector_t		reshape_safe;
591
	int			previous_raid_disks;
592 593
	int			prev_chunk_sectors;
	int			prev_algo;
594
	short			generation; /* increments with every reshape */
595
	seqcount_t		gen_lock;	/* lock against generation changes */
596 597
	unsigned long		reshape_checkpoint; /* Time we last updated
						     * metadata */
598 599 600 601 602 603
	long long		min_offset_diff; /* minimum difference between
						  * data_offset and
						  * new_data_offset across all
						  * devices.  May be negative,
						  * but is closest to zero.
						  */
604

Linus Torvalds's avatar
Linus Torvalds committed
605
	struct list_head	handle_list; /* stripes needing handling */
606
	struct list_head	loprio_list; /* low priority stripes */
607
	struct list_head	hold_list; /* preread ready stripes */
Linus Torvalds's avatar
Linus Torvalds committed
608
	struct list_head	delayed_list; /* stripes that have plugged requests */
609
	struct list_head	bitmap_list; /* stripes delaying awaiting bitmap update */
610
	struct bio		*retry_read_aligned; /* currently retrying aligned bios   */
611
	unsigned int		retry_read_offset; /* sector offset into retry_read_aligned */
612
	struct bio		*retry_read_aligned_list; /* aligned bios retry list  */
Linus Torvalds's avatar
Linus Torvalds committed
613
	atomic_t		preread_active_stripes; /* stripes with scheduled io */
614
	atomic_t		active_aligned_reads;
615 616 617
	atomic_t		pending_full_writes; /* full write backlog */
	int			bypass_count; /* bypassed prereads */
	int			bypass_threshold; /* preread nice */
618
	int			skip_copy; /* Don't copy data from bio to stripe cache */
619
	struct list_head	*last_hold; /* detect hold_list promotions */
Linus Torvalds's avatar
Linus Torvalds committed
620

621
	atomic_t		reshape_stripes; /* stripes with pending writes for reshape */
622 623 624 625
	/* unfortunately we need two cache names as we temporarily have
	 * two caches.
	 */
	int			active_name;
626
	char			cache_name[2][32];
627 628
	struct kmem_cache	*slab_cache; /* for allocating stripes */
	struct mutex		cache_size_mutex; /* Protect changes to cache size */
629 630 631 632 633 634 635 636

	int			seq_flush, seq_write;
	int			quiesce;

	int			fullsync;  /* set to 1 if a full sync is needed,
					    * (fresh device added).
					    * Cleared when a sync completes.
					    */
637
	int			recovery_disabled;
638 639 640
	/* per cpu variables */
	struct raid5_percpu {
		struct page	*spare_page; /* Used when checking P/Q in raid6 */
641
		struct flex_array *scribble;   /* space for constructing buffer
642 643 644
					      * lists and performing address
					      * conversions
					      */
645
	} __percpu *percpu;
646 647
	int scribble_disks;
	int scribble_sectors;
648
	struct hlist_node node;
649

Linus Torvalds's avatar
Linus Torvalds committed
650 651 652 653
	/*
	 * Free stripes pool
	 */
	atomic_t		active_stripes;
654
	struct list_head	inactive_list[NR_STRIPE_HASH_LOCKS];
655 656 657 658 659

	atomic_t		r5c_cached_full_stripes;
	struct list_head	r5c_full_stripe_list;
	atomic_t		r5c_cached_partial_stripes;
	struct list_head	r5c_partial_stripe_list;
660 661
	atomic_t		r5c_flushing_full_stripes;
	atomic_t		r5c_flushing_partial_stripes;
662

663
	atomic_t		empty_inactive_list_nr;
664
	struct llist_head	released_stripes;
665
	wait_queue_head_t	wait_for_quiescent;
666
	wait_queue_head_t	wait_for_stripe;
Linus Torvalds's avatar
Linus Torvalds committed
667
	wait_queue_head_t	wait_for_overlap;
668
	unsigned long		cache_state;
669
	struct shrinker		shrinker;
670
	int			pool_size; /* number of disks in stripeheads in pool */
Linus Torvalds's avatar
Linus Torvalds committed
671
	spinlock_t		device_lock;
672
	struct disk_info	*disks;
673
	struct bio_set		bio_split;
NeilBrown's avatar
NeilBrown committed
674 675 676 677

	/* When taking over an array from a different personality, we store
	 * the new thread here until we fully activate the array.
	 */
678
	struct md_thread	*thread;
679
	struct list_head	temp_inactive_list[NR_STRIPE_HASH_LOCKS];
680 681 682
	struct r5worker_group	*worker_groups;
	int			group_cnt;
	int			worker_cnt_per_group;
Shaohua Li's avatar
Shaohua Li committed
683
	struct r5l_log		*log;
684
	void			*log_private;
685 686 687

	spinlock_t		pending_bios_lock;
	bool			batch_bio_dispatch;
Shaohua Li's avatar
Shaohua Li committed
688 689 690 691 692
	struct r5pending_data	*pending_data;
	struct list_head	free_list;
	struct list_head	pending_list;
	int			pending_data_cnt;
	struct r5pending_data	*next_pending_data;
Linus Torvalds's avatar
Linus Torvalds committed
693 694
};

695

Linus Torvalds's avatar
Linus Torvalds committed
696 697 698
/*
 * Our supported algorithms
 */
699 700 701 702
#define ALGORITHM_LEFT_ASYMMETRIC	0 /* Rotating Parity N with Data Restart */
#define ALGORITHM_RIGHT_ASYMMETRIC	1 /* Rotating Parity 0 with Data Restart */
#define ALGORITHM_LEFT_SYMMETRIC	2 /* Rotating Parity N with Data Continuation */
#define ALGORITHM_RIGHT_SYMMETRIC	3 /* Rotating Parity 0 with Data Continuation */
Linus Torvalds's avatar
Linus Torvalds committed
703

704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746
/* Define non-rotating (raid4) algorithms.  These allow
 * conversion of raid4 to raid5.
 */
#define ALGORITHM_PARITY_0		4 /* P or P,Q are initial devices */
#define ALGORITHM_PARITY_N		5 /* P or P,Q are final devices. */

/* DDF RAID6 layouts differ from md/raid6 layouts in two ways.
 * Firstly, the exact positioning of the parity block is slightly
 * different between the 'LEFT_*' modes of md and the "_N_*" modes
 * of DDF.
 * Secondly, or order of datablocks over which the Q syndrome is computed
 * is different.
 * Consequently we have different layouts for DDF/raid6 than md/raid6.
 * These layouts are from the DDFv1.2 spec.
 * Interestingly DDFv1.2-Errata-A does not specify N_CONTINUE but
 * leaves RLQ=3 as 'Vendor Specific'
 */

#define ALGORITHM_ROTATING_ZERO_RESTART	8 /* DDF PRL=6 RLQ=1 */
#define ALGORITHM_ROTATING_N_RESTART	9 /* DDF PRL=6 RLQ=2 */
#define ALGORITHM_ROTATING_N_CONTINUE	10 /*DDF PRL=6 RLQ=3 */

/* For every RAID5 algorithm we define a RAID6 algorithm
 * with exactly the same layout for data and parity, and
 * with the Q block always on the last device (N-1).
 * This allows trivial conversion from RAID5 to RAID6
 */
#define ALGORITHM_LEFT_ASYMMETRIC_6	16
#define ALGORITHM_RIGHT_ASYMMETRIC_6	17
#define ALGORITHM_LEFT_SYMMETRIC_6	18
#define ALGORITHM_RIGHT_SYMMETRIC_6	19
#define ALGORITHM_PARITY_0_6		20
#define ALGORITHM_PARITY_N_6		ALGORITHM_PARITY_N

static inline int algorithm_valid_raid5(int layout)
{
	return (layout >= 0) &&
		(layout <= 5);
}
static inline int algorithm_valid_raid6(int layout)
{
	return (layout >= 0 && layout <= 5)
		||
747
		(layout >= 8 && layout <= 10)
748 749 750 751 752 753 754 755
		||
		(layout >= 16 && layout <= 20);
}

static inline int algorithm_is_DDF(int layout)
{
	return layout >= 8 && layout <= 10;
}
756

757
extern void md_raid5_kick_device(struct r5conf *conf);
758
extern int raid5_set_cache_size(struct mddev *mddev, int size);
Shaohua Li's avatar
Shaohua Li committed
759 760 761 762 763 764 765 766
extern sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous);
extern void raid5_release_stripe(struct stripe_head *sh);
extern sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
				     int previous, int *dd_idx,
				     struct stripe_head *sh);
extern struct stripe_head *
raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
			int previous, int noblock, int noquiesce);
767
extern int raid5_calc_degraded(struct r5conf *conf);
768
extern int r5c_journal_mode_set(struct mddev *mddev, int journal_mode);
Linus Torvalds's avatar
Linus Torvalds committed
769
#endif