dm.c 34.5 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2
/*
 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3
 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
Linus Torvalds's avatar
Linus Torvalds committed
4 5 6 7 8 9
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-list.h"
Mike Anderson's avatar
Mike Anderson committed
10
#include "dm-uevent.h"
Linus Torvalds's avatar
Linus Torvalds committed
11 12 13

#include <linux/init.h>
#include <linux/module.h>
14
#include <linux/mutex.h>
Linus Torvalds's avatar
Linus Torvalds committed
15 16 17 18 19 20 21
#include <linux/moduleparam.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
Darrick J. Wong's avatar
Darrick J. Wong committed
22
#include <linux/hdreg.h>
23
#include <linux/blktrace_api.h>
24
#include <linux/smp_lock.h>
Linus Torvalds's avatar
Linus Torvalds committed
25

26 27
#define DM_MSG_PREFIX "core"

Linus Torvalds's avatar
Linus Torvalds committed
28 29 30 31 32
static const char *_name = DM_NAME;

static unsigned int major = 0;
static unsigned int _major = 0;

33
static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds's avatar
Linus Torvalds committed
34 35 36 37 38 39 40
/*
 * One of these is allocated per bio.
 */
struct dm_io {
	struct mapped_device *md;
	int error;
	atomic_t io_count;
Richard Kennedy's avatar
Richard Kennedy committed
41
	struct bio *bio;
42
	unsigned long start_time;
Linus Torvalds's avatar
Linus Torvalds committed
43 44 45 46 47 48
};

/*
 * One of these is allocated per target within a bio.  Hopefully
 * this will be simplified out one day.
 */
Alasdair G Kergon's avatar
Alasdair G Kergon committed
49
struct dm_target_io {
Linus Torvalds's avatar
Linus Torvalds committed
50 51 52 53 54 55 56
	struct dm_io *io;
	struct dm_target *ti;
	union map_info info;
};

union map_info *dm_get_mapinfo(struct bio *bio)
{
Alasdair G Kergon's avatar
Alasdair G Kergon committed
57
	if (bio && bio->bi_private)
Alasdair G Kergon's avatar
Alasdair G Kergon committed
58
		return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon's avatar
Alasdair G Kergon committed
59
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
60 61
}

62 63
#define MINOR_ALLOCED ((void *)-1)

Linus Torvalds's avatar
Linus Torvalds committed
64 65 66 67 68
/*
 * Bits for the md->flags field.
 */
#define DMF_BLOCK_IO 0
#define DMF_SUSPENDED 1
69
#define DMF_FROZEN 2
Jeff Mahoney's avatar
Jeff Mahoney committed
70
#define DMF_FREEING 3
71
#define DMF_DELETING 4
72
#define DMF_NOFLUSH_SUSPENDING 5
Linus Torvalds's avatar
Linus Torvalds committed
73

74 75 76 77 78 79 80 81 82 83 84 85 86
/*
 * Work processed by per-device workqueue.
 */
struct dm_wq_req {
	enum {
		DM_WQ_FLUSH_ALL,
		DM_WQ_FLUSH_DEFERRED,
	} type;
	struct work_struct work;
	struct mapped_device *md;
	void *context;
};

Linus Torvalds's avatar
Linus Torvalds committed
87
struct mapped_device {
88
	struct rw_semaphore io_lock;
89
	struct mutex suspend_lock;
90
	spinlock_t pushback_lock;
Linus Torvalds's avatar
Linus Torvalds committed
91 92
	rwlock_t map_lock;
	atomic_t holders;
93
	atomic_t open_count;
Linus Torvalds's avatar
Linus Torvalds committed
94 95 96

	unsigned long flags;

97
	struct request_queue *queue;
Linus Torvalds's avatar
Linus Torvalds committed
98
	struct gendisk *disk;
Mike Anderson's avatar
Mike Anderson committed
99
	char name[16];
Linus Torvalds's avatar
Linus Torvalds committed
100 101 102 103 104 105 106 107

	void *interface_ptr;

	/*
	 * A list of ios that arrived while we were suspended.
	 */
	atomic_t pending;
	wait_queue_head_t wait;
108
	struct bio_list deferred;
109
	struct bio_list pushback;
Linus Torvalds's avatar
Linus Torvalds committed
110

111 112 113 114 115
	/*
	 * Processing queue (flush/barriers)
	 */
	struct workqueue_struct *wq;

Linus Torvalds's avatar
Linus Torvalds committed
116 117 118 119 120 121 122 123 124 125 126
	/*
	 * The current mapping.
	 */
	struct dm_table *map;

	/*
	 * io objects are allocated from here.
	 */
	mempool_t *io_pool;
	mempool_t *tio_pool;

Stefan Bader's avatar
Stefan Bader committed
127 128
	struct bio_set *bs;

Linus Torvalds's avatar
Linus Torvalds committed
129 130 131 132 133
	/*
	 * Event handling.
	 */
	atomic_t event_nr;
	wait_queue_head_t eventq;
Mike Anderson's avatar
Mike Anderson committed
134 135 136
	atomic_t uevent_seq;
	struct list_head uevent_list;
	spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds's avatar
Linus Torvalds committed
137 138 139 140 141

	/*
	 * freeze/thaw support require holding onto a super block
	 */
	struct super_block *frozen_sb;
142
	struct block_device *suspended_bdev;
Darrick J. Wong's avatar
Darrick J. Wong committed
143 144 145

	/* forced geometry settings */
	struct hd_geometry geometry;
Linus Torvalds's avatar
Linus Torvalds committed
146 147 148
};

#define MIN_IOS 256
149 150
static struct kmem_cache *_io_cache;
static struct kmem_cache *_tio_cache;
Linus Torvalds's avatar
Linus Torvalds committed
151 152 153 154 155 156

static int __init local_init(void)
{
	int r;

	/* allocate a slab for the dm_ios */
Alasdair G Kergon's avatar
Alasdair G Kergon committed
157
	_io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds's avatar
Linus Torvalds committed
158 159 160 161
	if (!_io_cache)
		return -ENOMEM;

	/* allocate a slab for the target ios */
Alasdair G Kergon's avatar
Alasdair G Kergon committed
162
	_tio_cache = KMEM_CACHE(dm_target_io, 0);
Linus Torvalds's avatar
Linus Torvalds committed
163 164 165 166 167
	if (!_tio_cache) {
		kmem_cache_destroy(_io_cache);
		return -ENOMEM;
	}

Mike Anderson's avatar
Mike Anderson committed
168 169 170 171 172 173 174
	r = dm_uevent_init();
	if (r) {
		kmem_cache_destroy(_tio_cache);
		kmem_cache_destroy(_io_cache);
		return r;
	}

Linus Torvalds's avatar
Linus Torvalds committed
175 176 177 178 179
	_major = major;
	r = register_blkdev(_major, _name);
	if (r < 0) {
		kmem_cache_destroy(_tio_cache);
		kmem_cache_destroy(_io_cache);
Mike Anderson's avatar
Mike Anderson committed
180
		dm_uevent_exit();
Linus Torvalds's avatar
Linus Torvalds committed
181 182 183 184 185 186 187 188 189 190 191 192 193
		return r;
	}

	if (!_major)
		_major = r;

	return 0;
}

static void local_exit(void)
{
	kmem_cache_destroy(_tio_cache);
	kmem_cache_destroy(_io_cache);
194
	unregister_blkdev(_major, _name);
Mike Anderson's avatar
Mike Anderson committed
195
	dm_uevent_exit();
Linus Torvalds's avatar
Linus Torvalds committed
196 197 198 199 200 201

	_major = 0;

	DMINFO("cleaned up");
}

202
static int (*_inits[])(void) __initdata = {
Linus Torvalds's avatar
Linus Torvalds committed
203 204 205 206
	local_init,
	dm_target_init,
	dm_linear_init,
	dm_stripe_init,
207
	dm_kcopyd_init,
Linus Torvalds's avatar
Linus Torvalds committed
208 209 210
	dm_interface_init,
};

211
static void (*_exits[])(void) = {
Linus Torvalds's avatar
Linus Torvalds committed
212 213 214 215
	local_exit,
	dm_target_exit,
	dm_linear_exit,
	dm_stripe_exit,
216
	dm_kcopyd_exit,
Linus Torvalds's avatar
Linus Torvalds committed
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
	dm_interface_exit,
};

static int __init dm_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * Block device functions
 */
static int dm_blk_open(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

Jeff Mahoney's avatar
Jeff Mahoney committed
256 257
	spin_lock(&_minor_lock);

Linus Torvalds's avatar
Linus Torvalds committed
258
	md = inode->i_bdev->bd_disk->private_data;
Jeff Mahoney's avatar
Jeff Mahoney committed
259 260 261
	if (!md)
		goto out;

262 263
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoney's avatar
Jeff Mahoney committed
264 265 266 267
		md = NULL;
		goto out;
	}

Linus Torvalds's avatar
Linus Torvalds committed
268
	dm_get(md);
269
	atomic_inc(&md->open_count);
Jeff Mahoney's avatar
Jeff Mahoney committed
270 271 272 273 274

out:
	spin_unlock(&_minor_lock);

	return md ? 0 : -ENXIO;
Linus Torvalds's avatar
Linus Torvalds committed
275 276 277 278 279 280 281
}

static int dm_blk_close(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

	md = inode->i_bdev->bd_disk->private_data;
282
	atomic_dec(&md->open_count);
Linus Torvalds's avatar
Linus Torvalds committed
283 284 285 286
	dm_put(md);
	return 0;
}

287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
int dm_open_count(struct mapped_device *md)
{
	return atomic_read(&md->open_count);
}

/*
 * Guarantees nothing is using the device before it's deleted.
 */
int dm_lock_for_deletion(struct mapped_device *md)
{
	int r = 0;

	spin_lock(&_minor_lock);

	if (dm_open_count(md))
		r = -EBUSY;
	else
		set_bit(DMF_DELETING, &md->flags);

	spin_unlock(&_minor_lock);

	return r;
}

Darrick J. Wong's avatar
Darrick J. Wong committed
311 312 313 314 315 316 317
static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct mapped_device *md = bdev->bd_disk->private_data;

	return dm_get_geometry(md, geo);
}

318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
static int dm_blk_ioctl(struct inode *inode, struct file *file,
			unsigned int cmd, unsigned long arg)
{
	struct mapped_device *md;
	struct dm_table *map;
	struct dm_target *tgt;
	int r = -ENOTTY;

	/* We don't really need this lock, but we do need 'inode'. */
	unlock_kernel();

	md = inode->i_bdev->bd_disk->private_data;

	map = dm_get_table(md);

	if (!map || !dm_table_get_size(map))
		goto out;

	/* We only support devices that have a single target */
	if (dm_table_get_num_targets(map) != 1)
		goto out;

	tgt = dm_table_get_target(map, 0);

	if (dm_suspended(md)) {
		r = -EAGAIN;
		goto out;
	}

	if (tgt->type->ioctl)
		r = tgt->type->ioctl(tgt, inode, file, cmd, arg);

out:
	dm_table_put(map);

	lock_kernel();
	return r;
}

Alasdair G Kergon's avatar
Alasdair G Kergon committed
357
static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds's avatar
Linus Torvalds committed
358 359 360 361
{
	return mempool_alloc(md->io_pool, GFP_NOIO);
}

Alasdair G Kergon's avatar
Alasdair G Kergon committed
362
static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds's avatar
Linus Torvalds committed
363 364 365 366
{
	mempool_free(io, md->io_pool);
}

Alasdair G Kergon's avatar
Alasdair G Kergon committed
367
static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds's avatar
Linus Torvalds committed
368 369 370 371
{
	return mempool_alloc(md->tio_pool, GFP_NOIO);
}

Alasdair G Kergon's avatar
Alasdair G Kergon committed
372
static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds's avatar
Linus Torvalds committed
373 374 375 376
{
	mempool_free(tio, md->tio_pool);
}

377 378 379
static void start_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
Tejun Heo's avatar
Tejun Heo committed
380
	int cpu;
381 382 383

	io->start_time = jiffies;

384 385 386 387
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_unlock();
	dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
388 389 390 391 392 393 394
}

static int end_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
	struct bio *bio = io->bio;
	unsigned long duration = jiffies - io->start_time;
Tejun Heo's avatar
Tejun Heo committed
395
	int pending, cpu;
396 397
	int rw = bio_data_dir(bio);

398 399 400 401
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
	part_stat_unlock();
402

403 404
	dm_disk(md)->part0.in_flight = pending =
		atomic_dec_return(&md->pending);
405 406 407 408

	return !pending;
}

Linus Torvalds's avatar
Linus Torvalds committed
409 410 411 412 413
/*
 * Add the bio to the list of deferred io.
 */
static int queue_io(struct mapped_device *md, struct bio *bio)
{
414
	down_write(&md->io_lock);
Linus Torvalds's avatar
Linus Torvalds committed
415 416

	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
417
		up_write(&md->io_lock);
Linus Torvalds's avatar
Linus Torvalds committed
418 419 420 421 422
		return 1;
	}

	bio_list_add(&md->deferred, bio);

423
	up_write(&md->io_lock);
Linus Torvalds's avatar
Linus Torvalds committed
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
	return 0;		/* deferred successfully */
}

/*
 * Everyone (including functions in this file), should use this
 * function to access the md->map field, and make sure they call
 * dm_table_put() when finished.
 */
struct dm_table *dm_get_table(struct mapped_device *md)
{
	struct dm_table *t;

	read_lock(&md->map_lock);
	t = md->map;
	if (t)
		dm_table_get(t);
	read_unlock(&md->map_lock);

	return t;
}

Darrick J. Wong's avatar
Darrick J. Wong committed
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/*
 * Get the geometry associated with a dm device
 */
int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	*geo = md->geometry;

	return 0;
}

/*
 * Set the geometry of a device.
 */
int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;

	if (geo->start > sz) {
		DMWARN("Start sector is beyond the geometry limits.");
		return -EINVAL;
	}

	md->geometry = *geo;

	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
472 473 474 475 476 477 478 479 480
/*-----------------------------------------------------------------
 * CRUD START:
 *   A more elegant soln is in the works that uses the queue
 *   merge fn, unfortunately there are a couple of changes to
 *   the block layer that I want to make for this.  So in the
 *   interests of getting something for people to use I give
 *   you this clearly demarcated crap.
 *---------------------------------------------------------------*/

481 482 483 484 485
static int __noflush_suspending(struct mapped_device *md)
{
	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
}

Linus Torvalds's avatar
Linus Torvalds committed
486 487 488 489
/*
 * Decrements the number of outstanding ios that a bio has been
 * cloned into, completing the original io if necc.
 */
490
static void dec_pending(struct dm_io *io, int error)
Linus Torvalds's avatar
Linus Torvalds committed
491
{
492 493 494 495
	unsigned long flags;

	/* Push-back supersedes any I/O errors */
	if (error && !(io->error > 0 && __noflush_suspending(io->md)))
Linus Torvalds's avatar
Linus Torvalds committed
496 497 498
		io->error = error;

	if (atomic_dec_and_test(&io->io_count)) {
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
		if (io->error == DM_ENDIO_REQUEUE) {
			/*
			 * Target requested pushing back the I/O.
			 * This must be handled before the sleeper on
			 * suspend queue merges the pushback list.
			 */
			spin_lock_irqsave(&io->md->pushback_lock, flags);
			if (__noflush_suspending(io->md))
				bio_list_add(&io->md->pushback, io->bio);
			else
				/* noflush suspend was interrupted. */
				io->error = -EIO;
			spin_unlock_irqrestore(&io->md->pushback_lock, flags);
		}

514
		if (end_io_acct(io))
Linus Torvalds's avatar
Linus Torvalds committed
515 516 517
			/* nudge anyone waiting on suspend queue */
			wake_up(&io->md->wait);

518 519 520 521
		if (io->error != DM_ENDIO_REQUEUE) {
			blk_add_trace_bio(io->md->queue, io->bio,
					  BLK_TA_COMPLETE);

522
			bio_endio(io->bio, io->error);
523
		}
524

Linus Torvalds's avatar
Linus Torvalds committed
525 526 527 528
		free_io(io->md, io);
	}
}

529
static void clone_endio(struct bio *bio, int error)
Linus Torvalds's avatar
Linus Torvalds committed
530 531
{
	int r = 0;
Alasdair G Kergon's avatar
Alasdair G Kergon committed
532
	struct dm_target_io *tio = bio->bi_private;
Stefan Bader's avatar
Stefan Bader committed
533
	struct mapped_device *md = tio->io->md;
Linus Torvalds's avatar
Linus Torvalds committed
534 535 536 537 538 539 540
	dm_endio_fn endio = tio->ti->type->end_io;

	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
		error = -EIO;

	if (endio) {
		r = endio(tio->ti, bio, error, &tio->info);
541 542 543 544 545
		if (r < 0 || r == DM_ENDIO_REQUEUE)
			/*
			 * error and requeue request are handled
			 * in dec_pending().
			 */
Linus Torvalds's avatar
Linus Torvalds committed
546
			error = r;
547 548
		else if (r == DM_ENDIO_INCOMPLETE)
			/* The target will handle the io */
549
			return;
550 551 552 553
		else if (r) {
			DMWARN("unimplemented target endio return value: %d", r);
			BUG();
		}
Linus Torvalds's avatar
Linus Torvalds committed
554 555
	}

Stefan Bader's avatar
Stefan Bader committed
556 557 558 559 560 561 562
	dec_pending(tio->io, error);

	/*
	 * Store md for cleanup instead of tio which is about to get freed.
	 */
	bio->bi_private = md->bs;

Linus Torvalds's avatar
Linus Torvalds committed
563
	bio_put(bio);
Stefan Bader's avatar
Stefan Bader committed
564
	free_tio(md, tio);
Linus Torvalds's avatar
Linus Torvalds committed
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
}

static sector_t max_io_len(struct mapped_device *md,
			   sector_t sector, struct dm_target *ti)
{
	sector_t offset = sector - ti->begin;
	sector_t len = ti->len - offset;

	/*
	 * Does the target need to split even further ?
	 */
	if (ti->split_io) {
		sector_t boundary;
		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
			   - offset;
		if (len > boundary)
			len = boundary;
	}

	return len;
}

static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon's avatar
Alasdair G Kergon committed
588
		      struct dm_target_io *tio)
Linus Torvalds's avatar
Linus Torvalds committed
589 590
{
	int r;
591
	sector_t sector;
Stefan Bader's avatar
Stefan Bader committed
592
	struct mapped_device *md;
Linus Torvalds's avatar
Linus Torvalds committed
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607

	/*
	 * Sanity checks.
	 */
	BUG_ON(!clone->bi_size);

	clone->bi_end_io = clone_endio;
	clone->bi_private = tio;

	/*
	 * Map the clone.  If r == 0 we don't need to do
	 * anything, the target has assumed ownership of
	 * this io.
	 */
	atomic_inc(&tio->io->io_count);
608
	sector = clone->bi_sector;
Linus Torvalds's avatar
Linus Torvalds committed
609
	r = ti->type->map(ti, clone, &tio->info);
610
	if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds's avatar
Linus Torvalds committed
611
		/* the bio has been remapped so dispatch it */
612

Alasdair G Kergon's avatar
Alasdair G Kergon committed
613
		blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
614 615
				    tio->io->bio->bi_bdev->bd_dev,
				    clone->bi_sector, sector);
616

Linus Torvalds's avatar
Linus Torvalds committed
617
		generic_make_request(clone);
618 619
	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
		/* error the io and bail out, or requeue it if needed */
Stefan Bader's avatar
Stefan Bader committed
620 621 622 623 624 625
		md = tio->io->md;
		dec_pending(tio->io, r);
		/*
		 * Store bio_set for cleanup.
		 */
		clone->bi_private = md->bs;
Linus Torvalds's avatar
Linus Torvalds committed
626
		bio_put(clone);
Stefan Bader's avatar
Stefan Bader committed
627
		free_tio(md, tio);
628 629 630
	} else if (r) {
		DMWARN("unimplemented target map return value: %d", r);
		BUG();
Linus Torvalds's avatar
Linus Torvalds committed
631 632 633 634 635 636 637 638 639 640 641 642 643
	}
}

struct clone_info {
	struct mapped_device *md;
	struct dm_table *map;
	struct bio *bio;
	struct dm_io *io;
	sector_t sector;
	sector_t sector_count;
	unsigned short idx;
};

Peter Osterlund's avatar
Peter Osterlund committed
644 645
static void dm_bio_destructor(struct bio *bio)
{
Stefan Bader's avatar
Stefan Bader committed
646 647 648
	struct bio_set *bs = bio->bi_private;

	bio_free(bio, bs);
Peter Osterlund's avatar
Peter Osterlund committed
649 650
}

Linus Torvalds's avatar
Linus Torvalds committed
651 652 653 654 655
/*
 * Creates a little bio that is just does part of a bvec.
 */
static struct bio *split_bvec(struct bio *bio, sector_t sector,
			      unsigned short idx, unsigned int offset,
Stefan Bader's avatar
Stefan Bader committed
656
			      unsigned int len, struct bio_set *bs)
Linus Torvalds's avatar
Linus Torvalds committed
657 658 659 660
{
	struct bio *clone;
	struct bio_vec *bv = bio->bi_io_vec + idx;

Stefan Bader's avatar
Stefan Bader committed
661
	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund's avatar
Peter Osterlund committed
662
	clone->bi_destructor = dm_bio_destructor;
Linus Torvalds's avatar
Linus Torvalds committed
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
	*clone->bi_io_vec = *bv;

	clone->bi_sector = sector;
	clone->bi_bdev = bio->bi_bdev;
	clone->bi_rw = bio->bi_rw;
	clone->bi_vcnt = 1;
	clone->bi_size = to_bytes(len);
	clone->bi_io_vec->bv_offset = offset;
	clone->bi_io_vec->bv_len = clone->bi_size;

	return clone;
}

/*
 * Creates a bio that consists of range of complete bvecs.
 */
static struct bio *clone_bio(struct bio *bio, sector_t sector,
			     unsigned short idx, unsigned short bv_count,
Stefan Bader's avatar
Stefan Bader committed
681
			     unsigned int len, struct bio_set *bs)
Linus Torvalds's avatar
Linus Torvalds committed
682 683 684
{
	struct bio *clone;

Stefan Bader's avatar
Stefan Bader committed
685 686 687
	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
	__bio_clone(clone, bio);
	clone->bi_destructor = dm_bio_destructor;
Linus Torvalds's avatar
Linus Torvalds committed
688 689 690 691 692 693 694 695 696
	clone->bi_sector = sector;
	clone->bi_idx = idx;
	clone->bi_vcnt = idx + bv_count;
	clone->bi_size = to_bytes(len);
	clone->bi_flags &= ~(1 << BIO_SEG_VALID);

	return clone;
}

697
static int __clone_and_map(struct clone_info *ci)
Linus Torvalds's avatar
Linus Torvalds committed
698 699
{
	struct bio *clone, *bio = ci->bio;
700 701
	struct dm_target *ti;
	sector_t len = 0, max;
Alasdair G Kergon's avatar
Alasdair G Kergon committed
702
	struct dm_target_io *tio;
Linus Torvalds's avatar
Linus Torvalds committed
703

704 705 706 707 708 709
	ti = dm_table_find_target(ci->map, ci->sector);
	if (!dm_target_is_valid(ti))
		return -EIO;

	max = max_io_len(ci->md, ci->sector, ti);

Linus Torvalds's avatar
Linus Torvalds committed
710 711 712 713 714 715 716 717 718 719 720 721 722 723
	/*
	 * Allocate a target io object.
	 */
	tio = alloc_tio(ci->md);
	tio->io = ci->io;
	tio->ti = ti;
	memset(&tio->info, 0, sizeof(tio->info));

	if (ci->sector_count <= max) {
		/*
		 * Optimise for the simple case where we can do all of
		 * the remaining io with a single clone.
		 */
		clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader's avatar
Stefan Bader committed
724 725
				  bio->bi_vcnt - ci->idx, ci->sector_count,
				  ci->md->bs);
Linus Torvalds's avatar
Linus Torvalds committed
726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
		__map_bio(ti, clone, tio);
		ci->sector_count = 0;

	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
		/*
		 * There are some bvecs that don't span targets.
		 * Do as many of these as possible.
		 */
		int i;
		sector_t remaining = max;
		sector_t bv_len;

		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
			bv_len = to_sector(bio->bi_io_vec[i].bv_len);

			if (bv_len > remaining)
				break;

			remaining -= bv_len;
			len += bv_len;
		}

Stefan Bader's avatar
Stefan Bader committed
748 749
		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
				  ci->md->bs);
Linus Torvalds's avatar
Linus Torvalds committed
750 751 752 753 754 755 756 757
		__map_bio(ti, clone, tio);

		ci->sector += len;
		ci->sector_count -= len;
		ci->idx = i;

	} else {
		/*
758
		 * Handle a bvec that must be split between two or more targets.
Linus Torvalds's avatar
Linus Torvalds committed
759 760
		 */
		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
761 762
		sector_t remaining = to_sector(bv->bv_len);
		unsigned int offset = 0;
Linus Torvalds's avatar
Linus Torvalds committed
763

764 765 766
		do {
			if (offset) {
				ti = dm_table_find_target(ci->map, ci->sector);
767 768 769
				if (!dm_target_is_valid(ti))
					return -EIO;

770
				max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds's avatar
Linus Torvalds committed
771

772 773 774 775 776 777 778 779 780
				tio = alloc_tio(ci->md);
				tio->io = ci->io;
				tio->ti = ti;
				memset(&tio->info, 0, sizeof(tio->info));
			}

			len = min(remaining, max);

			clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader's avatar
Stefan Bader committed
781 782
					   bv->bv_offset + offset, len,
					   ci->md->bs);
783 784 785 786 787 788 789

			__map_bio(ti, clone, tio);

			ci->sector += len;
			ci->sector_count -= len;
			offset += to_bytes(len);
		} while (remaining -= len);
Linus Torvalds's avatar
Linus Torvalds committed
790 791 792

		ci->idx++;
	}
793 794

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
795 796 797 798 799
}

/*
 * Split the bio into several clones.
 */
Milan Broz's avatar
Milan Broz committed
800
static int __split_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
801 802
{
	struct clone_info ci;
803
	int error = 0;
Linus Torvalds's avatar
Linus Torvalds committed
804 805

	ci.map = dm_get_table(md);
Milan Broz's avatar
Milan Broz committed
806 807
	if (unlikely(!ci.map))
		return -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
808 809 810 811 812 813 814 815 816 817 818 819

	ci.md = md;
	ci.bio = bio;
	ci.io = alloc_io(md);
	ci.io->error = 0;
	atomic_set(&ci.io->io_count, 1);
	ci.io->bio = bio;
	ci.io->md = md;
	ci.sector = bio->bi_sector;
	ci.sector_count = bio_sectors(bio);
	ci.idx = bio->bi_idx;

820
	start_io_acct(ci.io);
821 822
	while (ci.sector_count && !error)
		error = __clone_and_map(&ci);
Linus Torvalds's avatar
Linus Torvalds committed
823 824

	/* drop the extra reference count */
825
	dec_pending(ci.io, error);
Linus Torvalds's avatar
Linus Torvalds committed
826
	dm_table_put(ci.map);
Milan Broz's avatar
Milan Broz committed
827 828

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
829 830 831 832 833
}
/*-----------------------------------------------------------------
 * CRUD END
 *---------------------------------------------------------------*/

Milan Broz's avatar
Milan Broz committed
834 835 836 837 838 839 840 841
static int dm_merge_bvec(struct request_queue *q,
			 struct bvec_merge_data *bvm,
			 struct bio_vec *biovec)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);
	struct dm_target *ti;
	sector_t max_sectors;
842
	int max_size = 0;
Milan Broz's avatar
Milan Broz committed
843 844

	if (unlikely(!map))
845
		goto out;
Milan Broz's avatar
Milan Broz committed
846 847

	ti = dm_table_find_target(map, bvm->bi_sector);
848 849
	if (!dm_target_is_valid(ti))
		goto out_table;
Milan Broz's avatar
Milan Broz committed
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867

	/*
	 * Find maximum amount of I/O that won't need splitting
	 */
	max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
			  (sector_t) BIO_MAX_SECTORS);
	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
	if (max_size < 0)
		max_size = 0;

	/*
	 * merge_bvec_fn() returns number of bytes
	 * it can accept at this offset
	 * max is precomputed maximal io size
	 */
	if (max_size && ti->type->merge)
		max_size = ti->type->merge(ti, bvm, biovec, max_size);

868
out_table:
869 870 871
	dm_table_put(map);

out:
Milan Broz's avatar
Milan Broz committed
872 873 874 875 876 877 878 879 880
	/*
	 * Always allow an entire first page
	 */
	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
		max_size = biovec->bv_len;

	return max_size;
}

Linus Torvalds's avatar
Linus Torvalds committed
881 882 883 884
/*
 * The request function that just remaps the bio built up by
 * dm_merge_bvec.
 */
885
static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
886
{
Milan Broz's avatar
Milan Broz committed
887
	int r = -EIO;
888
	int rw = bio_data_dir(bio);
Linus Torvalds's avatar
Linus Torvalds committed
889
	struct mapped_device *md = q->queuedata;
Tejun Heo's avatar
Tejun Heo committed
890
	int cpu;
Linus Torvalds's avatar
Linus Torvalds committed
891

Stefan Bader's avatar
Stefan Bader committed
892 893 894 895 896
	/*
	 * There is no use in forwarding any barrier request since we can't
	 * guarantee it is (or can be) handled by the targets correctly.
	 */
	if (unlikely(bio_barrier(bio))) {
897
		bio_endio(bio, -EOPNOTSUPP);
Stefan Bader's avatar
Stefan Bader committed
898 899 900
		return 0;
	}

901
	down_read(&md->io_lock);
Linus Torvalds's avatar
Linus Torvalds committed
902

903 904 905 906
	cpu = part_stat_lock();
	part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
	part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
	part_stat_unlock();
907

Linus Torvalds's avatar
Linus Torvalds committed
908 909 910 911 912
	/*
	 * If we're suspended we have to queue
	 * this io for later.
	 */
	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
913
		up_read(&md->io_lock);
Linus Torvalds's avatar
Linus Torvalds committed
914

Milan Broz's avatar
Milan Broz committed
915 916
		if (bio_rw(bio) != READA)
			r = queue_io(md, bio);
Linus Torvalds's avatar
Linus Torvalds committed
917

Milan Broz's avatar
Milan Broz committed
918 919
		if (r <= 0)
			goto out_req;
Linus Torvalds's avatar
Linus Torvalds committed
920 921 922 923 924

		/*
		 * We're in a while loop, because someone could suspend
		 * before we get to the following read lock.
		 */
925
		down_read(&md->io_lock);
Linus Torvalds's avatar
Linus Torvalds committed
926 927
	}

Milan Broz's avatar
Milan Broz committed
928
	r = __split_bio(md, bio);
929
	up_read(&md->io_lock);
Milan Broz's avatar
Milan Broz committed
930 931 932 933 934

out_req:
	if (r < 0)
		bio_io_error(bio);

Linus Torvalds's avatar
Linus Torvalds committed
935 936 937
	return 0;
}

938
static void dm_unplug_all(struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);

	if (map) {
		dm_table_unplug_all(map);
		dm_table_put(map);
	}
}

static int dm_any_congested(void *congested_data, int bdi_bits)
{
	int r;
	struct mapped_device *md = (struct mapped_device *) congested_data;
	struct dm_table *map = dm_get_table(md);

	if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
		r = bdi_bits;
	else
		r = dm_table_any_congested(map, bdi_bits);

	dm_table_put(map);
	return r;
}

/*-----------------------------------------------------------------
 * An IDR is used to keep track of allocated minor numbers.
 *---------------------------------------------------------------*/
static DEFINE_IDR(_minor_idr);

969
static void free_minor(int minor)
Linus Torvalds's avatar
Linus Torvalds committed
970
{
971
	spin_lock(&_minor_lock);
Linus Torvalds's avatar
Linus Torvalds committed
972
	idr_remove(&_minor_idr, minor);
973
	spin_unlock(&_minor_lock);
Linus Torvalds's avatar
Linus Torvalds committed
974 975 976 977 978
}

/*
 * See if the device with a specific minor # is free.
 */
979
static int specific_minor(int minor)
Linus Torvalds's avatar
Linus Torvalds committed
980 981 982 983 984 985
{
	int r, m;

	if (minor >= (1 << MINORBITS))
		return -EINVAL;

Jeff Mahoney's avatar
Jeff Mahoney committed
986 987 988 989
	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
	if (!r)
		return -ENOMEM;

990
	spin_lock(&_minor_lock);
Linus Torvalds's avatar
Linus Torvalds committed
991 992 993 994 995 996

	if (idr_find(&_minor_idr, minor)) {
		r = -EBUSY;
		goto out;
	}

997
	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney's avatar
Jeff Mahoney committed
998
	if (r)
Linus Torvalds's avatar
Linus Torvalds committed
999 1000 1001 1002 1003 1004 1005 1006 1007
		goto out;

	if (m != minor) {
		idr_remove(&_minor_idr, m);
		r = -EBUSY;
		goto out;
	}

out:
1008
	spin_unlock(&_minor_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1009 1010 1011
	return r;
}

1012
static int next_free_minor(int *minor)
Linus Torvalds's avatar
Linus Torvalds committed
1013
{
1014
	int r, m;
Linus Torvalds's avatar
Linus Torvalds committed
1015 1016

	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney's avatar
Jeff Mahoney committed
1017 1018 1019
	if (!r)
		return -ENOMEM;

1020
	spin_lock(&_minor_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1021

1022
	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1023
	if (r)
Linus Torvalds's avatar
Linus Torvalds committed
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
		goto out;

	if (m >= (1 << MINORBITS)) {
		idr_remove(&_minor_idr, m);
		r = -ENOSPC;
		goto out;
	}

	*minor = m;

out:
1035
	spin_unlock(&_minor_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1036 1037 1038 1039 1040 1041 1042 1043
	return r;
}

static struct block_device_operations dm_blk_dops;

/*
 * Allocate and initialise a blank device with a given minor.
 */
1044
static struct mapped_device *alloc_dev(int minor)
Linus Torvalds's avatar
Linus Torvalds committed
1045 1046
{
	int r;
1047
	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1048
	void *old_md;
Linus Torvalds's avatar
Linus Torvalds committed
1049 1050 1051 1052 1053 1054

	if (!md) {
		DMWARN("unable to allocate device, out of memory.");
		return NULL;
	}

1055
	if (!try_module_get(THIS_MODULE))
Milan Broz's avatar
Milan Broz committed
1056
		goto bad_module_get;
1057

Linus Torvalds's avatar
Linus Torvalds committed
1058
	/* get a minor number for the dev */
1059
	if (minor == DM_ANY_MINOR)
1060
		r = next_free_minor(&minor);
1061
	else
1062
		r = specific_minor(minor);
Linus Torvalds's avatar
Linus Torvalds committed
1063
	if (r < 0)
Milan Broz's avatar
Milan Broz committed
1064
		goto bad_minor;
Linus Torvalds's avatar
Linus Torvalds committed
1065

1066
	init_rwsem(&md->io_lock);
1067
	mutex_init(&md->suspend_lock);
1068
	spin_lock_init(&md->pushback_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1069 1070
	rwlock_init(&md->map_lock);
	atomic_set(&md->holders, 1);
1071
	atomic_set(&md->open_count, 0);
Linus Torvalds's avatar
Linus Torvalds committed
1072
	atomic_set(&md->event_nr, 0);
Mike Anderson's avatar
Mike Anderson committed
1073 1074 1075
	atomic_set(&md->uevent_seq, 0);
	INIT_LIST_HEAD(&md->uevent_list);
	spin_lock_init(&md->uevent_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1076 1077 1078

	md->queue = blk_alloc_queue(GFP_KERNEL);
	if (!md->queue)
Milan Broz's avatar
Milan Broz committed
1079
		goto bad_queue;
Linus Torvalds's avatar
Linus Torvalds committed
1080 1081 1082 1083 1084

	md->queue->queuedata = md;
	md->queue->backing_dev_info.congested_fn = dm_any_congested;
	md->queue->backing_dev_info.congested_data = md;
	blk_queue_make_request(md->queue, dm_request);
1085
	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds's avatar
Linus Torvalds committed
1086
	md->queue->unplug_fn = dm_unplug_all;
Milan Broz's avatar
Milan Broz committed
1087
	blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Linus Torvalds's avatar
Linus Torvalds committed
1088

1089
	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
1090
	if (!md->io_pool)
Milan Broz's avatar
Milan Broz committed
1091
		goto bad_io_pool;
Linus Torvalds's avatar
Linus Torvalds committed
1092

1093
	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds's avatar
Linus Torvalds committed
1094
	if (!md->tio_pool)
Milan Broz's avatar
Milan Broz committed
1095
		goto bad_tio_pool;
Linus Torvalds's avatar
Linus Torvalds committed
1096

1097
	md->bs = bioset_create(16, 16);
Stefan Bader's avatar
Stefan Bader committed
1098 1099 1100
	if (!md->bs)
		goto bad_no_bioset;

Linus Torvalds's avatar
Linus Torvalds committed
1101 1102
	md->disk = alloc_disk(1);
	if (!md->disk)
Milan Broz's avatar
Milan Broz committed
1103
		goto bad_disk;
Linus Torvalds's avatar
Linus Torvalds committed
1104

1105 1106 1107 1108
	atomic_set(&md->pending, 0);
	init_waitqueue_head(&md->wait);
	init_waitqueue_head(&md->eventq);

Linus Torvalds's avatar
Linus Torvalds committed
1109 1110 1111 1112 1113 1114 1115
	md->disk->major = _major;
	md->disk->first_minor = minor;
	md->disk->fops = &dm_blk_dops;
	md->disk->queue = md->queue;
	md->disk->private_data = md;
	sprintf(md->disk->disk_name, "dm-%d", minor);
	add_disk(md->disk);
Mike Anderson's avatar
Mike Anderson committed
1116
	format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds's avatar
Linus Torvalds committed
1117

1118 1119 1120 1121
	md->wq = create_singlethread_workqueue("kdmflush");
	if (!md->wq)
		goto bad_thread;

1122
	/* Populate the mapping, nobody knows we exist yet */
1123
	spin_lock(&_minor_lock);
1124
	old_md = idr_replace(&_minor_idr, md, minor);
1125
	spin_unlock(&_minor_lock);
1126 1127 1128

	BUG_ON(old_md != MINOR_ALLOCED);

Linus Torvalds's avatar
Linus Torvalds committed
1129 1130
	return md;

1131 1132
bad_thread:
	put_disk(md->disk);
Milan Broz's avatar
Milan Broz committed
1133
bad_disk:
Stefan Bader's avatar
Stefan Bader committed
1134
	bioset_free(md->bs);
Milan Broz's avatar
Milan Broz committed
1135
bad_no_bioset:
Linus Torvalds's avatar
Linus Torvalds committed
1136
	mempool_destroy(md->tio_pool);
Milan Broz's avatar
Milan Broz committed
1137
bad_tio_pool:
Linus Torvalds's avatar
Linus Torvalds committed
1138
	mempool_destroy(md->io_pool);
Milan Broz's avatar
Milan Broz committed
1139
bad_io_pool:
1140
	blk_cleanup_queue(md->queue);
Milan Broz's avatar
Milan Broz committed
1141
bad_queue:
Linus Torvalds's avatar
Linus Torvalds committed
1142
	free_minor(minor);
Milan Broz's avatar
Milan Broz committed
1143
bad_minor:
1144
	module_put(THIS_MODULE);
Milan Broz's avatar
Milan Broz committed
1145
bad_module_get:
Linus Torvalds's avatar
Linus Torvalds committed
1146 1147 1148 1149
	kfree(md);
	return NULL;
}

Jun'ichi Nomura's avatar
Jun'ichi Nomura committed
1150 1151
static void unlock_fs(struct mapped_device *md);

Linus Torvalds's avatar
Linus Torvalds committed
1152 1153
static void free_dev(struct mapped_device *md)
{
1154
	int minor = MINOR(disk_devt(md->disk));
1155

1156
	if (md->suspended_bdev) {
Jun'ichi Nomura's avatar
Jun'ichi Nomura committed
1157
		unlock_fs(md);
1158 1159
		bdput(md->suspended_bdev);
	}
1160
	destroy_workqueue(md->wq);
Linus Torvalds's avatar
Linus Torvalds committed
1161 1162
	mempool_destroy(md->tio_pool);
	mempool_destroy(md->io_pool);
Stefan Bader's avatar
Stefan Bader committed
1163
	bioset_free(md->bs);
Linus Torvalds's avatar
Linus Torvalds committed
1164
	del_gendisk(md->disk);
1165
	free_minor(minor);
Jeff Mahoney's avatar
Jeff Mahoney committed
1166 1167 1168 1169 1170

	spin_lock(&_minor_lock);
	md->disk->private_data = NULL;
	spin_unlock(&_minor_lock);

Linus Torvalds's avatar
Linus Torvalds committed
1171
	put_disk(md->disk);
1172
	blk_cleanup_queue(md->queue);
1173
	module_put(THIS_MODULE);
Linus Torvalds's avatar
Linus Torvalds committed
1174 1175 1176 1177 1178 1179 1180 1181
	kfree(md);
}

/*
 * Bind a table to the device.
 */
static void event_callback(void *context)
{
Mike Anderson's avatar
Mike Anderson committed
1182 1183
	unsigned long flags;
	LIST_HEAD(uevents);
Linus Torvalds's avatar
Linus Torvalds committed
1184 1185
	struct mapped_device *md = (struct mapped_device *) context;

Mike Anderson's avatar
Mike Anderson committed
1186 1187 1188 1189
	spin_lock_irqsave(&md->uevent_lock, flags);
	list_splice_init(&md->uevent_list, &uevents);
	spin_unlock_irqrestore(&md->uevent_lock, flags);

1190
	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson's avatar
Mike Anderson committed
1191

Linus Torvalds's avatar
Linus Torvalds committed
1192 1193 1194 1195
	atomic_inc(&md->event_nr);
	wake_up(&md->eventq);
}

1196
static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds's avatar
Linus Torvalds committed
1197
{
1198
	set_capacity(md->disk, size);
Linus Torvalds's avatar
Linus Torvalds committed
1199

1200
	mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
1201
	i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1202
	mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1203 1204 1205 1206
}

static int __bind(struct mapped_device *md, struct dm_table *t)
{
1207
	struct request_queue *q = md->queue;
Linus Torvalds's avatar
Linus Torvalds committed
1208 1209 1210
	sector_t size;

	size = dm_table_get_size(t);
Darrick J. Wong's avatar
Darrick J. Wong committed
1211 1212 1213 1214 1215 1216 1217

	/*
	 * Wipe any geometry if the size of the table changed.
	 */
	if (size != get_capacity(md->disk))
		memset(&md->geometry, 0, sizeof(md->geometry));

1218 1219
	if (md->suspended_bdev)
		__set_size(md, size);
Linus Torvalds's avatar
Linus Torvalds committed
1220 1221 1222
	if (size == 0)
		return 0;

1223 1224 1225
	dm_table_get(t);
	dm_table_event_callback(t, event_callback, md);

Linus Torvalds's avatar
Linus Torvalds committed
1226 1227
	write_lock(&md->map_lock);
	md->map = t;
1228
	dm_table_set_restrictions(t, q);
Linus Torvalds's avatar
Linus Torvalds committed
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
	write_unlock(&md->map_lock);

	return 0;
}

static void __unbind(struct mapped_device *md)
{
	struct dm_table *map = md->map;

	if (!map)
		return;

	dm_table_event_callback(map, NULL, NULL);
	write_lock(&md->map_lock);
	md->map = NULL;
	write_unlock(&md->map_lock);
	dm_table_put(map);
}

/*
 * Constructor for a new device.
 */