mempolicy.c 70.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4
/*
 * Simple NUMA memory policy for the Linux kernel.
 *
 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5
 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
Linus Torvalds's avatar
Linus Torvalds committed
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 * Subject to the GNU Public License, version 2.
 *
 * NUMA policy allows the user to give hints in which node(s) memory should
 * be allocated.
 *
 * Support four policies per VMA and per process:
 *
 * The VMA policy has priority over the process policy for a page fault.
 *
 * interleave     Allocate memory interleaved over a set of nodes,
 *                with normal fallback if it fails.
 *                For VMA based allocations this interleaves based on the
 *                offset into the backing object or offset into the mapping
 *                for anonymous memory. For process policy an process counter
 *                is used.
21
 *
Linus Torvalds's avatar
Linus Torvalds committed
22 23
 * bind           Only allocate memory on a specific set of nodes,
 *                no fallback.
24 25 26 27
 *                FIXME: memory is allocated starting with the first node
 *                to the last. It would be better if bind would truly restrict
 *                the allocation to memory nodes instead
 *
Linus Torvalds's avatar
Linus Torvalds committed
28
 * preferred       Try a specific node first before normal fallback.
David Rientjes's avatar
David Rientjes committed
29
 *                As a special case NUMA_NO_NODE here means do the allocation
Linus Torvalds's avatar
Linus Torvalds committed
30 31 32
 *                on the local CPU. This is normally identical to default,
 *                but useful to set in a VMA when you have a non default
 *                process policy.
33
 *
Linus Torvalds's avatar
Linus Torvalds committed
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
 * default        Allocate on the local node first, or when on a VMA
 *                use the process policy. This is what Linux always did
 *		  in a NUMA aware kernel and still does by, ahem, default.
 *
 * The process policy is applied for most non interrupt memory allocations
 * in that process' context. Interrupts ignore the policies and always
 * try to allocate on the local CPU. The VMA policy is only applied for memory
 * allocations for a VMA in the VM.
 *
 * Currently there are a few corner cases in swapping where the policy
 * is not applied, but the majority should be handled. When process policy
 * is used it is not remembered over swap outs/swap ins.
 *
 * Only the highest zone in the zone hierarchy gets policied. Allocations
 * requesting a lower zone just use default policy. This implies that
 * on systems with highmem kernel lowmem allocation don't get policied.
 * Same with GFP_DMA allocations.
 *
 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
 * all users and remembered even when nobody has memory mapped.
 */

/* Notebook:
   fix mmap readahead to honour policy and enable policy for any page cache
   object
   statistics for bigpages
   global policy for page cache? currently it uses process policy. Requires
   first item above.
   handle mremap for shared memory (currently ignored for the policy)
   grows down?
   make bind policy root only? It can trigger oom much faster and the
   kernel is not always grateful with that.
*/

68 69
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

Linus Torvalds's avatar
Linus Torvalds committed
70 71 72 73 74 75 76 77 78 79
#include <linux/mempolicy.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/hugetlb.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/nodemask.h>
#include <linux/cpuset.h>
#include <linux/slab.h>
#include <linux/string.h>
80
#include <linux/export.h>
81
#include <linux/nsproxy.h>
Linus Torvalds's avatar
Linus Torvalds committed
82 83 84
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/compat.h>
85
#include <linux/swap.h>
86 87
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
88
#include <linux/migrate.h>
89
#include <linux/ksm.h>
90
#include <linux/rmap.h>
91
#include <linux/security.h>
Adrian Bunk's avatar
Adrian Bunk committed
92
#include <linux/syscalls.h>
93
#include <linux/ctype.h>
94
#include <linux/mm_inline.h>
95
#include <linux/mmu_notifier.h>
96
#include <linux/printk.h>
97

Linus Torvalds's avatar
Linus Torvalds committed
98 99 100
#include <asm/tlbflush.h>
#include <asm/uaccess.h>

101 102
#include "internal.h"

103
/* Internal flags */
104
#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
105
#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
106

107 108
static struct kmem_cache *policy_cache;
static struct kmem_cache *sn_cache;
Linus Torvalds's avatar
Linus Torvalds committed
109 110 111

/* Highest zone. An specific allocation for a zone below that is not
   policied. */
112
enum zone_type policy_zone = 0;
Linus Torvalds's avatar
Linus Torvalds committed
113

114 115 116
/*
 * run-time system-wide default policy => local allocation
 */
117
static struct mempolicy default_policy = {
Linus Torvalds's avatar
Linus Torvalds committed
118
	.refcnt = ATOMIC_INIT(1), /* never free it */
119
	.mode = MPOL_PREFERRED,
120
	.flags = MPOL_F_LOCAL,
Linus Torvalds's avatar
Linus Torvalds committed
121 122
};

123 124
static struct mempolicy preferred_node_policy[MAX_NUMNODES];

125
struct mempolicy *get_task_policy(struct task_struct *p)
126 127
{
	struct mempolicy *pol = p->mempolicy;
128
	int node;
129

130 131
	if (pol)
		return pol;
132

133 134 135 136 137 138
	node = numa_node_id();
	if (node != NUMA_NO_NODE) {
		pol = &preferred_node_policy[node];
		/* preferred_node_policy is not initialised early in boot */
		if (pol->mode)
			return pol;
139 140
	}

141
	return &default_policy;
142 143
}

144 145
static const struct mempolicy_operations {
	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
	/*
	 * If read-side task has no lock to protect task->mempolicy, write-side
	 * task will rebind the task->mempolicy by two step. The first step is
	 * setting all the newly nodes, and the second step is cleaning all the
	 * disallowed nodes. In this way, we can avoid finding no node to alloc
	 * page.
	 * If we have a lock to protect task->mempolicy in read-side, we do
	 * rebind directly.
	 *
	 * step:
	 * 	MPOL_REBIND_ONCE - do rebind work at once
	 * 	MPOL_REBIND_STEP1 - set all the newly nodes
	 * 	MPOL_REBIND_STEP2 - clean all the disallowed nodes
	 */
	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
			enum mpol_rebind_step step);
162 163
} mpol_ops[MPOL_MAX];

164 165
static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
{
Bob Liu's avatar
Bob Liu committed
166
	return pol->flags & MPOL_MODE_FLAGS;
167 168 169 170 171 172 173 174
}

static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
				   const nodemask_t *rel)
{
	nodemask_t tmp;
	nodes_fold(tmp, *orig, nodes_weight(*rel));
	nodes_onto(*ret, tmp, *rel);
175 176
}

177 178 179 180 181 182 183 184 185 186 187
static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
{
	if (nodes_empty(*nodes))
		return -EINVAL;
	pol->v.nodes = *nodes;
	return 0;
}

static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
{
	if (!nodes)
188
		pol->flags |= MPOL_F_LOCAL;	/* local allocation */
189 190 191 192 193 194 195 196 197
	else if (nodes_empty(*nodes))
		return -EINVAL;			/*  no allowed nodes */
	else
		pol->v.preferred_node = first_node(*nodes);
	return 0;
}

static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
{
198
	if (nodes_empty(*nodes))
199 200 201 202 203
		return -EINVAL;
	pol->v.nodes = *nodes;
	return 0;
}

204 205 206 207 208 209 210 211 212
/*
 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
 * any, for the new policy.  mpol_new() has already validated the nodes
 * parameter with respect to the policy mode and flags.  But, we need to
 * handle an empty nodemask with MPOL_PREFERRED here.
 *
 * Must be called holding task's alloc_lock to protect task's mems_allowed
 * and mempolicy.  May also be called holding the mmap_semaphore for write.
 */
213 214
static int mpol_set_nodemask(struct mempolicy *pol,
		     const nodemask_t *nodes, struct nodemask_scratch *nsc)
215 216 217 218 219 220
{
	int ret;

	/* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
	if (pol == NULL)
		return 0;
221
	/* Check N_MEMORY */
222
	nodes_and(nsc->mask1,
223
		  cpuset_current_mems_allowed, node_states[N_MEMORY]);
224 225 226 227 228 229

	VM_BUG_ON(!nodes);
	if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
		nodes = NULL;	/* explicit local allocation */
	else {
		if (pol->flags & MPOL_F_RELATIVE_NODES)
230
			mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
231
		else
232 233
			nodes_and(nsc->mask2, *nodes, nsc->mask1);

234 235 236 237 238 239 240
		if (mpol_store_user_nodemask(pol))
			pol->w.user_nodemask = *nodes;
		else
			pol->w.cpuset_mems_allowed =
						cpuset_current_mems_allowed;
	}

241 242 243 244
	if (nodes)
		ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
	else
		ret = mpol_ops[pol->mode].create(pol, NULL);
245 246 247 248 249 250 251
	return ret;
}

/*
 * This function just creates a new policy, does some check and simple
 * initialization. You must invoke mpol_set_nodemask() to set nodes.
 */
252 253
static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
				  nodemask_t *nodes)
Linus Torvalds's avatar
Linus Torvalds committed
254 255 256
{
	struct mempolicy *policy;

257
	pr_debug("setting mode %d flags %d nodes[0] %lx\n",
David Rientjes's avatar
David Rientjes committed
258
		 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
259

260 261
	if (mode == MPOL_DEFAULT) {
		if (nodes && !nodes_empty(*nodes))
262
			return ERR_PTR(-EINVAL);
263
		return NULL;
264
	}
265 266 267 268 269 270 271 272 273 274 275 276 277
	VM_BUG_ON(!nodes);

	/*
	 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
	 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
	 * All other modes require a valid pointer to a non-empty nodemask.
	 */
	if (mode == MPOL_PREFERRED) {
		if (nodes_empty(*nodes)) {
			if (((flags & MPOL_F_STATIC_NODES) ||
			     (flags & MPOL_F_RELATIVE_NODES)))
				return ERR_PTR(-EINVAL);
		}
278 279 280 281
	} else if (mode == MPOL_LOCAL) {
		if (!nodes_empty(*nodes))
			return ERR_PTR(-EINVAL);
		mode = MPOL_PREFERRED;
282 283
	} else if (nodes_empty(*nodes))
		return ERR_PTR(-EINVAL);
Linus Torvalds's avatar
Linus Torvalds committed
284 285 286 287
	policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
	if (!policy)
		return ERR_PTR(-ENOMEM);
	atomic_set(&policy->refcnt, 1);
288
	policy->mode = mode;
289
	policy->flags = flags;
290

Linus Torvalds's avatar
Linus Torvalds committed
291
	return policy;
292 293
}

294 295 296 297 298 299 300 301
/* Slow path of a mpol destructor. */
void __mpol_put(struct mempolicy *p)
{
	if (!atomic_dec_and_test(&p->refcnt))
		return;
	kmem_cache_free(policy_cache, p);
}

302 303
static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
				enum mpol_rebind_step step)
304 305 306
{
}

307 308 309 310 311 312 313 314
/*
 * step:
 * 	MPOL_REBIND_ONCE  - do rebind work at once
 * 	MPOL_REBIND_STEP1 - set all the newly nodes
 * 	MPOL_REBIND_STEP2 - clean all the disallowed nodes
 */
static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
				 enum mpol_rebind_step step)
315 316 317 318 319 320 321 322
{
	nodemask_t tmp;

	if (pol->flags & MPOL_F_STATIC_NODES)
		nodes_and(tmp, pol->w.user_nodemask, *nodes);
	else if (pol->flags & MPOL_F_RELATIVE_NODES)
		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
	else {
323 324 325 326 327 328 329 330 331 332 333 334 335
		/*
		 * if step == 1, we use ->w.cpuset_mems_allowed to cache the
		 * result
		 */
		if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
			nodes_remap(tmp, pol->v.nodes,
					pol->w.cpuset_mems_allowed, *nodes);
			pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
		} else if (step == MPOL_REBIND_STEP2) {
			tmp = pol->w.cpuset_mems_allowed;
			pol->w.cpuset_mems_allowed = *nodes;
		} else
			BUG();
336
	}
337

338 339 340 341 342 343 344 345 346 347
	if (nodes_empty(tmp))
		tmp = *nodes;

	if (step == MPOL_REBIND_STEP1)
		nodes_or(pol->v.nodes, pol->v.nodes, tmp);
	else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
		pol->v.nodes = tmp;
	else
		BUG();

348
	if (!node_isset(current->il_next, tmp)) {
349
		current->il_next = next_node_in(current->il_next, tmp);
350 351 352 353 354 355
		if (current->il_next >= MAX_NUMNODES)
			current->il_next = numa_node_id();
	}
}

static void mpol_rebind_preferred(struct mempolicy *pol,
356 357
				  const nodemask_t *nodes,
				  enum mpol_rebind_step step)
358 359 360 361 362 363
{
	nodemask_t tmp;

	if (pol->flags & MPOL_F_STATIC_NODES) {
		int node = first_node(pol->w.user_nodemask);

364
		if (node_isset(node, *nodes)) {
365
			pol->v.preferred_node = node;
366 367 368
			pol->flags &= ~MPOL_F_LOCAL;
		} else
			pol->flags |= MPOL_F_LOCAL;
369 370 371
	} else if (pol->flags & MPOL_F_RELATIVE_NODES) {
		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
		pol->v.preferred_node = first_node(tmp);
372
	} else if (!(pol->flags & MPOL_F_LOCAL)) {
373 374 375 376 377
		pol->v.preferred_node = node_remap(pol->v.preferred_node,
						   pol->w.cpuset_mems_allowed,
						   *nodes);
		pol->w.cpuset_mems_allowed = *nodes;
	}
Linus Torvalds's avatar
Linus Torvalds committed
378 379
}

380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
/*
 * mpol_rebind_policy - Migrate a policy to a different set of nodes
 *
 * If read-side task has no lock to protect task->mempolicy, write-side
 * task will rebind the task->mempolicy by two step. The first step is
 * setting all the newly nodes, and the second step is cleaning all the
 * disallowed nodes. In this way, we can avoid finding no node to alloc
 * page.
 * If we have a lock to protect task->mempolicy in read-side, we do
 * rebind directly.
 *
 * step:
 * 	MPOL_REBIND_ONCE  - do rebind work at once
 * 	MPOL_REBIND_STEP1 - set all the newly nodes
 * 	MPOL_REBIND_STEP2 - clean all the disallowed nodes
 */
static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
				enum mpol_rebind_step step)
398 399 400
{
	if (!pol)
		return;
401
	if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
402 403
	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
		return;
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418

	if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
		return;

	if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
		BUG();

	if (step == MPOL_REBIND_STEP1)
		pol->flags |= MPOL_F_REBINDING;
	else if (step == MPOL_REBIND_STEP2)
		pol->flags &= ~MPOL_F_REBINDING;
	else if (step >= MPOL_REBIND_NSTEP)
		BUG();

	mpol_ops[pol->mode].rebind(pol, newmask, step);
419 420 421 422 423
}

/*
 * Wrapper for mpol_rebind_policy() that just requires task
 * pointer, and updates task mempolicy.
424 425
 *
 * Called with task's alloc_lock held.
426 427
 */

428 429
void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
			enum mpol_rebind_step step)
430
{
431
	mpol_rebind_policy(tsk->mempolicy, new, step);
432 433 434 435 436 437 438 439 440 441 442 443 444 445
}

/*
 * Rebind each vma in mm to new nodemask.
 *
 * Call holding a reference to mm.  Takes mm->mmap_sem during call.
 */

void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
{
	struct vm_area_struct *vma;

	down_write(&mm->mmap_sem);
	for (vma = mm->mmap; vma; vma = vma->vm_next)
446
		mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
447 448 449
	up_write(&mm->mmap_sem);
}

450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
	[MPOL_DEFAULT] = {
		.rebind = mpol_rebind_default,
	},
	[MPOL_INTERLEAVE] = {
		.create = mpol_new_interleave,
		.rebind = mpol_rebind_nodemask,
	},
	[MPOL_PREFERRED] = {
		.create = mpol_new_preferred,
		.rebind = mpol_rebind_preferred,
	},
	[MPOL_BIND] = {
		.create = mpol_new_bind,
		.rebind = mpol_rebind_nodemask,
	},
};

468 469
static void migrate_page_add(struct page *page, struct list_head *pagelist,
				unsigned long flags);
470

471 472 473 474 475 476 477
struct queue_pages {
	struct list_head *pagelist;
	unsigned long flags;
	nodemask_t *nmask;
	struct vm_area_struct *prev;
};

478 479 480 481
/*
 * Scan through pages checking if pages follow certain conditions,
 * and move them to the pagelist if they do.
 */
482 483
static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
			unsigned long end, struct mm_walk *walk)
Linus Torvalds's avatar
Linus Torvalds committed
484
{
485 486 487 488
	struct vm_area_struct *vma = walk->vma;
	struct page *page;
	struct queue_pages *qp = walk->private;
	unsigned long flags = qp->flags;
489
	int nid, ret;
490
	pte_t *pte;
491
	spinlock_t *ptl;
492

493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
	if (pmd_trans_huge(*pmd)) {
		ptl = pmd_lock(walk->mm, pmd);
		if (pmd_trans_huge(*pmd)) {
			page = pmd_page(*pmd);
			if (is_huge_zero_page(page)) {
				spin_unlock(ptl);
				split_huge_pmd(vma, pmd, addr);
			} else {
				get_page(page);
				spin_unlock(ptl);
				lock_page(page);
				ret = split_huge_page(page);
				unlock_page(page);
				put_page(page);
				if (ret)
					return 0;
			}
		} else {
			spin_unlock(ptl);
		}
	}
514

515 516
	if (pmd_trans_unstable(pmd))
		return 0;
517
retry:
518 519
	pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE) {
520
		if (!pte_present(*pte))
Linus Torvalds's avatar
Linus Torvalds committed
521
			continue;
522 523
		page = vm_normal_page(vma, addr, *pte);
		if (!page)
Linus Torvalds's avatar
Linus Torvalds committed
524
			continue;
525
		/*
526 527
		 * vm_normal_page() filters out zero pages, but there might
		 * still be PageReserved pages to skip, perhaps in a VDSO.
528
		 */
Hugh Dickins's avatar
Hugh Dickins committed
529
		if (PageReserved(page))
530
			continue;
531
		nid = page_to_nid(page);
532
		if (node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT))
533
			continue;
534
		if (PageTransCompound(page)) {
535 536 537 538 539 540 541 542 543 544 545 546 547 548
			get_page(page);
			pte_unmap_unlock(pte, ptl);
			lock_page(page);
			ret = split_huge_page(page);
			unlock_page(page);
			put_page(page);
			/* Failed to split -- skip. */
			if (ret) {
				pte = pte_offset_map_lock(walk->mm, pmd,
						addr, &ptl);
				continue;
			}
			goto retry;
		}
549

550
		migrate_page_add(page, qp->pagelist, flags);
551 552 553 554
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();
	return 0;
555 556
}

557 558 559
static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask,
			       unsigned long addr, unsigned long end,
			       struct mm_walk *walk)
560 561
{
#ifdef CONFIG_HUGETLB_PAGE
562 563
	struct queue_pages *qp = walk->private;
	unsigned long flags = qp->flags;
564 565
	int nid;
	struct page *page;
566
	spinlock_t *ptl;
567
	pte_t entry;
568

569 570
	ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
	entry = huge_ptep_get(pte);
571 572 573
	if (!pte_present(entry))
		goto unlock;
	page = pte_page(entry);
574
	nid = page_to_nid(page);
575
	if (node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT))
576 577 578 579
		goto unlock;
	/* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
	if (flags & (MPOL_MF_MOVE_ALL) ||
	    (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
580
		isolate_huge_page(page, qp->pagelist);
581
unlock:
582
	spin_unlock(ptl);
583 584 585
#else
	BUG();
#endif
586
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
587 588
}

589
#ifdef CONFIG_NUMA_BALANCING
590
/*
591 592 593 594 595 596 597
 * This is used to mark a range of virtual addresses to be inaccessible.
 * These are later cleared by a NUMA hinting fault. Depending on these
 * faults, pages may be migrated for better NUMA placement.
 *
 * This is assuming that NUMA faults are handled using PROT_NONE. If
 * an architecture makes a different choice, it will need further
 * changes to the core.
598
 */
599 600
unsigned long change_prot_numa(struct vm_area_struct *vma,
			unsigned long addr, unsigned long end)
601
{
602
	int nr_updated;
603

604
	nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1);
605 606
	if (nr_updated)
		count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
607

608
	return nr_updated;
609 610 611 612 613 614 615
}
#else
static unsigned long change_prot_numa(struct vm_area_struct *vma,
			unsigned long addr, unsigned long end)
{
	return 0;
}
616
#endif /* CONFIG_NUMA_BALANCING */
617

618 619 620 621 622 623 624 625
static int queue_pages_test_walk(unsigned long start, unsigned long end,
				struct mm_walk *walk)
{
	struct vm_area_struct *vma = walk->vma;
	struct queue_pages *qp = walk->private;
	unsigned long endvma = vma->vm_end;
	unsigned long flags = qp->flags;

626
	if (!vma_migratable(vma))
627 628
		return 1;

629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
	if (endvma > end)
		endvma = end;
	if (vma->vm_start > start)
		start = vma->vm_start;

	if (!(flags & MPOL_MF_DISCONTIG_OK)) {
		if (!vma->vm_next && vma->vm_end < end)
			return -EFAULT;
		if (qp->prev && qp->prev->vm_end < vma->vm_start)
			return -EFAULT;
	}

	qp->prev = vma;

	if (flags & MPOL_MF_LAZY) {
		/* Similar to task_numa_work, skip inaccessible VMAs */
645 646 647
		if (!is_vm_hugetlb_page(vma) &&
			(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)) &&
			!(vma->vm_flags & VM_MIXEDMAP))
648 649 650 651
			change_prot_numa(vma, start, endvma);
		return 1;
	}

652 653
	/* queue pages from current vma */
	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
654 655 656 657
		return 0;
	return 1;
}

658
/*
659 660 661 662 663
 * Walk through page tables and collect pages to be migrated.
 *
 * If pages found in a given range are on a set of nodes (determined by
 * @nodes and @flags,) it's isolated and queued to the pagelist which is
 * passed via @private.)
664
 */
665
static int
666
queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
667 668
		nodemask_t *nodes, unsigned long flags,
		struct list_head *pagelist)
Linus Torvalds's avatar
Linus Torvalds committed
669
{
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
	struct queue_pages qp = {
		.pagelist = pagelist,
		.flags = flags,
		.nmask = nodes,
		.prev = NULL,
	};
	struct mm_walk queue_pages_walk = {
		.hugetlb_entry = queue_pages_hugetlb,
		.pmd_entry = queue_pages_pte_range,
		.test_walk = queue_pages_test_walk,
		.mm = mm,
		.private = &qp,
	};

	return walk_page_range(start, end, &queue_pages_walk);
Linus Torvalds's avatar
Linus Torvalds committed
685 686
}

687 688 689 690 691 692
/*
 * Apply policy to a single VMA
 * This must be called with the mmap_sem held for writing.
 */
static int vma_replace_policy(struct vm_area_struct *vma,
						struct mempolicy *pol)
693
{
694 695 696
	int err;
	struct mempolicy *old;
	struct mempolicy *new;
697 698 699 700 701 702

	pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
		 vma->vm_start, vma->vm_end, vma->vm_pgoff,
		 vma->vm_ops, vma->vm_file,
		 vma->vm_ops ? vma->vm_ops->set_policy : NULL);

703 704 705 706 707
	new = mpol_dup(pol);
	if (IS_ERR(new))
		return PTR_ERR(new);

	if (vma->vm_ops && vma->vm_ops->set_policy) {
708
		err = vma->vm_ops->set_policy(vma, new);
709 710
		if (err)
			goto err_out;
711
	}
712 713 714 715 716 717 718 719

	old = vma->vm_policy;
	vma->vm_policy = new; /* protected by mmap_sem */
	mpol_put(old);

	return 0;
 err_out:
	mpol_put(new);
720 721 722
	return err;
}

Linus Torvalds's avatar
Linus Torvalds committed
723
/* Step 2: apply policy to a range and do splits. */
724 725
static int mbind_range(struct mm_struct *mm, unsigned long start,
		       unsigned long end, struct mempolicy *new_pol)
Linus Torvalds's avatar
Linus Torvalds committed
726 727
{
	struct vm_area_struct *next;
728 729 730
	struct vm_area_struct *prev;
	struct vm_area_struct *vma;
	int err = 0;
731
	pgoff_t pgoff;
732 733
	unsigned long vmstart;
	unsigned long vmend;
Linus Torvalds's avatar
Linus Torvalds committed
734

735
	vma = find_vma(mm, start);
736 737 738
	if (!vma || vma->vm_start > start)
		return -EFAULT;

739
	prev = vma->vm_prev;
740 741 742
	if (start > vma->vm_start)
		prev = vma;

743
	for (; vma && vma->vm_start < end; prev = vma, vma = next) {
Linus Torvalds's avatar
Linus Torvalds committed
744
		next = vma->vm_next;
745 746 747
		vmstart = max(start, vma->vm_start);
		vmend   = min(end, vma->vm_end);

748 749 750 751 752
		if (mpol_equal(vma_policy(vma), new_pol))
			continue;

		pgoff = vma->vm_pgoff +
			((vmstart - vma->vm_start) >> PAGE_SHIFT);
753
		prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
754 755
				 vma->anon_vma, vma->vm_file, pgoff,
				 new_pol, vma->vm_userfaultfd_ctx);
756 757 758
		if (prev) {
			vma = prev;
			next = vma->vm_next;
759 760 761 762
			if (mpol_equal(vma_policy(vma), new_pol))
				continue;
			/* vma_merge() joined vma && vma->next, case 8 */
			goto replace;
763 764 765 766 767 768 769 770 771 772 773
		}
		if (vma->vm_start != vmstart) {
			err = split_vma(vma->vm_mm, vma, vmstart, 1);
			if (err)
				goto out;
		}
		if (vma->vm_end != vmend) {
			err = split_vma(vma->vm_mm, vma, vmend, 0);
			if (err)
				goto out;
		}
774
 replace:
775
		err = vma_replace_policy(vma, new_pol);
776 777
		if (err)
			goto out;
Linus Torvalds's avatar
Linus Torvalds committed
778
	}
779 780

 out:
Linus Torvalds's avatar
Linus Torvalds committed
781 782 783 784
	return err;
}

/* Set the process memory policy */
785 786
static long do_set_mempolicy(unsigned short mode, unsigned short flags,
			     nodemask_t *nodes)
Linus Torvalds's avatar
Linus Torvalds committed
787
{
788
	struct mempolicy *new, *old;
789
	NODEMASK_SCRATCH(scratch);
790
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
791

792 793
	if (!scratch)
		return -ENOMEM;
794

795 796 797 798 799
	new = mpol_new(mode, flags, nodes);
	if (IS_ERR(new)) {
		ret = PTR_ERR(new);
		goto out;
	}
800

801
	task_lock(current);
802
	ret = mpol_set_nodemask(new, nodes, scratch);
803 804 805
	if (ret) {
		task_unlock(current);
		mpol_put(new);
806
		goto out;
807 808
	}
	old = current->mempolicy;
Linus Torvalds's avatar
Linus Torvalds committed
809
	current->mempolicy = new;
810
	if (new && new->mode == MPOL_INTERLEAVE &&
811
	    nodes_weight(new->v.nodes))
812
		current->il_next = first_node(new->v.nodes);
813 814
	task_unlock(current);
	mpol_put(old);
815 816 817 818
	ret = 0;
out:
	NODEMASK_SCRATCH_FREE(scratch);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
819 820
}

821 822
/*
 * Return nodemask for policy for get_mempolicy() query
823 824
 *
 * Called with task's alloc_lock held
825 826
 */
static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
Linus Torvalds's avatar
Linus Torvalds committed
827
{
828
	nodes_clear(*nodes);
829 830 831
	if (p == &default_policy)
		return;

832
	switch (p->mode) {
833 834
	case MPOL_BIND:
		/* Fall through */
Linus Torvalds's avatar
Linus Torvalds committed
835
	case MPOL_INTERLEAVE:
836
		*nodes = p->v.nodes;
Linus Torvalds's avatar
Linus Torvalds committed
837 838
		break;
	case MPOL_PREFERRED:
839
		if (!(p->flags & MPOL_F_LOCAL))
840
			node_set(p->v.preferred_node, *nodes);
841
		/* else return empty node mask for local allocation */
Linus Torvalds's avatar
Linus Torvalds committed
842 843 844 845 846 847
		break;
	default:
		BUG();
	}
}

848
static int lookup_node(unsigned long addr)
Linus Torvalds's avatar
Linus Torvalds committed
849 850 851 852
{
	struct page *p;
	int err;

853
	err = get_user_pages(addr & PAGE_MASK, 1, 0, 0, &p, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
854 855 856 857 858 859 860 861
	if (err >= 0) {
		err = page_to_nid(p);
		put_page(p);
	}
	return err;
}

/* Retrieve NUMA policy */
Adrian Bunk's avatar
Adrian Bunk committed
862 863
static long do_get_mempolicy(int *policy, nodemask_t *nmask,
			     unsigned long addr, unsigned long flags)
Linus Torvalds's avatar
Linus Torvalds committed
864
{
865
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
866 867 868 869
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma = NULL;
	struct mempolicy *pol = current->mempolicy;

870 871
	if (flags &
		~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
Linus Torvalds's avatar
Linus Torvalds committed
872
		return -EINVAL;
873 874 875 876 877

	if (flags & MPOL_F_MEMS_ALLOWED) {
		if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
			return -EINVAL;
		*policy = 0;	/* just so it's initialized */
878
		task_lock(current);
879
		*nmask  = cpuset_current_mems_allowed;
880
		task_unlock(current);
881 882 883
		return 0;
	}

Linus Torvalds's avatar
Linus Torvalds committed
884
	if (flags & MPOL_F_ADDR) {
885 886 887 888 889
		/*
		 * Do NOT fall back to task policy if the
		 * vma/shared policy at addr is NULL.  We
		 * want to return MPOL_DEFAULT in this case.
		 */
Linus Torvalds's avatar
Linus Torvalds committed
890 891 892 893 894 895 896 897 898 899 900 901 902 903
		down_read(&mm->mmap_sem);
		vma = find_vma_intersection(mm, addr, addr+1);
		if (!vma) {
			up_read(&mm->mmap_sem);
			return -EFAULT;
		}
		if (vma->vm_ops && vma->vm_ops->get_policy)
			pol = vma->vm_ops->get_policy(vma, addr);
		else
			pol = vma->vm_policy;
	} else if (addr)
		return -EINVAL;

	if (!pol)
904
		pol = &default_policy;	/* indicates default behavior */
Linus Torvalds's avatar
Linus Torvalds committed
905 906 907

	if (flags & MPOL_F_NODE) {
		if (flags & MPOL_F_ADDR) {
908
			err = lookup_node(addr);
Linus Torvalds's avatar
Linus Torvalds committed
909 910
			if (err < 0)
				goto out;
911
			*policy = err;
Linus Torvalds's avatar
Linus Torvalds committed
912
		} else if (pol == current->mempolicy &&
913
				pol->mode == MPOL_INTERLEAVE) {
914
			*policy = current->il_next;
Linus Torvalds's avatar
Linus Torvalds committed
915 916 917 918
		} else {
			err = -EINVAL;
			goto out;
		}
919 920 921
	} else {
		*policy = pol == &default_policy ? MPOL_DEFAULT :
						pol->mode;
922 923 924 925 926
		/*
		 * Internal mempolicy flags must be masked off before exposing
		 * the policy to userspace.
		 */
		*policy |= (pol->flags & MPOL_MODE_FLAGS);
927
	}
Linus Torvalds's avatar
Linus Torvalds committed
928 929 930 931 932 933 934

	if (vma) {
		up_read(&current->mm->mmap_sem);
		vma = NULL;
	}

	err = 0;
935
	if (nmask) {
936 937 938 939 940 941 942
		if (mpol_store_user_nodemask(pol)) {
			*nmask = pol->w.user_nodemask;
		} else {
			task_lock(current);
			get_policy_nodemask(pol, nmask);
			task_unlock(current);
		}
943
	}
Linus Torvalds's avatar
Linus Torvalds committed
944 945

 out:
946
	mpol_cond_put(pol);
Linus Torvalds's avatar
Linus Torvalds committed
947 948 949 950 951
	if (vma)
		up_read(&current->mm->mmap_sem);
	return err;
}

952
#ifdef CONFIG_MIGRATION
953 954 955
/*
 * page migration
 */
956 957
static void migrate_page_add(struct page *page, struct list_head *pagelist,
				unsigned long flags)
958 959
{
	/*
960
	 * Avoid migrating a page that is shared with others.
961
	 */
962 963 964
	if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
		if (!isolate_lru_page(page)) {
			list_add_tail(&page->lru, pagelist);
965 966
			inc_zone_page_state(page, NR_ISOLATED_ANON +
					    page_is_file_cache(page));
967 968
		}
	}
969
}
970

971
static struct page *new_node_page(struct page *page, unsigned long node, int **x)
972
{
973 974 975 976
	if (PageHuge(page))
		return alloc_huge_page_node(page_hstate(compound_head(page)),
					node);
	else
977
		return __alloc_pages_node(node, GFP_HIGHUSER_MOVABLE |
978
						    __GFP_THISNODE, 0);
979 980
}

981 982 983 984
/*
 * Migrate pages from one node to a target node.
 * Returns error or the number of pages not migrated.
 */
Adrian Bunk's avatar
Adrian Bunk committed
985 986
static int migrate_to_node(struct mm_struct *mm, int source, int dest,
			   int flags)
987 988 989 990 991 992 993
{
	nodemask_t nmask;
	LIST_HEAD(pagelist);
	int err = 0;

	nodes_clear(nmask);
	node_set(source, nmask);
994

995 996 997 998 999 1000
	/*
	 * This does not "check" the range but isolates all pages that
	 * need migration.  Between passing in the full user address
	 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
	 */
	VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
1001
	queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
1002 1003
			flags | MPOL_MF_DISCONTIG_OK, &pagelist);

1004
	if (!list_empty(&pagelist)) {
1005
		err = migrate_pages(&pagelist, new_node_page, NULL, dest,
1006
					MIGRATE_SYNC, MR_SYSCALL);
1007
		if (err)
1008
			putback_movable_pages(&pagelist);
1009
	}
1010

1011
	return err;
1012 1013
}

1014
/*
1015 1016
 * Move pages between the two nodesets so as to preserve the physical
 * layout as much as possible.
1017 1018 1019
 *
 * Returns the number of page that could not be moved.
 */
1020 1021
int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
		     const nodemask_t *to, int flags)
1022
{
1023
	int busy = 0;
1024
	int err;
1025
	nodemask_t tmp;
1026

1027 1028 1029 1030
	err = migrate_prep();
	if (err)
		return err;

1031
	down_read(&mm->mmap_sem);
1032

1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
	/*
	 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
	 * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
	 * bit in 'tmp', and return that <source, dest> pair for migration.
	 * The pair of nodemasks 'to' and 'from' define the map.
	 *
	 * If no pair of bits is found that way, fallback to picking some
	 * pair of 'source' and 'dest' bits that are not the same.  If the
	 * 'source' and 'dest' bits are the same, this represents a node
	 * that will be migrating to itself, so no pages need move.
	 *
	 * If no bits are left in 'tmp', or if all remaining bits left
	 * in 'tmp' correspond to the same bit in 'to', return false
	 * (nothing left to migrate).
	 *
	 * This lets us pick a pair of nodes to migrate between, such that
	 * if possible the dest node is not already occupied by some other
	 * source node, minimizing the risk of overloading the memory on a
	 * node that would happen if we migrated incoming memory to a node
	 * before migrating outgoing memory source that same node.
	 *
	 * A single scan of tmp is sufficient.  As we go, we remember the
	 * most recent <s, d> pair that moved (s != d).  If we find a pair
	 * that not only moved, but what's better, moved to an empty slot
	 * (d is not set in tmp), then we break out then, with that pair.
1058
	 * Otherwise when we finish scanning from_tmp, we at least have the
1059 1060 1061 1062
	 * most recent <s, d> pair that moved.  If we get all the way through
	 * the scan of tmp without finding any node that moved, much less
	 * moved to an empty node, then there is nothing left worth migrating.
	 */
1063

1064
	tmp = *from;
1065 1066
	while (!nodes_empty(tmp)) {
		int s,d;
Jianguo Wu's avatar
Jianguo Wu committed
1067
		int source = NUMA_NO_NODE;
1068 1069 1070
		int dest = 0;

		for_each_node_mask(s, tmp) {
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086

			/*
			 * do_migrate_pages() tries to maintain the relative
			 * node relationship of the pages established between
			 * threads and memory areas.
                         *
			 * However if the number of source nodes is not equal to
			 * the number of destination nodes we can not preserve
			 * this node relative relationship.  In that case, skip
			 * copying memory from a node that is in the destination
			 * mask.
			 *
			 * Example: [2,3,4] -> [3,4,5] moves everything.
			 *          [0-7] - > [3,4,5] moves only 0,1,2,6,7.
			 */

1087 1088
			if ((nodes_weight(*from) != nodes_weight(*to)) &&
						(node_isset(s, *to)))
1089 1090
				continue;

1091
			d = node_remap(s, *from, *to);
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
			if (s == d)
				continue;

			source = s;	/* Node moved. Memorize */
			dest = d;

			/* dest not in remaining from nodes? */
			if (!node_isset(dest, tmp))
				break;
		}
Jianguo Wu's avatar
Jianguo Wu committed
1102
		if (source == NUMA_NO_NODE)
1103 1104 1105 1106 1107 1108 1109 1110
			break;

		node_clear(source, tmp);
		err = migrate_to_node(mm, source, dest, flags);
		if (err > 0)
			busy += err;
		if (err < 0)
			break;
1111 1112
	}
	up_read(&mm->mmap_sem);
1113 1114 1115
	if (err < 0)
		return err;
	return busy;