mspec.c 10.3 KB
Newer Older
Jes Sorensen's avatar
Jes Sorensen committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
/*
 * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights
 * reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU General Public License
 * as published by the Free Software Foundation.
 */

/*
 * SN Platform Special Memory (mspec) Support
 *
 * This driver exports the SN special memory (mspec) facility to user
 * processes.
 * There are three types of memory made available thru this driver:
 * fetchops, uncached and cached.
 *
 * Fetchops are atomic memory operations that are implemented in the
 * memory controller on SGI SN hardware.
 *
 * Uncached are used for memory write combining feature of the ia64
 * cpu.
 *
 * Cached are used for areas of memory that are used as cached addresses
 * on our partition and used as uncached addresses from other partitions.
 * Due to a design constraint of the SN2 Shub, you can not have processors
 * on the same FSB perform both a cached and uncached reference to the
 * same cache line.  These special memory cached regions prevent the
 * kernel from ever dropping in a TLB entry and therefore prevent the
 * processor from ever speculating a cache line from this page.
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/miscdevice.h>
#include <linux/spinlock.h>
#include <linux/mm.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
41
#include <linux/fs.h>
Jes Sorensen's avatar
Jes Sorensen committed
42 43 44 45
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/numa.h>
46
#include <linux/refcount.h>
Jes Sorensen's avatar
Jes Sorensen committed
47 48
#include <asm/page.h>
#include <asm/pgtable.h>
Arun Sharma's avatar
Arun Sharma committed
49
#include <linux/atomic.h>
Jes Sorensen's avatar
Jes Sorensen committed
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
#include <asm/tlbflush.h>
#include <asm/uncached.h>
#include <asm/sn/addrs.h>
#include <asm/sn/arch.h>
#include <asm/sn/mspec.h>
#include <asm/sn/sn_cpuid.h>
#include <asm/sn/io.h>
#include <asm/sn/bte.h>
#include <asm/sn/shubio.h>


#define FETCHOP_ID	"SGI Fetchop,"
#define CACHED_ID	"Cached,"
#define UNCACHED_ID	"Uncached"
#define REVISION	"4.0"
#define MSPEC_BASENAME	"mspec"

/*
 * Page types allocated by the device.
 */
70
enum mspec_page_type {
Jes Sorensen's avatar
Jes Sorensen committed
71 72 73 74 75
	MSPEC_FETCHOP = 1,
	MSPEC_CACHED,
	MSPEC_UNCACHED
};

76
#ifdef CONFIG_SGI_SN
Jes Sorensen's avatar
Jes Sorensen committed
77
static int is_sn2;
78 79 80
#else
#define is_sn2		0
#endif
Jes Sorensen's avatar
Jes Sorensen committed
81 82 83 84 85

/*
 * One of these structures is allocated when an mspec region is mmaped. The
 * structure is pointed to by the vma->vm_private_data field in the vma struct.
 * This structure is used to record the addresses of the mspec pages.
86 87 88 89 90
 * This structure is shared by all vma's that are split off from the
 * original vma when split_vma()'s are done.
 *
 * The refcnt is incremented atomically because mm->mmap_sem does not
 * protect in fork case where multiple tasks share the vma_data.
Jes Sorensen's avatar
Jes Sorensen committed
91 92
 */
struct vma_data {
93
	refcount_t refcnt;	/* Number of vmas sharing the data. */
94
	spinlock_t lock;	/* Serialize access to this structure. */
Jes Sorensen's avatar
Jes Sorensen committed
95
	int count;		/* Number of pages allocated. */
96 97 98
	enum mspec_page_type type; /* Type of pages allocated. */
	unsigned long vm_start;	/* Original (unsplit) base. */
	unsigned long vm_end;	/* Original (unsplit) end. */
Jes Sorensen's avatar
Jes Sorensen committed
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
	unsigned long maddr[0];	/* Array of MSPEC addresses. */
};

/* used on shub2 to clear FOP cache in the HUB */
static unsigned long scratch_page[MAX_NUMNODES];
#define SH2_AMO_CACHE_ENTRIES	4

static inline int
mspec_zero_block(unsigned long addr, int len)
{
	int status;

	if (is_sn2) {
		if (is_shub2()) {
			int nid;
			void *p;
			int i;

			nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
			p = (void *)TO_AMO(scratch_page[nid]);

			for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
				FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
				p += FETCHOP_VAR_SIZE;
			}
		}

		status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
				  BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
	} else {
		memset((char *) addr, 0, len);
		status = 0;
	}
	return status;
}

/*
 * mspec_open
 *
 * Called when a device mapping is created by a means other than mmap
139 140
 * (via fork, munmap, etc.).  Increments the reference count on the
 * underlying mspec data so it is not freed prematurely.
Jes Sorensen's avatar
Jes Sorensen committed
141 142 143 144 145 146 147
 */
static void
mspec_open(struct vm_area_struct *vma)
{
	struct vma_data *vdata;

	vdata = vma->vm_private_data;
148
	refcount_inc(&vdata->refcnt);
Jes Sorensen's avatar
Jes Sorensen committed
149 150 151 152 153 154
}

/*
 * mspec_close
 *
 * Called when unmapping a device mapping. Frees all mspec pages
155
 * belonging to all the vma's sharing this vma_data structure.
Jes Sorensen's avatar
Jes Sorensen committed
156 157 158 159 160
 */
static void
mspec_close(struct vm_area_struct *vma)
{
	struct vma_data *vdata;
161
	int index, last_index;
162
	unsigned long my_page;
Jes Sorensen's avatar
Jes Sorensen committed
163 164 165

	vdata = vma->vm_private_data;

166
	if (!refcount_dec_and_test(&vdata->refcnt))
167
		return;
168

169 170
	last_index = (vdata->vm_end - vdata->vm_start) >> PAGE_SHIFT;
	for (index = 0; index < last_index; index++) {
171
		if (vdata->maddr[index] == 0)
Jes Sorensen's avatar
Jes Sorensen committed
172 173 174 175 176
			continue;
		/*
		 * Clear the page before sticking it back
		 * into the pool.
		 */
177 178
		my_page = vdata->maddr[index];
		vdata->maddr[index] = 0;
179
		if (!mspec_zero_block(my_page, PAGE_SIZE))
180
			uncached_free_page(my_page, 1);
Jes Sorensen's avatar
Jes Sorensen committed
181 182
		else
			printk(KERN_WARNING "mspec_close(): "
183
			       "failed to zero page %ld\n", my_page);
Jes Sorensen's avatar
Jes Sorensen committed
184
	}
185

186
	kvfree(vdata);
Jes Sorensen's avatar
Jes Sorensen committed
187 188 189
}

/*
Nick Piggin's avatar
Nick Piggin committed
190
 * mspec_fault
Jes Sorensen's avatar
Jes Sorensen committed
191 192 193
 *
 * Creates a mspec page and maps it to user space.
 */
194
static vm_fault_t
195
mspec_fault(struct vm_fault *vmf)
Jes Sorensen's avatar
Jes Sorensen committed
196 197 198
{
	unsigned long paddr, maddr;
	unsigned long pfn;
Nick Piggin's avatar
Nick Piggin committed
199
	pgoff_t index = vmf->pgoff;
200
	struct vma_data *vdata = vmf->vma->vm_private_data;
Jes Sorensen's avatar
Jes Sorensen committed
201 202 203

	maddr = (volatile unsigned long) vdata->maddr[index];
	if (maddr == 0) {
204
		maddr = uncached_alloc_page(numa_node_id(), 1);
Jes Sorensen's avatar
Jes Sorensen committed
205
		if (maddr == 0)
Nick Piggin's avatar
Nick Piggin committed
206
			return VM_FAULT_OOM;
Jes Sorensen's avatar
Jes Sorensen committed
207 208 209 210 211 212

		spin_lock(&vdata->lock);
		if (vdata->maddr[index] == 0) {
			vdata->count++;
			vdata->maddr[index] = maddr;
		} else {
213
			uncached_free_page(maddr, 1);
Jes Sorensen's avatar
Jes Sorensen committed
214 215 216 217 218 219 220 221
			maddr = vdata->maddr[index];
		}
		spin_unlock(&vdata->lock);
	}

	if (vdata->type == MSPEC_FETCHOP)
		paddr = TO_AMO(maddr);
	else
222
		paddr = maddr & ~__IA64_UNCACHED_OFFSET;
Jes Sorensen's avatar
Jes Sorensen committed
223 224 225

	pfn = paddr >> PAGE_SHIFT;

226
	return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
Jes Sorensen's avatar
Jes Sorensen committed
227 228
}

229
static const struct vm_operations_struct mspec_vm_ops = {
Jes Sorensen's avatar
Jes Sorensen committed
230 231
	.open = mspec_open,
	.close = mspec_close,
Nick Piggin's avatar
Nick Piggin committed
232
	.fault = mspec_fault,
Jes Sorensen's avatar
Jes Sorensen committed
233 234 235 236 237
};

/*
 * mspec_mmap
 *
238
 * Called when mmapping the device.  Initializes the vma with a fault handler
Jes Sorensen's avatar
Jes Sorensen committed
239 240 241 242
 * and private data structure necessary to allocate, track, and free the
 * underlying pages.
 */
static int
243 244
mspec_mmap(struct file *file, struct vm_area_struct *vma,
					enum mspec_page_type type)
Jes Sorensen's avatar
Jes Sorensen committed
245 246
{
	struct vma_data *vdata;
247
	int pages, vdata_size;
Jes Sorensen's avatar
Jes Sorensen committed
248 249 250 251 252 253 254 255 256 257

	if (vma->vm_pgoff != 0)
		return -EINVAL;

	if ((vma->vm_flags & VM_SHARED) == 0)
		return -EINVAL;

	if ((vma->vm_flags & VM_WRITE) == 0)
		return -EPERM;

258
	pages = vma_pages(vma);
Jes Sorensen's avatar
Jes Sorensen committed
259 260
	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
	if (vdata_size <= PAGE_SIZE)
261
		vdata = kzalloc(vdata_size, GFP_KERNEL);
262
	else
263
		vdata = vzalloc(vdata_size);
Jes Sorensen's avatar
Jes Sorensen committed
264 265 266
	if (!vdata)
		return -ENOMEM;

267 268
	vdata->vm_start = vma->vm_start;
	vdata->vm_end = vma->vm_end;
Jes Sorensen's avatar
Jes Sorensen committed
269 270
	vdata->type = type;
	spin_lock_init(&vdata->lock);
271
	refcount_set(&vdata->refcnt, 1);
Jes Sorensen's avatar
Jes Sorensen committed
272 273
	vma->vm_private_data = vdata;

274
	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
Jes Sorensen's avatar
Jes Sorensen committed
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
	if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	vma->vm_ops = &mspec_vm_ops;

	return 0;
}

static int
fetchop_mmap(struct file *file, struct vm_area_struct *vma)
{
	return mspec_mmap(file, vma, MSPEC_FETCHOP);
}

static int
cached_mmap(struct file *file, struct vm_area_struct *vma)
{
	return mspec_mmap(file, vma, MSPEC_CACHED);
}

static int
uncached_mmap(struct file *file, struct vm_area_struct *vma)
{
	return mspec_mmap(file, vma, MSPEC_UNCACHED);
}

300
static const struct file_operations fetchop_fops = {
Jes Sorensen's avatar
Jes Sorensen committed
301
	.owner = THIS_MODULE,
302 303
	.mmap = fetchop_mmap,
	.llseek = noop_llseek,
Jes Sorensen's avatar
Jes Sorensen committed
304 305 306 307 308 309 310 311
};

static struct miscdevice fetchop_miscdev = {
	.minor = MISC_DYNAMIC_MINOR,
	.name = "sgi_fetchop",
	.fops = &fetchop_fops
};

312
static const struct file_operations cached_fops = {
Jes Sorensen's avatar
Jes Sorensen committed
313
	.owner = THIS_MODULE,
314 315
	.mmap = cached_mmap,
	.llseek = noop_llseek,
Jes Sorensen's avatar
Jes Sorensen committed
316 317 318 319 320 321 322 323
};

static struct miscdevice cached_miscdev = {
	.minor = MISC_DYNAMIC_MINOR,
	.name = "mspec_cached",
	.fops = &cached_fops
};

324
static const struct file_operations uncached_fops = {
Jes Sorensen's avatar
Jes Sorensen committed
325
	.owner = THIS_MODULE,
326 327
	.mmap = uncached_mmap,
	.llseek = noop_llseek,
Jes Sorensen's avatar
Jes Sorensen committed
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
};

static struct miscdevice uncached_miscdev = {
	.minor = MISC_DYNAMIC_MINOR,
	.name = "mspec_uncached",
	.fops = &uncached_fops
};

/*
 * mspec_init
 *
 * Called at boot time to initialize the mspec facility.
 */
static int __init
mspec_init(void)
{
	int ret;
	int nid;

	/*
	 * The fetchop device only works on SN2 hardware, uncached and cached
	 * memory drivers should both be valid on all ia64 hardware
	 */
351
#ifdef CONFIG_SGI_SN
Jes Sorensen's avatar
Jes Sorensen committed
352 353 354 355
	if (ia64_platform_is("sn2")) {
		is_sn2 = 1;
		if (is_shub2()) {
			ret = -ENOMEM;
356
			for_each_node_state(nid, N_ONLINE) {
Jes Sorensen's avatar
Jes Sorensen committed
357 358 359 360
				int actual_nid;
				int nasid;
				unsigned long phys;

361
				scratch_page[nid] = uncached_alloc_page(nid, 1);
Jes Sorensen's avatar
Jes Sorensen committed
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
				if (scratch_page[nid] == 0)
					goto free_scratch_pages;
				phys = __pa(scratch_page[nid]);
				nasid = get_node_number(phys);
				actual_nid = nasid_to_cnodeid(nasid);
				if (actual_nid != nid)
					goto free_scratch_pages;
			}
		}

		ret = misc_register(&fetchop_miscdev);
		if (ret) {
			printk(KERN_ERR
			       "%s: failed to register device %i\n",
			       FETCHOP_ID, ret);
			goto free_scratch_pages;
		}
	}
380
#endif
Jes Sorensen's avatar
Jes Sorensen committed
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
	ret = misc_register(&cached_miscdev);
	if (ret) {
		printk(KERN_ERR "%s: failed to register device %i\n",
		       CACHED_ID, ret);
		if (is_sn2)
			misc_deregister(&fetchop_miscdev);
		goto free_scratch_pages;
	}
	ret = misc_register(&uncached_miscdev);
	if (ret) {
		printk(KERN_ERR "%s: failed to register device %i\n",
		       UNCACHED_ID, ret);
		misc_deregister(&cached_miscdev);
		if (is_sn2)
			misc_deregister(&fetchop_miscdev);
		goto free_scratch_pages;
	}

	printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
	       MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
	       CACHED_ID, UNCACHED_ID);

	return 0;

 free_scratch_pages:
	for_each_node(nid) {
		if (scratch_page[nid] != 0)
408
			uncached_free_page(scratch_page[nid], 1);
Jes Sorensen's avatar
Jes Sorensen committed
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
	}
	return ret;
}

static void __exit
mspec_exit(void)
{
	int nid;

	misc_deregister(&uncached_miscdev);
	misc_deregister(&cached_miscdev);
	if (is_sn2) {
		misc_deregister(&fetchop_miscdev);

		for_each_node(nid) {
			if (scratch_page[nid] != 0)
425
				uncached_free_page(scratch_page[nid], 1);
Jes Sorensen's avatar
Jes Sorensen committed
426 427 428 429 430 431 432 433 434 435
		}
	}
}

module_init(mspec_init);
module_exit(mspec_exit);

MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
MODULE_LICENSE("GPL");