i915_gem.c 134 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drm.h"
#include "i915_drv.h"
Chris Wilson's avatar
Chris Wilson committed
32
#include "i915_trace.h"
33
#include "intel_drv.h"
34
#include <linux/slab.h>
35
#include <linux/swap.h>
36
#include <linux/pci.h>
37

38
39
40
41
42
43
44
45
46
static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
					     int write);
static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
						     uint64_t offset,
						     uint64_t size);
static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
47
static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
48
49
50
static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
					   unsigned alignment);
static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
51
static int i915_gem_evict_something(struct drm_device *dev, int min_size);
52
static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
53
54
55
static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
				struct drm_i915_gem_pwrite *args,
				struct drm_file *file_priv);
56

57
58
59
static LIST_HEAD(shrink_list);
static DEFINE_SPINLOCK(shrink_list_lock);

60
61
int i915_gem_do_init(struct drm_device *dev, unsigned long start,
		     unsigned long end)
62
63
64
{
	drm_i915_private_t *dev_priv = dev->dev_private;

65
66
67
	if (start >= end ||
	    (start & (PAGE_SIZE - 1)) != 0 ||
	    (end & (PAGE_SIZE - 1)) != 0) {
68
69
70
		return -EINVAL;
	}

71
72
	drm_mm_init(&dev_priv->mm.gtt_space, start,
		    end - start);
73

74
75
76
77
	dev->gtt_total = (uint32_t) (end - start);

	return 0;
}
78

79
80
81
82
83
84
85
86
87
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
		    struct drm_file *file_priv)
{
	struct drm_i915_gem_init *args = data;
	int ret;

	mutex_lock(&dev->struct_mutex);
	ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
88
89
	mutex_unlock(&dev->struct_mutex);

90
	return ret;
91
92
}

93
94
95
96
97
98
99
100
101
102
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
			    struct drm_file *file_priv)
{
	struct drm_i915_gem_get_aperture *args = data;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

	args->aper_size = dev->gtt_total;
103
104
	args->aper_available_size = (args->aper_size -
				     atomic_read(&dev->pin_memory));
105
106
107
108

	return 0;
}

109
110
111
112
113
114
115
116
117
118

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv)
{
	struct drm_i915_gem_create *args = data;
	struct drm_gem_object *obj;
119
120
	int ret;
	u32 handle;
121
122
123
124

	args->size = roundup(args->size, PAGE_SIZE);

	/* Allocate the new object */
125
	obj = i915_gem_alloc_object(dev, args->size);
126
127
128
129
	if (obj == NULL)
		return -ENOMEM;

	ret = drm_gem_handle_create(file_priv, obj, &handle);
130
	drm_gem_object_handle_unreference_unlocked(obj);
131
132
133
134
135
136
137
138
139

	if (ret)
		return ret;

	args->handle = handle;

	return 0;
}

140
141
142
143
144
145
146
static inline int
fast_shmem_read(struct page **pages,
		loff_t page_base, int page_offset,
		char __user *data,
		int length)
{
	char __iomem *vaddr;
147
	int unwritten;
148
149
150
151

	vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
	if (vaddr == NULL)
		return -ENOMEM;
152
	unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
153
154
	kunmap_atomic(vaddr, KM_USER0);

155
156
157
158
	if (unwritten)
		return -EFAULT;

	return 0;
159
160
}

161
162
163
static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
{
	drm_i915_private_t *dev_priv = obj->dev->dev_private;
164
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
165
166
167
168
169

	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		obj_priv->tiling_mode != I915_TILING_NONE;
}

170
static inline void
171
172
173
174
175
176
177
178
slow_shmem_copy(struct page *dst_page,
		int dst_offset,
		struct page *src_page,
		int src_offset,
		int length)
{
	char *dst_vaddr, *src_vaddr;

179
180
	dst_vaddr = kmap(dst_page);
	src_vaddr = kmap(src_page);
181
182
183

	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);

184
185
	kunmap(src_page);
	kunmap(dst_page);
186
187
}

188
static inline void
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
slow_shmem_bit17_copy(struct page *gpu_page,
		      int gpu_offset,
		      struct page *cpu_page,
		      int cpu_offset,
		      int length,
		      int is_read)
{
	char *gpu_vaddr, *cpu_vaddr;

	/* Use the unswizzled path if this page isn't affected. */
	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
		if (is_read)
			return slow_shmem_copy(cpu_page, cpu_offset,
					       gpu_page, gpu_offset, length);
		else
			return slow_shmem_copy(gpu_page, gpu_offset,
					       cpu_page, cpu_offset, length);
	}

208
209
	gpu_vaddr = kmap(gpu_page);
	cpu_vaddr = kmap(cpu_page);
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232

	/* Copy the data, XORing A6 with A17 (1). The user already knows he's
	 * XORing with the other bits (A9 for Y, A9 and A10 for X)
	 */
	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		if (is_read) {
			memcpy(cpu_vaddr + cpu_offset,
			       gpu_vaddr + swizzled_gpu_offset,
			       this_length);
		} else {
			memcpy(gpu_vaddr + swizzled_gpu_offset,
			       cpu_vaddr + cpu_offset,
			       this_length);
		}
		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

233
234
	kunmap(cpu_page);
	kunmap(gpu_page);
235
236
}

237
238
239
240
241
242
243
244
245
246
/**
 * This is the fast shmem pread path, which attempts to copy_from_user directly
 * from the backing pages of the object to the user's address space.  On a
 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 */
static int
i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
			  struct drm_i915_gem_pread *args,
			  struct drm_file *file_priv)
{
247
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
248
249
250
251
252
253
254
255
256
257
258
	ssize_t remain;
	loff_t offset, page_base;
	char __user *user_data;
	int page_offset, page_length;
	int ret;

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;

	mutex_lock(&dev->struct_mutex);

259
	ret = i915_gem_object_get_pages(obj, 0);
260
261
262
263
264
265
266
267
	if (ret != 0)
		goto fail_unlock;

	ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
							args->size);
	if (ret != 0)
		goto fail_put_pages;

268
	obj_priv = to_intel_bo(obj);
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
	offset = args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
		 */
		page_base = (offset & ~(PAGE_SIZE-1));
		page_offset = offset & (PAGE_SIZE-1);
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		ret = fast_shmem_read(obj_priv->pages,
				      page_base, page_offset,
				      user_data, page_length);
		if (ret)
			goto fail_put_pages;

		remain -= page_length;
		user_data += page_length;
		offset += page_length;
	}

fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

303
304
305
306
307
static int
i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
{
	int ret;

308
	ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
309
310
311
312
313
314
315
316
317
318
319

	/* If we've insufficient memory to map in the pages, attempt
	 * to make some space by throwing out some old buffers.
	 */
	if (ret == -ENOMEM) {
		struct drm_device *dev = obj->dev;

		ret = i915_gem_evict_something(dev, obj->size);
		if (ret)
			return ret;

320
		ret = i915_gem_object_get_pages(obj, 0);
321
322
323
324
325
	}

	return ret;
}

326
327
328
329
330
331
332
333
334
335
336
/**
 * This is the fallback shmem pread path, which allocates temporary storage
 * in kernel space to copy_to_user into outside of the struct_mutex, so we
 * can copy out of the object's backing pages while holding the struct mutex
 * and not take page faults.
 */
static int
i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
			  struct drm_i915_gem_pread *args,
			  struct drm_file *file_priv)
{
337
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
338
339
340
341
342
343
344
345
346
347
	struct mm_struct *mm = current->mm;
	struct page **user_pages;
	ssize_t remain;
	loff_t offset, pinned_pages, i;
	loff_t first_data_page, last_data_page, num_pages;
	int shmem_page_index, shmem_page_offset;
	int data_page_index,  data_page_offset;
	int page_length;
	int ret;
	uint64_t data_ptr = args->data_ptr;
348
	int do_bit17_swizzling;
349
350
351
352
353
354
355
356
357
358
359

	remain = args->size;

	/* Pin the user pages containing the data.  We can't fault while
	 * holding the struct mutex, yet we want to hold it while
	 * dereferencing the user data.
	 */
	first_data_page = data_ptr / PAGE_SIZE;
	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
	num_pages = last_data_page - first_data_page + 1;

360
	user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
361
362
363
364
365
	if (user_pages == NULL)
		return -ENOMEM;

	down_read(&mm->mmap_sem);
	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
366
				      num_pages, 1, 0, user_pages, NULL);
367
368
369
370
371
372
	up_read(&mm->mmap_sem);
	if (pinned_pages < num_pages) {
		ret = -EFAULT;
		goto fail_put_user_pages;
	}

373
374
	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);

375
376
	mutex_lock(&dev->struct_mutex);

377
378
	ret = i915_gem_object_get_pages_or_evict(obj);
	if (ret)
379
380
381
382
383
384
385
		goto fail_unlock;

	ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
							args->size);
	if (ret != 0)
		goto fail_put_pages;

386
	obj_priv = to_intel_bo(obj);
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
	offset = args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * shmem_page_index = page number within shmem file
		 * shmem_page_offset = offset within page in shmem file
		 * data_page_index = page number in get_user_pages return
		 * data_page_offset = offset with data_page_index page.
		 * page_length = bytes to copy for this page
		 */
		shmem_page_index = offset / PAGE_SIZE;
		shmem_page_offset = offset & ~PAGE_MASK;
		data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		data_page_offset = data_ptr & ~PAGE_MASK;

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;
		if ((data_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - data_page_offset;

409
		if (do_bit17_swizzling) {
410
			slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
411
					      shmem_page_offset,
412
413
414
415
416
417
418
419
420
421
					      user_pages[data_page_index],
					      data_page_offset,
					      page_length,
					      1);
		} else {
			slow_shmem_copy(user_pages[data_page_index],
					data_page_offset,
					obj_priv->pages[shmem_page_index],
					shmem_page_offset,
					page_length);
422
		}
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437

		remain -= page_length;
		data_ptr += page_length;
		offset += page_length;
	}

fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
	mutex_unlock(&dev->struct_mutex);
fail_put_user_pages:
	for (i = 0; i < pinned_pages; i++) {
		SetPageDirty(user_pages[i]);
		page_cache_release(user_pages[i]);
	}
438
	drm_free_large(user_pages);
439
440
441
442

	return ret;
}

443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
		     struct drm_file *file_priv)
{
	struct drm_i915_gem_pread *args = data;
	struct drm_gem_object *obj;
	struct drm_i915_gem_object *obj_priv;
	int ret;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return -EBADF;
460
	obj_priv = to_intel_bo(obj);
461
462
463
464
465
466
467

	/* Bounds check source.
	 *
	 * XXX: This could use review for overflow issues...
	 */
	if (args->offset > obj->size || args->size > obj->size ||
	    args->offset + args->size > obj->size) {
468
		drm_gem_object_unreference_unlocked(obj);
469
470
471
		return -EINVAL;
	}

472
	if (i915_gem_object_needs_bit17_swizzle(obj)) {
473
		ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
474
475
476
477
478
479
	} else {
		ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
		if (ret != 0)
			ret = i915_gem_shmem_pread_slow(dev, obj, args,
							file_priv);
	}
480

481
	drm_gem_object_unreference_unlocked(obj);
482

483
	return ret;
484
485
}

486
487
/* This is the fast write path which cannot handle
 * page faults in the source data
488
 */
489
490
491
492
493
494

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
495
496
{
	char *vaddr_atomic;
497
	unsigned long unwritten;
498

499
500
501
502
503
504
505
506
507
508
509
510
511
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
	unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
						      user_data, length);
	io_mapping_unmap_atomic(vaddr_atomic);
	if (unwritten)
		return -EFAULT;
	return 0;
}

/* Here's the write path which can sleep for
 * page faults
 */

512
static inline void
513
514
515
516
slow_kernel_write(struct io_mapping *mapping,
		  loff_t gtt_base, int gtt_offset,
		  struct page *user_page, int user_offset,
		  int length)
517
{
518
519
	char __iomem *dst_vaddr;
	char *src_vaddr;
520

521
522
523
524
525
526
527
528
529
	dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
	src_vaddr = kmap(user_page);

	memcpy_toio(dst_vaddr + gtt_offset,
		    src_vaddr + user_offset,
		    length);

	kunmap(user_page);
	io_mapping_unmap(dst_vaddr);
530
531
}

532
533
534
535
536
537
538
static inline int
fast_shmem_write(struct page **pages,
		 loff_t page_base, int page_offset,
		 char __user *data,
		 int length)
{
	char __iomem *vaddr;
539
	unsigned long unwritten;
540
541
542
543

	vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
	if (vaddr == NULL)
		return -ENOMEM;
544
	unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
545
546
	kunmap_atomic(vaddr, KM_USER0);

547
548
	if (unwritten)
		return -EFAULT;
549
550
551
	return 0;
}

552
553
554
555
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
556
static int
557
558
559
i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
			 struct drm_i915_gem_pwrite *args,
			 struct drm_file *file_priv)
560
{
561
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
562
	drm_i915_private_t *dev_priv = dev->dev_private;
563
	ssize_t remain;
564
	loff_t offset, page_base;
565
	char __user *user_data;
566
567
	int page_offset, page_length;
	int ret;
568
569
570
571
572
573
574
575
576
577
578
579
580

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;
	if (!access_ok(VERIFY_READ, user_data, remain))
		return -EFAULT;


	mutex_lock(&dev->struct_mutex);
	ret = i915_gem_object_pin(obj, 0);
	if (ret) {
		mutex_unlock(&dev->struct_mutex);
		return ret;
	}
581
	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
582
583
584
	if (ret)
		goto fail;

585
	obj_priv = to_intel_bo(obj);
586
587
588
589
590
	offset = obj_priv->gtt_offset + args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
591
592
593
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
594
		 */
595
596
597
598
599
600
601
602
603
604
		page_base = (offset & ~(PAGE_SIZE-1));
		page_offset = offset & (PAGE_SIZE-1);
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
				       page_offset, user_data, page_length);

		/* If we get a fault while copying data, then (presumably) our
605
606
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
607
		 */
608
609
		if (ret)
			goto fail;
610

611
612
613
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
614
615
616
617
618
619
620
621
622
	}

fail:
	i915_gem_object_unpin(obj);
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

623
624
625
626
627
628
629
/**
 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 * the memory and maps it using kmap_atomic for copying.
 *
 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 */
630
static int
631
632
633
i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
			 struct drm_i915_gem_pwrite *args,
			 struct drm_file *file_priv)
634
{
635
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
636
637
638
639
640
641
642
643
	drm_i915_private_t *dev_priv = dev->dev_private;
	ssize_t remain;
	loff_t gtt_page_base, offset;
	loff_t first_data_page, last_data_page, num_pages;
	loff_t pinned_pages, i;
	struct page **user_pages;
	struct mm_struct *mm = current->mm;
	int gtt_page_offset, data_page_offset, data_page_index, page_length;
644
	int ret;
645
646
647
648
649
650
651
652
653
654
655
656
	uint64_t data_ptr = args->data_ptr;

	remain = args->size;

	/* Pin the user pages containing the data.  We can't fault while
	 * holding the struct mutex, and all of the pwrite implementations
	 * want to hold it while dereferencing the user data.
	 */
	first_data_page = data_ptr / PAGE_SIZE;
	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
	num_pages = last_data_page - first_data_page + 1;

657
	user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
658
659
660
661
662
663
664
665
666
667
668
	if (user_pages == NULL)
		return -ENOMEM;

	down_read(&mm->mmap_sem);
	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
				      num_pages, 0, 0, user_pages, NULL);
	up_read(&mm->mmap_sem);
	if (pinned_pages < num_pages) {
		ret = -EFAULT;
		goto out_unpin_pages;
	}
669
670

	mutex_lock(&dev->struct_mutex);
671
672
673
674
675
676
677
678
	ret = i915_gem_object_pin(obj, 0);
	if (ret)
		goto out_unlock;

	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
	if (ret)
		goto out_unpin_object;

679
	obj_priv = to_intel_bo(obj);
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
	offset = obj_priv->gtt_offset + args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * gtt_page_base = page offset within aperture
		 * gtt_page_offset = offset within page in aperture
		 * data_page_index = page number in get_user_pages return
		 * data_page_offset = offset with data_page_index page.
		 * page_length = bytes to copy for this page
		 */
		gtt_page_base = offset & PAGE_MASK;
		gtt_page_offset = offset & ~PAGE_MASK;
		data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		data_page_offset = data_ptr & ~PAGE_MASK;

		page_length = remain;
		if ((gtt_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - gtt_page_offset;
		if ((data_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - data_page_offset;

702
703
704
705
706
		slow_kernel_write(dev_priv->mm.gtt_mapping,
				  gtt_page_base, gtt_page_offset,
				  user_pages[data_page_index],
				  data_page_offset,
				  page_length);
707
708
709
710
711
712
713
714
715
716
717
718
719

		remain -= page_length;
		offset += page_length;
		data_ptr += page_length;
	}

out_unpin_object:
	i915_gem_object_unpin(obj);
out_unlock:
	mutex_unlock(&dev->struct_mutex);
out_unpin_pages:
	for (i = 0; i < pinned_pages; i++)
		page_cache_release(user_pages[i]);
720
	drm_free_large(user_pages);
721
722
723
724

	return ret;
}

725
726
727
728
/**
 * This is the fast shmem pwrite path, which attempts to directly
 * copy_from_user into the kmapped pages backing the object.
 */
729
static int
730
731
732
i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
			   struct drm_i915_gem_pwrite *args,
			   struct drm_file *file_priv)
733
{
734
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
735
736
737
738
	ssize_t remain;
	loff_t offset, page_base;
	char __user *user_data;
	int page_offset, page_length;
739
	int ret;
740
741
742

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;
743
744
745

	mutex_lock(&dev->struct_mutex);

746
	ret = i915_gem_object_get_pages(obj, 0);
747
748
	if (ret != 0)
		goto fail_unlock;
749

750
	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
751
752
753
	if (ret != 0)
		goto fail_put_pages;

754
	obj_priv = to_intel_bo(obj);
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
	offset = args->offset;
	obj_priv->dirty = 1;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
		 */
		page_base = (offset & ~(PAGE_SIZE-1));
		page_offset = offset & (PAGE_SIZE-1);
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		ret = fast_shmem_write(obj_priv->pages,
				       page_base, page_offset,
				       user_data, page_length);
		if (ret)
			goto fail_put_pages;

		remain -= page_length;
		user_data += page_length;
		offset += page_length;
	}

fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

/**
 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 * the memory and maps it using kmap_atomic for copying.
 *
 * This avoids taking mmap_sem for faulting on the user's address while the
 * struct_mutex is held.
 */
static int
i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
			   struct drm_i915_gem_pwrite *args,
			   struct drm_file *file_priv)
{
802
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
803
804
805
806
807
808
809
810
811
812
	struct mm_struct *mm = current->mm;
	struct page **user_pages;
	ssize_t remain;
	loff_t offset, pinned_pages, i;
	loff_t first_data_page, last_data_page, num_pages;
	int shmem_page_index, shmem_page_offset;
	int data_page_index,  data_page_offset;
	int page_length;
	int ret;
	uint64_t data_ptr = args->data_ptr;
813
	int do_bit17_swizzling;
814
815
816
817
818
819
820
821
822
823
824

	remain = args->size;

	/* Pin the user pages containing the data.  We can't fault while
	 * holding the struct mutex, and all of the pwrite implementations
	 * want to hold it while dereferencing the user data.
	 */
	first_data_page = data_ptr / PAGE_SIZE;
	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
	num_pages = last_data_page - first_data_page + 1;

825
	user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
826
827
828
829
830
831
832
833
834
835
	if (user_pages == NULL)
		return -ENOMEM;

	down_read(&mm->mmap_sem);
	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
				      num_pages, 0, 0, user_pages, NULL);
	up_read(&mm->mmap_sem);
	if (pinned_pages < num_pages) {
		ret = -EFAULT;
		goto fail_put_user_pages;
836
837
	}

838
839
	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);

840
841
	mutex_lock(&dev->struct_mutex);

842
843
	ret = i915_gem_object_get_pages_or_evict(obj);
	if (ret)
844
845
846
847
848
849
		goto fail_unlock;

	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
	if (ret != 0)
		goto fail_put_pages;

850
	obj_priv = to_intel_bo(obj);
851
	offset = args->offset;
852
	obj_priv->dirty = 1;
853

854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
	while (remain > 0) {
		/* Operation in this page
		 *
		 * shmem_page_index = page number within shmem file
		 * shmem_page_offset = offset within page in shmem file
		 * data_page_index = page number in get_user_pages return
		 * data_page_offset = offset with data_page_index page.
		 * page_length = bytes to copy for this page
		 */
		shmem_page_index = offset / PAGE_SIZE;
		shmem_page_offset = offset & ~PAGE_MASK;
		data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		data_page_offset = data_ptr & ~PAGE_MASK;

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;
		if ((data_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - data_page_offset;

874
		if (do_bit17_swizzling) {
875
			slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
876
877
878
					      shmem_page_offset,
					      user_pages[data_page_index],
					      data_page_offset,
879
880
881
882
883
884
885
886
					      page_length,
					      0);
		} else {
			slow_shmem_copy(obj_priv->pages[shmem_page_index],
					shmem_page_offset,
					user_pages[data_page_index],
					data_page_offset,
					page_length);
887
		}
888
889
890
891

		remain -= page_length;
		data_ptr += page_length;
		offset += page_length;
892
893
	}

894
895
896
fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
897
	mutex_unlock(&dev->struct_mutex);
898
899
900
fail_put_user_pages:
	for (i = 0; i < pinned_pages; i++)
		page_cache_release(user_pages[i]);
901
	drm_free_large(user_pages);
902

903
	return ret;
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv)
{
	struct drm_i915_gem_pwrite *args = data;
	struct drm_gem_object *obj;
	struct drm_i915_gem_object *obj_priv;
	int ret = 0;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return -EBADF;
923
	obj_priv = to_intel_bo(obj);
924
925
926
927
928
929
930

	/* Bounds check destination.
	 *
	 * XXX: This could use review for overflow issues...
	 */
	if (args->offset > obj->size || args->size > obj->size ||
	    args->offset + args->size > obj->size) {
931
		drm_gem_object_unreference_unlocked(obj);
932
933
934
935
936
937
938
939
940
		return -EINVAL;
	}

	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
941
942
943
	if (obj_priv->phys_obj)
		ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
	else if (obj_priv->tiling_mode == I915_TILING_NONE &&
944
945
		 dev->gtt_total != 0 &&
		 obj->write_domain != I915_GEM_DOMAIN_CPU) {
946
947
948
949
950
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
		if (ret == -EFAULT) {
			ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
						       file_priv);
		}
951
952
	} else if (i915_gem_object_needs_bit17_swizzle(obj)) {
		ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
953
954
955
956
957
958
959
	} else {
		ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
		if (ret == -EFAULT) {
			ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
							 file_priv);
		}
	}
960
961
962
963
964
965

#if WATCH_PWRITE
	if (ret)
		DRM_INFO("pwrite failed %d\n", ret);
#endif

966
	drm_gem_object_unreference_unlocked(obj);
967
968
969
970
971

	return ret;
}

/**
972
973
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
974
975
976
977
978
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file_priv)
{
979
	struct drm_i915_private *dev_priv = dev->dev_private;
980
981
	struct drm_i915_gem_set_domain *args = data;
	struct drm_gem_object *obj;
982
	struct drm_i915_gem_object *obj_priv;
983
984
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
985
986
987
988
989
	int ret;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

990
	/* Only handle setting domains to types used by the CPU. */
991
	if (write_domain & I915_GEM_GPU_DOMAINS)
992
993
		return -EINVAL;

994
	if (read_domains & I915_GEM_GPU_DOMAINS)
995
996
997
998
999
1000
1001
1002
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1003
1004
1005
	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return -EBADF;
1006
	obj_priv = to_intel_bo(obj);
1007
1008

	mutex_lock(&dev->struct_mutex);
1009
1010
1011

	intel_mark_busy(dev, obj);

1012
#if WATCH_BUF
1013
	DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1014
		 obj, obj->size, read_domains, write_domain);
1015
#endif
1016
1017
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1018

1019
1020
1021
1022
		/* Update the LRU on the fence for the CPU access that's
		 * about to occur.
		 */
		if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1023
1024
1025
			struct drm_i915_fence_reg *reg =
				&dev_priv->fence_regs[obj_priv->fence_reg];
			list_move_tail(&reg->lru_list,
1026
1027
1028
				       &dev_priv->mm.fence_list);
		}

1029
1030
1031
1032
1033
1034
		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
1035
	} else {
1036
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1037
1038
	}

1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
	drm_gem_object_unreference(obj);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv)
{
	struct drm_i915_gem_sw_finish *args = data;
	struct drm_gem_object *obj;
	struct drm_i915_gem_object *obj_priv;
	int ret = 0;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

	mutex_lock(&dev->struct_mutex);
	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL) {
		mutex_unlock(&dev->struct_mutex);
		return -EBADF;
	}

#if WATCH_BUF
1067
	DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1068
1069
		 __func__, args->handle, obj, obj->size);
#endif
1070
	obj_priv = to_intel_bo(obj);
1071
1072

	/* Pinned buffers may be scanout, so flush the cache */
1073
1074
1075
	if (obj_priv->pin_count)
		i915_gem_object_flush_cpu_write_domain(obj);

1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
	drm_gem_object_unreference(obj);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
		   struct drm_file *file_priv)
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	loff_t offset;
	unsigned long addr;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return -EBADF;

	offset = args->offset;

	down_write(&current->mm->mmap_sem);
	addr = do_mmap(obj->filp, 0, args->size,
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
	up_write(&current->mm->mmap_sem);
1111
	drm_gem_object_unreference_unlocked(obj);
1112
1113
1114
1115
1116
1117
1118
1119
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
/**
 * i915_gem_fault - fault a page into the GTT
 * vma: VMA in question
 * vmf: fault info
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct drm_gem_object *obj = vma->vm_private_data;
	struct drm_device *dev = obj->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
1141
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1142
1143
1144
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1145
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1146
1147
1148
1149
1150
1151
1152
1153

	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

	/* Now bind it into the GTT if needed */
	mutex_lock(&dev->struct_mutex);
	if (!obj_priv->gtt_space) {
1154
		ret = i915_gem_object_bind_to_gtt(obj, 0);
1155
1156
		if (ret)
			goto unlock;
1157

1158
		list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1159
1160

		ret = i915_gem_object_set_to_gtt_domain(obj, write);
1161
1162
		if (ret)
			goto unlock;
1163
1164
1165
	}

	/* Need a new fence register? */
1166
	if (obj_priv->tiling_mode != I915_TILING_NONE) {
1167
		ret = i915_gem_object_get_fence_reg(obj);
1168
1169
		if (ret)
			goto unlock;
1170
	}
1171
1172
1173
1174
1175
1176

	pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
		page_offset;

	/* Finally, remap it using the new GTT offset */
	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1177
unlock:
1178
1179
1180
	mutex_unlock(&dev->struct_mutex);

	switch (ret) {
1181
1182
1183
	case 0:
	case -ERESTARTSYS:
		return VM_FAULT_NOPAGE;
1184
1185
1186
1187
	case -ENOMEM:
	case -EAGAIN:
		return VM_FAULT_OOM;
	default:
1188
		return VM_FAULT_SIGBUS;
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
	}
}

/**
 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
 * @obj: obj in question
 *
 * GEM memory mapping works by handing back to userspace a fake mmap offset
 * it can use in a subsequent mmap(2) call.  The DRM core code then looks
 * up the object based on the offset and sets up the various memory mapping
 * structures.
 *
 * This routine allocates and attaches a fake offset for @obj.
 */
static int
i915_gem_create_mmap_offset(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct drm_gem_mm *mm = dev->mm_private;
1208
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1209
	struct drm_map_list *list;
1210
	struct drm_local_map *map;
1211
1212
1213
1214
	int ret = 0;

	/* Set the object up for mmap'ing */
	list = &obj->map_list;
1215
	list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
	if (!list->map)
		return -ENOMEM;

	map = list->map;
	map->type = _DRM_GEM;
	map->size = obj->size;
	map->handle = obj;

	/* Get a DRM GEM mmap offset allocated... */
	list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
						    obj->size / PAGE_SIZE, 0, 0);
	if (!list->file_offset_node) {
		DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
		ret = -ENOMEM;
		goto out_free_list;
	}

	list->file_offset_node = drm_mm_get_block(list->file_offset_node,
						  obj->size / PAGE_SIZE, 0);
	if (!list->file_offset_node) {
		ret = -ENOMEM;
		goto out_free_list;
	}

	list->hash.key = list->file_offset_node->start;
	if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
		DRM_ERROR("failed to add to map hash\n");
1243
		ret = -ENOMEM;
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
		goto out_free_mm;
	}

	/* By now we should be all set, any drm_mmap request on the offset
	 * below will get to our mmap & fault handler */
	obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;

	return 0;

out_free_mm:
	drm_mm_put_block(list->file_offset_node);
out_free_list:
1256
	kfree(list->map);
1257
1258
1259
1260

	return ret;
}

1261
1262
1263
1264
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1265
 * Preserve the reservation of the mmapping with the DRM core code, but
1266
1267
1268
1269
1270
1271
1272
1273
1274
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1275
void
1276
1277
1278
i915_gem_release_mmap(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
1279
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1280
1281
1282
1283
1284
1285

	if (dev->dev_mapping)
		unmap_mapping_range(dev->dev_mapping,
				    obj_priv->mmap_offset, obj->size, 1);
}

1286
1287
1288
1289
static void
i915_gem_free_mmap_offset(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
1290
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
	struct drm_gem_mm *mm = dev->mm_private;
	struct drm_map_list *list;

	list = &obj->map_list;
	drm_ht_remove_item(&mm->offset_hash, &list->hash);

	if (list->file_offset_node) {
		drm_mm_put_block(list->file_offset_node);
		list->file_offset_node = NULL;
	}

	if (list->map) {
1303
		kfree(list->map);
1304
1305
1306
1307
1308
1309
		list->map = NULL;
	}

	obj_priv->mmap_offset = 0;
}

1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
/**
 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 * @obj: object to check
 *
 * Return the required GTT alignment for an object, taking into account
 * potential fence register mapping if needed.
 */
static uint32_t
i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
1321
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
	int start, i;

	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
	if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
		return 4096;

	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
	if (IS_I9XX(dev))
		start = 1024*1024;
	else
		start = 512*1024;

	for (i = start; i < obj->size; i <<= 1)
		;

	return i;
}

/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file_priv: GEM object info
 *
 * Simply returns the fake offset to userspace so it can mmap it.
 * The mmap call will end up in drm_gem_mmap(), which will set things
 * up so we can get faults in the handler above.
 *
 * The fault handler will take care of binding the object into the GTT
 * (since it may have been evicted to make room for something), allocating
 * a fence register, and mapping the appropriate aperture address into
 * userspace.
 */
int
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
	struct drm_i915_gem_mmap_gtt *args = data;
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_gem_object *obj;
	struct drm_i915_gem_object *obj_priv;
	int ret;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return -EBADF;

	mutex_lock(&dev->struct_mutex);

1380
	obj_priv = to_intel_bo(obj);
1381

1382
1383
1384
1385
1386
1387
1388
1389
	if (obj_priv->madv != I915_MADV_WILLNEED) {
		DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		drm_gem_object_unreference(obj);
		mutex_unlock(&dev->struct_mutex);
		return -EINVAL;
	}


1390
1391
	if (!obj_priv->mmap_offset) {
		ret = i915_gem_create_mmap_offset(obj);
1392
1393
1394
		if (ret) {
			drm_gem_object_unreference(obj);
			mutex_unlock(&dev->struct_mutex);
1395
			return ret;
1396
		}
1397
1398
1399
1400
1401
1402
1403
1404
1405
	}

	args->offset = obj_priv->mmap_offset;

	/*
	 * Pull it into the GTT so that we have a page list (makes the
	 * initial fault faster and any subsequent flushing possible).
	 */
	if (!obj_priv->agp_mem) {
1406
		ret = i915_gem_object_bind_to_gtt(obj, 0);
1407
1408
1409
1410
1411
		if (ret) {
			drm_gem_object_unreference(obj);
			mutex_unlock(&dev->struct_mutex);
			return ret;
		}
1412
		list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1413
1414
1415
1416
1417
1418
1419
1420
	}

	drm_gem_object_unreference(obj);
	mutex_unlock(&dev->struct_mutex);

	return 0;
}

1421
void
1422
i915_gem_object_put_pages(struct drm_gem_object *obj)
1423
{
1424
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1425
1426
1427
	int page_count = obj->size / PAGE_SIZE;
	int i;

1428
	BUG_ON(obj_priv->pages_refcount == 0);
Chris Wilson's avatar
Chris Wilson committed
1429
	BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1430

1431
1432
	if (--obj_priv->pages_refcount != 0)
		return;
1433

1434
1435
1436
	if (obj_priv->tiling_mode != I915_TILING_NONE)
		i915_gem_object_save_bit_17_swizzle(obj);

1437
	if (obj_priv->madv == I915_MADV_DONTNEED)
1438
		obj_priv->dirty = 0;
1439