lima_bo.c 14.1 KB
Newer Older
Qiang Yu's avatar
Qiang Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
/*
 * Copyright (C) 2017-2019 Lima Project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include <stdlib.h>
#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>

#include "xf86drm.h"
#include "drm-uapi/lima_drm.h"

#include "util/u_hash_table.h"
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
33
#include "util/u_math.h"
Qiang Yu's avatar
Qiang Yu committed
34
35
36
#include "util/os_time.h"
#include "os/os_mman.h"

37
#include "frontend/drm_driver.h"
Qiang Yu's avatar
Qiang Yu committed
38
39
40

#include "lima_screen.h"
#include "lima_bo.h"
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
41
#include "lima_util.h"
Qiang Yu's avatar
Qiang Yu committed
42
43
44

bool lima_bo_table_init(struct lima_screen *screen)
{
45
   screen->bo_handles = util_hash_table_create_ptr_keys();
Qiang Yu's avatar
Qiang Yu committed
46
47
48
   if (!screen->bo_handles)
      return false;

49
   screen->bo_flink_names = util_hash_table_create_ptr_keys();
Qiang Yu's avatar
Qiang Yu committed
50
51
52
53
54
55
56
   if (!screen->bo_flink_names)
      goto err_out0;

   mtx_init(&screen->bo_table_lock, mtx_plain);
   return true;

err_out0:
57
   _mesa_hash_table_destroy(screen->bo_handles, NULL);
Qiang Yu's avatar
Qiang Yu committed
58
59
60
   return false;
}

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
61
62
63
64
65
66
67
68
69
70
bool lima_bo_cache_init(struct lima_screen *screen)
{
   mtx_init(&screen->bo_cache_lock, mtx_plain);
   list_inithead(&screen->bo_cache_time);
   for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++)
      list_inithead(&screen->bo_cache_buckets[i]);

   return true;
}

Qiang Yu's avatar
Qiang Yu committed
71
72
73
void lima_bo_table_fini(struct lima_screen *screen)
{
   mtx_destroy(&screen->bo_table_lock);
74
75
   _mesa_hash_table_destroy(screen->bo_handles, NULL);
   _mesa_hash_table_destroy(screen->bo_flink_names, NULL);
Qiang Yu's avatar
Qiang Yu committed
76
77
}

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
78
79
80
81
82
83
84
static void
lima_bo_cache_remove(struct lima_bo *bo)
{
   list_del(&bo->size_list);
   list_del(&bo->time_list);
}

Qiang Yu's avatar
Qiang Yu committed
85
86
87
88
89
90
91
92
93
static void lima_close_kms_handle(struct lima_screen *screen, uint32_t handle)
{
   struct drm_gem_close args = {
      .handle = handle,
   };

   drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &args);
}

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
94
95
96
97
static void
lima_bo_free(struct lima_bo *bo)
{
   struct lima_screen *screen = bo->screen;
98
99
100
101
102

   if (lima_debug & LIMA_DEBUG_BO_CACHE)
      fprintf(stderr, "%s: %p (size=%d)\n", __func__,
              bo, bo->size);

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
103
   mtx_lock(&screen->bo_table_lock);
104
   _mesa_hash_table_remove_key(screen->bo_handles,
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
105
106
                          (void *)(uintptr_t)bo->handle);
   if (bo->flink_name)
107
      _mesa_hash_table_remove_key(screen->bo_flink_names,
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
                             (void *)(uintptr_t)bo->flink_name);
   mtx_unlock(&screen->bo_table_lock);

   if (bo->map)
      lima_bo_unmap(bo);

   lima_close_kms_handle(screen, bo->handle);
   free(bo);
}

void lima_bo_cache_fini(struct lima_screen *screen)
{
   mtx_destroy(&screen->bo_cache_lock);

   list_for_each_entry_safe(struct lima_bo, entry,
                            &screen->bo_cache_time, time_list) {
      lima_bo_cache_remove(entry);
      lima_bo_free(entry);
   }
}

Qiang Yu's avatar
Qiang Yu committed
129
130
131
132
133
134
135
136
137
138
139
140
141
142
static bool lima_bo_get_info(struct lima_bo *bo)
{
   struct drm_lima_gem_info req = {
      .handle = bo->handle,
   };

   if(drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_INFO, &req))
      return false;

   bo->offset = req.offset;
   bo->va = req.va;
   return true;
}

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
static unsigned
lima_bucket_index(unsigned size)
{
   /* Round down to POT to compute a bucket index */

   unsigned bucket_index = util_logbase2(size);

   /* Clamp the bucket index; all huge allocations will be
    * sorted into the largest bucket */
   bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET,
                        MAX_BO_CACHE_BUCKET);

   /* Reindex from 0 */
   return (bucket_index - MIN_BO_CACHE_BUCKET);
}

static struct list_head *
lima_bo_cache_get_bucket(struct lima_screen *screen, unsigned size)
{
   return &screen->bo_cache_buckets[lima_bucket_index(size)];
}

static void
lima_bo_cache_free_stale_bos(struct lima_screen *screen, time_t time)
{
168
   unsigned cnt = 0;
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
169
170
171
172
173
174
   list_for_each_entry_safe(struct lima_bo, entry,
                            &screen->bo_cache_time, time_list) {
      /* Free BOs that are sitting idle for longer than 5 seconds */
      if (time - entry->free_time > 6) {
         lima_bo_cache_remove(entry);
         lima_bo_free(entry);
175
         cnt++;
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
176
177
178
      } else
         break;
   }
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
   if ((lima_debug & LIMA_DEBUG_BO_CACHE) && cnt)
      fprintf(stderr, "%s: freed %d stale BOs\n", __func__, cnt);
}

static void
lima_bo_cache_print_stats(struct lima_screen *screen)
{
   fprintf(stderr, "===============\n");
   fprintf(stderr, "BO cache stats:\n");
   unsigned total_size = 0;
   for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++) {
      struct list_head *bucket = &screen->bo_cache_buckets[i];
      unsigned bucket_size = 0;
      list_for_each_entry(struct lima_bo, entry, bucket, size_list) {
         bucket_size += entry->size;
         total_size += entry->size;
      }
      fprintf(stderr, "Bucket #%d, BOs: %d, size: %u\n", i,
              list_length(bucket),
              bucket_size);
   }
   fprintf(stderr, "Total size: %u\n", total_size);
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
}

static bool
lima_bo_cache_put(struct lima_bo *bo)
{
   if (!bo->cacheable)
      return false;

   struct lima_screen *screen = bo->screen;

   mtx_lock(&screen->bo_cache_lock);
   struct list_head *bucket = lima_bo_cache_get_bucket(screen, bo->size);

   if (!bucket) {
      mtx_unlock(&screen->bo_cache_lock);
      return false;
   }

   struct timespec time;
   clock_gettime(CLOCK_MONOTONIC, &time);
   bo->free_time = time.tv_sec;
   list_addtail(&bo->size_list, bucket);
   list_addtail(&bo->time_list, &screen->bo_cache_time);
   lima_bo_cache_free_stale_bos(screen, time.tv_sec);
225
226
227
228
   if (lima_debug & LIMA_DEBUG_BO_CACHE) {
      fprintf(stderr, "%s: put BO: %p (size=%d)\n", __func__, bo, bo->size);
      lima_bo_cache_print_stats(screen);
   }
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
229
230
231
232
233
234
235
236
   mtx_unlock(&screen->bo_cache_lock);

   return true;
}

static struct lima_bo *
lima_bo_cache_get(struct lima_screen *screen, uint32_t size, uint32_t flags)
{
237
238
239
240
   /* we won't cache heap buffer */
   if (flags & LIMA_BO_FLAG_HEAP)
      return NULL;

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
241
242
243
244
245
246
247
248
249
250
   struct lima_bo *bo = NULL;
   mtx_lock(&screen->bo_cache_lock);
   struct list_head *bucket = lima_bo_cache_get_bucket(screen, size);

   if (!bucket) {
      mtx_unlock(&screen->bo_cache_lock);
      return false;
   }

   list_for_each_entry_safe(struct lima_bo, entry, bucket, size_list) {
251
      if (entry->size >= size) {
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
252
         /* Check if BO is idle. If it's not it's better to allocate new one */
253
254
255
256
257
         if (!lima_bo_wait(entry, LIMA_GEM_WAIT_WRITE, 0)) {
            if (lima_debug & LIMA_DEBUG_BO_CACHE) {
               fprintf(stderr, "%s: found BO %p but it's busy\n", __func__,
                       entry);
            }
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
258
            break;
259
         }
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
260
261
262

         lima_bo_cache_remove(entry);
         p_atomic_set(&entry->refcnt, 1);
263
         entry->flags = flags;
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
264
         bo = entry;
265
266
267
268
269
         if (lima_debug & LIMA_DEBUG_BO_CACHE) {
            fprintf(stderr, "%s: got BO: %p (size=%d), requested size %d\n",
                    __func__, bo, bo->size, size);
            lima_bo_cache_print_stats(screen);
         }
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
270
271
272
273
274
275
276
277
278
         break;
      }
   }

   mtx_unlock(&screen->bo_cache_lock);

   return bo;
}

Qiang Yu's avatar
Qiang Yu committed
279
280
281
282
struct lima_bo *lima_bo_create(struct lima_screen *screen,
                               uint32_t size, uint32_t flags)
{
   struct lima_bo *bo;
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
283

284
285
   size = align(size, LIMA_PAGE_SIZE);

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
286
287
288
289
290
   /* Try to get bo from cache first */
   bo = lima_bo_cache_get(screen, size, flags);
   if (bo)
      return bo;

Qiang Yu's avatar
Qiang Yu committed
291
292
293
294
295
296
297
298
   struct drm_lima_gem_create req = {
      .size = size,
      .flags = flags,
   };

   if (!(bo = calloc(1, sizeof(*bo))))
      return NULL;

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
299
300
301
   list_inithead(&bo->time_list);
   list_inithead(&bo->size_list);

Qiang Yu's avatar
Qiang Yu committed
302
303
304
305
306
   if (drmIoctl(screen->fd, DRM_IOCTL_LIMA_GEM_CREATE, &req))
      goto err_out0;

   bo->screen = screen;
   bo->size = req.size;
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
307
   bo->flags = req.flags;
Qiang Yu's avatar
Qiang Yu committed
308
   bo->handle = req.handle;
309
310
   bo->cacheable = !(lima_debug & LIMA_DEBUG_NO_BO_CACHE ||
                     flags & LIMA_BO_FLAG_HEAP);
Qiang Yu's avatar
Qiang Yu committed
311
312
313
314
315
   p_atomic_set(&bo->refcnt, 1);

   if (!lima_bo_get_info(bo))
      goto err_out1;

316
317
318
319
   if (lima_debug & LIMA_DEBUG_BO_CACHE)
      fprintf(stderr, "%s: %p (size=%d)\n", __func__,
              bo, bo->size);

Qiang Yu's avatar
Qiang Yu committed
320
321
322
323
324
325
326
327
328
   return bo;

err_out1:
   lima_close_kms_handle(screen, bo->handle);
err_out0:
   free(bo);
   return NULL;
}

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
329
void lima_bo_unreference(struct lima_bo *bo)
Qiang Yu's avatar
Qiang Yu committed
330
331
332
333
{
   if (!p_atomic_dec_zero(&bo->refcnt))
      return;

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
334
335
336
   /* Try to put it into cache */
   if (lima_bo_cache_put(bo))
      return;
Qiang Yu's avatar
Qiang Yu committed
337

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
338
   lima_bo_free(bo);
Qiang Yu's avatar
Qiang Yu committed
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
}

void *lima_bo_map(struct lima_bo *bo)
{
   if (!bo->map) {
      bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE,
                        MAP_SHARED, bo->screen->fd, bo->offset);
      if (bo->map == MAP_FAILED)
          bo->map = NULL;
   }

   return bo->map;
}

void lima_bo_unmap(struct lima_bo *bo)
{
   if (bo->map) {
      os_munmap(bo->map, bo->size);
      bo->map = NULL;
   }
}

bool lima_bo_export(struct lima_bo *bo, struct winsys_handle *handle)
{
   struct lima_screen *screen = bo->screen;

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
365
366
367
   /* Don't cache exported BOs */
   bo->cacheable = false;

Qiang Yu's avatar
Qiang Yu committed
368
369
370
371
372
373
374
375
376
377
378
379
380
   switch (handle->type) {
   case WINSYS_HANDLE_TYPE_SHARED:
      if (!bo->flink_name) {
         struct drm_gem_flink flink = {
            .handle = bo->handle,
            .name = 0,
         };
         if (drmIoctl(screen->fd, DRM_IOCTL_GEM_FLINK, &flink))
            return false;

         bo->flink_name = flink.name;

         mtx_lock(&screen->bo_table_lock);
381
         _mesa_hash_table_insert(screen->bo_flink_names,
Qiang Yu's avatar
Qiang Yu committed
382
383
384
385
386
387
388
389
                             (void *)(uintptr_t)bo->flink_name, bo);
         mtx_unlock(&screen->bo_table_lock);
      }
      handle->handle = bo->flink_name;
      return true;

   case WINSYS_HANDLE_TYPE_KMS:
      mtx_lock(&screen->bo_table_lock);
390
      _mesa_hash_table_insert(screen->bo_handles,
Qiang Yu's avatar
Qiang Yu committed
391
392
393
394
395
396
397
398
399
400
401
402
                          (void *)(uintptr_t)bo->handle, bo);
      mtx_unlock(&screen->bo_table_lock);

      handle->handle = bo->handle;
      return true;

   case WINSYS_HANDLE_TYPE_FD:
      if (drmPrimeHandleToFD(screen->fd, bo->handle, DRM_CLOEXEC,
                             (int*)&handle->handle))
         return false;

      mtx_lock(&screen->bo_table_lock);
403
      _mesa_hash_table_insert(screen->bo_handles,
Qiang Yu's avatar
Qiang Yu committed
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
                          (void *)(uintptr_t)bo->handle, bo);
      mtx_unlock(&screen->bo_table_lock);
      return true;

   default:
      return false;
   }
}

struct lima_bo *lima_bo_import(struct lima_screen *screen,
                               struct winsys_handle *handle)
{
   struct lima_bo *bo = NULL;
   struct drm_gem_open req = {0};
   uint32_t dma_buf_size = 0;
   unsigned h = handle->handle;

   mtx_lock(&screen->bo_table_lock);

   /* Convert a DMA buf handle to a KMS handle now. */
   if (handle->type == WINSYS_HANDLE_TYPE_FD) {
      uint32_t prime_handle;
      off_t size;

      /* Get a KMS handle. */
      if (drmPrimeFDToHandle(screen->fd, h, &prime_handle)) {
         mtx_unlock(&screen->bo_table_lock);
         return NULL;
      }

      /* Query the buffer size. */
      size = lseek(h, 0, SEEK_END);
      if (size == (off_t)-1) {
         mtx_unlock(&screen->bo_table_lock);
         lima_close_kms_handle(screen, prime_handle);
         return NULL;
      }
      lseek(h, 0, SEEK_SET);

      dma_buf_size = size;
      h = prime_handle;
   }

   switch (handle->type) {
   case WINSYS_HANDLE_TYPE_SHARED:
      bo = util_hash_table_get(screen->bo_flink_names,
                               (void *)(uintptr_t)h);
      break;
   case WINSYS_HANDLE_TYPE_KMS:
   case WINSYS_HANDLE_TYPE_FD:
      bo = util_hash_table_get(screen->bo_handles,
                               (void *)(uintptr_t)h);
      break;
   default:
      mtx_unlock(&screen->bo_table_lock);
      return NULL;
   }

   if (bo) {
      p_atomic_inc(&bo->refcnt);
Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
464
465
      /* Don't cache imported BOs */
      bo->cacheable = false;
Qiang Yu's avatar
Qiang Yu committed
466
467
468
469
470
471
472
473
474
475
476
      mtx_unlock(&screen->bo_table_lock);
      return bo;
   }

   if (!(bo = calloc(1, sizeof(*bo)))) {
      mtx_unlock(&screen->bo_table_lock);
      if (handle->type == WINSYS_HANDLE_TYPE_FD)
         lima_close_kms_handle(screen, h);
      return NULL;
   }

Vasily Khoruzhick's avatar
Vasily Khoruzhick committed
477
478
479
480
   /* Don't cache imported BOs */
   bo->cacheable = false;
   list_inithead(&bo->time_list);
   list_inithead(&bo->size_list);
Qiang Yu's avatar
Qiang Yu committed
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
   bo->screen = screen;
   p_atomic_set(&bo->refcnt, 1);

   switch (handle->type) {
   case WINSYS_HANDLE_TYPE_SHARED:
      req.name = h;
      if (drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &req)) {
         mtx_unlock(&screen->bo_table_lock);
         free(bo);
         return NULL;
      }
      bo->handle = req.handle;
      bo->flink_name = h;
      bo->size = req.size;
      break;
   case WINSYS_HANDLE_TYPE_FD:
      bo->handle = h;
      bo->size = dma_buf_size;
      break;
   default:
      /* not possible */
      assert(0);
   }

   if (lima_bo_get_info(bo)) {
      if (handle->type == WINSYS_HANDLE_TYPE_SHARED)
507
         _mesa_hash_table_insert(screen->bo_flink_names,
Qiang Yu's avatar
Qiang Yu committed
508
                          (void *)(uintptr_t)bo->flink_name, bo);
509
      _mesa_hash_table_insert(screen->bo_handles,
Qiang Yu's avatar
Qiang Yu committed
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
                          (void*)(uintptr_t)bo->handle, bo);
   }
   else {
      lima_close_kms_handle(screen, bo->handle);
      free(bo);
      bo = NULL;
   }

   mtx_unlock(&screen->bo_table_lock);

   return bo;
}

bool lima_bo_wait(struct lima_bo *bo, uint32_t op, uint64_t timeout_ns)
{
525
526
527
528
529
530
531
   int64_t abs_timeout;

   if (timeout_ns == 0)
      abs_timeout = 0;
   else
      abs_timeout = os_time_get_absolute_timeout(timeout_ns);

532
533
534
   if (abs_timeout == OS_TIMEOUT_INFINITE)
      abs_timeout = INT64_MAX;

Qiang Yu's avatar
Qiang Yu committed
535
536
537
538
539
540
541
542
   struct drm_lima_gem_wait req = {
      .handle = bo->handle,
      .op = op,
      .timeout_ns = abs_timeout,
   };

   return drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_WAIT, &req) == 0;
}