Line data Source code
1 : /*
2 : * Copyright © 2012-2014 Intel Corporation
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a
5 : * copy of this software and associated documentation files (the "Software"),
6 : * to deal in the Software without restriction, including without limitation
7 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : * and/or sell copies of the Software, and to permit persons to whom the
9 : * Software is furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice (including the next
12 : * paragraph) shall be included in all copies or substantial portions of the
13 : * Software.
14 : *
15 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 : * IN THE SOFTWARE.
22 : *
23 : */
24 :
25 : #include <dev/pci/drm/drmP.h>
26 : #include <dev/pci/drm/i915_drm.h>
27 : #include "i915_drv.h"
28 : #include "i915_trace.h"
29 : #include "intel_drv.h"
30 : #ifdef __linux__
31 : #include <linux/mmu_context.h>
32 : #include <linux/mmu_notifier.h>
33 : #include <linux/mempolicy.h>
34 : #include <linux/swap.h>
35 : #endif
36 :
37 : #ifdef __linux__
38 :
39 : struct i915_mm_struct {
40 : struct mm_struct *mm;
41 : struct drm_device *dev;
42 : struct i915_mmu_notifier *mn;
43 : struct hlist_node node;
44 : struct kref kref;
45 : struct work_struct work;
46 : };
47 :
48 : #if defined(CONFIG_MMU_NOTIFIER)
49 : #include <linux/interval_tree.h>
50 :
51 : struct i915_mmu_notifier {
52 : spinlock_t lock;
53 : struct hlist_node node;
54 : struct mmu_notifier mn;
55 : struct rb_root objects;
56 : struct list_head linear;
57 : bool has_linear;
58 : };
59 :
60 : struct i915_mmu_object {
61 : struct i915_mmu_notifier *mn;
62 : struct interval_tree_node it;
63 : struct list_head link;
64 : struct drm_i915_gem_object *obj;
65 : struct work_struct work;
66 : bool active;
67 : bool is_linear;
68 : };
69 :
70 : static void __cancel_userptr__worker(struct work_struct *work)
71 : {
72 : struct i915_mmu_object *mo = container_of(work, typeof(*mo), work);
73 : struct drm_i915_gem_object *obj = mo->obj;
74 : struct drm_device *dev = obj->base.dev;
75 :
76 : mutex_lock(&dev->struct_mutex);
77 : /* Cancel any active worker and force us to re-evaluate gup */
78 : obj->userptr.work = NULL;
79 :
80 : if (obj->pages != NULL) {
81 : struct drm_i915_private *dev_priv = to_i915(dev);
82 : struct i915_vma *vma, *tmp;
83 : bool was_interruptible;
84 :
85 : was_interruptible = dev_priv->mm.interruptible;
86 : dev_priv->mm.interruptible = false;
87 :
88 : list_for_each_entry_safe(vma, tmp, &obj->vma_list, vma_link) {
89 : int ret = i915_vma_unbind(vma);
90 : WARN_ON(ret && ret != -EIO);
91 : }
92 : WARN_ON(i915_gem_object_put_pages(obj));
93 :
94 : dev_priv->mm.interruptible = was_interruptible;
95 : }
96 :
97 : drm_gem_object_unreference(&obj->base);
98 : mutex_unlock(&dev->struct_mutex);
99 : }
100 :
101 : static unsigned long cancel_userptr(struct i915_mmu_object *mo)
102 : {
103 : unsigned long end = mo->obj->userptr.ptr + mo->obj->base.size;
104 :
105 : /* The mmu_object is released late when destroying the
106 : * GEM object so it is entirely possible to gain a
107 : * reference on an object in the process of being freed
108 : * since our serialisation is via the spinlock and not
109 : * the struct_mutex - and consequently use it after it
110 : * is freed and then double free it.
111 : */
112 : if (mo->active && kref_get_unless_zero(&mo->obj->base.refcount)) {
113 : schedule_work(&mo->work);
114 : /* only schedule one work packet to avoid the refleak */
115 : mo->active = false;
116 : }
117 :
118 : return end;
119 : }
120 :
121 : static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
122 : struct mm_struct *mm,
123 : unsigned long start,
124 : unsigned long end)
125 : {
126 : struct i915_mmu_notifier *mn =
127 : container_of(_mn, struct i915_mmu_notifier, mn);
128 : struct i915_mmu_object *mo;
129 :
130 : /* interval ranges are inclusive, but invalidate range is exclusive */
131 : end--;
132 :
133 : spin_lock(&mn->lock);
134 : if (mn->has_linear) {
135 : list_for_each_entry(mo, &mn->linear, link) {
136 : if (mo->it.last < start || mo->it.start > end)
137 : continue;
138 :
139 : cancel_userptr(mo);
140 : }
141 : } else {
142 : struct interval_tree_node *it;
143 :
144 : it = interval_tree_iter_first(&mn->objects, start, end);
145 : while (it) {
146 : mo = container_of(it, struct i915_mmu_object, it);
147 : start = cancel_userptr(mo);
148 : it = interval_tree_iter_next(it, start, end);
149 : }
150 : }
151 : spin_unlock(&mn->lock);
152 : }
153 :
154 : static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
155 : .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start,
156 : };
157 :
158 : static struct i915_mmu_notifier *
159 : i915_mmu_notifier_create(struct mm_struct *mm)
160 : {
161 : struct i915_mmu_notifier *mn;
162 : int ret;
163 :
164 : mn = kmalloc(sizeof(*mn), GFP_KERNEL);
165 : if (mn == NULL)
166 : return ERR_PTR(-ENOMEM);
167 :
168 : spin_lock_init(&mn->lock);
169 : mn->mn.ops = &i915_gem_userptr_notifier;
170 : mn->objects = RB_ROOT;
171 : INIT_LIST_HEAD(&mn->linear);
172 : mn->has_linear = false;
173 :
174 : /* Protected by mmap_sem (write-lock) */
175 : ret = __mmu_notifier_register(&mn->mn, mm);
176 : if (ret) {
177 : kfree(mn);
178 : return ERR_PTR(ret);
179 : }
180 :
181 : return mn;
182 : }
183 :
184 : static int
185 : i915_mmu_notifier_add(struct drm_device *dev,
186 : struct i915_mmu_notifier *mn,
187 : struct i915_mmu_object *mo)
188 : {
189 : struct interval_tree_node *it;
190 : int ret = 0;
191 :
192 : /* By this point we have already done a lot of expensive setup that
193 : * we do not want to repeat just because the caller (e.g. X) has a
194 : * signal pending (and partly because of that expensive setup, X
195 : * using an interrupt timer is likely to get stuck in an EINTR loop).
196 : */
197 : mutex_lock(&dev->struct_mutex);
198 :
199 : /* Make sure we drop the final active reference (and thereby
200 : * remove the objects from the interval tree) before we do
201 : * the check for overlapping objects.
202 : */
203 : i915_gem_retire_requests(dev);
204 :
205 : spin_lock(&mn->lock);
206 : it = interval_tree_iter_first(&mn->objects,
207 : mo->it.start, mo->it.last);
208 : if (it) {
209 : struct drm_i915_gem_object *obj;
210 :
211 : /* We only need to check the first object in the range as it
212 : * either has cancelled gup work queued and we need to
213 : * return back to the user to give time for the gup-workers
214 : * to flush their object references upon which the object will
215 : * be removed from the interval-tree, or the the range is
216 : * still in use by another client and the overlap is invalid.
217 : *
218 : * If we do have an overlap, we cannot use the interval tree
219 : * for fast range invalidation.
220 : */
221 :
222 : obj = container_of(it, struct i915_mmu_object, it)->obj;
223 : if (!obj->userptr.workers)
224 : mn->has_linear = mo->is_linear = true;
225 : else
226 : ret = -EAGAIN;
227 : } else
228 : interval_tree_insert(&mo->it, &mn->objects);
229 :
230 : if (ret == 0)
231 : list_add(&mo->link, &mn->linear);
232 :
233 : spin_unlock(&mn->lock);
234 : mutex_unlock(&dev->struct_mutex);
235 :
236 : return ret;
237 : }
238 :
239 : static bool i915_mmu_notifier_has_linear(struct i915_mmu_notifier *mn)
240 : {
241 : struct i915_mmu_object *mo;
242 :
243 : list_for_each_entry(mo, &mn->linear, link)
244 : if (mo->is_linear)
245 : return true;
246 :
247 : return false;
248 : }
249 :
250 : static void
251 : i915_mmu_notifier_del(struct i915_mmu_notifier *mn,
252 : struct i915_mmu_object *mo)
253 : {
254 : spin_lock(&mn->lock);
255 : list_del(&mo->link);
256 : if (mo->is_linear)
257 : mn->has_linear = i915_mmu_notifier_has_linear(mn);
258 : else
259 : interval_tree_remove(&mo->it, &mn->objects);
260 : spin_unlock(&mn->lock);
261 : }
262 :
263 : static void
264 : i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
265 : {
266 : struct i915_mmu_object *mo;
267 :
268 : mo = obj->userptr.mmu_object;
269 : if (mo == NULL)
270 : return;
271 :
272 : i915_mmu_notifier_del(mo->mn, mo);
273 : kfree(mo);
274 :
275 : obj->userptr.mmu_object = NULL;
276 : }
277 :
278 : static struct i915_mmu_notifier *
279 : i915_mmu_notifier_find(struct i915_mm_struct *mm)
280 : {
281 : struct i915_mmu_notifier *mn = mm->mn;
282 :
283 : mn = mm->mn;
284 : if (mn)
285 : return mn;
286 :
287 : down_write(&mm->mm->mmap_sem);
288 : mutex_lock(&to_i915(mm->dev)->mm_lock);
289 : if ((mn = mm->mn) == NULL) {
290 : mn = i915_mmu_notifier_create(mm->mm);
291 : if (!IS_ERR(mn))
292 : mm->mn = mn;
293 : }
294 : mutex_unlock(&to_i915(mm->dev)->mm_lock);
295 : up_write(&mm->mm->mmap_sem);
296 :
297 : return mn;
298 : }
299 :
300 : static int
301 : i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
302 : unsigned flags)
303 : {
304 : struct i915_mmu_notifier *mn;
305 : struct i915_mmu_object *mo;
306 : int ret;
307 :
308 : if (flags & I915_USERPTR_UNSYNCHRONIZED)
309 : return capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
310 :
311 : if (WARN_ON(obj->userptr.mm == NULL))
312 : return -EINVAL;
313 :
314 : mn = i915_mmu_notifier_find(obj->userptr.mm);
315 : if (IS_ERR(mn))
316 : return PTR_ERR(mn);
317 :
318 : mo = kzalloc(sizeof(*mo), GFP_KERNEL);
319 : if (mo == NULL)
320 : return -ENOMEM;
321 :
322 : mo->mn = mn;
323 : mo->it.start = obj->userptr.ptr;
324 : mo->it.last = mo->it.start + obj->base.size - 1;
325 : mo->obj = obj;
326 : INIT_WORK(&mo->work, __cancel_userptr__worker);
327 :
328 : ret = i915_mmu_notifier_add(obj->base.dev, mn, mo);
329 : if (ret) {
330 : kfree(mo);
331 : return ret;
332 : }
333 :
334 : obj->userptr.mmu_object = mo;
335 : return 0;
336 : }
337 :
338 : static void
339 : i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
340 : struct mm_struct *mm)
341 : {
342 : if (mn == NULL)
343 : return;
344 :
345 : mmu_notifier_unregister(&mn->mn, mm);
346 : kfree(mn);
347 : }
348 :
349 : #else
350 :
351 : static void
352 : i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
353 : {
354 : }
355 :
356 : static int
357 : i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
358 : unsigned flags)
359 : {
360 : if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0)
361 : return -ENODEV;
362 :
363 : if (!capable(CAP_SYS_ADMIN))
364 : return -EPERM;
365 :
366 : return 0;
367 : }
368 :
369 : static void
370 : i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
371 : struct mm_struct *mm)
372 : {
373 : }
374 :
375 : #endif
376 :
377 : static struct i915_mm_struct *
378 : __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real)
379 : {
380 : struct i915_mm_struct *mm;
381 :
382 : /* Protected by dev_priv->mm_lock */
383 : hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real)
384 : if (mm->mm == real)
385 : return mm;
386 :
387 : return NULL;
388 : }
389 :
390 : static int
391 : i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj)
392 : {
393 : struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
394 : struct i915_mm_struct *mm;
395 : int ret = 0;
396 :
397 : /* During release of the GEM object we hold the struct_mutex. This
398 : * precludes us from calling mmput() at that time as that may be
399 : * the last reference and so call exit_mmap(). exit_mmap() will
400 : * attempt to reap the vma, and if we were holding a GTT mmap
401 : * would then call drm_gem_vm_close() and attempt to reacquire
402 : * the struct mutex. So in order to avoid that recursion, we have
403 : * to defer releasing the mm reference until after we drop the
404 : * struct_mutex, i.e. we need to schedule a worker to do the clean
405 : * up.
406 : */
407 : mutex_lock(&dev_priv->mm_lock);
408 : mm = __i915_mm_struct_find(dev_priv, current->mm);
409 : if (mm == NULL) {
410 : mm = kmalloc(sizeof(*mm), GFP_KERNEL);
411 : if (mm == NULL) {
412 : ret = -ENOMEM;
413 : goto out;
414 : }
415 :
416 : kref_init(&mm->kref);
417 : mm->dev = obj->base.dev;
418 :
419 : mm->mm = current->mm;
420 : atomic_inc(¤t->mm->mm_count);
421 :
422 : mm->mn = NULL;
423 :
424 : /* Protected by dev_priv->mm_lock */
425 : hash_add(dev_priv->mm_structs,
426 : &mm->node, (unsigned long)mm->mm);
427 : } else
428 : kref_get(&mm->kref);
429 :
430 : obj->userptr.mm = mm;
431 : out:
432 : mutex_unlock(&dev_priv->mm_lock);
433 : return ret;
434 : }
435 :
436 : static void
437 : __i915_mm_struct_free__worker(struct work_struct *work)
438 : {
439 : struct i915_mm_struct *mm = container_of(work, typeof(*mm), work);
440 : i915_mmu_notifier_free(mm->mn, mm->mm);
441 : mmdrop(mm->mm);
442 : kfree(mm);
443 : }
444 :
445 : static void
446 : __i915_mm_struct_free(struct kref *kref)
447 : {
448 : struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref);
449 :
450 : /* Protected by dev_priv->mm_lock */
451 : hash_del(&mm->node);
452 : mutex_unlock(&to_i915(mm->dev)->mm_lock);
453 :
454 : INIT_WORK(&mm->work, __i915_mm_struct_free__worker);
455 : schedule_work(&mm->work);
456 : }
457 :
458 : static void
459 : i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj)
460 : {
461 : if (obj->userptr.mm == NULL)
462 : return;
463 :
464 : kref_put_mutex(&obj->userptr.mm->kref,
465 : __i915_mm_struct_free,
466 : &to_i915(obj->base.dev)->mm_lock);
467 : obj->userptr.mm = NULL;
468 : }
469 :
470 : struct get_pages_work {
471 : struct work_struct work;
472 : struct drm_i915_gem_object *obj;
473 : struct task_struct *task;
474 : };
475 :
476 : #if IS_ENABLED(CONFIG_SWIOTLB)
477 : #define swiotlb_active() swiotlb_nr_tbl()
478 : #else
479 : #define swiotlb_active() 0
480 : #endif
481 :
482 : static int
483 : st_set_pages(struct sg_table **st, struct page **pvec, int num_pages)
484 : {
485 : struct scatterlist *sg;
486 : int ret, n;
487 :
488 : *st = kmalloc(sizeof(**st), GFP_KERNEL);
489 : if (*st == NULL)
490 : return -ENOMEM;
491 :
492 : if (swiotlb_active()) {
493 : ret = sg_alloc_table(*st, num_pages, GFP_KERNEL);
494 : if (ret)
495 : goto err;
496 :
497 : for_each_sg((*st)->sgl, sg, num_pages, n)
498 : sg_set_page(sg, pvec[n], PAGE_SIZE, 0);
499 : } else {
500 : ret = sg_alloc_table_from_pages(*st, pvec, num_pages,
501 : 0, num_pages << PAGE_SHIFT,
502 : GFP_KERNEL);
503 : if (ret)
504 : goto err;
505 : }
506 :
507 : return 0;
508 :
509 : err:
510 : kfree(*st);
511 : *st = NULL;
512 : return ret;
513 : }
514 :
515 : static int
516 : __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
517 : struct page **pvec, int num_pages)
518 : {
519 : int ret;
520 :
521 : ret = st_set_pages(&obj->pages, pvec, num_pages);
522 : if (ret)
523 : return ret;
524 :
525 : ret = i915_gem_gtt_prepare_object(obj);
526 : if (ret) {
527 : sg_free_table(obj->pages);
528 : kfree(obj->pages);
529 : obj->pages = NULL;
530 : }
531 :
532 : return ret;
533 : }
534 :
535 : static int
536 : __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj,
537 : bool value)
538 : {
539 : int ret = 0;
540 :
541 : /* During mm_invalidate_range we need to cancel any userptr that
542 : * overlaps the range being invalidated. Doing so requires the
543 : * struct_mutex, and that risks recursion. In order to cause
544 : * recursion, the user must alias the userptr address space with
545 : * a GTT mmapping (possible with a MAP_FIXED) - then when we have
546 : * to invalidate that mmaping, mm_invalidate_range is called with
547 : * the userptr address *and* the struct_mutex held. To prevent that
548 : * we set a flag under the i915_mmu_notifier spinlock to indicate
549 : * whether this object is valid.
550 : */
551 : #if defined(CONFIG_MMU_NOTIFIER)
552 : if (obj->userptr.mmu_object == NULL)
553 : return 0;
554 :
555 : spin_lock(&obj->userptr.mmu_object->mn->lock);
556 : /* In order to serialise get_pages with an outstanding
557 : * cancel_userptr, we must drop the struct_mutex and try again.
558 : */
559 : if (!value || !work_pending(&obj->userptr.mmu_object->work))
560 : obj->userptr.mmu_object->active = value;
561 : else
562 : ret = -EAGAIN;
563 : spin_unlock(&obj->userptr.mmu_object->mn->lock);
564 : #endif
565 :
566 : return ret;
567 : }
568 :
569 : static void
570 : __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
571 : {
572 : struct get_pages_work *work = container_of(_work, typeof(*work), work);
573 : struct drm_i915_gem_object *obj = work->obj;
574 : struct drm_device *dev = obj->base.dev;
575 : const int npages = obj->base.size >> PAGE_SHIFT;
576 : struct page **pvec;
577 : int pinned, ret;
578 :
579 : ret = -ENOMEM;
580 : pinned = 0;
581 :
582 : pvec = kmalloc(npages*sizeof(struct page *),
583 : GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
584 : if (pvec == NULL)
585 : pvec = drm_malloc_ab(npages, sizeof(struct page *));
586 : if (pvec != NULL) {
587 : struct mm_struct *mm = obj->userptr.mm->mm;
588 :
589 : down_read(&mm->mmap_sem);
590 : while (pinned < npages) {
591 : ret = get_user_pages(work->task, mm,
592 : obj->userptr.ptr + pinned * PAGE_SIZE,
593 : npages - pinned,
594 : !obj->userptr.read_only, 0,
595 : pvec + pinned, NULL);
596 : if (ret < 0)
597 : break;
598 :
599 : pinned += ret;
600 : }
601 : up_read(&mm->mmap_sem);
602 : }
603 :
604 : mutex_lock(&dev->struct_mutex);
605 : if (obj->userptr.work == &work->work) {
606 : if (pinned == npages) {
607 : ret = __i915_gem_userptr_set_pages(obj, pvec, npages);
608 : if (ret == 0) {
609 : list_add_tail(&obj->global_list,
610 : &to_i915(dev)->mm.unbound_list);
611 : obj->get_page.sg = obj->pages->sgl;
612 : obj->get_page.last = 0;
613 : pinned = 0;
614 : }
615 : }
616 : obj->userptr.work = ERR_PTR(ret);
617 : if (ret)
618 : __i915_gem_userptr_set_active(obj, false);
619 : }
620 :
621 : obj->userptr.workers--;
622 : drm_gem_object_unreference(&obj->base);
623 : mutex_unlock(&dev->struct_mutex);
624 :
625 : release_pages(pvec, pinned, 0);
626 : drm_free_large(pvec);
627 :
628 : put_task_struct(work->task);
629 : kfree(work);
630 : }
631 :
632 : static int
633 : __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
634 : bool *active)
635 : {
636 : struct get_pages_work *work;
637 :
638 : /* Spawn a worker so that we can acquire the
639 : * user pages without holding our mutex. Access
640 : * to the user pages requires mmap_sem, and we have
641 : * a strict lock ordering of mmap_sem, struct_mutex -
642 : * we already hold struct_mutex here and so cannot
643 : * call gup without encountering a lock inversion.
644 : *
645 : * Userspace will keep on repeating the operation
646 : * (thanks to EAGAIN) until either we hit the fast
647 : * path or the worker completes. If the worker is
648 : * cancelled or superseded, the task is still run
649 : * but the results ignored. (This leads to
650 : * complications that we may have a stray object
651 : * refcount that we need to be wary of when
652 : * checking for existing objects during creation.)
653 : * If the worker encounters an error, it reports
654 : * that error back to this function through
655 : * obj->userptr.work = ERR_PTR.
656 : */
657 : if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS)
658 : return -EAGAIN;
659 :
660 : work = kmalloc(sizeof(*work), GFP_KERNEL);
661 : if (work == NULL)
662 : return -ENOMEM;
663 :
664 : obj->userptr.work = &work->work;
665 : obj->userptr.workers++;
666 :
667 : work->obj = obj;
668 : drm_gem_object_reference(&obj->base);
669 :
670 : work->task = current;
671 : get_task_struct(work->task);
672 :
673 : INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
674 : schedule_work(&work->work);
675 :
676 : *active = true;
677 : return -EAGAIN;
678 : }
679 :
680 : static int
681 : i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
682 : {
683 : const int num_pages = obj->base.size >> PAGE_SHIFT;
684 : struct page **pvec;
685 : int pinned, ret;
686 : bool active;
687 :
688 : /* If userspace should engineer that these pages are replaced in
689 : * the vma between us binding this page into the GTT and completion
690 : * of rendering... Their loss. If they change the mapping of their
691 : * pages they need to create a new bo to point to the new vma.
692 : *
693 : * However, that still leaves open the possibility of the vma
694 : * being copied upon fork. Which falls under the same userspace
695 : * synchronisation issue as a regular bo, except that this time
696 : * the process may not be expecting that a particular piece of
697 : * memory is tied to the GPU.
698 : *
699 : * Fortunately, we can hook into the mmu_notifier in order to
700 : * discard the page references prior to anything nasty happening
701 : * to the vma (discard or cloning) which should prevent the more
702 : * egregious cases from causing harm.
703 : */
704 : if (IS_ERR(obj->userptr.work)) {
705 : /* active flag will have been dropped already by the worker */
706 : ret = PTR_ERR(obj->userptr.work);
707 : obj->userptr.work = NULL;
708 : return ret;
709 : }
710 : if (obj->userptr.work)
711 : /* active flag should still be held for the pending work */
712 : return -EAGAIN;
713 :
714 : /* Let the mmu-notifier know that we have begun and need cancellation */
715 : ret = __i915_gem_userptr_set_active(obj, true);
716 : if (ret)
717 : return ret;
718 :
719 : pvec = NULL;
720 : pinned = 0;
721 : if (obj->userptr.mm->mm == current->mm) {
722 : pvec = kmalloc(num_pages*sizeof(struct page *),
723 : GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
724 : if (pvec == NULL) {
725 : pvec = drm_malloc_ab(num_pages, sizeof(struct page *));
726 : if (pvec == NULL) {
727 : __i915_gem_userptr_set_active(obj, false);
728 : return -ENOMEM;
729 : }
730 : }
731 :
732 : pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages,
733 : !obj->userptr.read_only, pvec);
734 : }
735 :
736 : active = false;
737 : if (pinned < 0)
738 : ret = pinned, pinned = 0;
739 : else if (pinned < num_pages)
740 : ret = __i915_gem_userptr_get_pages_schedule(obj, &active);
741 : else
742 : ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
743 : if (ret) {
744 : __i915_gem_userptr_set_active(obj, active);
745 : release_pages(pvec, pinned, 0);
746 : }
747 : drm_free_large(pvec);
748 : return ret;
749 : }
750 :
751 : static void
752 : i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
753 : {
754 : struct sg_page_iter sg_iter;
755 :
756 : BUG_ON(obj->userptr.work != NULL);
757 : __i915_gem_userptr_set_active(obj, false);
758 :
759 : if (obj->madv != I915_MADV_WILLNEED)
760 : obj->dirty = 0;
761 :
762 : i915_gem_gtt_finish_object(obj);
763 :
764 : for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
765 : struct page *page = sg_page_iter_page(&sg_iter);
766 :
767 : if (obj->dirty)
768 : set_page_dirty(page);
769 :
770 : mark_page_accessed(page);
771 : page_cache_release(page);
772 : }
773 : obj->dirty = 0;
774 :
775 : sg_free_table(obj->pages);
776 : kfree(obj->pages);
777 : }
778 :
779 : static void
780 : i915_gem_userptr_release(struct drm_i915_gem_object *obj)
781 : {
782 : i915_gem_userptr_release__mmu_notifier(obj);
783 : i915_gem_userptr_release__mm_struct(obj);
784 : }
785 :
786 : static int
787 : i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
788 : {
789 : if (obj->userptr.mmu_object)
790 : return 0;
791 :
792 : return i915_gem_userptr_init__mmu_notifier(obj, 0);
793 : }
794 :
795 : static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
796 : .dmabuf_export = i915_gem_userptr_dmabuf_export,
797 : .get_pages = i915_gem_userptr_get_pages,
798 : .put_pages = i915_gem_userptr_put_pages,
799 : .release = i915_gem_userptr_release,
800 : };
801 :
802 : /**
803 : * Creates a new mm object that wraps some normal memory from the process
804 : * context - user memory.
805 : *
806 : * We impose several restrictions upon the memory being mapped
807 : * into the GPU.
808 : * 1. It must be page aligned (both start/end addresses, i.e ptr and size).
809 : * 2. It must be normal system memory, not a pointer into another map of IO
810 : * space (e.g. it must not be a GTT mmapping of another object).
811 : * 3. We only allow a bo as large as we could in theory map into the GTT,
812 : * that is we limit the size to the total size of the GTT.
813 : * 4. The bo is marked as being snoopable. The backing pages are left
814 : * accessible directly by the CPU, but reads and writes by the GPU may
815 : * incur the cost of a snoop (unless you have an LLC architecture).
816 : *
817 : * Synchronisation between multiple users and the GPU is left to userspace
818 : * through the normal set-domain-ioctl. The kernel will enforce that the
819 : * GPU relinquishes the VMA before it is returned back to the system
820 : * i.e. upon free(), munmap() or process termination. However, the userspace
821 : * malloc() library may not immediately relinquish the VMA after free() and
822 : * instead reuse it whilst the GPU is still reading and writing to the VMA.
823 : * Caveat emptor.
824 : *
825 : * Also note, that the object created here is not currently a "first class"
826 : * object, in that several ioctls are banned. These are the CPU access
827 : * ioctls: mmap(), pwrite and pread. In practice, you are expected to use
828 : * direct access via your pointer rather than use those ioctls. Another
829 : * restriction is that we do not allow userptr surfaces to be pinned to the
830 : * hardware and so we reject any attempt to create a framebuffer out of a
831 : * userptr.
832 : *
833 : * If you think this is a good interface to use to pass GPU memory between
834 : * drivers, please use dma-buf instead. In fact, wherever possible use
835 : * dma-buf instead.
836 : */
837 : int
838 : i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
839 : {
840 : struct drm_i915_gem_userptr *args = data;
841 : struct drm_i915_gem_object *obj;
842 : int ret;
843 : u32 handle;
844 :
845 : if (args->flags & ~(I915_USERPTR_READ_ONLY |
846 : I915_USERPTR_UNSYNCHRONIZED))
847 : return -EINVAL;
848 :
849 : if (!args->user_size)
850 : return -EINVAL;
851 :
852 : if (offset_in_page(args->user_ptr | args->user_size))
853 : return -EINVAL;
854 :
855 : if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE,
856 : (char __user *)(unsigned long)args->user_ptr, args->user_size))
857 : return -EFAULT;
858 :
859 : if (args->flags & I915_USERPTR_READ_ONLY) {
860 : /* On almost all of the current hw, we cannot tell the GPU that a
861 : * page is readonly, so this is just a placeholder in the uAPI.
862 : */
863 : return -ENODEV;
864 : }
865 :
866 : obj = i915_gem_object_alloc(dev);
867 : if (obj == NULL)
868 : return -ENOMEM;
869 :
870 : drm_gem_private_object_init(dev, &obj->base, args->user_size);
871 : i915_gem_object_init(obj, &i915_gem_userptr_ops);
872 : obj->cache_level = I915_CACHE_LLC;
873 : obj->base.write_domain = I915_GEM_DOMAIN_CPU;
874 : obj->base.read_domains = I915_GEM_DOMAIN_CPU;
875 :
876 : obj->userptr.ptr = args->user_ptr;
877 : obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY);
878 :
879 : /* And keep a pointer to the current->mm for resolving the user pages
880 : * at binding. This means that we need to hook into the mmu_notifier
881 : * in order to detect if the mmu is destroyed.
882 : */
883 : ret = i915_gem_userptr_init__mm_struct(obj);
884 : if (ret == 0)
885 : ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags);
886 : if (ret == 0)
887 : ret = drm_gem_handle_create(file, &obj->base, &handle);
888 :
889 : /* drop reference from allocate - handle holds it now */
890 : drm_gem_object_unreference_unlocked(&obj->base);
891 : if (ret)
892 : return ret;
893 :
894 : args->handle = handle;
895 : return 0;
896 : }
897 :
898 : int
899 : i915_gem_init_userptr(struct drm_device *dev)
900 : {
901 : struct drm_i915_private *dev_priv = to_i915(dev);
902 : mutex_init(&dev_priv->mm_lock);
903 : hash_init(dev_priv->mm_structs);
904 : return 0;
905 : }
906 :
907 : #else
908 :
909 : int
910 0 : i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
911 : {
912 0 : return -ENODEV;
913 : }
914 :
915 : int
916 0 : i915_gem_init_userptr(struct drm_device *dev)
917 : {
918 0 : return 0;
919 : }
920 :
921 : #endif
|
