Line data Source code
1 : /*
2 : * Copyright © 2008,2010 Intel Corporation
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a
5 : * copy of this software and associated documentation files (the "Software"),
6 : * to deal in the Software without restriction, including without limitation
7 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : * and/or sell copies of the Software, and to permit persons to whom the
9 : * Software is furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice (including the next
12 : * paragraph) shall be included in all copies or substantial portions of the
13 : * Software.
14 : *
15 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 : * IN THE SOFTWARE.
22 : *
23 : * Authors:
24 : * Eric Anholt <eric@anholt.net>
25 : * Chris Wilson <chris@chris-wilson.co.uk>
26 : *
27 : */
28 :
29 : #include <dev/pci/drm/drmP.h>
30 : #include <dev/pci/drm/i915_drm.h>
31 : #include "i915_drv.h"
32 : #include "i915_trace.h"
33 : #include "intel_drv.h"
34 : #ifdef __linux__
35 : #include <linux/dma_remapping.h>
36 : #include <linux/uaccess.h>
37 : #endif
38 :
39 : #define __EXEC_OBJECT_HAS_PIN (1<<31)
40 : #define __EXEC_OBJECT_HAS_FENCE (1<<30)
41 : #define __EXEC_OBJECT_NEEDS_MAP (1<<29)
42 : #define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
43 :
44 : #define BATCH_OFFSET_BIAS (256*1024)
45 :
46 : struct eb_vmas {
47 : struct list_head vmas;
48 : int and;
49 : union {
50 : struct i915_vma *lut[0];
51 : struct hlist_head buckets[0];
52 : };
53 : };
54 :
55 : static struct eb_vmas *
56 0 : eb_create(struct drm_i915_gem_execbuffer2 *args)
57 : {
58 : struct eb_vmas *eb = NULL;
59 :
60 0 : if (args->flags & I915_EXEC_HANDLE_LUT) {
61 0 : unsigned size = args->buffer_count;
62 0 : size *= sizeof(struct i915_vma *);
63 0 : size += sizeof(struct eb_vmas);
64 0 : eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
65 0 : }
66 :
67 0 : if (eb == NULL) {
68 0 : unsigned size = args->buffer_count;
69 : unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
70 : BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
71 0 : while (count > 2*size)
72 0 : count >>= 1;
73 0 : eb = kzalloc(count*sizeof(struct hlist_head) +
74 : sizeof(struct eb_vmas),
75 : GFP_TEMPORARY);
76 0 : if (eb == NULL)
77 0 : return eb;
78 :
79 0 : eb->and = count - 1;
80 0 : } else
81 0 : eb->and = -args->buffer_count;
82 :
83 0 : INIT_LIST_HEAD(&eb->vmas);
84 0 : return eb;
85 0 : }
86 :
87 : static void
88 0 : eb_reset(struct eb_vmas *eb)
89 : {
90 0 : if (eb->and >= 0)
91 0 : memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
92 0 : }
93 :
94 : static int
95 0 : eb_lookup_vmas(struct eb_vmas *eb,
96 : struct drm_i915_gem_exec_object2 *exec,
97 : const struct drm_i915_gem_execbuffer2 *args,
98 : struct i915_address_space *vm,
99 : struct drm_file *file)
100 : {
101 : struct drm_i915_gem_object *obj;
102 0 : struct list_head objects;
103 : int i, ret;
104 :
105 0 : INIT_LIST_HEAD(&objects);
106 0 : spin_lock(&file->table_lock);
107 : /* Grab a reference to the object and release the lock so we can lookup
108 : * or create the VMA without using GFP_ATOMIC */
109 0 : for (i = 0; i < args->buffer_count; i++) {
110 0 : obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
111 0 : if (obj == NULL) {
112 0 : spin_unlock(&file->table_lock);
113 : DRM_DEBUG("Invalid object handle %d at index %d\n",
114 : exec[i].handle, i);
115 : ret = -ENOENT;
116 0 : goto err;
117 : }
118 :
119 0 : if (!list_empty(&obj->obj_exec_link)) {
120 0 : spin_unlock(&file->table_lock);
121 : DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
122 : obj, exec[i].handle, i);
123 : ret = -EINVAL;
124 0 : goto err;
125 : }
126 :
127 0 : drm_gem_object_reference(&obj->base);
128 0 : list_add_tail(&obj->obj_exec_link, &objects);
129 : }
130 0 : spin_unlock(&file->table_lock);
131 :
132 : i = 0;
133 0 : while (!list_empty(&objects)) {
134 : struct i915_vma *vma;
135 :
136 0 : obj = list_first_entry(&objects,
137 : struct drm_i915_gem_object,
138 : obj_exec_link);
139 :
140 : /*
141 : * NOTE: We can leak any vmas created here when something fails
142 : * later on. But that's no issue since vma_unbind can deal with
143 : * vmas which are not actually bound. And since only
144 : * lookup_or_create exists as an interface to get at the vma
145 : * from the (obj, vm) we don't run the risk of creating
146 : * duplicated vmas for the same vm.
147 : */
148 0 : vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
149 0 : if (IS_ERR(vma)) {
150 : DRM_DEBUG("Failed to lookup VMA\n");
151 0 : ret = PTR_ERR(vma);
152 0 : goto err;
153 : }
154 :
155 : /* Transfer ownership from the objects list to the vmas list. */
156 0 : list_add_tail(&vma->exec_list, &eb->vmas);
157 0 : list_del_init(&obj->obj_exec_link);
158 :
159 0 : vma->exec_entry = &exec[i];
160 0 : if (eb->and < 0) {
161 0 : eb->lut[i] = vma;
162 0 : } else {
163 0 : uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
164 0 : vma->exec_handle = handle;
165 0 : hlist_add_head(&vma->exec_node,
166 0 : &eb->buckets[handle & eb->and]);
167 : }
168 0 : ++i;
169 0 : }
170 :
171 0 : return 0;
172 :
173 :
174 : err:
175 0 : while (!list_empty(&objects)) {
176 0 : obj = list_first_entry(&objects,
177 : struct drm_i915_gem_object,
178 : obj_exec_link);
179 0 : list_del_init(&obj->obj_exec_link);
180 0 : drm_gem_object_unreference(&obj->base);
181 : }
182 : /*
183 : * Objects already transfered to the vmas list will be unreferenced by
184 : * eb_destroy.
185 : */
186 :
187 0 : return ret;
188 0 : }
189 :
190 0 : static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
191 : {
192 0 : if (eb->and < 0) {
193 0 : if (handle >= -eb->and)
194 0 : return NULL;
195 0 : return eb->lut[handle];
196 : } else {
197 : struct hlist_head *head;
198 : struct hlist_node *node;
199 :
200 0 : head = &eb->buckets[handle & eb->and];
201 0 : hlist_for_each(node, head) {
202 : struct i915_vma *vma;
203 :
204 0 : vma = hlist_entry(node, struct i915_vma, exec_node);
205 0 : if (vma->exec_handle == handle)
206 0 : return vma;
207 0 : }
208 0 : return NULL;
209 : }
210 0 : }
211 :
212 : static void
213 0 : i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
214 : {
215 : struct drm_i915_gem_exec_object2 *entry;
216 0 : struct drm_i915_gem_object *obj = vma->obj;
217 :
218 0 : if (!drm_mm_node_allocated(&vma->node))
219 0 : return;
220 :
221 0 : entry = vma->exec_entry;
222 :
223 0 : if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
224 0 : i915_gem_object_unpin_fence(obj);
225 :
226 0 : if (entry->flags & __EXEC_OBJECT_HAS_PIN)
227 0 : vma->pin_count--;
228 :
229 0 : entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
230 0 : }
231 :
232 0 : static void eb_destroy(struct eb_vmas *eb)
233 : {
234 0 : while (!list_empty(&eb->vmas)) {
235 : struct i915_vma *vma;
236 :
237 0 : vma = list_first_entry(&eb->vmas,
238 : struct i915_vma,
239 : exec_list);
240 0 : list_del_init(&vma->exec_list);
241 0 : i915_gem_execbuffer_unreserve_vma(vma);
242 0 : drm_gem_object_unreference(&vma->obj->base);
243 : }
244 0 : kfree(eb);
245 0 : }
246 :
247 0 : static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
248 : {
249 0 : return (HAS_LLC(obj->base.dev) ||
250 0 : obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
251 0 : obj->cache_level != I915_CACHE_NONE);
252 : }
253 :
254 : static int
255 0 : relocate_entry_cpu(struct drm_i915_gem_object *obj,
256 : struct drm_i915_gem_relocation_entry *reloc,
257 : uint64_t target_offset)
258 : {
259 0 : struct drm_device *dev = obj->base.dev;
260 0 : uint32_t page_offset = offset_in_page(reloc->offset);
261 0 : uint64_t delta = reloc->delta + target_offset;
262 : char *vaddr;
263 : int ret;
264 :
265 0 : ret = i915_gem_object_set_to_cpu_domain(obj, true);
266 0 : if (ret)
267 0 : return ret;
268 :
269 0 : vaddr = kmap_atomic(i915_gem_object_get_page(obj,
270 0 : reloc->offset >> PAGE_SHIFT));
271 0 : *(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
272 :
273 0 : if (INTEL_INFO(dev)->gen >= 8) {
274 0 : page_offset = offset_in_page(page_offset + sizeof(uint32_t));
275 :
276 0 : if (page_offset == 0) {
277 0 : kunmap_atomic(vaddr);
278 0 : vaddr = kmap_atomic(i915_gem_object_get_page(obj,
279 0 : (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
280 0 : }
281 :
282 0 : *(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);
283 0 : }
284 :
285 0 : kunmap_atomic(vaddr);
286 :
287 0 : return 0;
288 0 : }
289 :
290 : static int
291 0 : relocate_entry_gtt(struct drm_i915_gem_object *obj,
292 : struct drm_i915_gem_relocation_entry *reloc,
293 : uint64_t target_offset)
294 : {
295 0 : struct drm_device *dev = obj->base.dev;
296 0 : struct drm_i915_private *dev_priv = dev->dev_private;
297 0 : uint64_t delta = reloc->delta + target_offset;
298 : uint64_t offset;
299 : void __iomem *reloc_page;
300 0 : bus_space_handle_t bsh;
301 : int ret;
302 :
303 0 : ret = i915_gem_object_set_to_gtt_domain(obj, true);
304 0 : if (ret)
305 0 : return ret;
306 :
307 0 : ret = i915_gem_object_put_fence(obj);
308 0 : if (ret)
309 0 : return ret;
310 :
311 : /* Map the page containing the relocation we're going to perform. */
312 0 : offset = i915_gem_obj_ggtt_offset(obj);
313 0 : offset += reloc->offset;
314 : #ifdef __linux__
315 : reloc_page = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
316 : offset & PAGE_MASK);
317 : #else
318 0 : agp_map_atomic(dev_priv->agph, trunc_page(offset), &bsh);
319 0 : reloc_page = bus_space_vaddr(dev_priv->bst, bsh);
320 : #endif
321 0 : iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
322 :
323 0 : if (INTEL_INFO(dev)->gen >= 8) {
324 0 : offset += sizeof(uint32_t);
325 :
326 0 : if (offset_in_page(offset) == 0) {
327 : #ifdef __linux__
328 : io_mapping_unmap_atomic(reloc_page);
329 : reloc_page =
330 : io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
331 : offset);
332 : #else
333 0 : agp_unmap_atomic(dev_priv->agph, bsh);
334 0 : agp_map_atomic(dev_priv->agph, offset, &bsh);
335 0 : reloc_page = bus_space_vaddr(dev_priv->bst, bsh);
336 : #endif
337 0 : }
338 :
339 0 : iowrite32(upper_32_bits(delta),
340 0 : reloc_page + offset_in_page(offset));
341 0 : }
342 :
343 : #ifdef __linux__
344 : io_mapping_unmap_atomic(reloc_page);
345 : #else
346 0 : agp_unmap_atomic(dev_priv->agph, bsh);
347 : #endif
348 :
349 0 : return 0;
350 0 : }
351 :
352 : static void
353 0 : clflush_write32(void *addr, uint32_t value)
354 : {
355 : /* This is not a fast path, so KISS. */
356 0 : drm_clflush_virt_range(addr, sizeof(uint32_t));
357 0 : *(uint32_t *)addr = value;
358 0 : drm_clflush_virt_range(addr, sizeof(uint32_t));
359 0 : }
360 :
361 : static int
362 0 : relocate_entry_clflush(struct drm_i915_gem_object *obj,
363 : struct drm_i915_gem_relocation_entry *reloc,
364 : uint64_t target_offset)
365 : {
366 0 : struct drm_device *dev = obj->base.dev;
367 0 : uint32_t page_offset = offset_in_page(reloc->offset);
368 0 : uint64_t delta = (int)reloc->delta + target_offset;
369 : char *vaddr;
370 : int ret;
371 :
372 0 : ret = i915_gem_object_set_to_gtt_domain(obj, true);
373 0 : if (ret)
374 0 : return ret;
375 :
376 0 : vaddr = kmap_atomic(i915_gem_object_get_page(obj,
377 0 : reloc->offset >> PAGE_SHIFT));
378 0 : clflush_write32(vaddr + page_offset, lower_32_bits(delta));
379 :
380 0 : if (INTEL_INFO(dev)->gen >= 8) {
381 0 : page_offset = offset_in_page(page_offset + sizeof(uint32_t));
382 :
383 0 : if (page_offset == 0) {
384 0 : kunmap_atomic(vaddr);
385 0 : vaddr = kmap_atomic(i915_gem_object_get_page(obj,
386 0 : (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
387 0 : }
388 :
389 0 : clflush_write32(vaddr + page_offset, upper_32_bits(delta));
390 0 : }
391 :
392 0 : kunmap_atomic(vaddr);
393 :
394 0 : return 0;
395 0 : }
396 :
397 : static int
398 0 : i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
399 : struct eb_vmas *eb,
400 : struct drm_i915_gem_relocation_entry *reloc)
401 : {
402 0 : struct drm_device *dev = obj->base.dev;
403 : struct drm_gem_object *target_obj;
404 : struct drm_i915_gem_object *target_i915_obj;
405 : struct i915_vma *target_vma;
406 : uint64_t target_offset;
407 : int ret;
408 :
409 : /* we've already hold a reference to all valid objects */
410 0 : target_vma = eb_get_vma(eb, reloc->target_handle);
411 0 : if (unlikely(target_vma == NULL))
412 0 : return -ENOENT;
413 0 : target_i915_obj = target_vma->obj;
414 0 : target_obj = &target_vma->obj->base;
415 :
416 0 : target_offset = target_vma->node.start;
417 :
418 : /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
419 : * pipe_control writes because the gpu doesn't properly redirect them
420 : * through the ppgtt for non_secure batchbuffers. */
421 0 : if (unlikely(IS_GEN6(dev) &&
422 : reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
423 0 : ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
424 : PIN_GLOBAL);
425 0 : if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
426 0 : return ret;
427 : }
428 :
429 : /* Validate that the target is in a valid r/w GPU domain */
430 0 : if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
431 : DRM_DEBUG("reloc with multiple write domains: "
432 : "obj %p target %d offset %d "
433 : "read %08x write %08x",
434 : obj, reloc->target_handle,
435 : (int) reloc->offset,
436 : reloc->read_domains,
437 : reloc->write_domain);
438 0 : return -EINVAL;
439 : }
440 0 : if (unlikely((reloc->write_domain | reloc->read_domains)
441 : & ~I915_GEM_GPU_DOMAINS)) {
442 : DRM_DEBUG("reloc with read/write non-GPU domains: "
443 : "obj %p target %d offset %d "
444 : "read %08x write %08x",
445 : obj, reloc->target_handle,
446 : (int) reloc->offset,
447 : reloc->read_domains,
448 : reloc->write_domain);
449 0 : return -EINVAL;
450 : }
451 :
452 0 : target_obj->pending_read_domains |= reloc->read_domains;
453 0 : target_obj->pending_write_domain |= reloc->write_domain;
454 :
455 : /* If the relocation already has the right value in it, no
456 : * more work needs to be done.
457 : */
458 0 : if (target_offset == reloc->presumed_offset)
459 0 : return 0;
460 :
461 : /* Check that the relocation address is valid... */
462 0 : if (unlikely(reloc->offset >
463 : obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
464 : DRM_DEBUG("Relocation beyond object bounds: "
465 : "obj %p target %d offset %d size %d.\n",
466 : obj, reloc->target_handle,
467 : (int) reloc->offset,
468 : (int) obj->base.size);
469 0 : return -EINVAL;
470 : }
471 0 : if (unlikely(reloc->offset & 3)) {
472 : DRM_DEBUG("Relocation not 4-byte aligned: "
473 : "obj %p target %d offset %d.\n",
474 : obj, reloc->target_handle,
475 : (int) reloc->offset);
476 0 : return -EINVAL;
477 : }
478 :
479 : /* We can't wait for rendering with pagefaults disabled */
480 0 : if (obj->active && pagefault_disabled())
481 0 : return -EFAULT;
482 :
483 0 : if (use_cpu_reloc(obj))
484 0 : ret = relocate_entry_cpu(obj, reloc, target_offset);
485 0 : else if (obj->map_and_fenceable)
486 0 : ret = relocate_entry_gtt(obj, reloc, target_offset);
487 : else if (cpu_has_clflush)
488 0 : ret = relocate_entry_clflush(obj, reloc, target_offset);
489 : else {
490 : WARN_ONCE(1, "Impossible case in relocation handling\n");
491 : ret = -ENODEV;
492 : }
493 :
494 0 : if (ret)
495 0 : return ret;
496 :
497 : /* and update the user's relocation entry */
498 0 : reloc->presumed_offset = target_offset;
499 :
500 0 : return 0;
501 0 : }
502 :
503 : static int
504 0 : i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
505 : struct eb_vmas *eb)
506 : {
507 : #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
508 0 : struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
509 : struct drm_i915_gem_relocation_entry __user *user_relocs;
510 0 : struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
511 : int remain, ret;
512 :
513 0 : user_relocs = to_user_ptr(entry->relocs_ptr);
514 :
515 0 : remain = entry->relocation_count;
516 0 : while (remain) {
517 0 : struct drm_i915_gem_relocation_entry *r = stack_reloc;
518 : int count = remain;
519 0 : if (count > ARRAY_SIZE(stack_reloc))
520 : count = ARRAY_SIZE(stack_reloc);
521 0 : remain -= count;
522 :
523 0 : if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
524 0 : return -EFAULT;
525 :
526 0 : do {
527 0 : u64 offset = r->presumed_offset;
528 :
529 0 : ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
530 0 : if (ret)
531 0 : return ret;
532 :
533 0 : if (r->presumed_offset != offset &&
534 0 : __copy_to_user_inatomic(&user_relocs->presumed_offset,
535 0 : &r->presumed_offset,
536 : sizeof(r->presumed_offset))) {
537 0 : return -EFAULT;
538 : }
539 :
540 0 : user_relocs++;
541 0 : r++;
542 0 : } while (--count);
543 0 : }
544 :
545 0 : return 0;
546 : #undef N_RELOC
547 0 : }
548 :
549 : static int
550 0 : i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
551 : struct eb_vmas *eb,
552 : struct drm_i915_gem_relocation_entry *relocs)
553 : {
554 0 : const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
555 : int i, ret;
556 :
557 0 : for (i = 0; i < entry->relocation_count; i++) {
558 0 : ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
559 0 : if (ret)
560 0 : return ret;
561 : }
562 :
563 0 : return 0;
564 0 : }
565 :
566 : static int
567 0 : i915_gem_execbuffer_relocate(struct eb_vmas *eb)
568 : {
569 : struct i915_vma *vma;
570 : int ret = 0;
571 :
572 : /* This is the fast path and we cannot handle a pagefault whilst
573 : * holding the struct mutex lest the user pass in the relocations
574 : * contained within a mmaped bo. For in such a case we, the page
575 : * fault handler would call i915_gem_fault() and we would try to
576 : * acquire the struct mutex again. Obviously this is bad and so
577 : * lockdep complains vehemently.
578 : */
579 0 : pagefault_disable();
580 0 : list_for_each_entry(vma, &eb->vmas, exec_list) {
581 0 : ret = i915_gem_execbuffer_relocate_vma(vma, eb);
582 0 : if (ret)
583 : break;
584 : }
585 0 : pagefault_enable();
586 :
587 0 : return ret;
588 : }
589 :
590 0 : static bool only_mappable_for_reloc(unsigned int flags)
591 : {
592 0 : return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
593 : __EXEC_OBJECT_NEEDS_MAP;
594 : }
595 :
596 : static int
597 0 : i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
598 : struct intel_engine_cs *ring,
599 : bool *need_reloc)
600 : {
601 0 : struct drm_i915_gem_object *obj = vma->obj;
602 0 : struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
603 : uint64_t flags;
604 : int ret;
605 :
606 : flags = PIN_USER;
607 0 : if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
608 0 : flags |= PIN_GLOBAL;
609 :
610 0 : if (!drm_mm_node_allocated(&vma->node)) {
611 : /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
612 : * limit address to the first 4GBs for unflagged objects.
613 : */
614 0 : if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
615 0 : flags |= PIN_ZONE_4G;
616 0 : if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
617 0 : flags |= PIN_GLOBAL | PIN_MAPPABLE;
618 0 : if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
619 0 : flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
620 0 : if ((flags & PIN_MAPPABLE) == 0)
621 0 : flags |= PIN_HIGH;
622 : }
623 :
624 0 : ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
625 0 : if ((ret == -ENOSPC || ret == -E2BIG) &&
626 0 : only_mappable_for_reloc(entry->flags))
627 0 : ret = i915_gem_object_pin(obj, vma->vm,
628 0 : entry->alignment,
629 0 : flags & ~PIN_MAPPABLE);
630 0 : if (ret)
631 0 : return ret;
632 :
633 0 : entry->flags |= __EXEC_OBJECT_HAS_PIN;
634 :
635 0 : if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
636 0 : ret = i915_gem_object_get_fence(obj);
637 0 : if (ret)
638 0 : return ret;
639 :
640 0 : if (i915_gem_object_pin_fence(obj))
641 0 : entry->flags |= __EXEC_OBJECT_HAS_FENCE;
642 : }
643 :
644 0 : if (entry->offset != vma->node.start) {
645 0 : entry->offset = vma->node.start;
646 0 : *need_reloc = true;
647 0 : }
648 :
649 0 : if (entry->flags & EXEC_OBJECT_WRITE) {
650 0 : obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
651 0 : obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
652 0 : }
653 :
654 0 : return 0;
655 0 : }
656 :
657 : static bool
658 0 : need_reloc_mappable(struct i915_vma *vma)
659 : {
660 0 : struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
661 :
662 0 : if (entry->relocation_count == 0)
663 0 : return false;
664 :
665 0 : if (!i915_is_ggtt(vma->vm))
666 0 : return false;
667 :
668 : /* See also use_cpu_reloc() */
669 0 : if (HAS_LLC(vma->obj->base.dev))
670 0 : return false;
671 :
672 0 : if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
673 0 : return false;
674 :
675 0 : return true;
676 0 : }
677 :
678 : static bool
679 0 : eb_vma_misplaced(struct i915_vma *vma)
680 : {
681 0 : struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
682 0 : struct drm_i915_gem_object *obj = vma->obj;
683 :
684 0 : WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
685 : !i915_is_ggtt(vma->vm));
686 :
687 0 : if (entry->alignment &&
688 0 : vma->node.start & (entry->alignment - 1))
689 0 : return true;
690 :
691 0 : if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
692 0 : vma->node.start < BATCH_OFFSET_BIAS)
693 0 : return true;
694 :
695 : /* avoid costly ping-pong once a batch bo ended up non-mappable */
696 0 : if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
697 0 : return !only_mappable_for_reloc(entry->flags);
698 :
699 0 : if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
700 0 : (vma->node.start + vma->node.size - 1) >> 32)
701 0 : return true;
702 :
703 0 : return false;
704 0 : }
705 :
706 : static int
707 0 : i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
708 : struct list_head *vmas,
709 : struct intel_context *ctx,
710 : bool *need_relocs)
711 : {
712 : struct drm_i915_gem_object *obj;
713 : struct i915_vma *vma;
714 : struct i915_address_space *vm;
715 0 : struct list_head ordered_vmas;
716 0 : bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
717 : int retry;
718 :
719 0 : i915_gem_retire_requests_ring(ring);
720 :
721 0 : vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
722 :
723 0 : INIT_LIST_HEAD(&ordered_vmas);
724 0 : while (!list_empty(vmas)) {
725 : struct drm_i915_gem_exec_object2 *entry;
726 : bool need_fence, need_mappable;
727 :
728 0 : vma = list_first_entry(vmas, struct i915_vma, exec_list);
729 0 : obj = vma->obj;
730 0 : entry = vma->exec_entry;
731 :
732 0 : if (ctx->flags & CONTEXT_NO_ZEROMAP)
733 0 : entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
734 :
735 0 : if (!has_fenced_gpu_access)
736 0 : entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
737 0 : need_fence =
738 0 : entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
739 0 : obj->tiling_mode != I915_TILING_NONE;
740 0 : need_mappable = need_fence || need_reloc_mappable(vma);
741 :
742 0 : if (need_mappable) {
743 0 : entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
744 0 : list_move(&vma->exec_list, &ordered_vmas);
745 0 : } else
746 0 : list_move_tail(&vma->exec_list, &ordered_vmas);
747 :
748 0 : obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
749 0 : obj->base.pending_write_domain = 0;
750 : }
751 0 : list_splice(&ordered_vmas, vmas);
752 :
753 : /* Attempt to pin all of the buffers into the GTT.
754 : * This is done in 3 phases:
755 : *
756 : * 1a. Unbind all objects that do not match the GTT constraints for
757 : * the execbuffer (fenceable, mappable, alignment etc).
758 : * 1b. Increment pin count for already bound objects.
759 : * 2. Bind new objects.
760 : * 3. Decrement pin count.
761 : *
762 : * This avoid unnecessary unbinding of later objects in order to make
763 : * room for the earlier objects *unless* we need to defragment.
764 : */
765 : retry = 0;
766 0 : do {
767 : int ret = 0;
768 :
769 : /* Unbind any ill-fitting objects or pin. */
770 0 : list_for_each_entry(vma, vmas, exec_list) {
771 0 : if (!drm_mm_node_allocated(&vma->node))
772 : continue;
773 :
774 0 : if (eb_vma_misplaced(vma))
775 0 : ret = i915_vma_unbind(vma);
776 : else
777 0 : ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
778 0 : if (ret)
779 : goto err;
780 : }
781 :
782 : /* Bind fresh objects */
783 0 : list_for_each_entry(vma, vmas, exec_list) {
784 0 : if (drm_mm_node_allocated(&vma->node))
785 : continue;
786 :
787 0 : ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
788 0 : if (ret)
789 : goto err;
790 : }
791 :
792 : err:
793 0 : if (ret != -ENOSPC || retry++)
794 0 : return ret;
795 :
796 : /* Decrement pin count for bound objects */
797 0 : list_for_each_entry(vma, vmas, exec_list)
798 0 : i915_gem_execbuffer_unreserve_vma(vma);
799 :
800 0 : ret = i915_gem_evict_vm(vm, true);
801 0 : if (ret)
802 0 : return ret;
803 0 : } while (1);
804 0 : }
805 :
806 : static int
807 0 : i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
808 : struct drm_i915_gem_execbuffer2 *args,
809 : struct drm_file *file,
810 : struct intel_engine_cs *ring,
811 : struct eb_vmas *eb,
812 : struct drm_i915_gem_exec_object2 *exec,
813 : struct intel_context *ctx)
814 : {
815 : struct drm_i915_gem_relocation_entry *reloc;
816 : struct i915_address_space *vm;
817 : struct i915_vma *vma;
818 0 : bool need_relocs;
819 : int *reloc_offset;
820 : int i, total, ret;
821 0 : unsigned count = args->buffer_count;
822 :
823 0 : vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
824 :
825 : /* We may process another execbuffer during the unlock... */
826 0 : while (!list_empty(&eb->vmas)) {
827 0 : vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
828 0 : list_del_init(&vma->exec_list);
829 0 : i915_gem_execbuffer_unreserve_vma(vma);
830 0 : drm_gem_object_unreference(&vma->obj->base);
831 : }
832 :
833 0 : mutex_unlock(&dev->struct_mutex);
834 :
835 : total = 0;
836 0 : for (i = 0; i < count; i++)
837 0 : total += exec[i].relocation_count;
838 :
839 0 : reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
840 0 : reloc = drm_malloc_ab(total, sizeof(*reloc));
841 0 : if (reloc == NULL || reloc_offset == NULL) {
842 0 : drm_free_large(reloc);
843 0 : drm_free_large(reloc_offset);
844 0 : mutex_lock(&dev->struct_mutex);
845 0 : return -ENOMEM;
846 : }
847 :
848 : total = 0;
849 0 : for (i = 0; i < count; i++) {
850 : struct drm_i915_gem_relocation_entry __user *user_relocs;
851 0 : u64 invalid_offset = (u64)-1;
852 : int j;
853 :
854 0 : user_relocs = to_user_ptr(exec[i].relocs_ptr);
855 :
856 0 : if (copy_from_user(reloc+total, user_relocs,
857 0 : exec[i].relocation_count * sizeof(*reloc))) {
858 : ret = -EFAULT;
859 0 : mutex_lock(&dev->struct_mutex);
860 0 : goto err;
861 : }
862 :
863 : /* As we do not update the known relocation offsets after
864 : * relocating (due to the complexities in lock handling),
865 : * we need to mark them as invalid now so that we force the
866 : * relocation processing next time. Just in case the target
867 : * object is evicted and then rebound into its old
868 : * presumed_offset before the next execbuffer - if that
869 : * happened we would make the mistake of assuming that the
870 : * relocations were valid.
871 : */
872 0 : for (j = 0; j < exec[i].relocation_count; j++) {
873 0 : if (__copy_to_user(&user_relocs[j].presumed_offset,
874 : &invalid_offset,
875 : sizeof(invalid_offset))) {
876 : ret = -EFAULT;
877 0 : mutex_lock(&dev->struct_mutex);
878 0 : goto err;
879 : }
880 : }
881 :
882 0 : reloc_offset[i] = total;
883 0 : total += exec[i].relocation_count;
884 0 : }
885 :
886 0 : ret = i915_mutex_lock_interruptible(dev);
887 0 : if (ret) {
888 0 : mutex_lock(&dev->struct_mutex);
889 0 : goto err;
890 : }
891 :
892 : /* reacquire the objects */
893 0 : eb_reset(eb);
894 0 : ret = eb_lookup_vmas(eb, exec, args, vm, file);
895 0 : if (ret)
896 : goto err;
897 :
898 0 : need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
899 0 : ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
900 0 : if (ret)
901 : goto err;
902 :
903 0 : list_for_each_entry(vma, &eb->vmas, exec_list) {
904 0 : int offset = vma->exec_entry - exec;
905 0 : ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
906 0 : reloc + reloc_offset[offset]);
907 0 : if (ret)
908 0 : goto err;
909 0 : }
910 :
911 : /* Leave the user relocations as are, this is the painfully slow path,
912 : * and we want to avoid the complication of dropping the lock whilst
913 : * having buffers reserved in the aperture and so causing spurious
914 : * ENOSPC for random operations.
915 : */
916 :
917 : err:
918 0 : drm_free_large(reloc);
919 0 : drm_free_large(reloc_offset);
920 0 : return ret;
921 0 : }
922 :
923 : static int
924 0 : i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
925 : struct list_head *vmas)
926 : {
927 0 : const unsigned other_rings = ~intel_ring_flag(req->ring);
928 : struct i915_vma *vma;
929 : uint32_t flush_domains = 0;
930 : bool flush_chipset = false;
931 : int ret;
932 :
933 0 : list_for_each_entry(vma, vmas, exec_list) {
934 0 : struct drm_i915_gem_object *obj = vma->obj;
935 :
936 0 : if (obj->active & other_rings) {
937 0 : ret = i915_gem_object_sync(obj, req->ring, &req);
938 0 : if (ret)
939 0 : return ret;
940 : }
941 :
942 0 : if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
943 0 : flush_chipset |= i915_gem_clflush_object(obj, false);
944 :
945 0 : flush_domains |= obj->base.write_domain;
946 0 : }
947 :
948 0 : if (flush_chipset)
949 0 : i915_gem_chipset_flush(req->ring->dev);
950 :
951 0 : if (flush_domains & I915_GEM_DOMAIN_GTT)
952 0 : wmb();
953 :
954 : /* Unconditionally invalidate gpu caches and ensure that we do flush
955 : * any residual writes from the previous batch.
956 : */
957 0 : return intel_ring_invalidate_all_caches(req);
958 0 : }
959 :
960 : static bool
961 0 : i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
962 : {
963 0 : if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
964 0 : return false;
965 :
966 : /* Kernel clipping was a DRI1 misfeature */
967 0 : if (exec->num_cliprects || exec->cliprects_ptr)
968 0 : return false;
969 :
970 0 : if (exec->DR4 == 0xffffffff) {
971 : DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
972 0 : exec->DR4 = 0;
973 0 : }
974 0 : if (exec->DR1 || exec->DR4)
975 0 : return false;
976 :
977 0 : if ((exec->batch_start_offset | exec->batch_len) & 0x7)
978 0 : return false;
979 :
980 0 : return true;
981 0 : }
982 :
983 : static int
984 0 : validate_exec_list(struct drm_device *dev,
985 : struct drm_i915_gem_exec_object2 *exec,
986 : int count)
987 : {
988 : unsigned relocs_total = 0;
989 : unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
990 : unsigned invalid_flags;
991 : int i;
992 :
993 : invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
994 0 : if (USES_FULL_PPGTT(dev))
995 0 : invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
996 :
997 0 : for (i = 0; i < count; i++) {
998 0 : char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
999 : int length; /* limited by fault_in_pages_readable() */
1000 :
1001 0 : if (exec[i].flags & invalid_flags)
1002 0 : return -EINVAL;
1003 :
1004 0 : if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
1005 0 : return -EINVAL;
1006 :
1007 : /* First check for malicious input causing overflow in
1008 : * the worst case where we need to allocate the entire
1009 : * relocation tree as a single array.
1010 : */
1011 0 : if (exec[i].relocation_count > relocs_max - relocs_total)
1012 0 : return -EINVAL;
1013 0 : relocs_total += exec[i].relocation_count;
1014 :
1015 0 : length = exec[i].relocation_count *
1016 : sizeof(struct drm_i915_gem_relocation_entry);
1017 : /*
1018 : * We must check that the entire relocation array is safe
1019 : * to read, but since we may need to update the presumed
1020 : * offsets during execution, check for full write access.
1021 : */
1022 0 : if (!access_ok(VERIFY_WRITE, ptr, length))
1023 0 : return -EFAULT;
1024 :
1025 : #ifdef __linux__
1026 : if (likely(!i915.prefault_disable)) {
1027 : if (fault_in_multipages_readable(ptr, length))
1028 : return -EFAULT;
1029 : }
1030 : #endif
1031 0 : }
1032 :
1033 0 : return 0;
1034 0 : }
1035 :
1036 : static struct intel_context *
1037 0 : i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
1038 : struct intel_engine_cs *ring, const u32 ctx_id)
1039 : {
1040 : struct intel_context *ctx = NULL;
1041 : struct i915_ctx_hang_stats *hs;
1042 :
1043 0 : if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
1044 0 : return ERR_PTR(-EINVAL);
1045 :
1046 0 : ctx = i915_gem_context_get(file->driver_priv, ctx_id);
1047 0 : if (IS_ERR(ctx))
1048 0 : return ctx;
1049 :
1050 0 : hs = &ctx->hang_stats;
1051 0 : if (hs->banned) {
1052 : DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
1053 0 : return ERR_PTR(-EIO);
1054 : }
1055 :
1056 0 : if (i915.enable_execlists && !ctx->engine[ring->id].state) {
1057 0 : int ret = intel_lr_context_deferred_alloc(ctx, ring);
1058 0 : if (ret) {
1059 : DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);
1060 0 : return ERR_PTR(ret);
1061 : }
1062 0 : }
1063 :
1064 0 : return ctx;
1065 0 : }
1066 :
1067 : void
1068 0 : i915_gem_execbuffer_move_to_active(struct list_head *vmas,
1069 : struct drm_i915_gem_request *req)
1070 : {
1071 0 : struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1072 : struct i915_vma *vma;
1073 :
1074 0 : list_for_each_entry(vma, vmas, exec_list) {
1075 0 : struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
1076 0 : struct drm_i915_gem_object *obj = vma->obj;
1077 0 : u32 old_read = obj->base.read_domains;
1078 0 : u32 old_write = obj->base.write_domain;
1079 :
1080 0 : obj->dirty = 1; /* be paranoid */
1081 0 : obj->base.write_domain = obj->base.pending_write_domain;
1082 0 : if (obj->base.write_domain == 0)
1083 0 : obj->base.pending_read_domains |= obj->base.read_domains;
1084 0 : obj->base.read_domains = obj->base.pending_read_domains;
1085 :
1086 0 : i915_vma_move_to_active(vma, req);
1087 0 : if (obj->base.write_domain) {
1088 0 : i915_gem_request_assign(&obj->last_write_req, req);
1089 :
1090 0 : intel_fb_obj_invalidate(obj, ORIGIN_CS);
1091 :
1092 : /* update for the implicit flush after a batch */
1093 0 : obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1094 0 : }
1095 0 : if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
1096 0 : i915_gem_request_assign(&obj->last_fenced_req, req);
1097 0 : if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
1098 0 : struct drm_i915_private *dev_priv = to_i915(ring->dev);
1099 0 : list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
1100 0 : &dev_priv->mm.fence_list);
1101 0 : }
1102 : }
1103 :
1104 0 : trace_i915_gem_object_change_domain(obj, old_read, old_write);
1105 : }
1106 0 : }
1107 :
1108 : void
1109 0 : i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
1110 : {
1111 : /* Unconditionally force add_request to emit a full flush. */
1112 0 : params->ring->gpu_caches_dirty = true;
1113 :
1114 : /* Add a breadcrumb for the completion of the batch buffer */
1115 0 : __i915_add_request(params->request, params->batch_obj, true);
1116 0 : }
1117 :
1118 : static int
1119 0 : i915_reset_gen7_sol_offsets(struct drm_device *dev,
1120 : struct drm_i915_gem_request *req)
1121 : {
1122 0 : struct intel_engine_cs *ring = req->ring;
1123 0 : struct drm_i915_private *dev_priv = dev->dev_private;
1124 : int ret, i;
1125 :
1126 0 : if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS]) {
1127 : DRM_DEBUG("sol reset is gen7/rcs only\n");
1128 0 : return -EINVAL;
1129 : }
1130 :
1131 0 : ret = intel_ring_begin(req, 4 * 3);
1132 0 : if (ret)
1133 0 : return ret;
1134 :
1135 0 : for (i = 0; i < 4; i++) {
1136 0 : intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1137 0 : intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
1138 0 : intel_ring_emit(ring, 0);
1139 : }
1140 :
1141 0 : intel_ring_advance(ring);
1142 :
1143 0 : return 0;
1144 0 : }
1145 :
1146 : static struct drm_i915_gem_object*
1147 0 : i915_gem_execbuffer_parse(struct intel_engine_cs *ring,
1148 : struct drm_i915_gem_exec_object2 *shadow_exec_entry,
1149 : struct eb_vmas *eb,
1150 : struct drm_i915_gem_object *batch_obj,
1151 : u32 batch_start_offset,
1152 : u32 batch_len,
1153 : bool is_master)
1154 : {
1155 : struct drm_i915_gem_object *shadow_batch_obj;
1156 : struct i915_vma *vma;
1157 : int ret;
1158 :
1159 0 : shadow_batch_obj = i915_gem_batch_pool_get(&ring->batch_pool,
1160 0 : PAGE_ALIGN(batch_len));
1161 0 : if (IS_ERR(shadow_batch_obj))
1162 0 : return shadow_batch_obj;
1163 :
1164 0 : ret = i915_parse_cmds(ring,
1165 : batch_obj,
1166 : shadow_batch_obj,
1167 : batch_start_offset,
1168 : batch_len,
1169 : is_master);
1170 0 : if (ret)
1171 : goto err;
1172 :
1173 0 : ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
1174 0 : if (ret)
1175 : goto err;
1176 :
1177 0 : i915_gem_object_unpin_pages(shadow_batch_obj);
1178 :
1179 0 : memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
1180 :
1181 0 : vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
1182 0 : vma->exec_entry = shadow_exec_entry;
1183 0 : vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
1184 0 : drm_gem_object_reference(&shadow_batch_obj->base);
1185 0 : list_add_tail(&vma->exec_list, &eb->vmas);
1186 :
1187 0 : shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
1188 :
1189 0 : return shadow_batch_obj;
1190 :
1191 : err:
1192 0 : i915_gem_object_unpin_pages(shadow_batch_obj);
1193 0 : if (ret == -EACCES) /* unhandled chained batch */
1194 0 : return batch_obj;
1195 : else
1196 0 : return ERR_PTR(ret);
1197 0 : }
1198 :
1199 : int
1200 0 : i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
1201 : struct drm_i915_gem_execbuffer2 *args,
1202 : struct list_head *vmas)
1203 : {
1204 0 : struct drm_device *dev = params->dev;
1205 0 : struct intel_engine_cs *ring = params->ring;
1206 0 : struct drm_i915_private *dev_priv = dev->dev_private;
1207 : u64 exec_start, exec_len;
1208 : int instp_mode;
1209 : u32 instp_mask;
1210 : int ret;
1211 :
1212 0 : ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
1213 0 : if (ret)
1214 0 : return ret;
1215 :
1216 0 : ret = i915_switch_context(params->request);
1217 0 : if (ret)
1218 0 : return ret;
1219 :
1220 0 : WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<<ring->id),
1221 : "%s didn't clear reload\n", ring->name);
1222 :
1223 0 : instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1224 : instp_mask = I915_EXEC_CONSTANTS_MASK;
1225 0 : switch (instp_mode) {
1226 : case I915_EXEC_CONSTANTS_REL_GENERAL:
1227 : case I915_EXEC_CONSTANTS_ABSOLUTE:
1228 : case I915_EXEC_CONSTANTS_REL_SURFACE:
1229 0 : if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
1230 : DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
1231 0 : return -EINVAL;
1232 : }
1233 :
1234 0 : if (instp_mode != dev_priv->relative_constants_mode) {
1235 0 : if (INTEL_INFO(dev)->gen < 4) {
1236 : DRM_DEBUG("no rel constants on pre-gen4\n");
1237 0 : return -EINVAL;
1238 : }
1239 :
1240 0 : if (INTEL_INFO(dev)->gen > 5 &&
1241 0 : instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
1242 : DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
1243 0 : return -EINVAL;
1244 : }
1245 :
1246 : /* The HW changed the meaning on this bit on gen6 */
1247 0 : if (INTEL_INFO(dev)->gen >= 6)
1248 0 : instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
1249 : }
1250 : break;
1251 : default:
1252 : DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
1253 0 : return -EINVAL;
1254 : }
1255 :
1256 0 : if (ring == &dev_priv->ring[RCS] &&
1257 0 : instp_mode != dev_priv->relative_constants_mode) {
1258 0 : ret = intel_ring_begin(params->request, 4);
1259 0 : if (ret)
1260 0 : return ret;
1261 :
1262 0 : intel_ring_emit(ring, MI_NOOP);
1263 0 : intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1264 0 : intel_ring_emit(ring, INSTPM);
1265 0 : intel_ring_emit(ring, instp_mask << 16 | instp_mode);
1266 0 : intel_ring_advance(ring);
1267 :
1268 0 : dev_priv->relative_constants_mode = instp_mode;
1269 0 : }
1270 :
1271 0 : if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1272 0 : ret = i915_reset_gen7_sol_offsets(dev, params->request);
1273 0 : if (ret)
1274 0 : return ret;
1275 : }
1276 :
1277 0 : exec_len = args->batch_len;
1278 0 : exec_start = params->batch_obj_vm_offset +
1279 0 : params->args_batch_start_offset;
1280 :
1281 0 : ret = ring->dispatch_execbuffer(params->request,
1282 : exec_start, exec_len,
1283 0 : params->dispatch_flags);
1284 0 : if (ret)
1285 0 : return ret;
1286 :
1287 0 : trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
1288 :
1289 0 : i915_gem_execbuffer_move_to_active(vmas, params->request);
1290 0 : i915_gem_execbuffer_retire_commands(params);
1291 :
1292 0 : return 0;
1293 0 : }
1294 :
1295 : /**
1296 : * Find one BSD ring to dispatch the corresponding BSD command.
1297 : * The Ring ID is returned.
1298 : */
1299 0 : static int gen8_dispatch_bsd_ring(struct drm_device *dev,
1300 : struct drm_file *file)
1301 : {
1302 0 : struct drm_i915_private *dev_priv = dev->dev_private;
1303 0 : struct drm_i915_file_private *file_priv = file->driver_priv;
1304 :
1305 : /* Check whether the file_priv is using one ring */
1306 0 : if (file_priv->bsd_ring)
1307 0 : return file_priv->bsd_ring->id;
1308 : else {
1309 : /* If no, use the ping-pong mechanism to select one ring */
1310 : int ring_id;
1311 :
1312 0 : mutex_lock(&dev->struct_mutex);
1313 0 : if (dev_priv->mm.bsd_ring_dispatch_index == 0) {
1314 : ring_id = VCS;
1315 0 : dev_priv->mm.bsd_ring_dispatch_index = 1;
1316 0 : } else {
1317 : ring_id = VCS2;
1318 0 : dev_priv->mm.bsd_ring_dispatch_index = 0;
1319 : }
1320 0 : file_priv->bsd_ring = &dev_priv->ring[ring_id];
1321 0 : mutex_unlock(&dev->struct_mutex);
1322 : return ring_id;
1323 : }
1324 0 : }
1325 :
1326 : static struct drm_i915_gem_object *
1327 0 : eb_get_batch(struct eb_vmas *eb)
1328 : {
1329 0 : struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
1330 :
1331 : /*
1332 : * SNA is doing fancy tricks with compressing batch buffers, which leads
1333 : * to negative relocation deltas. Usually that works out ok since the
1334 : * relocate address is still positive, except when the batch is placed
1335 : * very low in the GTT. Ensure this doesn't happen.
1336 : *
1337 : * Note that actual hangs have only been observed on gen7, but for
1338 : * paranoia do it everywhere.
1339 : */
1340 0 : vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
1341 :
1342 0 : return vma->obj;
1343 : }
1344 :
1345 : static int
1346 0 : i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1347 : struct drm_file *file,
1348 : struct drm_i915_gem_execbuffer2 *args,
1349 : struct drm_i915_gem_exec_object2 *exec)
1350 : {
1351 0 : struct drm_i915_private *dev_priv = dev->dev_private;
1352 : struct eb_vmas *eb;
1353 : struct drm_i915_gem_object *batch_obj;
1354 0 : struct drm_i915_gem_exec_object2 shadow_exec_entry;
1355 : struct intel_engine_cs *ring;
1356 : struct intel_context *ctx;
1357 : struct i915_address_space *vm;
1358 0 : struct i915_execbuffer_params params_master; /* XXX: will be removed later */
1359 : struct i915_execbuffer_params *params = ¶ms_master;
1360 0 : const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
1361 : u32 dispatch_flags;
1362 : int ret;
1363 0 : bool need_relocs;
1364 :
1365 0 : if (!i915_gem_check_execbuffer(args))
1366 0 : return -EINVAL;
1367 :
1368 0 : ret = validate_exec_list(dev, exec, args->buffer_count);
1369 0 : if (ret)
1370 0 : return ret;
1371 :
1372 : dispatch_flags = 0;
1373 0 : if (args->flags & I915_EXEC_SECURE) {
1374 0 : if (!file->is_master || !capable(CAP_SYS_ADMIN))
1375 0 : return -EPERM;
1376 :
1377 : dispatch_flags |= I915_DISPATCH_SECURE;
1378 0 : }
1379 0 : if (args->flags & I915_EXEC_IS_PINNED)
1380 0 : dispatch_flags |= I915_DISPATCH_PINNED;
1381 :
1382 0 : if ((args->flags & I915_EXEC_RING_MASK) > LAST_USER_RING) {
1383 : DRM_DEBUG("execbuf with unknown ring: %d\n",
1384 : (int)(args->flags & I915_EXEC_RING_MASK));
1385 0 : return -EINVAL;
1386 : }
1387 :
1388 0 : if (((args->flags & I915_EXEC_RING_MASK) != I915_EXEC_BSD) &&
1389 0 : ((args->flags & I915_EXEC_BSD_MASK) != 0)) {
1390 : DRM_DEBUG("execbuf with non bsd ring but with invalid "
1391 : "bsd dispatch flags: %d\n", (int)(args->flags));
1392 0 : return -EINVAL;
1393 : }
1394 :
1395 0 : if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT)
1396 0 : ring = &dev_priv->ring[RCS];
1397 0 : else if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_BSD) {
1398 0 : if (HAS_BSD2(dev)) {
1399 : int ring_id;
1400 :
1401 0 : switch (args->flags & I915_EXEC_BSD_MASK) {
1402 : case I915_EXEC_BSD_DEFAULT:
1403 0 : ring_id = gen8_dispatch_bsd_ring(dev, file);
1404 0 : ring = &dev_priv->ring[ring_id];
1405 0 : break;
1406 : case I915_EXEC_BSD_RING1:
1407 0 : ring = &dev_priv->ring[VCS];
1408 0 : break;
1409 : case I915_EXEC_BSD_RING2:
1410 0 : ring = &dev_priv->ring[VCS2];
1411 0 : break;
1412 : default:
1413 : DRM_DEBUG("execbuf with unknown bsd ring: %d\n",
1414 : (int)(args->flags & I915_EXEC_BSD_MASK));
1415 0 : return -EINVAL;
1416 : }
1417 0 : } else
1418 0 : ring = &dev_priv->ring[VCS];
1419 : } else
1420 0 : ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1];
1421 :
1422 0 : if (!intel_ring_initialized(ring)) {
1423 : DRM_DEBUG("execbuf with invalid ring: %d\n",
1424 : (int)(args->flags & I915_EXEC_RING_MASK));
1425 0 : return -EINVAL;
1426 : }
1427 :
1428 0 : if (args->buffer_count < 1) {
1429 : DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1430 0 : return -EINVAL;
1431 : }
1432 :
1433 0 : if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
1434 0 : if (!HAS_RESOURCE_STREAMER(dev)) {
1435 : DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
1436 0 : return -EINVAL;
1437 : }
1438 0 : if (ring->id != RCS) {
1439 : DRM_DEBUG("RS is not available on %s\n",
1440 : ring->name);
1441 0 : return -EINVAL;
1442 : }
1443 :
1444 0 : dispatch_flags |= I915_DISPATCH_RS;
1445 0 : }
1446 :
1447 0 : intel_runtime_pm_get(dev_priv);
1448 :
1449 0 : ret = i915_mutex_lock_interruptible(dev);
1450 0 : if (ret)
1451 : goto pre_mutex_err;
1452 :
1453 0 : ctx = i915_gem_validate_context(dev, file, ring, ctx_id);
1454 0 : if (IS_ERR(ctx)) {
1455 0 : mutex_unlock(&dev->struct_mutex);
1456 0 : ret = PTR_ERR(ctx);
1457 0 : goto pre_mutex_err;
1458 : }
1459 :
1460 0 : i915_gem_context_reference(ctx);
1461 :
1462 0 : if (ctx->ppgtt)
1463 0 : vm = &ctx->ppgtt->base;
1464 : else
1465 0 : vm = &dev_priv->gtt.base;
1466 :
1467 0 : memset(¶ms_master, 0x00, sizeof(params_master));
1468 :
1469 0 : eb = eb_create(args);
1470 0 : if (eb == NULL) {
1471 0 : i915_gem_context_unreference(ctx);
1472 0 : mutex_unlock(&dev->struct_mutex);
1473 : ret = -ENOMEM;
1474 0 : goto pre_mutex_err;
1475 : }
1476 :
1477 : /* Look up object handles */
1478 0 : ret = eb_lookup_vmas(eb, exec, args, vm, file);
1479 0 : if (ret)
1480 : goto err;
1481 :
1482 : /* take note of the batch buffer before we might reorder the lists */
1483 0 : batch_obj = eb_get_batch(eb);
1484 :
1485 : /* Move the objects en-masse into the GTT, evicting if necessary. */
1486 0 : need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1487 0 : ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
1488 0 : if (ret)
1489 : goto err;
1490 :
1491 : /* The objects are in their final locations, apply the relocations. */
1492 0 : if (need_relocs)
1493 0 : ret = i915_gem_execbuffer_relocate(eb);
1494 0 : if (ret) {
1495 0 : if (ret == -EFAULT) {
1496 0 : ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
1497 : eb, exec, ctx);
1498 0 : BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1499 : }
1500 0 : if (ret)
1501 : goto err;
1502 : }
1503 :
1504 : /* Set the pending read domains for the batch buffer to COMMAND */
1505 0 : if (batch_obj->base.pending_write_domain) {
1506 : DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1507 : ret = -EINVAL;
1508 0 : goto err;
1509 : }
1510 :
1511 0 : params->args_batch_start_offset = args->batch_start_offset;
1512 0 : if (i915_needs_cmd_parser(ring) && args->batch_len) {
1513 : struct drm_i915_gem_object *parsed_batch_obj;
1514 :
1515 0 : parsed_batch_obj = i915_gem_execbuffer_parse(ring,
1516 : &shadow_exec_entry,
1517 : eb,
1518 : batch_obj,
1519 0 : args->batch_start_offset,
1520 : args->batch_len,
1521 0 : file->is_master);
1522 0 : if (IS_ERR(parsed_batch_obj)) {
1523 0 : ret = PTR_ERR(parsed_batch_obj);
1524 0 : goto err;
1525 : }
1526 :
1527 : /*
1528 : * parsed_batch_obj == batch_obj means batch not fully parsed:
1529 : * Accept, but don't promote to secure.
1530 : */
1531 :
1532 0 : if (parsed_batch_obj != batch_obj) {
1533 : /*
1534 : * Batch parsed and accepted:
1535 : *
1536 : * Set the DISPATCH_SECURE bit to remove the NON_SECURE
1537 : * bit from MI_BATCH_BUFFER_START commands issued in
1538 : * the dispatch_execbuffer implementations. We
1539 : * specifically don't want that set on batches the
1540 : * command parser has accepted.
1541 : */
1542 0 : dispatch_flags |= I915_DISPATCH_SECURE;
1543 0 : params->args_batch_start_offset = 0;
1544 : batch_obj = parsed_batch_obj;
1545 0 : }
1546 0 : }
1547 :
1548 0 : batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1549 :
1550 : /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
1551 : * batch" bit. Hence we need to pin secure batches into the global gtt.
1552 : * hsw should have this fixed, but bdw mucks it up again. */
1553 0 : if (dispatch_flags & I915_DISPATCH_SECURE) {
1554 : /*
1555 : * So on first glance it looks freaky that we pin the batch here
1556 : * outside of the reservation loop. But:
1557 : * - The batch is already pinned into the relevant ppgtt, so we
1558 : * already have the backing storage fully allocated.
1559 : * - No other BO uses the global gtt (well contexts, but meh),
1560 : * so we don't really have issues with multiple objects not
1561 : * fitting due to fragmentation.
1562 : * So this is actually safe.
1563 : */
1564 0 : ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
1565 0 : if (ret)
1566 : goto err;
1567 :
1568 0 : params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
1569 0 : } else
1570 0 : params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
1571 :
1572 : /* Allocate a request for this batch buffer nice and early. */
1573 0 : ret = i915_gem_request_alloc(ring, ctx, ¶ms->request);
1574 0 : if (ret)
1575 : goto err_batch_unpin;
1576 :
1577 0 : ret = i915_gem_request_add_to_client(params->request, file);
1578 0 : if (ret)
1579 : goto err_batch_unpin;
1580 :
1581 : /*
1582 : * Save assorted stuff away to pass through to *_submission().
1583 : * NB: This data should be 'persistent' and not local as it will
1584 : * kept around beyond the duration of the IOCTL once the GPU
1585 : * scheduler arrives.
1586 : */
1587 0 : params->dev = dev;
1588 0 : params->file = file;
1589 0 : params->ring = ring;
1590 0 : params->dispatch_flags = dispatch_flags;
1591 0 : params->batch_obj = batch_obj;
1592 0 : params->ctx = ctx;
1593 :
1594 0 : ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
1595 :
1596 : err_batch_unpin:
1597 : /*
1598 : * FIXME: We crucially rely upon the active tracking for the (ppgtt)
1599 : * batch vma for correctness. For less ugly and less fragility this
1600 : * needs to be adjusted to also track the ggtt batch vma properly as
1601 : * active.
1602 : */
1603 0 : if (dispatch_flags & I915_DISPATCH_SECURE)
1604 0 : i915_gem_object_ggtt_unpin(batch_obj);
1605 :
1606 : err:
1607 : /* the request owns the ref now */
1608 0 : i915_gem_context_unreference(ctx);
1609 0 : eb_destroy(eb);
1610 :
1611 : /*
1612 : * If the request was created but not successfully submitted then it
1613 : * must be freed again. If it was submitted then it is being tracked
1614 : * on the active request list and no clean up is required here.
1615 : */
1616 0 : if (ret && params->request)
1617 0 : i915_gem_request_cancel(params->request);
1618 :
1619 0 : mutex_unlock(&dev->struct_mutex);
1620 :
1621 : pre_mutex_err:
1622 : /* intel_gpu_busy should also get a ref, so it will free when the device
1623 : * is really idle. */
1624 0 : intel_runtime_pm_put(dev_priv);
1625 0 : return ret;
1626 0 : }
1627 :
1628 : #ifdef __linux__
1629 : /*
1630 : * Legacy execbuffer just creates an exec2 list from the original exec object
1631 : * list array and passes it to the real function.
1632 : */
1633 : int
1634 : i915_gem_execbuffer(struct drm_device *dev, void *data,
1635 : struct drm_file *file)
1636 : {
1637 : struct drm_i915_gem_execbuffer *args = data;
1638 : struct drm_i915_gem_execbuffer2 exec2;
1639 : struct drm_i915_gem_exec_object *exec_list = NULL;
1640 : struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1641 : int ret, i;
1642 :
1643 : if (args->buffer_count < 1) {
1644 : DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1645 : return -EINVAL;
1646 : }
1647 :
1648 : /* Copy in the exec list from userland */
1649 : exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1650 : exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1651 : if (exec_list == NULL || exec2_list == NULL) {
1652 : DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1653 : args->buffer_count);
1654 : drm_free_large(exec_list);
1655 : drm_free_large(exec2_list);
1656 : return -ENOMEM;
1657 : }
1658 : ret = copy_from_user(exec_list,
1659 : to_user_ptr(args->buffers_ptr),
1660 : sizeof(*exec_list) * args->buffer_count);
1661 : if (ret != 0) {
1662 : DRM_DEBUG("copy %d exec entries failed %d\n",
1663 : args->buffer_count, ret);
1664 : drm_free_large(exec_list);
1665 : drm_free_large(exec2_list);
1666 : return -EFAULT;
1667 : }
1668 :
1669 : for (i = 0; i < args->buffer_count; i++) {
1670 : exec2_list[i].handle = exec_list[i].handle;
1671 : exec2_list[i].relocation_count = exec_list[i].relocation_count;
1672 : exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1673 : exec2_list[i].alignment = exec_list[i].alignment;
1674 : exec2_list[i].offset = exec_list[i].offset;
1675 : if (INTEL_INFO(dev)->gen < 4)
1676 : exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1677 : else
1678 : exec2_list[i].flags = 0;
1679 : }
1680 :
1681 : exec2.buffers_ptr = args->buffers_ptr;
1682 : exec2.buffer_count = args->buffer_count;
1683 : exec2.batch_start_offset = args->batch_start_offset;
1684 : exec2.batch_len = args->batch_len;
1685 : exec2.DR1 = args->DR1;
1686 : exec2.DR4 = args->DR4;
1687 : exec2.num_cliprects = args->num_cliprects;
1688 : exec2.cliprects_ptr = args->cliprects_ptr;
1689 : exec2.flags = I915_EXEC_RENDER;
1690 : i915_execbuffer2_set_context_id(exec2, 0);
1691 :
1692 : ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1693 : if (!ret) {
1694 : struct drm_i915_gem_exec_object __user *user_exec_list =
1695 : to_user_ptr(args->buffers_ptr);
1696 :
1697 : /* Copy the new buffer offsets back to the user's exec list. */
1698 : for (i = 0; i < args->buffer_count; i++) {
1699 : ret = __copy_to_user(&user_exec_list[i].offset,
1700 : &exec2_list[i].offset,
1701 : sizeof(user_exec_list[i].offset));
1702 : if (ret) {
1703 : ret = -EFAULT;
1704 : DRM_DEBUG("failed to copy %d exec entries "
1705 : "back to user (%d)\n",
1706 : args->buffer_count, ret);
1707 : break;
1708 : }
1709 : }
1710 : }
1711 :
1712 : drm_free_large(exec_list);
1713 : drm_free_large(exec2_list);
1714 : return ret;
1715 : }
1716 : #endif /* __linux__ */
1717 :
1718 : int
1719 0 : i915_gem_execbuffer2(struct drm_device *dev, void *data,
1720 : struct drm_file *file)
1721 : {
1722 0 : struct drm_i915_gem_execbuffer2 *args = data;
1723 : struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1724 : int ret;
1725 :
1726 0 : if (args->buffer_count < 1 ||
1727 0 : args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1728 : DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1729 0 : return -EINVAL;
1730 : }
1731 :
1732 0 : if (args->rsvd2 != 0) {
1733 : DRM_DEBUG("dirty rvsd2 field\n");
1734 0 : return -EINVAL;
1735 : }
1736 :
1737 0 : exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
1738 : GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
1739 0 : if (exec2_list == NULL)
1740 0 : exec2_list = drm_malloc_ab(sizeof(*exec2_list),
1741 0 : args->buffer_count);
1742 0 : if (exec2_list == NULL) {
1743 : DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1744 : args->buffer_count);
1745 0 : return -ENOMEM;
1746 : }
1747 0 : ret = copy_from_user(exec2_list,
1748 0 : to_user_ptr(args->buffers_ptr),
1749 0 : sizeof(*exec2_list) * args->buffer_count);
1750 0 : if (ret != 0) {
1751 : DRM_DEBUG("copy %d exec entries failed %d\n",
1752 : args->buffer_count, ret);
1753 0 : drm_free_large(exec2_list);
1754 0 : return -EFAULT;
1755 : }
1756 :
1757 0 : ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1758 0 : if (!ret) {
1759 : /* Copy the new buffer offsets back to the user's exec list. */
1760 : struct drm_i915_gem_exec_object2 __user *user_exec_list =
1761 0 : to_user_ptr(args->buffers_ptr);
1762 : int i;
1763 :
1764 0 : for (i = 0; i < args->buffer_count; i++) {
1765 0 : ret = __copy_to_user(&user_exec_list[i].offset,
1766 0 : &exec2_list[i].offset,
1767 : sizeof(user_exec_list[i].offset));
1768 0 : if (ret) {
1769 : ret = -EFAULT;
1770 : DRM_DEBUG("failed to copy %d exec entries "
1771 : "back to user\n",
1772 : args->buffer_count);
1773 0 : break;
1774 : }
1775 : }
1776 0 : }
1777 :
1778 0 : drm_free_large(exec2_list);
1779 0 : return ret;
1780 0 : }
|
