Line data Source code
1 : /*
2 : * Copyright 2011 Red Hat Inc.
3 : * All Rights Reserved.
4 : *
5 : * Permission is hereby granted, free of charge, to any person obtaining a
6 : * copy of this software and associated documentation files (the
7 : * "Software"), to deal in the Software without restriction, including
8 : * without limitation the rights to use, copy, modify, merge, publish,
9 : * distribute, sub license, and/or sell copies of the Software, and to
10 : * permit persons to whom the Software is furnished to do so, subject to
11 : * the following conditions:
12 : *
13 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 : * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 : * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 : * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 : * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 : * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 : *
21 : * The above copyright notice and this permission notice (including the
22 : * next paragraph) shall be included in all copies or substantial portions
23 : * of the Software.
24 : *
25 : */
26 : /*
27 : * Authors:
28 : * Jerome Glisse <glisse@freedesktop.org>
29 : */
30 : /* Algorithm:
31 : *
32 : * We store the last allocated bo in "hole", we always try to allocate
33 : * after the last allocated bo. Principle is that in a linear GPU ring
34 : * progression was is after last is the oldest bo we allocated and thus
35 : * the first one that should no longer be in use by the GPU.
36 : *
37 : * If it's not the case we skip over the bo after last to the closest
38 : * done bo if such one exist. If none exist and we are not asked to
39 : * block we report failure to allocate.
40 : *
41 : * If we are asked to block we wait on all the oldest fence of all
42 : * rings. We just wait for any of those fence to complete.
43 : */
44 : #include <dev/pci/drm/drmP.h>
45 : #include "radeon.h"
46 :
47 : static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo);
48 : static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager);
49 :
50 0 : int radeon_sa_bo_manager_init(struct radeon_device *rdev,
51 : struct radeon_sa_manager *sa_manager,
52 : unsigned size, u32 align, u32 domain, u32 flags)
53 : {
54 : int i, r;
55 :
56 0 : init_waitqueue_head(&sa_manager->wq);
57 0 : sa_manager->bo = NULL;
58 0 : sa_manager->size = size;
59 0 : sa_manager->domain = domain;
60 0 : sa_manager->align = align;
61 0 : sa_manager->hole = &sa_manager->olist;
62 0 : INIT_LIST_HEAD(&sa_manager->olist);
63 0 : for (i = 0; i < RADEON_NUM_RINGS; ++i) {
64 0 : INIT_LIST_HEAD(&sa_manager->flist[i]);
65 : }
66 :
67 0 : r = radeon_bo_create(rdev, size, align, true,
68 : domain, flags, NULL, NULL, &sa_manager->bo);
69 0 : if (r) {
70 0 : dev_err(rdev->dev, "(%d) failed to allocate bo for manager\n", r);
71 0 : return r;
72 : }
73 :
74 0 : return r;
75 0 : }
76 :
77 0 : void radeon_sa_bo_manager_fini(struct radeon_device *rdev,
78 : struct radeon_sa_manager *sa_manager)
79 : {
80 : struct radeon_sa_bo *sa_bo, *tmp;
81 :
82 0 : if (!list_empty(&sa_manager->olist)) {
83 0 : sa_manager->hole = &sa_manager->olist,
84 0 : radeon_sa_bo_try_free(sa_manager);
85 0 : if (!list_empty(&sa_manager->olist)) {
86 0 : dev_err(rdev->dev, "sa_manager is not empty, clearing anyway\n");
87 0 : }
88 : }
89 0 : list_for_each_entry_safe(sa_bo, tmp, &sa_manager->olist, olist) {
90 0 : radeon_sa_bo_remove_locked(sa_bo);
91 : }
92 0 : radeon_bo_unref(&sa_manager->bo);
93 0 : sa_manager->size = 0;
94 0 : }
95 :
96 0 : int radeon_sa_bo_manager_start(struct radeon_device *rdev,
97 : struct radeon_sa_manager *sa_manager)
98 : {
99 : int r;
100 :
101 0 : if (sa_manager->bo == NULL) {
102 0 : dev_err(rdev->dev, "no bo for sa manager\n");
103 0 : return -EINVAL;
104 : }
105 :
106 : /* map the buffer */
107 0 : r = radeon_bo_reserve(sa_manager->bo, false);
108 0 : if (r) {
109 0 : dev_err(rdev->dev, "(%d) failed to reserve manager bo\n", r);
110 0 : return r;
111 : }
112 0 : r = radeon_bo_pin(sa_manager->bo, sa_manager->domain, &sa_manager->gpu_addr);
113 0 : if (r) {
114 0 : radeon_bo_unreserve(sa_manager->bo);
115 0 : dev_err(rdev->dev, "(%d) failed to pin manager bo\n", r);
116 0 : return r;
117 : }
118 0 : r = radeon_bo_kmap(sa_manager->bo, &sa_manager->cpu_ptr);
119 0 : radeon_bo_unreserve(sa_manager->bo);
120 0 : return r;
121 0 : }
122 :
123 0 : int radeon_sa_bo_manager_suspend(struct radeon_device *rdev,
124 : struct radeon_sa_manager *sa_manager)
125 : {
126 : int r;
127 :
128 0 : if (sa_manager->bo == NULL) {
129 0 : dev_err(rdev->dev, "no bo for sa manager\n");
130 0 : return -EINVAL;
131 : }
132 :
133 0 : r = radeon_bo_reserve(sa_manager->bo, false);
134 0 : if (!r) {
135 0 : radeon_bo_kunmap(sa_manager->bo);
136 0 : radeon_bo_unpin(sa_manager->bo);
137 0 : radeon_bo_unreserve(sa_manager->bo);
138 0 : }
139 0 : return r;
140 0 : }
141 :
142 0 : static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo)
143 : {
144 0 : struct radeon_sa_manager *sa_manager = sa_bo->manager;
145 0 : if (sa_manager->hole == &sa_bo->olist) {
146 0 : sa_manager->hole = sa_bo->olist.prev;
147 0 : }
148 0 : list_del_init(&sa_bo->olist);
149 0 : list_del_init(&sa_bo->flist);
150 0 : radeon_fence_unref(&sa_bo->fence);
151 0 : kfree(sa_bo);
152 0 : }
153 :
154 0 : static void radeon_sa_bo_try_free(struct radeon_sa_manager *sa_manager)
155 : {
156 : struct radeon_sa_bo *sa_bo, *tmp;
157 :
158 0 : if (sa_manager->hole->next == &sa_manager->olist)
159 0 : return;
160 :
161 0 : sa_bo = list_entry(sa_manager->hole->next, struct radeon_sa_bo, olist);
162 0 : list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
163 0 : if (sa_bo->fence == NULL || !radeon_fence_signaled(sa_bo->fence)) {
164 0 : return;
165 : }
166 0 : radeon_sa_bo_remove_locked(sa_bo);
167 : }
168 0 : }
169 :
170 0 : static inline unsigned radeon_sa_bo_hole_soffset(struct radeon_sa_manager *sa_manager)
171 : {
172 0 : struct list_head *hole = sa_manager->hole;
173 :
174 0 : if (hole != &sa_manager->olist) {
175 0 : return list_entry(hole, struct radeon_sa_bo, olist)->eoffset;
176 : }
177 0 : return 0;
178 0 : }
179 :
180 0 : static inline unsigned radeon_sa_bo_hole_eoffset(struct radeon_sa_manager *sa_manager)
181 : {
182 0 : struct list_head *hole = sa_manager->hole;
183 :
184 0 : if (hole->next != &sa_manager->olist) {
185 0 : return list_entry(hole->next, struct radeon_sa_bo, olist)->soffset;
186 : }
187 0 : return sa_manager->size;
188 0 : }
189 :
190 0 : static bool radeon_sa_bo_try_alloc(struct radeon_sa_manager *sa_manager,
191 : struct radeon_sa_bo *sa_bo,
192 : unsigned size, unsigned align)
193 : {
194 : unsigned soffset, eoffset, wasted;
195 :
196 0 : soffset = radeon_sa_bo_hole_soffset(sa_manager);
197 0 : eoffset = radeon_sa_bo_hole_eoffset(sa_manager);
198 0 : wasted = (align - (soffset % align)) % align;
199 :
200 0 : if ((eoffset - soffset) >= (size + wasted)) {
201 0 : soffset += wasted;
202 :
203 0 : sa_bo->manager = sa_manager;
204 0 : sa_bo->soffset = soffset;
205 0 : sa_bo->eoffset = soffset + size;
206 0 : list_add(&sa_bo->olist, sa_manager->hole);
207 0 : INIT_LIST_HEAD(&sa_bo->flist);
208 0 : sa_manager->hole = &sa_bo->olist;
209 0 : return true;
210 : }
211 0 : return false;
212 0 : }
213 :
214 : /**
215 : * radeon_sa_event - Check if we can stop waiting
216 : *
217 : * @sa_manager: pointer to the sa_manager
218 : * @size: number of bytes we want to allocate
219 : * @align: alignment we need to match
220 : *
221 : * Check if either there is a fence we can wait for or
222 : * enough free memory to satisfy the allocation directly
223 : */
224 0 : static bool radeon_sa_event(struct radeon_sa_manager *sa_manager,
225 : unsigned size, unsigned align)
226 : {
227 : unsigned soffset, eoffset, wasted;
228 : int i;
229 :
230 0 : for (i = 0; i < RADEON_NUM_RINGS; ++i) {
231 0 : if (!list_empty(&sa_manager->flist[i])) {
232 0 : return true;
233 : }
234 : }
235 :
236 0 : soffset = radeon_sa_bo_hole_soffset(sa_manager);
237 0 : eoffset = radeon_sa_bo_hole_eoffset(sa_manager);
238 0 : wasted = (align - (soffset % align)) % align;
239 :
240 0 : if ((eoffset - soffset) >= (size + wasted)) {
241 0 : return true;
242 : }
243 :
244 0 : return false;
245 0 : }
246 :
247 0 : static bool radeon_sa_bo_next_hole(struct radeon_sa_manager *sa_manager,
248 : struct radeon_fence **fences,
249 : unsigned *tries)
250 : {
251 : struct radeon_sa_bo *best_bo = NULL;
252 : unsigned i, soffset, best, tmp;
253 :
254 : /* if hole points to the end of the buffer */
255 0 : if (sa_manager->hole->next == &sa_manager->olist) {
256 : /* try again with its beginning */
257 0 : sa_manager->hole = &sa_manager->olist;
258 0 : return true;
259 : }
260 :
261 0 : soffset = radeon_sa_bo_hole_soffset(sa_manager);
262 : /* to handle wrap around we add sa_manager->size */
263 0 : best = sa_manager->size * 2;
264 : /* go over all fence list and try to find the closest sa_bo
265 : * of the current last
266 : */
267 0 : for (i = 0; i < RADEON_NUM_RINGS; ++i) {
268 : struct radeon_sa_bo *sa_bo;
269 :
270 0 : if (list_empty(&sa_manager->flist[i])) {
271 0 : continue;
272 : }
273 :
274 0 : sa_bo = list_first_entry(&sa_manager->flist[i],
275 : struct radeon_sa_bo, flist);
276 :
277 0 : if (!radeon_fence_signaled(sa_bo->fence)) {
278 0 : fences[i] = sa_bo->fence;
279 0 : continue;
280 : }
281 :
282 : /* limit the number of tries each ring gets */
283 0 : if (tries[i] > 2) {
284 0 : continue;
285 : }
286 :
287 0 : tmp = sa_bo->soffset;
288 0 : if (tmp < soffset) {
289 : /* wrap around, pretend it's after */
290 0 : tmp += sa_manager->size;
291 0 : }
292 0 : tmp -= soffset;
293 0 : if (tmp < best) {
294 : /* this sa bo is the closest one */
295 : best = tmp;
296 : best_bo = sa_bo;
297 0 : }
298 0 : }
299 :
300 0 : if (best_bo) {
301 0 : ++tries[best_bo->fence->ring];
302 0 : sa_manager->hole = best_bo->olist.prev;
303 :
304 : /* we knew that this one is signaled,
305 : so it's save to remote it */
306 0 : radeon_sa_bo_remove_locked(best_bo);
307 0 : return true;
308 : }
309 0 : return false;
310 0 : }
311 :
312 0 : int radeon_sa_bo_new(struct radeon_device *rdev,
313 : struct radeon_sa_manager *sa_manager,
314 : struct radeon_sa_bo **sa_bo,
315 : unsigned size, unsigned align)
316 : {
317 0 : struct radeon_fence *fences[RADEON_NUM_RINGS];
318 0 : unsigned tries[RADEON_NUM_RINGS];
319 : int i, r;
320 :
321 0 : BUG_ON(align > sa_manager->align);
322 0 : BUG_ON(size > sa_manager->size);
323 :
324 0 : *sa_bo = kmalloc(sizeof(struct radeon_sa_bo), GFP_KERNEL);
325 0 : if ((*sa_bo) == NULL) {
326 0 : return -ENOMEM;
327 : }
328 0 : (*sa_bo)->manager = sa_manager;
329 0 : (*sa_bo)->fence = NULL;
330 0 : INIT_LIST_HEAD(&(*sa_bo)->olist);
331 0 : INIT_LIST_HEAD(&(*sa_bo)->flist);
332 :
333 0 : spin_lock(&sa_manager->wq.lock);
334 0 : do {
335 0 : for (i = 0; i < RADEON_NUM_RINGS; ++i) {
336 0 : fences[i] = NULL;
337 0 : tries[i] = 0;
338 : }
339 :
340 0 : do {
341 0 : radeon_sa_bo_try_free(sa_manager);
342 :
343 0 : if (radeon_sa_bo_try_alloc(sa_manager, *sa_bo,
344 : size, align)) {
345 0 : spin_unlock(&sa_manager->wq.lock);
346 0 : return 0;
347 : }
348 :
349 : /* see if we can skip over some allocations */
350 0 : } while (radeon_sa_bo_next_hole(sa_manager, fences, tries));
351 :
352 0 : for (i = 0; i < RADEON_NUM_RINGS; ++i)
353 0 : radeon_fence_ref(fences[i]);
354 :
355 0 : spin_unlock(&sa_manager->wq.lock);
356 0 : r = radeon_fence_wait_any(rdev, fences, false);
357 0 : for (i = 0; i < RADEON_NUM_RINGS; ++i)
358 0 : radeon_fence_unref(&fences[i]);
359 0 : spin_lock(&sa_manager->wq.lock);
360 : /* if we have nothing to wait for block */
361 0 : if (r == -ENOENT) {
362 0 : r = wait_event_interruptible_locked(
363 : sa_manager->wq,
364 : radeon_sa_event(sa_manager, size, align)
365 : );
366 0 : }
367 :
368 0 : } while (!r);
369 :
370 0 : spin_unlock(&sa_manager->wq.lock);
371 0 : kfree(*sa_bo);
372 0 : *sa_bo = NULL;
373 0 : return r;
374 0 : }
375 :
376 0 : void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **sa_bo,
377 : struct radeon_fence *fence)
378 : {
379 : struct radeon_sa_manager *sa_manager;
380 :
381 0 : if (sa_bo == NULL || *sa_bo == NULL) {
382 0 : return;
383 : }
384 :
385 0 : sa_manager = (*sa_bo)->manager;
386 0 : spin_lock(&sa_manager->wq.lock);
387 0 : if (fence && !radeon_fence_signaled(fence)) {
388 0 : (*sa_bo)->fence = radeon_fence_ref(fence);
389 0 : list_add_tail(&(*sa_bo)->flist,
390 0 : &sa_manager->flist[fence->ring]);
391 0 : } else {
392 0 : radeon_sa_bo_remove_locked(*sa_bo);
393 : }
394 0 : wake_up_all_locked(&sa_manager->wq);
395 0 : spin_unlock(&sa_manager->wq.lock);
396 0 : *sa_bo = NULL;
397 0 : }
398 :
399 : #if defined(CONFIG_DEBUG_FS)
400 : void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager,
401 : struct seq_file *m)
402 : {
403 : struct radeon_sa_bo *i;
404 :
405 : spin_lock(&sa_manager->wq.lock);
406 : list_for_each_entry(i, &sa_manager->olist, olist) {
407 : uint64_t soffset = i->soffset + sa_manager->gpu_addr;
408 : uint64_t eoffset = i->eoffset + sa_manager->gpu_addr;
409 : if (&i->olist == sa_manager->hole) {
410 : seq_printf(m, ">");
411 : } else {
412 : seq_printf(m, " ");
413 : }
414 : seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
415 : soffset, eoffset, eoffset - soffset);
416 : if (i->fence) {
417 : seq_printf(m, " protected by 0x%016llx on ring %d",
418 : i->fence->seq, i->fence->ring);
419 : }
420 : seq_printf(m, "\n");
421 : }
422 : spin_unlock(&sa_manager->wq.lock);
423 : }
424 : #endif
|
