Line data Source code
1 : /*
2 : * Copyright 2011 Advanced Micro Devices, Inc.
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a
5 : * copy of this software and associated documentation files (the "Software"),
6 : * to deal in the Software without restriction, including without limitation
7 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : * and/or sell copies of the Software, and to permit persons to whom the
9 : * Software is furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice shall be included in
12 : * all copies or substantial portions of the Software.
13 : *
14 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 : * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 : * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 : * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 : * OTHER DEALINGS IN THE SOFTWARE.
21 : *
22 : * Authors: Alex Deucher
23 : */
24 : #include <dev/pci/drm/drmP.h>
25 : #include "radeon.h"
26 : #include "radeon_asic.h"
27 : #include "radeon_audio.h"
28 : #include <dev/pci/drm/radeon_drm.h>
29 : #include "sid.h"
30 : #include "atom.h"
31 : #include "si_blit_shaders.h"
32 : #include "clearstate_si.h"
33 : #include "radeon_ucode.h"
34 :
35 :
36 : MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
37 : MODULE_FIRMWARE("radeon/TAHITI_me.bin");
38 : MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
39 : MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
40 : MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
41 : MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
42 : MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
43 :
44 : MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
45 : MODULE_FIRMWARE("radeon/tahiti_me.bin");
46 : MODULE_FIRMWARE("radeon/tahiti_ce.bin");
47 : MODULE_FIRMWARE("radeon/tahiti_mc.bin");
48 : MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
49 : MODULE_FIRMWARE("radeon/tahiti_smc.bin");
50 :
51 : MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
52 : MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
53 : MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
54 : MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
55 : MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
56 : MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
57 : MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
58 :
59 : MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
60 : MODULE_FIRMWARE("radeon/pitcairn_me.bin");
61 : MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
62 : MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
63 : MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
64 : MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
65 :
66 : MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
67 : MODULE_FIRMWARE("radeon/VERDE_me.bin");
68 : MODULE_FIRMWARE("radeon/VERDE_ce.bin");
69 : MODULE_FIRMWARE("radeon/VERDE_mc.bin");
70 : MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
71 : MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
72 : MODULE_FIRMWARE("radeon/VERDE_smc.bin");
73 :
74 : MODULE_FIRMWARE("radeon/verde_pfp.bin");
75 : MODULE_FIRMWARE("radeon/verde_me.bin");
76 : MODULE_FIRMWARE("radeon/verde_ce.bin");
77 : MODULE_FIRMWARE("radeon/verde_mc.bin");
78 : MODULE_FIRMWARE("radeon/verde_rlc.bin");
79 : MODULE_FIRMWARE("radeon/verde_smc.bin");
80 :
81 : MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
82 : MODULE_FIRMWARE("radeon/OLAND_me.bin");
83 : MODULE_FIRMWARE("radeon/OLAND_ce.bin");
84 : MODULE_FIRMWARE("radeon/OLAND_mc.bin");
85 : MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
86 : MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
87 : MODULE_FIRMWARE("radeon/OLAND_smc.bin");
88 :
89 : MODULE_FIRMWARE("radeon/oland_pfp.bin");
90 : MODULE_FIRMWARE("radeon/oland_me.bin");
91 : MODULE_FIRMWARE("radeon/oland_ce.bin");
92 : MODULE_FIRMWARE("radeon/oland_mc.bin");
93 : MODULE_FIRMWARE("radeon/oland_rlc.bin");
94 : MODULE_FIRMWARE("radeon/oland_smc.bin");
95 :
96 : MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
97 : MODULE_FIRMWARE("radeon/HAINAN_me.bin");
98 : MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
99 : MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
100 : MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
101 : MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
102 : MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
103 :
104 : MODULE_FIRMWARE("radeon/hainan_pfp.bin");
105 : MODULE_FIRMWARE("radeon/hainan_me.bin");
106 : MODULE_FIRMWARE("radeon/hainan_ce.bin");
107 : MODULE_FIRMWARE("radeon/hainan_mc.bin");
108 : MODULE_FIRMWARE("radeon/hainan_rlc.bin");
109 : MODULE_FIRMWARE("radeon/hainan_smc.bin");
110 :
111 : static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
112 : static void si_pcie_gen3_enable(struct radeon_device *rdev);
113 : static void si_program_aspm(struct radeon_device *rdev);
114 : extern void sumo_rlc_fini(struct radeon_device *rdev);
115 : extern int sumo_rlc_init(struct radeon_device *rdev);
116 : extern int r600_ih_ring_alloc(struct radeon_device *rdev);
117 : extern void r600_ih_ring_fini(struct radeon_device *rdev);
118 : extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
119 : extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120 : extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
121 : extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
122 : extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
123 : extern bool evergreen_is_display_hung(struct radeon_device *rdev);
124 : static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
125 : bool enable);
126 : static void si_init_pg(struct radeon_device *rdev);
127 : static void si_init_cg(struct radeon_device *rdev);
128 : static void si_fini_pg(struct radeon_device *rdev);
129 : static void si_fini_cg(struct radeon_device *rdev);
130 : static void si_rlc_stop(struct radeon_device *rdev);
131 :
132 : static const u32 verde_rlc_save_restore_register_list[] =
133 : {
134 : (0x8000 << 16) | (0x98f4 >> 2),
135 : 0x00000000,
136 : (0x8040 << 16) | (0x98f4 >> 2),
137 : 0x00000000,
138 : (0x8000 << 16) | (0xe80 >> 2),
139 : 0x00000000,
140 : (0x8040 << 16) | (0xe80 >> 2),
141 : 0x00000000,
142 : (0x8000 << 16) | (0x89bc >> 2),
143 : 0x00000000,
144 : (0x8040 << 16) | (0x89bc >> 2),
145 : 0x00000000,
146 : (0x8000 << 16) | (0x8c1c >> 2),
147 : 0x00000000,
148 : (0x8040 << 16) | (0x8c1c >> 2),
149 : 0x00000000,
150 : (0x9c00 << 16) | (0x98f0 >> 2),
151 : 0x00000000,
152 : (0x9c00 << 16) | (0xe7c >> 2),
153 : 0x00000000,
154 : (0x8000 << 16) | (0x9148 >> 2),
155 : 0x00000000,
156 : (0x8040 << 16) | (0x9148 >> 2),
157 : 0x00000000,
158 : (0x9c00 << 16) | (0x9150 >> 2),
159 : 0x00000000,
160 : (0x9c00 << 16) | (0x897c >> 2),
161 : 0x00000000,
162 : (0x9c00 << 16) | (0x8d8c >> 2),
163 : 0x00000000,
164 : (0x9c00 << 16) | (0xac54 >> 2),
165 : 0X00000000,
166 : 0x3,
167 : (0x9c00 << 16) | (0x98f8 >> 2),
168 : 0x00000000,
169 : (0x9c00 << 16) | (0x9910 >> 2),
170 : 0x00000000,
171 : (0x9c00 << 16) | (0x9914 >> 2),
172 : 0x00000000,
173 : (0x9c00 << 16) | (0x9918 >> 2),
174 : 0x00000000,
175 : (0x9c00 << 16) | (0x991c >> 2),
176 : 0x00000000,
177 : (0x9c00 << 16) | (0x9920 >> 2),
178 : 0x00000000,
179 : (0x9c00 << 16) | (0x9924 >> 2),
180 : 0x00000000,
181 : (0x9c00 << 16) | (0x9928 >> 2),
182 : 0x00000000,
183 : (0x9c00 << 16) | (0x992c >> 2),
184 : 0x00000000,
185 : (0x9c00 << 16) | (0x9930 >> 2),
186 : 0x00000000,
187 : (0x9c00 << 16) | (0x9934 >> 2),
188 : 0x00000000,
189 : (0x9c00 << 16) | (0x9938 >> 2),
190 : 0x00000000,
191 : (0x9c00 << 16) | (0x993c >> 2),
192 : 0x00000000,
193 : (0x9c00 << 16) | (0x9940 >> 2),
194 : 0x00000000,
195 : (0x9c00 << 16) | (0x9944 >> 2),
196 : 0x00000000,
197 : (0x9c00 << 16) | (0x9948 >> 2),
198 : 0x00000000,
199 : (0x9c00 << 16) | (0x994c >> 2),
200 : 0x00000000,
201 : (0x9c00 << 16) | (0x9950 >> 2),
202 : 0x00000000,
203 : (0x9c00 << 16) | (0x9954 >> 2),
204 : 0x00000000,
205 : (0x9c00 << 16) | (0x9958 >> 2),
206 : 0x00000000,
207 : (0x9c00 << 16) | (0x995c >> 2),
208 : 0x00000000,
209 : (0x9c00 << 16) | (0x9960 >> 2),
210 : 0x00000000,
211 : (0x9c00 << 16) | (0x9964 >> 2),
212 : 0x00000000,
213 : (0x9c00 << 16) | (0x9968 >> 2),
214 : 0x00000000,
215 : (0x9c00 << 16) | (0x996c >> 2),
216 : 0x00000000,
217 : (0x9c00 << 16) | (0x9970 >> 2),
218 : 0x00000000,
219 : (0x9c00 << 16) | (0x9974 >> 2),
220 : 0x00000000,
221 : (0x9c00 << 16) | (0x9978 >> 2),
222 : 0x00000000,
223 : (0x9c00 << 16) | (0x997c >> 2),
224 : 0x00000000,
225 : (0x9c00 << 16) | (0x9980 >> 2),
226 : 0x00000000,
227 : (0x9c00 << 16) | (0x9984 >> 2),
228 : 0x00000000,
229 : (0x9c00 << 16) | (0x9988 >> 2),
230 : 0x00000000,
231 : (0x9c00 << 16) | (0x998c >> 2),
232 : 0x00000000,
233 : (0x9c00 << 16) | (0x8c00 >> 2),
234 : 0x00000000,
235 : (0x9c00 << 16) | (0x8c14 >> 2),
236 : 0x00000000,
237 : (0x9c00 << 16) | (0x8c04 >> 2),
238 : 0x00000000,
239 : (0x9c00 << 16) | (0x8c08 >> 2),
240 : 0x00000000,
241 : (0x8000 << 16) | (0x9b7c >> 2),
242 : 0x00000000,
243 : (0x8040 << 16) | (0x9b7c >> 2),
244 : 0x00000000,
245 : (0x8000 << 16) | (0xe84 >> 2),
246 : 0x00000000,
247 : (0x8040 << 16) | (0xe84 >> 2),
248 : 0x00000000,
249 : (0x8000 << 16) | (0x89c0 >> 2),
250 : 0x00000000,
251 : (0x8040 << 16) | (0x89c0 >> 2),
252 : 0x00000000,
253 : (0x8000 << 16) | (0x914c >> 2),
254 : 0x00000000,
255 : (0x8040 << 16) | (0x914c >> 2),
256 : 0x00000000,
257 : (0x8000 << 16) | (0x8c20 >> 2),
258 : 0x00000000,
259 : (0x8040 << 16) | (0x8c20 >> 2),
260 : 0x00000000,
261 : (0x8000 << 16) | (0x9354 >> 2),
262 : 0x00000000,
263 : (0x8040 << 16) | (0x9354 >> 2),
264 : 0x00000000,
265 : (0x9c00 << 16) | (0x9060 >> 2),
266 : 0x00000000,
267 : (0x9c00 << 16) | (0x9364 >> 2),
268 : 0x00000000,
269 : (0x9c00 << 16) | (0x9100 >> 2),
270 : 0x00000000,
271 : (0x9c00 << 16) | (0x913c >> 2),
272 : 0x00000000,
273 : (0x8000 << 16) | (0x90e0 >> 2),
274 : 0x00000000,
275 : (0x8000 << 16) | (0x90e4 >> 2),
276 : 0x00000000,
277 : (0x8000 << 16) | (0x90e8 >> 2),
278 : 0x00000000,
279 : (0x8040 << 16) | (0x90e0 >> 2),
280 : 0x00000000,
281 : (0x8040 << 16) | (0x90e4 >> 2),
282 : 0x00000000,
283 : (0x8040 << 16) | (0x90e8 >> 2),
284 : 0x00000000,
285 : (0x9c00 << 16) | (0x8bcc >> 2),
286 : 0x00000000,
287 : (0x9c00 << 16) | (0x8b24 >> 2),
288 : 0x00000000,
289 : (0x9c00 << 16) | (0x88c4 >> 2),
290 : 0x00000000,
291 : (0x9c00 << 16) | (0x8e50 >> 2),
292 : 0x00000000,
293 : (0x9c00 << 16) | (0x8c0c >> 2),
294 : 0x00000000,
295 : (0x9c00 << 16) | (0x8e58 >> 2),
296 : 0x00000000,
297 : (0x9c00 << 16) | (0x8e5c >> 2),
298 : 0x00000000,
299 : (0x9c00 << 16) | (0x9508 >> 2),
300 : 0x00000000,
301 : (0x9c00 << 16) | (0x950c >> 2),
302 : 0x00000000,
303 : (0x9c00 << 16) | (0x9494 >> 2),
304 : 0x00000000,
305 : (0x9c00 << 16) | (0xac0c >> 2),
306 : 0x00000000,
307 : (0x9c00 << 16) | (0xac10 >> 2),
308 : 0x00000000,
309 : (0x9c00 << 16) | (0xac14 >> 2),
310 : 0x00000000,
311 : (0x9c00 << 16) | (0xae00 >> 2),
312 : 0x00000000,
313 : (0x9c00 << 16) | (0xac08 >> 2),
314 : 0x00000000,
315 : (0x9c00 << 16) | (0x88d4 >> 2),
316 : 0x00000000,
317 : (0x9c00 << 16) | (0x88c8 >> 2),
318 : 0x00000000,
319 : (0x9c00 << 16) | (0x88cc >> 2),
320 : 0x00000000,
321 : (0x9c00 << 16) | (0x89b0 >> 2),
322 : 0x00000000,
323 : (0x9c00 << 16) | (0x8b10 >> 2),
324 : 0x00000000,
325 : (0x9c00 << 16) | (0x8a14 >> 2),
326 : 0x00000000,
327 : (0x9c00 << 16) | (0x9830 >> 2),
328 : 0x00000000,
329 : (0x9c00 << 16) | (0x9834 >> 2),
330 : 0x00000000,
331 : (0x9c00 << 16) | (0x9838 >> 2),
332 : 0x00000000,
333 : (0x9c00 << 16) | (0x9a10 >> 2),
334 : 0x00000000,
335 : (0x8000 << 16) | (0x9870 >> 2),
336 : 0x00000000,
337 : (0x8000 << 16) | (0x9874 >> 2),
338 : 0x00000000,
339 : (0x8001 << 16) | (0x9870 >> 2),
340 : 0x00000000,
341 : (0x8001 << 16) | (0x9874 >> 2),
342 : 0x00000000,
343 : (0x8040 << 16) | (0x9870 >> 2),
344 : 0x00000000,
345 : (0x8040 << 16) | (0x9874 >> 2),
346 : 0x00000000,
347 : (0x8041 << 16) | (0x9870 >> 2),
348 : 0x00000000,
349 : (0x8041 << 16) | (0x9874 >> 2),
350 : 0x00000000,
351 : 0x00000000
352 : };
353 :
354 : static const u32 tahiti_golden_rlc_registers[] =
355 : {
356 : 0xc424, 0xffffffff, 0x00601005,
357 : 0xc47c, 0xffffffff, 0x10104040,
358 : 0xc488, 0xffffffff, 0x0100000a,
359 : 0xc314, 0xffffffff, 0x00000800,
360 : 0xc30c, 0xffffffff, 0x800000f4,
361 : 0xf4a8, 0xffffffff, 0x00000000
362 : };
363 :
364 : static const u32 tahiti_golden_registers[] =
365 : {
366 : 0x9a10, 0x00010000, 0x00018208,
367 : 0x9830, 0xffffffff, 0x00000000,
368 : 0x9834, 0xf00fffff, 0x00000400,
369 : 0x9838, 0x0002021c, 0x00020200,
370 : 0xc78, 0x00000080, 0x00000000,
371 : 0xd030, 0x000300c0, 0x00800040,
372 : 0xd830, 0x000300c0, 0x00800040,
373 : 0x5bb0, 0x000000f0, 0x00000070,
374 : 0x5bc0, 0x00200000, 0x50100000,
375 : 0x7030, 0x31000311, 0x00000011,
376 : 0x277c, 0x00000003, 0x000007ff,
377 : 0x240c, 0x000007ff, 0x00000000,
378 : 0x8a14, 0xf000001f, 0x00000007,
379 : 0x8b24, 0xffffffff, 0x00ffffff,
380 : 0x8b10, 0x0000ff0f, 0x00000000,
381 : 0x28a4c, 0x07ffffff, 0x4e000000,
382 : 0x28350, 0x3f3f3fff, 0x2a00126a,
383 : 0x30, 0x000000ff, 0x0040,
384 : 0x34, 0x00000040, 0x00004040,
385 : 0x9100, 0x07ffffff, 0x03000000,
386 : 0x8e88, 0x01ff1f3f, 0x00000000,
387 : 0x8e84, 0x01ff1f3f, 0x00000000,
388 : 0x9060, 0x0000007f, 0x00000020,
389 : 0x9508, 0x00010000, 0x00010000,
390 : 0xac14, 0x00000200, 0x000002fb,
391 : 0xac10, 0xffffffff, 0x0000543b,
392 : 0xac0c, 0xffffffff, 0xa9210876,
393 : 0x88d0, 0xffffffff, 0x000fff40,
394 : 0x88d4, 0x0000001f, 0x00000010,
395 : 0x1410, 0x20000000, 0x20fffed8,
396 : 0x15c0, 0x000c0fc0, 0x000c0400
397 : };
398 :
399 : static const u32 tahiti_golden_registers2[] =
400 : {
401 : 0xc64, 0x00000001, 0x00000001
402 : };
403 :
404 : static const u32 pitcairn_golden_rlc_registers[] =
405 : {
406 : 0xc424, 0xffffffff, 0x00601004,
407 : 0xc47c, 0xffffffff, 0x10102020,
408 : 0xc488, 0xffffffff, 0x01000020,
409 : 0xc314, 0xffffffff, 0x00000800,
410 : 0xc30c, 0xffffffff, 0x800000a4
411 : };
412 :
413 : static const u32 pitcairn_golden_registers[] =
414 : {
415 : 0x9a10, 0x00010000, 0x00018208,
416 : 0x9830, 0xffffffff, 0x00000000,
417 : 0x9834, 0xf00fffff, 0x00000400,
418 : 0x9838, 0x0002021c, 0x00020200,
419 : 0xc78, 0x00000080, 0x00000000,
420 : 0xd030, 0x000300c0, 0x00800040,
421 : 0xd830, 0x000300c0, 0x00800040,
422 : 0x5bb0, 0x000000f0, 0x00000070,
423 : 0x5bc0, 0x00200000, 0x50100000,
424 : 0x7030, 0x31000311, 0x00000011,
425 : 0x2ae4, 0x00073ffe, 0x000022a2,
426 : 0x240c, 0x000007ff, 0x00000000,
427 : 0x8a14, 0xf000001f, 0x00000007,
428 : 0x8b24, 0xffffffff, 0x00ffffff,
429 : 0x8b10, 0x0000ff0f, 0x00000000,
430 : 0x28a4c, 0x07ffffff, 0x4e000000,
431 : 0x28350, 0x3f3f3fff, 0x2a00126a,
432 : 0x30, 0x000000ff, 0x0040,
433 : 0x34, 0x00000040, 0x00004040,
434 : 0x9100, 0x07ffffff, 0x03000000,
435 : 0x9060, 0x0000007f, 0x00000020,
436 : 0x9508, 0x00010000, 0x00010000,
437 : 0xac14, 0x000003ff, 0x000000f7,
438 : 0xac10, 0xffffffff, 0x00000000,
439 : 0xac0c, 0xffffffff, 0x32761054,
440 : 0x88d4, 0x0000001f, 0x00000010,
441 : 0x15c0, 0x000c0fc0, 0x000c0400
442 : };
443 :
444 : static const u32 verde_golden_rlc_registers[] =
445 : {
446 : 0xc424, 0xffffffff, 0x033f1005,
447 : 0xc47c, 0xffffffff, 0x10808020,
448 : 0xc488, 0xffffffff, 0x00800008,
449 : 0xc314, 0xffffffff, 0x00001000,
450 : 0xc30c, 0xffffffff, 0x80010014
451 : };
452 :
453 : static const u32 verde_golden_registers[] =
454 : {
455 : 0x9a10, 0x00010000, 0x00018208,
456 : 0x9830, 0xffffffff, 0x00000000,
457 : 0x9834, 0xf00fffff, 0x00000400,
458 : 0x9838, 0x0002021c, 0x00020200,
459 : 0xc78, 0x00000080, 0x00000000,
460 : 0xd030, 0x000300c0, 0x00800040,
461 : 0xd030, 0x000300c0, 0x00800040,
462 : 0xd830, 0x000300c0, 0x00800040,
463 : 0xd830, 0x000300c0, 0x00800040,
464 : 0x5bb0, 0x000000f0, 0x00000070,
465 : 0x5bc0, 0x00200000, 0x50100000,
466 : 0x7030, 0x31000311, 0x00000011,
467 : 0x2ae4, 0x00073ffe, 0x000022a2,
468 : 0x2ae4, 0x00073ffe, 0x000022a2,
469 : 0x2ae4, 0x00073ffe, 0x000022a2,
470 : 0x240c, 0x000007ff, 0x00000000,
471 : 0x240c, 0x000007ff, 0x00000000,
472 : 0x240c, 0x000007ff, 0x00000000,
473 : 0x8a14, 0xf000001f, 0x00000007,
474 : 0x8a14, 0xf000001f, 0x00000007,
475 : 0x8a14, 0xf000001f, 0x00000007,
476 : 0x8b24, 0xffffffff, 0x00ffffff,
477 : 0x8b10, 0x0000ff0f, 0x00000000,
478 : 0x28a4c, 0x07ffffff, 0x4e000000,
479 : 0x28350, 0x3f3f3fff, 0x0000124a,
480 : 0x28350, 0x3f3f3fff, 0x0000124a,
481 : 0x28350, 0x3f3f3fff, 0x0000124a,
482 : 0x30, 0x000000ff, 0x0040,
483 : 0x34, 0x00000040, 0x00004040,
484 : 0x9100, 0x07ffffff, 0x03000000,
485 : 0x9100, 0x07ffffff, 0x03000000,
486 : 0x8e88, 0x01ff1f3f, 0x00000000,
487 : 0x8e88, 0x01ff1f3f, 0x00000000,
488 : 0x8e88, 0x01ff1f3f, 0x00000000,
489 : 0x8e84, 0x01ff1f3f, 0x00000000,
490 : 0x8e84, 0x01ff1f3f, 0x00000000,
491 : 0x8e84, 0x01ff1f3f, 0x00000000,
492 : 0x9060, 0x0000007f, 0x00000020,
493 : 0x9508, 0x00010000, 0x00010000,
494 : 0xac14, 0x000003ff, 0x00000003,
495 : 0xac14, 0x000003ff, 0x00000003,
496 : 0xac14, 0x000003ff, 0x00000003,
497 : 0xac10, 0xffffffff, 0x00000000,
498 : 0xac10, 0xffffffff, 0x00000000,
499 : 0xac10, 0xffffffff, 0x00000000,
500 : 0xac0c, 0xffffffff, 0x00001032,
501 : 0xac0c, 0xffffffff, 0x00001032,
502 : 0xac0c, 0xffffffff, 0x00001032,
503 : 0x88d4, 0x0000001f, 0x00000010,
504 : 0x88d4, 0x0000001f, 0x00000010,
505 : 0x88d4, 0x0000001f, 0x00000010,
506 : 0x15c0, 0x000c0fc0, 0x000c0400
507 : };
508 :
509 : static const u32 oland_golden_rlc_registers[] =
510 : {
511 : 0xc424, 0xffffffff, 0x00601005,
512 : 0xc47c, 0xffffffff, 0x10104040,
513 : 0xc488, 0xffffffff, 0x0100000a,
514 : 0xc314, 0xffffffff, 0x00000800,
515 : 0xc30c, 0xffffffff, 0x800000f4
516 : };
517 :
518 : static const u32 oland_golden_registers[] =
519 : {
520 : 0x9a10, 0x00010000, 0x00018208,
521 : 0x9830, 0xffffffff, 0x00000000,
522 : 0x9834, 0xf00fffff, 0x00000400,
523 : 0x9838, 0x0002021c, 0x00020200,
524 : 0xc78, 0x00000080, 0x00000000,
525 : 0xd030, 0x000300c0, 0x00800040,
526 : 0xd830, 0x000300c0, 0x00800040,
527 : 0x5bb0, 0x000000f0, 0x00000070,
528 : 0x5bc0, 0x00200000, 0x50100000,
529 : 0x7030, 0x31000311, 0x00000011,
530 : 0x2ae4, 0x00073ffe, 0x000022a2,
531 : 0x240c, 0x000007ff, 0x00000000,
532 : 0x8a14, 0xf000001f, 0x00000007,
533 : 0x8b24, 0xffffffff, 0x00ffffff,
534 : 0x8b10, 0x0000ff0f, 0x00000000,
535 : 0x28a4c, 0x07ffffff, 0x4e000000,
536 : 0x28350, 0x3f3f3fff, 0x00000082,
537 : 0x30, 0x000000ff, 0x0040,
538 : 0x34, 0x00000040, 0x00004040,
539 : 0x9100, 0x07ffffff, 0x03000000,
540 : 0x9060, 0x0000007f, 0x00000020,
541 : 0x9508, 0x00010000, 0x00010000,
542 : 0xac14, 0x000003ff, 0x000000f3,
543 : 0xac10, 0xffffffff, 0x00000000,
544 : 0xac0c, 0xffffffff, 0x00003210,
545 : 0x88d4, 0x0000001f, 0x00000010,
546 : 0x15c0, 0x000c0fc0, 0x000c0400
547 : };
548 :
549 : static const u32 hainan_golden_registers[] =
550 : {
551 : 0x9a10, 0x00010000, 0x00018208,
552 : 0x9830, 0xffffffff, 0x00000000,
553 : 0x9834, 0xf00fffff, 0x00000400,
554 : 0x9838, 0x0002021c, 0x00020200,
555 : 0xd0c0, 0xff000fff, 0x00000100,
556 : 0xd030, 0x000300c0, 0x00800040,
557 : 0xd8c0, 0xff000fff, 0x00000100,
558 : 0xd830, 0x000300c0, 0x00800040,
559 : 0x2ae4, 0x00073ffe, 0x000022a2,
560 : 0x240c, 0x000007ff, 0x00000000,
561 : 0x8a14, 0xf000001f, 0x00000007,
562 : 0x8b24, 0xffffffff, 0x00ffffff,
563 : 0x8b10, 0x0000ff0f, 0x00000000,
564 : 0x28a4c, 0x07ffffff, 0x4e000000,
565 : 0x28350, 0x3f3f3fff, 0x00000000,
566 : 0x30, 0x000000ff, 0x0040,
567 : 0x34, 0x00000040, 0x00004040,
568 : 0x9100, 0x03e00000, 0x03600000,
569 : 0x9060, 0x0000007f, 0x00000020,
570 : 0x9508, 0x00010000, 0x00010000,
571 : 0xac14, 0x000003ff, 0x000000f1,
572 : 0xac10, 0xffffffff, 0x00000000,
573 : 0xac0c, 0xffffffff, 0x00003210,
574 : 0x88d4, 0x0000001f, 0x00000010,
575 : 0x15c0, 0x000c0fc0, 0x000c0400
576 : };
577 :
578 : static const u32 hainan_golden_registers2[] =
579 : {
580 : 0x98f8, 0xffffffff, 0x02010001
581 : };
582 :
583 : static const u32 tahiti_mgcg_cgcg_init[] =
584 : {
585 : 0xc400, 0xffffffff, 0xfffffffc,
586 : 0x802c, 0xffffffff, 0xe0000000,
587 : 0x9a60, 0xffffffff, 0x00000100,
588 : 0x92a4, 0xffffffff, 0x00000100,
589 : 0xc164, 0xffffffff, 0x00000100,
590 : 0x9774, 0xffffffff, 0x00000100,
591 : 0x8984, 0xffffffff, 0x06000100,
592 : 0x8a18, 0xffffffff, 0x00000100,
593 : 0x92a0, 0xffffffff, 0x00000100,
594 : 0xc380, 0xffffffff, 0x00000100,
595 : 0x8b28, 0xffffffff, 0x00000100,
596 : 0x9144, 0xffffffff, 0x00000100,
597 : 0x8d88, 0xffffffff, 0x00000100,
598 : 0x8d8c, 0xffffffff, 0x00000100,
599 : 0x9030, 0xffffffff, 0x00000100,
600 : 0x9034, 0xffffffff, 0x00000100,
601 : 0x9038, 0xffffffff, 0x00000100,
602 : 0x903c, 0xffffffff, 0x00000100,
603 : 0xad80, 0xffffffff, 0x00000100,
604 : 0xac54, 0xffffffff, 0x00000100,
605 : 0x897c, 0xffffffff, 0x06000100,
606 : 0x9868, 0xffffffff, 0x00000100,
607 : 0x9510, 0xffffffff, 0x00000100,
608 : 0xaf04, 0xffffffff, 0x00000100,
609 : 0xae04, 0xffffffff, 0x00000100,
610 : 0x949c, 0xffffffff, 0x00000100,
611 : 0x802c, 0xffffffff, 0xe0000000,
612 : 0x9160, 0xffffffff, 0x00010000,
613 : 0x9164, 0xffffffff, 0x00030002,
614 : 0x9168, 0xffffffff, 0x00040007,
615 : 0x916c, 0xffffffff, 0x00060005,
616 : 0x9170, 0xffffffff, 0x00090008,
617 : 0x9174, 0xffffffff, 0x00020001,
618 : 0x9178, 0xffffffff, 0x00040003,
619 : 0x917c, 0xffffffff, 0x00000007,
620 : 0x9180, 0xffffffff, 0x00060005,
621 : 0x9184, 0xffffffff, 0x00090008,
622 : 0x9188, 0xffffffff, 0x00030002,
623 : 0x918c, 0xffffffff, 0x00050004,
624 : 0x9190, 0xffffffff, 0x00000008,
625 : 0x9194, 0xffffffff, 0x00070006,
626 : 0x9198, 0xffffffff, 0x000a0009,
627 : 0x919c, 0xffffffff, 0x00040003,
628 : 0x91a0, 0xffffffff, 0x00060005,
629 : 0x91a4, 0xffffffff, 0x00000009,
630 : 0x91a8, 0xffffffff, 0x00080007,
631 : 0x91ac, 0xffffffff, 0x000b000a,
632 : 0x91b0, 0xffffffff, 0x00050004,
633 : 0x91b4, 0xffffffff, 0x00070006,
634 : 0x91b8, 0xffffffff, 0x0008000b,
635 : 0x91bc, 0xffffffff, 0x000a0009,
636 : 0x91c0, 0xffffffff, 0x000d000c,
637 : 0x91c4, 0xffffffff, 0x00060005,
638 : 0x91c8, 0xffffffff, 0x00080007,
639 : 0x91cc, 0xffffffff, 0x0000000b,
640 : 0x91d0, 0xffffffff, 0x000a0009,
641 : 0x91d4, 0xffffffff, 0x000d000c,
642 : 0x91d8, 0xffffffff, 0x00070006,
643 : 0x91dc, 0xffffffff, 0x00090008,
644 : 0x91e0, 0xffffffff, 0x0000000c,
645 : 0x91e4, 0xffffffff, 0x000b000a,
646 : 0x91e8, 0xffffffff, 0x000e000d,
647 : 0x91ec, 0xffffffff, 0x00080007,
648 : 0x91f0, 0xffffffff, 0x000a0009,
649 : 0x91f4, 0xffffffff, 0x0000000d,
650 : 0x91f8, 0xffffffff, 0x000c000b,
651 : 0x91fc, 0xffffffff, 0x000f000e,
652 : 0x9200, 0xffffffff, 0x00090008,
653 : 0x9204, 0xffffffff, 0x000b000a,
654 : 0x9208, 0xffffffff, 0x000c000f,
655 : 0x920c, 0xffffffff, 0x000e000d,
656 : 0x9210, 0xffffffff, 0x00110010,
657 : 0x9214, 0xffffffff, 0x000a0009,
658 : 0x9218, 0xffffffff, 0x000c000b,
659 : 0x921c, 0xffffffff, 0x0000000f,
660 : 0x9220, 0xffffffff, 0x000e000d,
661 : 0x9224, 0xffffffff, 0x00110010,
662 : 0x9228, 0xffffffff, 0x000b000a,
663 : 0x922c, 0xffffffff, 0x000d000c,
664 : 0x9230, 0xffffffff, 0x00000010,
665 : 0x9234, 0xffffffff, 0x000f000e,
666 : 0x9238, 0xffffffff, 0x00120011,
667 : 0x923c, 0xffffffff, 0x000c000b,
668 : 0x9240, 0xffffffff, 0x000e000d,
669 : 0x9244, 0xffffffff, 0x00000011,
670 : 0x9248, 0xffffffff, 0x0010000f,
671 : 0x924c, 0xffffffff, 0x00130012,
672 : 0x9250, 0xffffffff, 0x000d000c,
673 : 0x9254, 0xffffffff, 0x000f000e,
674 : 0x9258, 0xffffffff, 0x00100013,
675 : 0x925c, 0xffffffff, 0x00120011,
676 : 0x9260, 0xffffffff, 0x00150014,
677 : 0x9264, 0xffffffff, 0x000e000d,
678 : 0x9268, 0xffffffff, 0x0010000f,
679 : 0x926c, 0xffffffff, 0x00000013,
680 : 0x9270, 0xffffffff, 0x00120011,
681 : 0x9274, 0xffffffff, 0x00150014,
682 : 0x9278, 0xffffffff, 0x000f000e,
683 : 0x927c, 0xffffffff, 0x00110010,
684 : 0x9280, 0xffffffff, 0x00000014,
685 : 0x9284, 0xffffffff, 0x00130012,
686 : 0x9288, 0xffffffff, 0x00160015,
687 : 0x928c, 0xffffffff, 0x0010000f,
688 : 0x9290, 0xffffffff, 0x00120011,
689 : 0x9294, 0xffffffff, 0x00000015,
690 : 0x9298, 0xffffffff, 0x00140013,
691 : 0x929c, 0xffffffff, 0x00170016,
692 : 0x9150, 0xffffffff, 0x96940200,
693 : 0x8708, 0xffffffff, 0x00900100,
694 : 0xc478, 0xffffffff, 0x00000080,
695 : 0xc404, 0xffffffff, 0x0020003f,
696 : 0x30, 0xffffffff, 0x0000001c,
697 : 0x34, 0x000f0000, 0x000f0000,
698 : 0x160c, 0xffffffff, 0x00000100,
699 : 0x1024, 0xffffffff, 0x00000100,
700 : 0x102c, 0x00000101, 0x00000000,
701 : 0x20a8, 0xffffffff, 0x00000104,
702 : 0x264c, 0x000c0000, 0x000c0000,
703 : 0x2648, 0x000c0000, 0x000c0000,
704 : 0x55e4, 0xff000fff, 0x00000100,
705 : 0x55e8, 0x00000001, 0x00000001,
706 : 0x2f50, 0x00000001, 0x00000001,
707 : 0x30cc, 0xc0000fff, 0x00000104,
708 : 0xc1e4, 0x00000001, 0x00000001,
709 : 0xd0c0, 0xfffffff0, 0x00000100,
710 : 0xd8c0, 0xfffffff0, 0x00000100
711 : };
712 :
713 : static const u32 pitcairn_mgcg_cgcg_init[] =
714 : {
715 : 0xc400, 0xffffffff, 0xfffffffc,
716 : 0x802c, 0xffffffff, 0xe0000000,
717 : 0x9a60, 0xffffffff, 0x00000100,
718 : 0x92a4, 0xffffffff, 0x00000100,
719 : 0xc164, 0xffffffff, 0x00000100,
720 : 0x9774, 0xffffffff, 0x00000100,
721 : 0x8984, 0xffffffff, 0x06000100,
722 : 0x8a18, 0xffffffff, 0x00000100,
723 : 0x92a0, 0xffffffff, 0x00000100,
724 : 0xc380, 0xffffffff, 0x00000100,
725 : 0x8b28, 0xffffffff, 0x00000100,
726 : 0x9144, 0xffffffff, 0x00000100,
727 : 0x8d88, 0xffffffff, 0x00000100,
728 : 0x8d8c, 0xffffffff, 0x00000100,
729 : 0x9030, 0xffffffff, 0x00000100,
730 : 0x9034, 0xffffffff, 0x00000100,
731 : 0x9038, 0xffffffff, 0x00000100,
732 : 0x903c, 0xffffffff, 0x00000100,
733 : 0xad80, 0xffffffff, 0x00000100,
734 : 0xac54, 0xffffffff, 0x00000100,
735 : 0x897c, 0xffffffff, 0x06000100,
736 : 0x9868, 0xffffffff, 0x00000100,
737 : 0x9510, 0xffffffff, 0x00000100,
738 : 0xaf04, 0xffffffff, 0x00000100,
739 : 0xae04, 0xffffffff, 0x00000100,
740 : 0x949c, 0xffffffff, 0x00000100,
741 : 0x802c, 0xffffffff, 0xe0000000,
742 : 0x9160, 0xffffffff, 0x00010000,
743 : 0x9164, 0xffffffff, 0x00030002,
744 : 0x9168, 0xffffffff, 0x00040007,
745 : 0x916c, 0xffffffff, 0x00060005,
746 : 0x9170, 0xffffffff, 0x00090008,
747 : 0x9174, 0xffffffff, 0x00020001,
748 : 0x9178, 0xffffffff, 0x00040003,
749 : 0x917c, 0xffffffff, 0x00000007,
750 : 0x9180, 0xffffffff, 0x00060005,
751 : 0x9184, 0xffffffff, 0x00090008,
752 : 0x9188, 0xffffffff, 0x00030002,
753 : 0x918c, 0xffffffff, 0x00050004,
754 : 0x9190, 0xffffffff, 0x00000008,
755 : 0x9194, 0xffffffff, 0x00070006,
756 : 0x9198, 0xffffffff, 0x000a0009,
757 : 0x919c, 0xffffffff, 0x00040003,
758 : 0x91a0, 0xffffffff, 0x00060005,
759 : 0x91a4, 0xffffffff, 0x00000009,
760 : 0x91a8, 0xffffffff, 0x00080007,
761 : 0x91ac, 0xffffffff, 0x000b000a,
762 : 0x91b0, 0xffffffff, 0x00050004,
763 : 0x91b4, 0xffffffff, 0x00070006,
764 : 0x91b8, 0xffffffff, 0x0008000b,
765 : 0x91bc, 0xffffffff, 0x000a0009,
766 : 0x91c0, 0xffffffff, 0x000d000c,
767 : 0x9200, 0xffffffff, 0x00090008,
768 : 0x9204, 0xffffffff, 0x000b000a,
769 : 0x9208, 0xffffffff, 0x000c000f,
770 : 0x920c, 0xffffffff, 0x000e000d,
771 : 0x9210, 0xffffffff, 0x00110010,
772 : 0x9214, 0xffffffff, 0x000a0009,
773 : 0x9218, 0xffffffff, 0x000c000b,
774 : 0x921c, 0xffffffff, 0x0000000f,
775 : 0x9220, 0xffffffff, 0x000e000d,
776 : 0x9224, 0xffffffff, 0x00110010,
777 : 0x9228, 0xffffffff, 0x000b000a,
778 : 0x922c, 0xffffffff, 0x000d000c,
779 : 0x9230, 0xffffffff, 0x00000010,
780 : 0x9234, 0xffffffff, 0x000f000e,
781 : 0x9238, 0xffffffff, 0x00120011,
782 : 0x923c, 0xffffffff, 0x000c000b,
783 : 0x9240, 0xffffffff, 0x000e000d,
784 : 0x9244, 0xffffffff, 0x00000011,
785 : 0x9248, 0xffffffff, 0x0010000f,
786 : 0x924c, 0xffffffff, 0x00130012,
787 : 0x9250, 0xffffffff, 0x000d000c,
788 : 0x9254, 0xffffffff, 0x000f000e,
789 : 0x9258, 0xffffffff, 0x00100013,
790 : 0x925c, 0xffffffff, 0x00120011,
791 : 0x9260, 0xffffffff, 0x00150014,
792 : 0x9150, 0xffffffff, 0x96940200,
793 : 0x8708, 0xffffffff, 0x00900100,
794 : 0xc478, 0xffffffff, 0x00000080,
795 : 0xc404, 0xffffffff, 0x0020003f,
796 : 0x30, 0xffffffff, 0x0000001c,
797 : 0x34, 0x000f0000, 0x000f0000,
798 : 0x160c, 0xffffffff, 0x00000100,
799 : 0x1024, 0xffffffff, 0x00000100,
800 : 0x102c, 0x00000101, 0x00000000,
801 : 0x20a8, 0xffffffff, 0x00000104,
802 : 0x55e4, 0xff000fff, 0x00000100,
803 : 0x55e8, 0x00000001, 0x00000001,
804 : 0x2f50, 0x00000001, 0x00000001,
805 : 0x30cc, 0xc0000fff, 0x00000104,
806 : 0xc1e4, 0x00000001, 0x00000001,
807 : 0xd0c0, 0xfffffff0, 0x00000100,
808 : 0xd8c0, 0xfffffff0, 0x00000100
809 : };
810 :
811 : static const u32 verde_mgcg_cgcg_init[] =
812 : {
813 : 0xc400, 0xffffffff, 0xfffffffc,
814 : 0x802c, 0xffffffff, 0xe0000000,
815 : 0x9a60, 0xffffffff, 0x00000100,
816 : 0x92a4, 0xffffffff, 0x00000100,
817 : 0xc164, 0xffffffff, 0x00000100,
818 : 0x9774, 0xffffffff, 0x00000100,
819 : 0x8984, 0xffffffff, 0x06000100,
820 : 0x8a18, 0xffffffff, 0x00000100,
821 : 0x92a0, 0xffffffff, 0x00000100,
822 : 0xc380, 0xffffffff, 0x00000100,
823 : 0x8b28, 0xffffffff, 0x00000100,
824 : 0x9144, 0xffffffff, 0x00000100,
825 : 0x8d88, 0xffffffff, 0x00000100,
826 : 0x8d8c, 0xffffffff, 0x00000100,
827 : 0x9030, 0xffffffff, 0x00000100,
828 : 0x9034, 0xffffffff, 0x00000100,
829 : 0x9038, 0xffffffff, 0x00000100,
830 : 0x903c, 0xffffffff, 0x00000100,
831 : 0xad80, 0xffffffff, 0x00000100,
832 : 0xac54, 0xffffffff, 0x00000100,
833 : 0x897c, 0xffffffff, 0x06000100,
834 : 0x9868, 0xffffffff, 0x00000100,
835 : 0x9510, 0xffffffff, 0x00000100,
836 : 0xaf04, 0xffffffff, 0x00000100,
837 : 0xae04, 0xffffffff, 0x00000100,
838 : 0x949c, 0xffffffff, 0x00000100,
839 : 0x802c, 0xffffffff, 0xe0000000,
840 : 0x9160, 0xffffffff, 0x00010000,
841 : 0x9164, 0xffffffff, 0x00030002,
842 : 0x9168, 0xffffffff, 0x00040007,
843 : 0x916c, 0xffffffff, 0x00060005,
844 : 0x9170, 0xffffffff, 0x00090008,
845 : 0x9174, 0xffffffff, 0x00020001,
846 : 0x9178, 0xffffffff, 0x00040003,
847 : 0x917c, 0xffffffff, 0x00000007,
848 : 0x9180, 0xffffffff, 0x00060005,
849 : 0x9184, 0xffffffff, 0x00090008,
850 : 0x9188, 0xffffffff, 0x00030002,
851 : 0x918c, 0xffffffff, 0x00050004,
852 : 0x9190, 0xffffffff, 0x00000008,
853 : 0x9194, 0xffffffff, 0x00070006,
854 : 0x9198, 0xffffffff, 0x000a0009,
855 : 0x919c, 0xffffffff, 0x00040003,
856 : 0x91a0, 0xffffffff, 0x00060005,
857 : 0x91a4, 0xffffffff, 0x00000009,
858 : 0x91a8, 0xffffffff, 0x00080007,
859 : 0x91ac, 0xffffffff, 0x000b000a,
860 : 0x91b0, 0xffffffff, 0x00050004,
861 : 0x91b4, 0xffffffff, 0x00070006,
862 : 0x91b8, 0xffffffff, 0x0008000b,
863 : 0x91bc, 0xffffffff, 0x000a0009,
864 : 0x91c0, 0xffffffff, 0x000d000c,
865 : 0x9200, 0xffffffff, 0x00090008,
866 : 0x9204, 0xffffffff, 0x000b000a,
867 : 0x9208, 0xffffffff, 0x000c000f,
868 : 0x920c, 0xffffffff, 0x000e000d,
869 : 0x9210, 0xffffffff, 0x00110010,
870 : 0x9214, 0xffffffff, 0x000a0009,
871 : 0x9218, 0xffffffff, 0x000c000b,
872 : 0x921c, 0xffffffff, 0x0000000f,
873 : 0x9220, 0xffffffff, 0x000e000d,
874 : 0x9224, 0xffffffff, 0x00110010,
875 : 0x9228, 0xffffffff, 0x000b000a,
876 : 0x922c, 0xffffffff, 0x000d000c,
877 : 0x9230, 0xffffffff, 0x00000010,
878 : 0x9234, 0xffffffff, 0x000f000e,
879 : 0x9238, 0xffffffff, 0x00120011,
880 : 0x923c, 0xffffffff, 0x000c000b,
881 : 0x9240, 0xffffffff, 0x000e000d,
882 : 0x9244, 0xffffffff, 0x00000011,
883 : 0x9248, 0xffffffff, 0x0010000f,
884 : 0x924c, 0xffffffff, 0x00130012,
885 : 0x9250, 0xffffffff, 0x000d000c,
886 : 0x9254, 0xffffffff, 0x000f000e,
887 : 0x9258, 0xffffffff, 0x00100013,
888 : 0x925c, 0xffffffff, 0x00120011,
889 : 0x9260, 0xffffffff, 0x00150014,
890 : 0x9150, 0xffffffff, 0x96940200,
891 : 0x8708, 0xffffffff, 0x00900100,
892 : 0xc478, 0xffffffff, 0x00000080,
893 : 0xc404, 0xffffffff, 0x0020003f,
894 : 0x30, 0xffffffff, 0x0000001c,
895 : 0x34, 0x000f0000, 0x000f0000,
896 : 0x160c, 0xffffffff, 0x00000100,
897 : 0x1024, 0xffffffff, 0x00000100,
898 : 0x102c, 0x00000101, 0x00000000,
899 : 0x20a8, 0xffffffff, 0x00000104,
900 : 0x264c, 0x000c0000, 0x000c0000,
901 : 0x2648, 0x000c0000, 0x000c0000,
902 : 0x55e4, 0xff000fff, 0x00000100,
903 : 0x55e8, 0x00000001, 0x00000001,
904 : 0x2f50, 0x00000001, 0x00000001,
905 : 0x30cc, 0xc0000fff, 0x00000104,
906 : 0xc1e4, 0x00000001, 0x00000001,
907 : 0xd0c0, 0xfffffff0, 0x00000100,
908 : 0xd8c0, 0xfffffff0, 0x00000100
909 : };
910 :
911 : static const u32 oland_mgcg_cgcg_init[] =
912 : {
913 : 0xc400, 0xffffffff, 0xfffffffc,
914 : 0x802c, 0xffffffff, 0xe0000000,
915 : 0x9a60, 0xffffffff, 0x00000100,
916 : 0x92a4, 0xffffffff, 0x00000100,
917 : 0xc164, 0xffffffff, 0x00000100,
918 : 0x9774, 0xffffffff, 0x00000100,
919 : 0x8984, 0xffffffff, 0x06000100,
920 : 0x8a18, 0xffffffff, 0x00000100,
921 : 0x92a0, 0xffffffff, 0x00000100,
922 : 0xc380, 0xffffffff, 0x00000100,
923 : 0x8b28, 0xffffffff, 0x00000100,
924 : 0x9144, 0xffffffff, 0x00000100,
925 : 0x8d88, 0xffffffff, 0x00000100,
926 : 0x8d8c, 0xffffffff, 0x00000100,
927 : 0x9030, 0xffffffff, 0x00000100,
928 : 0x9034, 0xffffffff, 0x00000100,
929 : 0x9038, 0xffffffff, 0x00000100,
930 : 0x903c, 0xffffffff, 0x00000100,
931 : 0xad80, 0xffffffff, 0x00000100,
932 : 0xac54, 0xffffffff, 0x00000100,
933 : 0x897c, 0xffffffff, 0x06000100,
934 : 0x9868, 0xffffffff, 0x00000100,
935 : 0x9510, 0xffffffff, 0x00000100,
936 : 0xaf04, 0xffffffff, 0x00000100,
937 : 0xae04, 0xffffffff, 0x00000100,
938 : 0x949c, 0xffffffff, 0x00000100,
939 : 0x802c, 0xffffffff, 0xe0000000,
940 : 0x9160, 0xffffffff, 0x00010000,
941 : 0x9164, 0xffffffff, 0x00030002,
942 : 0x9168, 0xffffffff, 0x00040007,
943 : 0x916c, 0xffffffff, 0x00060005,
944 : 0x9170, 0xffffffff, 0x00090008,
945 : 0x9174, 0xffffffff, 0x00020001,
946 : 0x9178, 0xffffffff, 0x00040003,
947 : 0x917c, 0xffffffff, 0x00000007,
948 : 0x9180, 0xffffffff, 0x00060005,
949 : 0x9184, 0xffffffff, 0x00090008,
950 : 0x9188, 0xffffffff, 0x00030002,
951 : 0x918c, 0xffffffff, 0x00050004,
952 : 0x9190, 0xffffffff, 0x00000008,
953 : 0x9194, 0xffffffff, 0x00070006,
954 : 0x9198, 0xffffffff, 0x000a0009,
955 : 0x919c, 0xffffffff, 0x00040003,
956 : 0x91a0, 0xffffffff, 0x00060005,
957 : 0x91a4, 0xffffffff, 0x00000009,
958 : 0x91a8, 0xffffffff, 0x00080007,
959 : 0x91ac, 0xffffffff, 0x000b000a,
960 : 0x91b0, 0xffffffff, 0x00050004,
961 : 0x91b4, 0xffffffff, 0x00070006,
962 : 0x91b8, 0xffffffff, 0x0008000b,
963 : 0x91bc, 0xffffffff, 0x000a0009,
964 : 0x91c0, 0xffffffff, 0x000d000c,
965 : 0x91c4, 0xffffffff, 0x00060005,
966 : 0x91c8, 0xffffffff, 0x00080007,
967 : 0x91cc, 0xffffffff, 0x0000000b,
968 : 0x91d0, 0xffffffff, 0x000a0009,
969 : 0x91d4, 0xffffffff, 0x000d000c,
970 : 0x9150, 0xffffffff, 0x96940200,
971 : 0x8708, 0xffffffff, 0x00900100,
972 : 0xc478, 0xffffffff, 0x00000080,
973 : 0xc404, 0xffffffff, 0x0020003f,
974 : 0x30, 0xffffffff, 0x0000001c,
975 : 0x34, 0x000f0000, 0x000f0000,
976 : 0x160c, 0xffffffff, 0x00000100,
977 : 0x1024, 0xffffffff, 0x00000100,
978 : 0x102c, 0x00000101, 0x00000000,
979 : 0x20a8, 0xffffffff, 0x00000104,
980 : 0x264c, 0x000c0000, 0x000c0000,
981 : 0x2648, 0x000c0000, 0x000c0000,
982 : 0x55e4, 0xff000fff, 0x00000100,
983 : 0x55e8, 0x00000001, 0x00000001,
984 : 0x2f50, 0x00000001, 0x00000001,
985 : 0x30cc, 0xc0000fff, 0x00000104,
986 : 0xc1e4, 0x00000001, 0x00000001,
987 : 0xd0c0, 0xfffffff0, 0x00000100,
988 : 0xd8c0, 0xfffffff0, 0x00000100
989 : };
990 :
991 : static const u32 hainan_mgcg_cgcg_init[] =
992 : {
993 : 0xc400, 0xffffffff, 0xfffffffc,
994 : 0x802c, 0xffffffff, 0xe0000000,
995 : 0x9a60, 0xffffffff, 0x00000100,
996 : 0x92a4, 0xffffffff, 0x00000100,
997 : 0xc164, 0xffffffff, 0x00000100,
998 : 0x9774, 0xffffffff, 0x00000100,
999 : 0x8984, 0xffffffff, 0x06000100,
1000 : 0x8a18, 0xffffffff, 0x00000100,
1001 : 0x92a0, 0xffffffff, 0x00000100,
1002 : 0xc380, 0xffffffff, 0x00000100,
1003 : 0x8b28, 0xffffffff, 0x00000100,
1004 : 0x9144, 0xffffffff, 0x00000100,
1005 : 0x8d88, 0xffffffff, 0x00000100,
1006 : 0x8d8c, 0xffffffff, 0x00000100,
1007 : 0x9030, 0xffffffff, 0x00000100,
1008 : 0x9034, 0xffffffff, 0x00000100,
1009 : 0x9038, 0xffffffff, 0x00000100,
1010 : 0x903c, 0xffffffff, 0x00000100,
1011 : 0xad80, 0xffffffff, 0x00000100,
1012 : 0xac54, 0xffffffff, 0x00000100,
1013 : 0x897c, 0xffffffff, 0x06000100,
1014 : 0x9868, 0xffffffff, 0x00000100,
1015 : 0x9510, 0xffffffff, 0x00000100,
1016 : 0xaf04, 0xffffffff, 0x00000100,
1017 : 0xae04, 0xffffffff, 0x00000100,
1018 : 0x949c, 0xffffffff, 0x00000100,
1019 : 0x802c, 0xffffffff, 0xe0000000,
1020 : 0x9160, 0xffffffff, 0x00010000,
1021 : 0x9164, 0xffffffff, 0x00030002,
1022 : 0x9168, 0xffffffff, 0x00040007,
1023 : 0x916c, 0xffffffff, 0x00060005,
1024 : 0x9170, 0xffffffff, 0x00090008,
1025 : 0x9174, 0xffffffff, 0x00020001,
1026 : 0x9178, 0xffffffff, 0x00040003,
1027 : 0x917c, 0xffffffff, 0x00000007,
1028 : 0x9180, 0xffffffff, 0x00060005,
1029 : 0x9184, 0xffffffff, 0x00090008,
1030 : 0x9188, 0xffffffff, 0x00030002,
1031 : 0x918c, 0xffffffff, 0x00050004,
1032 : 0x9190, 0xffffffff, 0x00000008,
1033 : 0x9194, 0xffffffff, 0x00070006,
1034 : 0x9198, 0xffffffff, 0x000a0009,
1035 : 0x919c, 0xffffffff, 0x00040003,
1036 : 0x91a0, 0xffffffff, 0x00060005,
1037 : 0x91a4, 0xffffffff, 0x00000009,
1038 : 0x91a8, 0xffffffff, 0x00080007,
1039 : 0x91ac, 0xffffffff, 0x000b000a,
1040 : 0x91b0, 0xffffffff, 0x00050004,
1041 : 0x91b4, 0xffffffff, 0x00070006,
1042 : 0x91b8, 0xffffffff, 0x0008000b,
1043 : 0x91bc, 0xffffffff, 0x000a0009,
1044 : 0x91c0, 0xffffffff, 0x000d000c,
1045 : 0x91c4, 0xffffffff, 0x00060005,
1046 : 0x91c8, 0xffffffff, 0x00080007,
1047 : 0x91cc, 0xffffffff, 0x0000000b,
1048 : 0x91d0, 0xffffffff, 0x000a0009,
1049 : 0x91d4, 0xffffffff, 0x000d000c,
1050 : 0x9150, 0xffffffff, 0x96940200,
1051 : 0x8708, 0xffffffff, 0x00900100,
1052 : 0xc478, 0xffffffff, 0x00000080,
1053 : 0xc404, 0xffffffff, 0x0020003f,
1054 : 0x30, 0xffffffff, 0x0000001c,
1055 : 0x34, 0x000f0000, 0x000f0000,
1056 : 0x160c, 0xffffffff, 0x00000100,
1057 : 0x1024, 0xffffffff, 0x00000100,
1058 : 0x20a8, 0xffffffff, 0x00000104,
1059 : 0x264c, 0x000c0000, 0x000c0000,
1060 : 0x2648, 0x000c0000, 0x000c0000,
1061 : 0x2f50, 0x00000001, 0x00000001,
1062 : 0x30cc, 0xc0000fff, 0x00000104,
1063 : 0xc1e4, 0x00000001, 0x00000001,
1064 : 0xd0c0, 0xfffffff0, 0x00000100,
1065 : 0xd8c0, 0xfffffff0, 0x00000100
1066 : };
1067 :
1068 : static u32 verde_pg_init[] =
1069 : {
1070 : 0x353c, 0xffffffff, 0x40000,
1071 : 0x3538, 0xffffffff, 0x200010ff,
1072 : 0x353c, 0xffffffff, 0x0,
1073 : 0x353c, 0xffffffff, 0x0,
1074 : 0x353c, 0xffffffff, 0x0,
1075 : 0x353c, 0xffffffff, 0x0,
1076 : 0x353c, 0xffffffff, 0x0,
1077 : 0x353c, 0xffffffff, 0x7007,
1078 : 0x3538, 0xffffffff, 0x300010ff,
1079 : 0x353c, 0xffffffff, 0x0,
1080 : 0x353c, 0xffffffff, 0x0,
1081 : 0x353c, 0xffffffff, 0x0,
1082 : 0x353c, 0xffffffff, 0x0,
1083 : 0x353c, 0xffffffff, 0x0,
1084 : 0x353c, 0xffffffff, 0x400000,
1085 : 0x3538, 0xffffffff, 0x100010ff,
1086 : 0x353c, 0xffffffff, 0x0,
1087 : 0x353c, 0xffffffff, 0x0,
1088 : 0x353c, 0xffffffff, 0x0,
1089 : 0x353c, 0xffffffff, 0x0,
1090 : 0x353c, 0xffffffff, 0x0,
1091 : 0x353c, 0xffffffff, 0x120200,
1092 : 0x3538, 0xffffffff, 0x500010ff,
1093 : 0x353c, 0xffffffff, 0x0,
1094 : 0x353c, 0xffffffff, 0x0,
1095 : 0x353c, 0xffffffff, 0x0,
1096 : 0x353c, 0xffffffff, 0x0,
1097 : 0x353c, 0xffffffff, 0x0,
1098 : 0x353c, 0xffffffff, 0x1e1e16,
1099 : 0x3538, 0xffffffff, 0x600010ff,
1100 : 0x353c, 0xffffffff, 0x0,
1101 : 0x353c, 0xffffffff, 0x0,
1102 : 0x353c, 0xffffffff, 0x0,
1103 : 0x353c, 0xffffffff, 0x0,
1104 : 0x353c, 0xffffffff, 0x0,
1105 : 0x353c, 0xffffffff, 0x171f1e,
1106 : 0x3538, 0xffffffff, 0x700010ff,
1107 : 0x353c, 0xffffffff, 0x0,
1108 : 0x353c, 0xffffffff, 0x0,
1109 : 0x353c, 0xffffffff, 0x0,
1110 : 0x353c, 0xffffffff, 0x0,
1111 : 0x353c, 0xffffffff, 0x0,
1112 : 0x353c, 0xffffffff, 0x0,
1113 : 0x3538, 0xffffffff, 0x9ff,
1114 : 0x3500, 0xffffffff, 0x0,
1115 : 0x3504, 0xffffffff, 0x10000800,
1116 : 0x3504, 0xffffffff, 0xf,
1117 : 0x3504, 0xffffffff, 0xf,
1118 : 0x3500, 0xffffffff, 0x4,
1119 : 0x3504, 0xffffffff, 0x1000051e,
1120 : 0x3504, 0xffffffff, 0xffff,
1121 : 0x3504, 0xffffffff, 0xffff,
1122 : 0x3500, 0xffffffff, 0x8,
1123 : 0x3504, 0xffffffff, 0x80500,
1124 : 0x3500, 0xffffffff, 0x12,
1125 : 0x3504, 0xffffffff, 0x9050c,
1126 : 0x3500, 0xffffffff, 0x1d,
1127 : 0x3504, 0xffffffff, 0xb052c,
1128 : 0x3500, 0xffffffff, 0x2a,
1129 : 0x3504, 0xffffffff, 0x1053e,
1130 : 0x3500, 0xffffffff, 0x2d,
1131 : 0x3504, 0xffffffff, 0x10546,
1132 : 0x3500, 0xffffffff, 0x30,
1133 : 0x3504, 0xffffffff, 0xa054e,
1134 : 0x3500, 0xffffffff, 0x3c,
1135 : 0x3504, 0xffffffff, 0x1055f,
1136 : 0x3500, 0xffffffff, 0x3f,
1137 : 0x3504, 0xffffffff, 0x10567,
1138 : 0x3500, 0xffffffff, 0x42,
1139 : 0x3504, 0xffffffff, 0x1056f,
1140 : 0x3500, 0xffffffff, 0x45,
1141 : 0x3504, 0xffffffff, 0x10572,
1142 : 0x3500, 0xffffffff, 0x48,
1143 : 0x3504, 0xffffffff, 0x20575,
1144 : 0x3500, 0xffffffff, 0x4c,
1145 : 0x3504, 0xffffffff, 0x190801,
1146 : 0x3500, 0xffffffff, 0x67,
1147 : 0x3504, 0xffffffff, 0x1082a,
1148 : 0x3500, 0xffffffff, 0x6a,
1149 : 0x3504, 0xffffffff, 0x1b082d,
1150 : 0x3500, 0xffffffff, 0x87,
1151 : 0x3504, 0xffffffff, 0x310851,
1152 : 0x3500, 0xffffffff, 0xba,
1153 : 0x3504, 0xffffffff, 0x891,
1154 : 0x3500, 0xffffffff, 0xbc,
1155 : 0x3504, 0xffffffff, 0x893,
1156 : 0x3500, 0xffffffff, 0xbe,
1157 : 0x3504, 0xffffffff, 0x20895,
1158 : 0x3500, 0xffffffff, 0xc2,
1159 : 0x3504, 0xffffffff, 0x20899,
1160 : 0x3500, 0xffffffff, 0xc6,
1161 : 0x3504, 0xffffffff, 0x2089d,
1162 : 0x3500, 0xffffffff, 0xca,
1163 : 0x3504, 0xffffffff, 0x8a1,
1164 : 0x3500, 0xffffffff, 0xcc,
1165 : 0x3504, 0xffffffff, 0x8a3,
1166 : 0x3500, 0xffffffff, 0xce,
1167 : 0x3504, 0xffffffff, 0x308a5,
1168 : 0x3500, 0xffffffff, 0xd3,
1169 : 0x3504, 0xffffffff, 0x6d08cd,
1170 : 0x3500, 0xffffffff, 0x142,
1171 : 0x3504, 0xffffffff, 0x2000095a,
1172 : 0x3504, 0xffffffff, 0x1,
1173 : 0x3500, 0xffffffff, 0x144,
1174 : 0x3504, 0xffffffff, 0x301f095b,
1175 : 0x3500, 0xffffffff, 0x165,
1176 : 0x3504, 0xffffffff, 0xc094d,
1177 : 0x3500, 0xffffffff, 0x173,
1178 : 0x3504, 0xffffffff, 0xf096d,
1179 : 0x3500, 0xffffffff, 0x184,
1180 : 0x3504, 0xffffffff, 0x15097f,
1181 : 0x3500, 0xffffffff, 0x19b,
1182 : 0x3504, 0xffffffff, 0xc0998,
1183 : 0x3500, 0xffffffff, 0x1a9,
1184 : 0x3504, 0xffffffff, 0x409a7,
1185 : 0x3500, 0xffffffff, 0x1af,
1186 : 0x3504, 0xffffffff, 0xcdc,
1187 : 0x3500, 0xffffffff, 0x1b1,
1188 : 0x3504, 0xffffffff, 0x800,
1189 : 0x3508, 0xffffffff, 0x6c9b2000,
1190 : 0x3510, 0xfc00, 0x2000,
1191 : 0x3544, 0xffffffff, 0xfc0,
1192 : 0x28d4, 0x00000100, 0x100
1193 : };
1194 :
1195 0 : static void si_init_golden_registers(struct radeon_device *rdev)
1196 : {
1197 0 : switch (rdev->family) {
1198 : case CHIP_TAHITI:
1199 0 : radeon_program_register_sequence(rdev,
1200 : tahiti_golden_registers,
1201 : (const u32)ARRAY_SIZE(tahiti_golden_registers));
1202 0 : radeon_program_register_sequence(rdev,
1203 : tahiti_golden_rlc_registers,
1204 : (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1205 0 : radeon_program_register_sequence(rdev,
1206 : tahiti_mgcg_cgcg_init,
1207 : (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1208 0 : radeon_program_register_sequence(rdev,
1209 : tahiti_golden_registers2,
1210 : (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1211 0 : break;
1212 : case CHIP_PITCAIRN:
1213 0 : radeon_program_register_sequence(rdev,
1214 : pitcairn_golden_registers,
1215 : (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1216 0 : radeon_program_register_sequence(rdev,
1217 : pitcairn_golden_rlc_registers,
1218 : (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1219 0 : radeon_program_register_sequence(rdev,
1220 : pitcairn_mgcg_cgcg_init,
1221 : (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1222 0 : break;
1223 : case CHIP_VERDE:
1224 0 : radeon_program_register_sequence(rdev,
1225 : verde_golden_registers,
1226 : (const u32)ARRAY_SIZE(verde_golden_registers));
1227 0 : radeon_program_register_sequence(rdev,
1228 : verde_golden_rlc_registers,
1229 : (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1230 0 : radeon_program_register_sequence(rdev,
1231 : verde_mgcg_cgcg_init,
1232 : (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1233 0 : radeon_program_register_sequence(rdev,
1234 : verde_pg_init,
1235 : (const u32)ARRAY_SIZE(verde_pg_init));
1236 0 : break;
1237 : case CHIP_OLAND:
1238 0 : radeon_program_register_sequence(rdev,
1239 : oland_golden_registers,
1240 : (const u32)ARRAY_SIZE(oland_golden_registers));
1241 0 : radeon_program_register_sequence(rdev,
1242 : oland_golden_rlc_registers,
1243 : (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1244 0 : radeon_program_register_sequence(rdev,
1245 : oland_mgcg_cgcg_init,
1246 : (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1247 0 : break;
1248 : case CHIP_HAINAN:
1249 0 : radeon_program_register_sequence(rdev,
1250 : hainan_golden_registers,
1251 : (const u32)ARRAY_SIZE(hainan_golden_registers));
1252 0 : radeon_program_register_sequence(rdev,
1253 : hainan_golden_registers2,
1254 : (const u32)ARRAY_SIZE(hainan_golden_registers2));
1255 0 : radeon_program_register_sequence(rdev,
1256 : hainan_mgcg_cgcg_init,
1257 : (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1258 0 : break;
1259 : default:
1260 : break;
1261 : }
1262 0 : }
1263 :
1264 : /**
1265 : * si_get_allowed_info_register - fetch the register for the info ioctl
1266 : *
1267 : * @rdev: radeon_device pointer
1268 : * @reg: register offset in bytes
1269 : * @val: register value
1270 : *
1271 : * Returns 0 for success or -EINVAL for an invalid register
1272 : *
1273 : */
1274 0 : int si_get_allowed_info_register(struct radeon_device *rdev,
1275 : u32 reg, u32 *val)
1276 : {
1277 0 : switch (reg) {
1278 : case GRBM_STATUS:
1279 : case GRBM_STATUS2:
1280 : case GRBM_STATUS_SE0:
1281 : case GRBM_STATUS_SE1:
1282 : case SRBM_STATUS:
1283 : case SRBM_STATUS2:
1284 : case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1285 : case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1286 : case UVD_STATUS:
1287 0 : *val = RREG32(reg);
1288 0 : return 0;
1289 : default:
1290 0 : return -EINVAL;
1291 : }
1292 0 : }
1293 :
1294 : #define PCIE_BUS_CLK 10000
1295 : #define TCLK (PCIE_BUS_CLK / 10)
1296 :
1297 : /**
1298 : * si_get_xclk - get the xclk
1299 : *
1300 : * @rdev: radeon_device pointer
1301 : *
1302 : * Returns the reference clock used by the gfx engine
1303 : * (SI).
1304 : */
1305 0 : u32 si_get_xclk(struct radeon_device *rdev)
1306 : {
1307 0 : u32 reference_clock = rdev->clock.spll.reference_freq;
1308 : u32 tmp;
1309 :
1310 0 : tmp = RREG32(CG_CLKPIN_CNTL_2);
1311 0 : if (tmp & MUX_TCLK_TO_XCLK)
1312 0 : return TCLK;
1313 :
1314 0 : tmp = RREG32(CG_CLKPIN_CNTL);
1315 0 : if (tmp & XTALIN_DIVIDE)
1316 0 : return reference_clock / 4;
1317 :
1318 0 : return reference_clock;
1319 0 : }
1320 :
1321 : /* get temperature in millidegrees */
1322 0 : int si_get_temp(struct radeon_device *rdev)
1323 : {
1324 : u32 temp;
1325 : int actual_temp = 0;
1326 :
1327 0 : temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1328 : CTF_TEMP_SHIFT;
1329 :
1330 0 : if (temp & 0x200)
1331 0 : actual_temp = 255;
1332 : else
1333 0 : actual_temp = temp & 0x1ff;
1334 :
1335 0 : actual_temp = (actual_temp * 1000);
1336 :
1337 0 : return actual_temp;
1338 : }
1339 :
1340 : #define TAHITI_IO_MC_REGS_SIZE 36
1341 :
1342 : static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1343 : {0x0000006f, 0x03044000},
1344 : {0x00000070, 0x0480c018},
1345 : {0x00000071, 0x00000040},
1346 : {0x00000072, 0x01000000},
1347 : {0x00000074, 0x000000ff},
1348 : {0x00000075, 0x00143400},
1349 : {0x00000076, 0x08ec0800},
1350 : {0x00000077, 0x040000cc},
1351 : {0x00000079, 0x00000000},
1352 : {0x0000007a, 0x21000409},
1353 : {0x0000007c, 0x00000000},
1354 : {0x0000007d, 0xe8000000},
1355 : {0x0000007e, 0x044408a8},
1356 : {0x0000007f, 0x00000003},
1357 : {0x00000080, 0x00000000},
1358 : {0x00000081, 0x01000000},
1359 : {0x00000082, 0x02000000},
1360 : {0x00000083, 0x00000000},
1361 : {0x00000084, 0xe3f3e4f4},
1362 : {0x00000085, 0x00052024},
1363 : {0x00000087, 0x00000000},
1364 : {0x00000088, 0x66036603},
1365 : {0x00000089, 0x01000000},
1366 : {0x0000008b, 0x1c0a0000},
1367 : {0x0000008c, 0xff010000},
1368 : {0x0000008e, 0xffffefff},
1369 : {0x0000008f, 0xfff3efff},
1370 : {0x00000090, 0xfff3efbf},
1371 : {0x00000094, 0x00101101},
1372 : {0x00000095, 0x00000fff},
1373 : {0x00000096, 0x00116fff},
1374 : {0x00000097, 0x60010000},
1375 : {0x00000098, 0x10010000},
1376 : {0x00000099, 0x00006000},
1377 : {0x0000009a, 0x00001000},
1378 : {0x0000009f, 0x00a77400}
1379 : };
1380 :
1381 : static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1382 : {0x0000006f, 0x03044000},
1383 : {0x00000070, 0x0480c018},
1384 : {0x00000071, 0x00000040},
1385 : {0x00000072, 0x01000000},
1386 : {0x00000074, 0x000000ff},
1387 : {0x00000075, 0x00143400},
1388 : {0x00000076, 0x08ec0800},
1389 : {0x00000077, 0x040000cc},
1390 : {0x00000079, 0x00000000},
1391 : {0x0000007a, 0x21000409},
1392 : {0x0000007c, 0x00000000},
1393 : {0x0000007d, 0xe8000000},
1394 : {0x0000007e, 0x044408a8},
1395 : {0x0000007f, 0x00000003},
1396 : {0x00000080, 0x00000000},
1397 : {0x00000081, 0x01000000},
1398 : {0x00000082, 0x02000000},
1399 : {0x00000083, 0x00000000},
1400 : {0x00000084, 0xe3f3e4f4},
1401 : {0x00000085, 0x00052024},
1402 : {0x00000087, 0x00000000},
1403 : {0x00000088, 0x66036603},
1404 : {0x00000089, 0x01000000},
1405 : {0x0000008b, 0x1c0a0000},
1406 : {0x0000008c, 0xff010000},
1407 : {0x0000008e, 0xffffefff},
1408 : {0x0000008f, 0xfff3efff},
1409 : {0x00000090, 0xfff3efbf},
1410 : {0x00000094, 0x00101101},
1411 : {0x00000095, 0x00000fff},
1412 : {0x00000096, 0x00116fff},
1413 : {0x00000097, 0x60010000},
1414 : {0x00000098, 0x10010000},
1415 : {0x00000099, 0x00006000},
1416 : {0x0000009a, 0x00001000},
1417 : {0x0000009f, 0x00a47400}
1418 : };
1419 :
1420 : static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1421 : {0x0000006f, 0x03044000},
1422 : {0x00000070, 0x0480c018},
1423 : {0x00000071, 0x00000040},
1424 : {0x00000072, 0x01000000},
1425 : {0x00000074, 0x000000ff},
1426 : {0x00000075, 0x00143400},
1427 : {0x00000076, 0x08ec0800},
1428 : {0x00000077, 0x040000cc},
1429 : {0x00000079, 0x00000000},
1430 : {0x0000007a, 0x21000409},
1431 : {0x0000007c, 0x00000000},
1432 : {0x0000007d, 0xe8000000},
1433 : {0x0000007e, 0x044408a8},
1434 : {0x0000007f, 0x00000003},
1435 : {0x00000080, 0x00000000},
1436 : {0x00000081, 0x01000000},
1437 : {0x00000082, 0x02000000},
1438 : {0x00000083, 0x00000000},
1439 : {0x00000084, 0xe3f3e4f4},
1440 : {0x00000085, 0x00052024},
1441 : {0x00000087, 0x00000000},
1442 : {0x00000088, 0x66036603},
1443 : {0x00000089, 0x01000000},
1444 : {0x0000008b, 0x1c0a0000},
1445 : {0x0000008c, 0xff010000},
1446 : {0x0000008e, 0xffffefff},
1447 : {0x0000008f, 0xfff3efff},
1448 : {0x00000090, 0xfff3efbf},
1449 : {0x00000094, 0x00101101},
1450 : {0x00000095, 0x00000fff},
1451 : {0x00000096, 0x00116fff},
1452 : {0x00000097, 0x60010000},
1453 : {0x00000098, 0x10010000},
1454 : {0x00000099, 0x00006000},
1455 : {0x0000009a, 0x00001000},
1456 : {0x0000009f, 0x00a37400}
1457 : };
1458 :
1459 : static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1460 : {0x0000006f, 0x03044000},
1461 : {0x00000070, 0x0480c018},
1462 : {0x00000071, 0x00000040},
1463 : {0x00000072, 0x01000000},
1464 : {0x00000074, 0x000000ff},
1465 : {0x00000075, 0x00143400},
1466 : {0x00000076, 0x08ec0800},
1467 : {0x00000077, 0x040000cc},
1468 : {0x00000079, 0x00000000},
1469 : {0x0000007a, 0x21000409},
1470 : {0x0000007c, 0x00000000},
1471 : {0x0000007d, 0xe8000000},
1472 : {0x0000007e, 0x044408a8},
1473 : {0x0000007f, 0x00000003},
1474 : {0x00000080, 0x00000000},
1475 : {0x00000081, 0x01000000},
1476 : {0x00000082, 0x02000000},
1477 : {0x00000083, 0x00000000},
1478 : {0x00000084, 0xe3f3e4f4},
1479 : {0x00000085, 0x00052024},
1480 : {0x00000087, 0x00000000},
1481 : {0x00000088, 0x66036603},
1482 : {0x00000089, 0x01000000},
1483 : {0x0000008b, 0x1c0a0000},
1484 : {0x0000008c, 0xff010000},
1485 : {0x0000008e, 0xffffefff},
1486 : {0x0000008f, 0xfff3efff},
1487 : {0x00000090, 0xfff3efbf},
1488 : {0x00000094, 0x00101101},
1489 : {0x00000095, 0x00000fff},
1490 : {0x00000096, 0x00116fff},
1491 : {0x00000097, 0x60010000},
1492 : {0x00000098, 0x10010000},
1493 : {0x00000099, 0x00006000},
1494 : {0x0000009a, 0x00001000},
1495 : {0x0000009f, 0x00a17730}
1496 : };
1497 :
1498 : static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1499 : {0x0000006f, 0x03044000},
1500 : {0x00000070, 0x0480c018},
1501 : {0x00000071, 0x00000040},
1502 : {0x00000072, 0x01000000},
1503 : {0x00000074, 0x000000ff},
1504 : {0x00000075, 0x00143400},
1505 : {0x00000076, 0x08ec0800},
1506 : {0x00000077, 0x040000cc},
1507 : {0x00000079, 0x00000000},
1508 : {0x0000007a, 0x21000409},
1509 : {0x0000007c, 0x00000000},
1510 : {0x0000007d, 0xe8000000},
1511 : {0x0000007e, 0x044408a8},
1512 : {0x0000007f, 0x00000003},
1513 : {0x00000080, 0x00000000},
1514 : {0x00000081, 0x01000000},
1515 : {0x00000082, 0x02000000},
1516 : {0x00000083, 0x00000000},
1517 : {0x00000084, 0xe3f3e4f4},
1518 : {0x00000085, 0x00052024},
1519 : {0x00000087, 0x00000000},
1520 : {0x00000088, 0x66036603},
1521 : {0x00000089, 0x01000000},
1522 : {0x0000008b, 0x1c0a0000},
1523 : {0x0000008c, 0xff010000},
1524 : {0x0000008e, 0xffffefff},
1525 : {0x0000008f, 0xfff3efff},
1526 : {0x00000090, 0xfff3efbf},
1527 : {0x00000094, 0x00101101},
1528 : {0x00000095, 0x00000fff},
1529 : {0x00000096, 0x00116fff},
1530 : {0x00000097, 0x60010000},
1531 : {0x00000098, 0x10010000},
1532 : {0x00000099, 0x00006000},
1533 : {0x0000009a, 0x00001000},
1534 : {0x0000009f, 0x00a07730}
1535 : };
1536 :
1537 : /* ucode loading */
1538 0 : int si_mc_load_microcode(struct radeon_device *rdev)
1539 : {
1540 : const __be32 *fw_data = NULL;
1541 : const __le32 *new_fw_data = NULL;
1542 : u32 running, blackout = 0;
1543 : u32 *io_mc_regs = NULL;
1544 : const __le32 *new_io_mc_regs = NULL;
1545 : int i, regs_size, ucode_size;
1546 :
1547 0 : if (!rdev->mc_fw)
1548 0 : return -EINVAL;
1549 :
1550 0 : if (rdev->new_fw) {
1551 : const struct mc_firmware_header_v1_0 *hdr =
1552 0 : (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1553 :
1554 0 : radeon_ucode_print_mc_hdr(&hdr->header);
1555 0 : regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1556 0 : new_io_mc_regs = (const __le32 *)
1557 0 : (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1558 0 : ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1559 0 : new_fw_data = (const __le32 *)
1560 0 : (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1561 0 : } else {
1562 0 : ucode_size = rdev->mc_fw->size / 4;
1563 :
1564 0 : switch (rdev->family) {
1565 : case CHIP_TAHITI:
1566 : io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1567 : regs_size = TAHITI_IO_MC_REGS_SIZE;
1568 0 : break;
1569 : case CHIP_PITCAIRN:
1570 : io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1571 : regs_size = TAHITI_IO_MC_REGS_SIZE;
1572 0 : break;
1573 : case CHIP_VERDE:
1574 : default:
1575 : io_mc_regs = (u32 *)&verde_io_mc_regs;
1576 : regs_size = TAHITI_IO_MC_REGS_SIZE;
1577 0 : break;
1578 : case CHIP_OLAND:
1579 : io_mc_regs = (u32 *)&oland_io_mc_regs;
1580 : regs_size = TAHITI_IO_MC_REGS_SIZE;
1581 0 : break;
1582 : case CHIP_HAINAN:
1583 : io_mc_regs = (u32 *)&hainan_io_mc_regs;
1584 : regs_size = TAHITI_IO_MC_REGS_SIZE;
1585 0 : break;
1586 : }
1587 0 : fw_data = (const __be32 *)rdev->mc_fw->data;
1588 : }
1589 :
1590 0 : running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1591 :
1592 0 : if (running == 0) {
1593 0 : if (running) {
1594 0 : blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1595 0 : WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1596 0 : }
1597 :
1598 : /* reset the engine and set to writable */
1599 0 : WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1600 0 : WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1601 :
1602 : /* load mc io regs */
1603 0 : for (i = 0; i < regs_size; i++) {
1604 0 : if (rdev->new_fw) {
1605 0 : WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1606 0 : WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1607 0 : } else {
1608 0 : WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1609 0 : WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1610 : }
1611 : }
1612 : /* load the MC ucode */
1613 0 : for (i = 0; i < ucode_size; i++) {
1614 0 : if (rdev->new_fw)
1615 0 : WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1616 : else
1617 0 : WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1618 : }
1619 :
1620 : /* put the engine back into the active state */
1621 0 : WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1622 0 : WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1623 0 : WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1624 :
1625 : /* wait for training to complete */
1626 0 : for (i = 0; i < rdev->usec_timeout; i++) {
1627 0 : if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1628 : break;
1629 0 : udelay(1);
1630 : }
1631 0 : for (i = 0; i < rdev->usec_timeout; i++) {
1632 0 : if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1633 : break;
1634 0 : udelay(1);
1635 : }
1636 :
1637 0 : if (running)
1638 0 : WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1639 : }
1640 :
1641 0 : return 0;
1642 0 : }
1643 :
1644 0 : static int si_init_microcode(struct radeon_device *rdev)
1645 : {
1646 : const char *chip_name;
1647 : const char *new_chip_name;
1648 : size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1649 : size_t smc_req_size, mc2_req_size;
1650 0 : char fw_name[30];
1651 : int err;
1652 : int new_fw = 0;
1653 :
1654 : DRM_DEBUG("\n");
1655 :
1656 0 : switch (rdev->family) {
1657 : case CHIP_TAHITI:
1658 : chip_name = "TAHITI";
1659 : new_chip_name = "tahiti";
1660 : pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1661 : me_req_size = SI_PM4_UCODE_SIZE * 4;
1662 : ce_req_size = SI_CE_UCODE_SIZE * 4;
1663 : rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1664 : mc_req_size = SI_MC_UCODE_SIZE * 4;
1665 : mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1666 : smc_req_size = roundup2(TAHITI_SMC_UCODE_SIZE, 4);
1667 0 : break;
1668 : case CHIP_PITCAIRN:
1669 : chip_name = "PITCAIRN";
1670 : new_chip_name = "pitcairn";
1671 : pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1672 : me_req_size = SI_PM4_UCODE_SIZE * 4;
1673 : ce_req_size = SI_CE_UCODE_SIZE * 4;
1674 : rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1675 : mc_req_size = SI_MC_UCODE_SIZE * 4;
1676 : mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1677 : smc_req_size = roundup2(PITCAIRN_SMC_UCODE_SIZE, 4);
1678 0 : break;
1679 : case CHIP_VERDE:
1680 : chip_name = "VERDE";
1681 : new_chip_name = "verde";
1682 : pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1683 : me_req_size = SI_PM4_UCODE_SIZE * 4;
1684 : ce_req_size = SI_CE_UCODE_SIZE * 4;
1685 : rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1686 : mc_req_size = SI_MC_UCODE_SIZE * 4;
1687 : mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1688 : smc_req_size = roundup2(VERDE_SMC_UCODE_SIZE, 4);
1689 0 : break;
1690 : case CHIP_OLAND:
1691 : chip_name = "OLAND";
1692 : new_chip_name = "oland";
1693 : pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1694 : me_req_size = SI_PM4_UCODE_SIZE * 4;
1695 : ce_req_size = SI_CE_UCODE_SIZE * 4;
1696 : rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1697 : mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1698 : smc_req_size = roundup2(OLAND_SMC_UCODE_SIZE, 4);
1699 0 : break;
1700 : case CHIP_HAINAN:
1701 : chip_name = "HAINAN";
1702 : new_chip_name = "hainan";
1703 : pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1704 : me_req_size = SI_PM4_UCODE_SIZE * 4;
1705 : ce_req_size = SI_CE_UCODE_SIZE * 4;
1706 : rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1707 : mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1708 : smc_req_size = roundup2(HAINAN_SMC_UCODE_SIZE, 4);
1709 0 : break;
1710 0 : default: BUG();
1711 : }
1712 :
1713 : DRM_INFO("Loading %s Microcode\n", new_chip_name);
1714 :
1715 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1716 0 : err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1717 0 : if (err) {
1718 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1719 0 : err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1720 0 : if (err)
1721 : goto out;
1722 0 : if (rdev->pfp_fw->size != pfp_req_size) {
1723 0 : printk(KERN_ERR
1724 : "si_cp: Bogus length %zu in firmware \"%s\"\n",
1725 : rdev->pfp_fw->size, fw_name);
1726 : err = -EINVAL;
1727 0 : goto out;
1728 : }
1729 : } else {
1730 0 : err = radeon_ucode_validate(rdev->pfp_fw);
1731 0 : if (err) {
1732 0 : printk(KERN_ERR
1733 : "si_cp: validation failed for firmware \"%s\"\n",
1734 : fw_name);
1735 0 : goto out;
1736 : } else {
1737 : new_fw++;
1738 : }
1739 : }
1740 :
1741 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1742 0 : err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1743 0 : if (err) {
1744 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1745 0 : err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1746 0 : if (err)
1747 : goto out;
1748 0 : if (rdev->me_fw->size != me_req_size) {
1749 0 : printk(KERN_ERR
1750 : "si_cp: Bogus length %zu in firmware \"%s\"\n",
1751 : rdev->me_fw->size, fw_name);
1752 : err = -EINVAL;
1753 0 : }
1754 : } else {
1755 0 : err = radeon_ucode_validate(rdev->me_fw);
1756 0 : if (err) {
1757 0 : printk(KERN_ERR
1758 : "si_cp: validation failed for firmware \"%s\"\n",
1759 : fw_name);
1760 0 : goto out;
1761 : } else {
1762 0 : new_fw++;
1763 : }
1764 : }
1765 :
1766 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1767 0 : err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1768 0 : if (err) {
1769 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1770 0 : err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1771 0 : if (err)
1772 : goto out;
1773 0 : if (rdev->ce_fw->size != ce_req_size) {
1774 0 : printk(KERN_ERR
1775 : "si_cp: Bogus length %zu in firmware \"%s\"\n",
1776 : rdev->ce_fw->size, fw_name);
1777 : err = -EINVAL;
1778 0 : }
1779 : } else {
1780 0 : err = radeon_ucode_validate(rdev->ce_fw);
1781 0 : if (err) {
1782 0 : printk(KERN_ERR
1783 : "si_cp: validation failed for firmware \"%s\"\n",
1784 : fw_name);
1785 0 : goto out;
1786 : } else {
1787 0 : new_fw++;
1788 : }
1789 : }
1790 :
1791 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1792 0 : err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1793 0 : if (err) {
1794 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1795 0 : err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1796 0 : if (err)
1797 : goto out;
1798 0 : if (rdev->rlc_fw->size != rlc_req_size) {
1799 0 : printk(KERN_ERR
1800 : "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1801 : rdev->rlc_fw->size, fw_name);
1802 : err = -EINVAL;
1803 0 : }
1804 : } else {
1805 0 : err = radeon_ucode_validate(rdev->rlc_fw);
1806 0 : if (err) {
1807 0 : printk(KERN_ERR
1808 : "si_cp: validation failed for firmware \"%s\"\n",
1809 : fw_name);
1810 0 : goto out;
1811 : } else {
1812 0 : new_fw++;
1813 : }
1814 : }
1815 :
1816 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1817 0 : err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1818 0 : if (err) {
1819 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1820 0 : err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1821 0 : if (err) {
1822 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1823 0 : err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1824 0 : if (err)
1825 : goto out;
1826 : }
1827 0 : if ((rdev->mc_fw->size != mc_req_size) &&
1828 0 : (rdev->mc_fw->size != mc2_req_size)) {
1829 0 : printk(KERN_ERR
1830 : "si_mc: Bogus length %zu in firmware \"%s\"\n",
1831 : rdev->mc_fw->size, fw_name);
1832 : err = -EINVAL;
1833 0 : }
1834 : DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1835 : } else {
1836 0 : err = radeon_ucode_validate(rdev->mc_fw);
1837 0 : if (err) {
1838 0 : printk(KERN_ERR
1839 : "si_cp: validation failed for firmware \"%s\"\n",
1840 : fw_name);
1841 0 : goto out;
1842 : } else {
1843 0 : new_fw++;
1844 : }
1845 : }
1846 :
1847 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1848 0 : err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1849 0 : if (err) {
1850 0 : snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1851 0 : err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1852 0 : if (err) {
1853 0 : printk(KERN_ERR
1854 : "smc: error loading firmware \"%s\"\n",
1855 : fw_name);
1856 0 : release_firmware(rdev->smc_fw);
1857 0 : rdev->smc_fw = NULL;
1858 : err = 0;
1859 0 : } else if (rdev->smc_fw->size != smc_req_size) {
1860 0 : printk(KERN_ERR
1861 : "si_smc: Bogus length %zu in firmware \"%s\"\n",
1862 : rdev->smc_fw->size, fw_name);
1863 : err = -EINVAL;
1864 0 : }
1865 : } else {
1866 0 : err = radeon_ucode_validate(rdev->smc_fw);
1867 0 : if (err) {
1868 0 : printk(KERN_ERR
1869 : "si_cp: validation failed for firmware \"%s\"\n",
1870 : fw_name);
1871 0 : goto out;
1872 : } else {
1873 0 : new_fw++;
1874 : }
1875 : }
1876 :
1877 0 : if (new_fw == 0) {
1878 0 : rdev->new_fw = false;
1879 0 : } else if (new_fw < 6) {
1880 0 : printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1881 : err = -EINVAL;
1882 0 : } else {
1883 0 : rdev->new_fw = true;
1884 : }
1885 : out:
1886 0 : if (err) {
1887 0 : if (err != -EINVAL)
1888 0 : printk(KERN_ERR
1889 : "si_cp: Failed to load firmware \"%s\"\n",
1890 : fw_name);
1891 0 : release_firmware(rdev->pfp_fw);
1892 0 : rdev->pfp_fw = NULL;
1893 0 : release_firmware(rdev->me_fw);
1894 0 : rdev->me_fw = NULL;
1895 0 : release_firmware(rdev->ce_fw);
1896 0 : rdev->ce_fw = NULL;
1897 0 : release_firmware(rdev->rlc_fw);
1898 0 : rdev->rlc_fw = NULL;
1899 0 : release_firmware(rdev->mc_fw);
1900 0 : rdev->mc_fw = NULL;
1901 0 : release_firmware(rdev->smc_fw);
1902 0 : rdev->smc_fw = NULL;
1903 0 : }
1904 0 : return err;
1905 0 : }
1906 :
1907 : /* watermark setup */
1908 0 : static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1909 : struct radeon_crtc *radeon_crtc,
1910 : struct drm_display_mode *mode,
1911 : struct drm_display_mode *other_mode)
1912 : {
1913 : u32 tmp, buffer_alloc, i;
1914 0 : u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1915 : /*
1916 : * Line Buffer Setup
1917 : * There are 3 line buffers, each one shared by 2 display controllers.
1918 : * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1919 : * the display controllers. The paritioning is done via one of four
1920 : * preset allocations specified in bits 21:20:
1921 : * 0 - half lb
1922 : * 2 - whole lb, other crtc must be disabled
1923 : */
1924 : /* this can get tricky if we have two large displays on a paired group
1925 : * of crtcs. Ideally for multiple large displays we'd assign them to
1926 : * non-linked crtcs for maximum line buffer allocation.
1927 : */
1928 0 : if (radeon_crtc->base.enabled && mode) {
1929 0 : if (other_mode) {
1930 : tmp = 0; /* 1/2 */
1931 : buffer_alloc = 1;
1932 0 : } else {
1933 : tmp = 2; /* whole */
1934 : buffer_alloc = 2;
1935 : }
1936 : } else {
1937 : tmp = 0;
1938 : buffer_alloc = 0;
1939 : }
1940 :
1941 0 : WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1942 : DC_LB_MEMORY_CONFIG(tmp));
1943 :
1944 0 : WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1945 : DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1946 0 : for (i = 0; i < rdev->usec_timeout; i++) {
1947 0 : if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1948 : DMIF_BUFFERS_ALLOCATED_COMPLETED)
1949 : break;
1950 0 : udelay(1);
1951 : }
1952 :
1953 0 : if (radeon_crtc->base.enabled && mode) {
1954 0 : switch (tmp) {
1955 : case 0:
1956 : default:
1957 0 : return 4096 * 2;
1958 : case 2:
1959 0 : return 8192 * 2;
1960 : }
1961 : }
1962 :
1963 : /* controller not enabled, so no lb used */
1964 0 : return 0;
1965 0 : }
1966 :
1967 0 : static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1968 : {
1969 0 : u32 tmp = RREG32(MC_SHARED_CHMAP);
1970 :
1971 0 : switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1972 : case 0:
1973 : default:
1974 0 : return 1;
1975 : case 1:
1976 0 : return 2;
1977 : case 2:
1978 0 : return 4;
1979 : case 3:
1980 0 : return 8;
1981 : case 4:
1982 0 : return 3;
1983 : case 5:
1984 0 : return 6;
1985 : case 6:
1986 0 : return 10;
1987 : case 7:
1988 0 : return 12;
1989 : case 8:
1990 0 : return 16;
1991 : }
1992 0 : }
1993 :
1994 : struct dce6_wm_params {
1995 : u32 dram_channels; /* number of dram channels */
1996 : u32 yclk; /* bandwidth per dram data pin in kHz */
1997 : u32 sclk; /* engine clock in kHz */
1998 : u32 disp_clk; /* display clock in kHz */
1999 : u32 src_width; /* viewport width */
2000 : u32 active_time; /* active display time in ns */
2001 : u32 blank_time; /* blank time in ns */
2002 : bool interlaced; /* mode is interlaced */
2003 : fixed20_12 vsc; /* vertical scale ratio */
2004 : u32 num_heads; /* number of active crtcs */
2005 : u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2006 : u32 lb_size; /* line buffer allocated to pipe */
2007 : u32 vtaps; /* vertical scaler taps */
2008 : };
2009 :
2010 0 : static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2011 : {
2012 : /* Calculate raw DRAM Bandwidth */
2013 : fixed20_12 dram_efficiency; /* 0.7 */
2014 : fixed20_12 yclk, dram_channels, bandwidth;
2015 : fixed20_12 a;
2016 :
2017 : a.full = dfixed_const(1000);
2018 0 : yclk.full = dfixed_const(wm->yclk);
2019 0 : yclk.full = dfixed_div(yclk, a);
2020 0 : dram_channels.full = dfixed_const(wm->dram_channels * 4);
2021 : a.full = dfixed_const(10);
2022 : dram_efficiency.full = dfixed_const(7);
2023 0 : dram_efficiency.full = dfixed_div(dram_efficiency, a);
2024 0 : bandwidth.full = dfixed_mul(dram_channels, yclk);
2025 0 : bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2026 :
2027 0 : return dfixed_trunc(bandwidth);
2028 : }
2029 :
2030 0 : static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2031 : {
2032 : /* Calculate DRAM Bandwidth and the part allocated to display. */
2033 : fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2034 : fixed20_12 yclk, dram_channels, bandwidth;
2035 : fixed20_12 a;
2036 :
2037 : a.full = dfixed_const(1000);
2038 0 : yclk.full = dfixed_const(wm->yclk);
2039 0 : yclk.full = dfixed_div(yclk, a);
2040 0 : dram_channels.full = dfixed_const(wm->dram_channels * 4);
2041 : a.full = dfixed_const(10);
2042 : disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2043 0 : disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2044 0 : bandwidth.full = dfixed_mul(dram_channels, yclk);
2045 0 : bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2046 :
2047 0 : return dfixed_trunc(bandwidth);
2048 : }
2049 :
2050 0 : static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2051 : {
2052 : /* Calculate the display Data return Bandwidth */
2053 : fixed20_12 return_efficiency; /* 0.8 */
2054 : fixed20_12 sclk, bandwidth;
2055 : fixed20_12 a;
2056 :
2057 : a.full = dfixed_const(1000);
2058 0 : sclk.full = dfixed_const(wm->sclk);
2059 0 : sclk.full = dfixed_div(sclk, a);
2060 : a.full = dfixed_const(10);
2061 : return_efficiency.full = dfixed_const(8);
2062 0 : return_efficiency.full = dfixed_div(return_efficiency, a);
2063 : a.full = dfixed_const(32);
2064 0 : bandwidth.full = dfixed_mul(a, sclk);
2065 0 : bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2066 :
2067 0 : return dfixed_trunc(bandwidth);
2068 : }
2069 :
2070 0 : static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2071 : {
2072 0 : return 32;
2073 : }
2074 :
2075 0 : static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2076 : {
2077 : /* Calculate the DMIF Request Bandwidth */
2078 : fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2079 : fixed20_12 disp_clk, sclk, bandwidth;
2080 : fixed20_12 a, b1, b2;
2081 : u32 min_bandwidth;
2082 :
2083 : a.full = dfixed_const(1000);
2084 0 : disp_clk.full = dfixed_const(wm->disp_clk);
2085 0 : disp_clk.full = dfixed_div(disp_clk, a);
2086 0 : a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2087 0 : b1.full = dfixed_mul(a, disp_clk);
2088 :
2089 : a.full = dfixed_const(1000);
2090 0 : sclk.full = dfixed_const(wm->sclk);
2091 0 : sclk.full = dfixed_div(sclk, a);
2092 0 : a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2093 0 : b2.full = dfixed_mul(a, sclk);
2094 :
2095 : a.full = dfixed_const(10);
2096 : disp_clk_request_efficiency.full = dfixed_const(8);
2097 0 : disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2098 :
2099 0 : min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2100 :
2101 0 : a.full = dfixed_const(min_bandwidth);
2102 0 : bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2103 :
2104 0 : return dfixed_trunc(bandwidth);
2105 : }
2106 :
2107 0 : static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2108 : {
2109 : /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2110 0 : u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2111 0 : u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2112 0 : u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2113 :
2114 0 : return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2115 : }
2116 :
2117 0 : static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2118 : {
2119 : /* Calculate the display mode Average Bandwidth
2120 : * DisplayMode should contain the source and destination dimensions,
2121 : * timing, etc.
2122 : */
2123 : fixed20_12 bpp;
2124 : fixed20_12 line_time;
2125 : fixed20_12 src_width;
2126 : fixed20_12 bandwidth;
2127 : fixed20_12 a;
2128 :
2129 : a.full = dfixed_const(1000);
2130 0 : line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2131 0 : line_time.full = dfixed_div(line_time, a);
2132 0 : bpp.full = dfixed_const(wm->bytes_per_pixel);
2133 0 : src_width.full = dfixed_const(wm->src_width);
2134 0 : bandwidth.full = dfixed_mul(src_width, bpp);
2135 0 : bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2136 0 : bandwidth.full = dfixed_div(bandwidth, line_time);
2137 :
2138 0 : return dfixed_trunc(bandwidth);
2139 : }
2140 :
2141 0 : static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2142 : {
2143 : /* First calcualte the latency in ns */
2144 : u32 mc_latency = 2000; /* 2000 ns. */
2145 0 : u32 available_bandwidth = dce6_available_bandwidth(wm);
2146 0 : u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2147 0 : u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2148 0 : u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2149 0 : u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2150 0 : (wm->num_heads * cursor_line_pair_return_time);
2151 0 : u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2152 : u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2153 : u32 tmp, dmif_size = 12288;
2154 : fixed20_12 a, b, c;
2155 :
2156 0 : if (wm->num_heads == 0)
2157 0 : return 0;
2158 :
2159 : a.full = dfixed_const(2);
2160 : b.full = dfixed_const(1);
2161 0 : if ((wm->vsc.full > a.full) ||
2162 0 : ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2163 0 : (wm->vtaps >= 5) ||
2164 0 : ((wm->vsc.full >= a.full) && wm->interlaced))
2165 0 : max_src_lines_per_dst_line = 4;
2166 : else
2167 : max_src_lines_per_dst_line = 2;
2168 :
2169 0 : a.full = dfixed_const(available_bandwidth);
2170 0 : b.full = dfixed_const(wm->num_heads);
2171 0 : a.full = dfixed_div(a, b);
2172 :
2173 : b.full = dfixed_const(mc_latency + 512);
2174 0 : c.full = dfixed_const(wm->disp_clk);
2175 0 : b.full = dfixed_div(b, c);
2176 :
2177 : c.full = dfixed_const(dmif_size);
2178 0 : b.full = dfixed_div(c, b);
2179 :
2180 0 : tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2181 :
2182 : b.full = dfixed_const(1000);
2183 0 : c.full = dfixed_const(wm->disp_clk);
2184 0 : b.full = dfixed_div(c, b);
2185 0 : c.full = dfixed_const(wm->bytes_per_pixel);
2186 0 : b.full = dfixed_mul(b, c);
2187 :
2188 0 : lb_fill_bw = min(tmp, dfixed_trunc(b));
2189 :
2190 0 : a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2191 : b.full = dfixed_const(1000);
2192 0 : c.full = dfixed_const(lb_fill_bw);
2193 0 : b.full = dfixed_div(c, b);
2194 0 : a.full = dfixed_div(a, b);
2195 0 : line_fill_time = dfixed_trunc(a);
2196 :
2197 0 : if (line_fill_time < wm->active_time)
2198 0 : return latency;
2199 : else
2200 0 : return latency + (line_fill_time - wm->active_time);
2201 :
2202 0 : }
2203 :
2204 0 : static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2205 : {
2206 0 : if (dce6_average_bandwidth(wm) <=
2207 0 : (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2208 0 : return true;
2209 : else
2210 0 : return false;
2211 0 : };
2212 :
2213 0 : static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2214 : {
2215 0 : if (dce6_average_bandwidth(wm) <=
2216 0 : (dce6_available_bandwidth(wm) / wm->num_heads))
2217 0 : return true;
2218 : else
2219 0 : return false;
2220 0 : };
2221 :
2222 0 : static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2223 : {
2224 0 : u32 lb_partitions = wm->lb_size / wm->src_width;
2225 0 : u32 line_time = wm->active_time + wm->blank_time;
2226 : u32 latency_tolerant_lines;
2227 : u32 latency_hiding;
2228 : fixed20_12 a;
2229 :
2230 : a.full = dfixed_const(1);
2231 0 : if (wm->vsc.full > a.full)
2232 0 : latency_tolerant_lines = 1;
2233 : else {
2234 0 : if (lb_partitions <= (wm->vtaps + 1))
2235 0 : latency_tolerant_lines = 1;
2236 : else
2237 : latency_tolerant_lines = 2;
2238 : }
2239 :
2240 0 : latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2241 :
2242 0 : if (dce6_latency_watermark(wm) <= latency_hiding)
2243 0 : return true;
2244 : else
2245 0 : return false;
2246 0 : }
2247 :
2248 0 : static void dce6_program_watermarks(struct radeon_device *rdev,
2249 : struct radeon_crtc *radeon_crtc,
2250 : u32 lb_size, u32 num_heads)
2251 : {
2252 0 : struct drm_display_mode *mode = &radeon_crtc->base.mode;
2253 0 : struct dce6_wm_params wm_low, wm_high;
2254 : u32 dram_channels;
2255 : u32 pixel_period;
2256 : u32 line_time = 0;
2257 : u32 latency_watermark_a = 0, latency_watermark_b = 0;
2258 : u32 priority_a_mark = 0, priority_b_mark = 0;
2259 : u32 priority_a_cnt = PRIORITY_OFF;
2260 : u32 priority_b_cnt = PRIORITY_OFF;
2261 : u32 tmp, arb_control3;
2262 : fixed20_12 a, b, c;
2263 :
2264 0 : if (radeon_crtc->base.enabled && num_heads && mode) {
2265 0 : pixel_period = 1000000 / (u32)mode->clock;
2266 0 : line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2267 : priority_a_cnt = 0;
2268 : priority_b_cnt = 0;
2269 :
2270 0 : if (rdev->family == CHIP_ARUBA)
2271 0 : dram_channels = evergreen_get_number_of_dram_channels(rdev);
2272 : else
2273 0 : dram_channels = si_get_number_of_dram_channels(rdev);
2274 :
2275 : /* watermark for high clocks */
2276 0 : if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2277 0 : wm_high.yclk =
2278 0 : radeon_dpm_get_mclk(rdev, false) * 10;
2279 0 : wm_high.sclk =
2280 0 : radeon_dpm_get_sclk(rdev, false) * 10;
2281 0 : } else {
2282 0 : wm_high.yclk = rdev->pm.current_mclk * 10;
2283 0 : wm_high.sclk = rdev->pm.current_sclk * 10;
2284 : }
2285 :
2286 0 : wm_high.disp_clk = mode->clock;
2287 0 : wm_high.src_width = mode->crtc_hdisplay;
2288 0 : wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2289 0 : wm_high.blank_time = line_time - wm_high.active_time;
2290 0 : wm_high.interlaced = false;
2291 0 : if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2292 0 : wm_high.interlaced = true;
2293 0 : wm_high.vsc = radeon_crtc->vsc;
2294 0 : wm_high.vtaps = 1;
2295 0 : if (radeon_crtc->rmx_type != RMX_OFF)
2296 0 : wm_high.vtaps = 2;
2297 0 : wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2298 0 : wm_high.lb_size = lb_size;
2299 0 : wm_high.dram_channels = dram_channels;
2300 0 : wm_high.num_heads = num_heads;
2301 :
2302 : /* watermark for low clocks */
2303 0 : if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2304 0 : wm_low.yclk =
2305 0 : radeon_dpm_get_mclk(rdev, true) * 10;
2306 0 : wm_low.sclk =
2307 0 : radeon_dpm_get_sclk(rdev, true) * 10;
2308 0 : } else {
2309 0 : wm_low.yclk = rdev->pm.current_mclk * 10;
2310 0 : wm_low.sclk = rdev->pm.current_sclk * 10;
2311 : }
2312 :
2313 0 : wm_low.disp_clk = mode->clock;
2314 0 : wm_low.src_width = mode->crtc_hdisplay;
2315 0 : wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2316 0 : wm_low.blank_time = line_time - wm_low.active_time;
2317 0 : wm_low.interlaced = false;
2318 0 : if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2319 0 : wm_low.interlaced = true;
2320 0 : wm_low.vsc = radeon_crtc->vsc;
2321 0 : wm_low.vtaps = 1;
2322 0 : if (radeon_crtc->rmx_type != RMX_OFF)
2323 0 : wm_low.vtaps = 2;
2324 0 : wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2325 0 : wm_low.lb_size = lb_size;
2326 0 : wm_low.dram_channels = dram_channels;
2327 0 : wm_low.num_heads = num_heads;
2328 :
2329 : /* set for high clocks */
2330 0 : latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2331 : /* set for low clocks */
2332 0 : latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2333 :
2334 : /* possibly force display priority to high */
2335 : /* should really do this at mode validation time... */
2336 0 : if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2337 0 : !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2338 0 : !dce6_check_latency_hiding(&wm_high) ||
2339 0 : (rdev->disp_priority == 2)) {
2340 : DRM_DEBUG_KMS("force priority to high\n");
2341 : priority_a_cnt |= PRIORITY_ALWAYS_ON;
2342 : priority_b_cnt |= PRIORITY_ALWAYS_ON;
2343 0 : }
2344 0 : if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2345 0 : !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2346 0 : !dce6_check_latency_hiding(&wm_low) ||
2347 0 : (rdev->disp_priority == 2)) {
2348 : DRM_DEBUG_KMS("force priority to high\n");
2349 : priority_a_cnt |= PRIORITY_ALWAYS_ON;
2350 : priority_b_cnt |= PRIORITY_ALWAYS_ON;
2351 0 : }
2352 :
2353 : a.full = dfixed_const(1000);
2354 0 : b.full = dfixed_const(mode->clock);
2355 0 : b.full = dfixed_div(b, a);
2356 0 : c.full = dfixed_const(latency_watermark_a);
2357 0 : c.full = dfixed_mul(c, b);
2358 0 : c.full = dfixed_mul(c, radeon_crtc->hsc);
2359 0 : c.full = dfixed_div(c, a);
2360 : a.full = dfixed_const(16);
2361 0 : c.full = dfixed_div(c, a);
2362 0 : priority_a_mark = dfixed_trunc(c);
2363 0 : priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2364 :
2365 : a.full = dfixed_const(1000);
2366 0 : b.full = dfixed_const(mode->clock);
2367 0 : b.full = dfixed_div(b, a);
2368 0 : c.full = dfixed_const(latency_watermark_b);
2369 0 : c.full = dfixed_mul(c, b);
2370 0 : c.full = dfixed_mul(c, radeon_crtc->hsc);
2371 0 : c.full = dfixed_div(c, a);
2372 : a.full = dfixed_const(16);
2373 0 : c.full = dfixed_div(c, a);
2374 0 : priority_b_mark = dfixed_trunc(c);
2375 0 : priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2376 :
2377 : /* Save number of lines the linebuffer leads before the scanout */
2378 0 : radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2379 0 : }
2380 :
2381 : /* select wm A */
2382 0 : arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2383 : tmp = arb_control3;
2384 0 : tmp &= ~LATENCY_WATERMARK_MASK(3);
2385 0 : tmp |= LATENCY_WATERMARK_MASK(1);
2386 0 : WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2387 0 : WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2388 : (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2389 : LATENCY_HIGH_WATERMARK(line_time)));
2390 : /* select wm B */
2391 0 : tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2392 0 : tmp &= ~LATENCY_WATERMARK_MASK(3);
2393 0 : tmp |= LATENCY_WATERMARK_MASK(2);
2394 0 : WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2395 0 : WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2396 : (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2397 : LATENCY_HIGH_WATERMARK(line_time)));
2398 : /* restore original selection */
2399 0 : WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2400 :
2401 : /* write the priority marks */
2402 0 : WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2403 0 : WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2404 :
2405 : /* save values for DPM */
2406 0 : radeon_crtc->line_time = line_time;
2407 0 : radeon_crtc->wm_high = latency_watermark_a;
2408 0 : radeon_crtc->wm_low = latency_watermark_b;
2409 0 : }
2410 :
2411 0 : void dce6_bandwidth_update(struct radeon_device *rdev)
2412 : {
2413 : struct drm_display_mode *mode0 = NULL;
2414 : struct drm_display_mode *mode1 = NULL;
2415 : u32 num_heads = 0, lb_size;
2416 : int i;
2417 :
2418 0 : if (!rdev->mode_info.mode_config_initialized)
2419 0 : return;
2420 :
2421 0 : radeon_update_display_priority(rdev);
2422 :
2423 0 : for (i = 0; i < rdev->num_crtc; i++) {
2424 0 : if (rdev->mode_info.crtcs[i]->base.enabled)
2425 0 : num_heads++;
2426 : }
2427 0 : for (i = 0; i < rdev->num_crtc; i += 2) {
2428 0 : mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2429 0 : mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2430 0 : lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2431 0 : dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2432 0 : lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2433 0 : dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2434 : }
2435 0 : }
2436 :
2437 : /*
2438 : * Core functions
2439 : */
2440 0 : static void si_tiling_mode_table_init(struct radeon_device *rdev)
2441 : {
2442 : const u32 num_tile_mode_states = 32;
2443 : u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2444 :
2445 0 : switch (rdev->config.si.mem_row_size_in_kb) {
2446 : case 1:
2447 : split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2448 0 : break;
2449 : case 2:
2450 : default:
2451 : split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2452 0 : break;
2453 : case 4:
2454 : split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2455 0 : break;
2456 : }
2457 :
2458 0 : if ((rdev->family == CHIP_TAHITI) ||
2459 0 : (rdev->family == CHIP_PITCAIRN)) {
2460 0 : for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2461 0 : switch (reg_offset) {
2462 : case 0: /* non-AA compressed depth or any compressed stencil */
2463 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2465 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2466 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2467 : NUM_BANKS(ADDR_SURF_16_BANK) |
2468 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2470 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2471 0 : break;
2472 : case 1: /* 2xAA/4xAA compressed depth only */
2473 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2474 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2475 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2476 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2477 : NUM_BANKS(ADDR_SURF_16_BANK) |
2478 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2479 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2480 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2481 0 : break;
2482 : case 2: /* 8xAA compressed depth only */
2483 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2484 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2485 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2486 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2487 : NUM_BANKS(ADDR_SURF_16_BANK) |
2488 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2490 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2491 0 : break;
2492 : case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2493 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2494 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2495 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2496 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2497 : NUM_BANKS(ADDR_SURF_16_BANK) |
2498 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2499 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2500 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2501 0 : break;
2502 : case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2503 : gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2504 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2505 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2506 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2507 : NUM_BANKS(ADDR_SURF_16_BANK) |
2508 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2510 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2511 0 : break;
2512 : case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2513 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2515 0 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2516 0 : TILE_SPLIT(split_equal_to_row_size) |
2517 : NUM_BANKS(ADDR_SURF_16_BANK) |
2518 0 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2519 0 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2520 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2521 0 : break;
2522 : case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2523 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 0 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2526 0 : TILE_SPLIT(split_equal_to_row_size) |
2527 : NUM_BANKS(ADDR_SURF_16_BANK) |
2528 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2530 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2531 0 : break;
2532 : case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2533 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535 0 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2536 0 : TILE_SPLIT(split_equal_to_row_size) |
2537 : NUM_BANKS(ADDR_SURF_16_BANK) |
2538 0 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539 0 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2541 0 : break;
2542 : case 8: /* 1D and 1D Array Surfaces */
2543 : gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2544 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2545 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2546 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2547 : NUM_BANKS(ADDR_SURF_16_BANK) |
2548 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2549 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2550 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2551 0 : break;
2552 : case 9: /* Displayable maps. */
2553 : gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2554 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2555 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2556 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2557 : NUM_BANKS(ADDR_SURF_16_BANK) |
2558 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2559 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2560 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2561 0 : break;
2562 : case 10: /* Display 8bpp. */
2563 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2565 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2566 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2567 : NUM_BANKS(ADDR_SURF_16_BANK) |
2568 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2569 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2570 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2571 0 : break;
2572 : case 11: /* Display 16bpp. */
2573 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2574 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2575 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2576 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2577 : NUM_BANKS(ADDR_SURF_16_BANK) |
2578 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2579 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2580 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2581 0 : break;
2582 : case 12: /* Display 32bpp. */
2583 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2584 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2585 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2586 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2587 : NUM_BANKS(ADDR_SURF_16_BANK) |
2588 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2589 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2590 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2591 0 : break;
2592 : case 13: /* Thin. */
2593 : gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2595 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2596 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2597 : NUM_BANKS(ADDR_SURF_16_BANK) |
2598 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2599 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2600 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2601 0 : break;
2602 : case 14: /* Thin 8 bpp. */
2603 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2605 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2606 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2607 : NUM_BANKS(ADDR_SURF_16_BANK) |
2608 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2610 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2611 0 : break;
2612 : case 15: /* Thin 16 bpp. */
2613 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2615 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2616 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2617 : NUM_BANKS(ADDR_SURF_16_BANK) |
2618 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2621 0 : break;
2622 : case 16: /* Thin 32 bpp. */
2623 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2625 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2626 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2627 : NUM_BANKS(ADDR_SURF_16_BANK) |
2628 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2631 0 : break;
2632 : case 17: /* Thin 64 bpp. */
2633 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2634 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2635 0 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 0 : TILE_SPLIT(split_equal_to_row_size) |
2637 : NUM_BANKS(ADDR_SURF_16_BANK) |
2638 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2639 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2640 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2641 0 : break;
2642 : case 21: /* 8 bpp PRT. */
2643 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2645 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2646 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2647 : NUM_BANKS(ADDR_SURF_16_BANK) |
2648 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2649 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2650 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2651 0 : break;
2652 : case 22: /* 16 bpp PRT */
2653 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2655 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2656 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2657 : NUM_BANKS(ADDR_SURF_16_BANK) |
2658 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2659 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2660 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2661 0 : break;
2662 : case 23: /* 32 bpp PRT */
2663 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2664 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2665 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2666 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2667 : NUM_BANKS(ADDR_SURF_16_BANK) |
2668 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2670 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2671 0 : break;
2672 : case 24: /* 64 bpp PRT */
2673 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2675 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2676 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2677 : NUM_BANKS(ADDR_SURF_16_BANK) |
2678 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2681 0 : break;
2682 : case 25: /* 128 bpp PRT */
2683 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2684 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2685 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2686 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2687 : NUM_BANKS(ADDR_SURF_8_BANK) |
2688 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2689 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2690 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2691 0 : break;
2692 : default:
2693 : gb_tile_moden = 0;
2694 0 : break;
2695 : }
2696 0 : rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2697 0 : WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2698 : }
2699 0 : } else if ((rdev->family == CHIP_VERDE) ||
2700 0 : (rdev->family == CHIP_OLAND) ||
2701 0 : (rdev->family == CHIP_HAINAN)) {
2702 0 : for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2703 0 : switch (reg_offset) {
2704 : case 0: /* non-AA compressed depth or any compressed stencil */
2705 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2706 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2707 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2708 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2709 : NUM_BANKS(ADDR_SURF_16_BANK) |
2710 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2712 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2713 0 : break;
2714 : case 1: /* 2xAA/4xAA compressed depth only */
2715 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2716 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2717 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2718 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2719 : NUM_BANKS(ADDR_SURF_16_BANK) |
2720 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2722 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2723 0 : break;
2724 : case 2: /* 8xAA compressed depth only */
2725 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2726 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2727 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2728 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2729 : NUM_BANKS(ADDR_SURF_16_BANK) |
2730 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2731 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2732 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2733 0 : break;
2734 : case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2735 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2736 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2737 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2738 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2739 : NUM_BANKS(ADDR_SURF_16_BANK) |
2740 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2741 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2742 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2743 0 : break;
2744 : case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2745 : gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2746 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2747 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2749 : NUM_BANKS(ADDR_SURF_16_BANK) |
2750 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2753 0 : break;
2754 : case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2755 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 0 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2758 0 : TILE_SPLIT(split_equal_to_row_size) |
2759 : NUM_BANKS(ADDR_SURF_16_BANK) |
2760 0 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2761 0 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2762 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2763 0 : break;
2764 : case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2765 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2766 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2767 0 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 0 : TILE_SPLIT(split_equal_to_row_size) |
2769 : NUM_BANKS(ADDR_SURF_16_BANK) |
2770 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771 0 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2772 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2773 0 : break;
2774 : case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2775 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776 : MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2777 0 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778 0 : TILE_SPLIT(split_equal_to_row_size) |
2779 : NUM_BANKS(ADDR_SURF_16_BANK) |
2780 0 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781 0 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2782 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2783 0 : break;
2784 : case 8: /* 1D and 1D Array Surfaces */
2785 : gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2786 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2787 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2788 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2789 : NUM_BANKS(ADDR_SURF_16_BANK) |
2790 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2791 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2792 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2793 0 : break;
2794 : case 9: /* Displayable maps. */
2795 : gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2796 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2797 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2799 : NUM_BANKS(ADDR_SURF_16_BANK) |
2800 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2801 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2802 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2803 0 : break;
2804 : case 10: /* Display 8bpp. */
2805 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2808 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2809 : NUM_BANKS(ADDR_SURF_16_BANK) |
2810 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2811 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2812 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2813 0 : break;
2814 : case 11: /* Display 16bpp. */
2815 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2816 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2817 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2818 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2819 : NUM_BANKS(ADDR_SURF_16_BANK) |
2820 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2822 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2823 0 : break;
2824 : case 12: /* Display 32bpp. */
2825 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2826 : MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2827 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2828 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2829 : NUM_BANKS(ADDR_SURF_16_BANK) |
2830 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2831 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2832 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2833 0 : break;
2834 : case 13: /* Thin. */
2835 : gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2836 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2837 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2838 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2839 : NUM_BANKS(ADDR_SURF_16_BANK) |
2840 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2842 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2843 0 : break;
2844 : case 14: /* Thin 8 bpp. */
2845 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2846 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2847 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2848 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2849 : NUM_BANKS(ADDR_SURF_16_BANK) |
2850 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2851 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2852 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2853 0 : break;
2854 : case 15: /* Thin 16 bpp. */
2855 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2856 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2857 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2858 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2859 : NUM_BANKS(ADDR_SURF_16_BANK) |
2860 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2862 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2863 0 : break;
2864 : case 16: /* Thin 32 bpp. */
2865 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2867 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2868 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2869 : NUM_BANKS(ADDR_SURF_16_BANK) |
2870 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2872 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2873 0 : break;
2874 : case 17: /* Thin 64 bpp. */
2875 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2876 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2877 0 : PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2878 0 : TILE_SPLIT(split_equal_to_row_size) |
2879 : NUM_BANKS(ADDR_SURF_16_BANK) |
2880 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2881 0 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2882 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2883 0 : break;
2884 : case 21: /* 8 bpp PRT. */
2885 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2887 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2888 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2889 : NUM_BANKS(ADDR_SURF_16_BANK) |
2890 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2891 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2892 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2893 0 : break;
2894 : case 22: /* 16 bpp PRT */
2895 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2896 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2897 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2898 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2899 : NUM_BANKS(ADDR_SURF_16_BANK) |
2900 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2901 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2902 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2903 0 : break;
2904 : case 23: /* 32 bpp PRT */
2905 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2906 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2907 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2908 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2909 : NUM_BANKS(ADDR_SURF_16_BANK) |
2910 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2911 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2912 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2913 0 : break;
2914 : case 24: /* 64 bpp PRT */
2915 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2916 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2917 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2918 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2919 : NUM_BANKS(ADDR_SURF_16_BANK) |
2920 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2922 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2923 0 : break;
2924 : case 25: /* 128 bpp PRT */
2925 : gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2926 : MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2927 : PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2928 : TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2929 : NUM_BANKS(ADDR_SURF_8_BANK) |
2930 : BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931 : BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932 : MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2933 0 : break;
2934 : default:
2935 : gb_tile_moden = 0;
2936 0 : break;
2937 : }
2938 0 : rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2939 0 : WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2940 : }
2941 : } else
2942 0 : DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2943 0 : }
2944 :
2945 0 : static void si_select_se_sh(struct radeon_device *rdev,
2946 : u32 se_num, u32 sh_num)
2947 : {
2948 : u32 data = INSTANCE_BROADCAST_WRITES;
2949 :
2950 0 : if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2951 0 : data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2952 0 : else if (se_num == 0xffffffff)
2953 0 : data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2954 0 : else if (sh_num == 0xffffffff)
2955 0 : data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2956 : else
2957 0 : data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2958 0 : WREG32(GRBM_GFX_INDEX, data);
2959 0 : }
2960 :
2961 0 : static u32 si_create_bitmask(u32 bit_width)
2962 : {
2963 : u32 i, mask = 0;
2964 :
2965 0 : for (i = 0; i < bit_width; i++) {
2966 0 : mask <<= 1;
2967 0 : mask |= 1;
2968 : }
2969 0 : return mask;
2970 : }
2971 :
2972 0 : static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2973 : {
2974 : u32 data, mask;
2975 :
2976 0 : data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2977 0 : if (data & 1)
2978 0 : data &= INACTIVE_CUS_MASK;
2979 : else
2980 : data = 0;
2981 0 : data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2982 :
2983 0 : data >>= INACTIVE_CUS_SHIFT;
2984 :
2985 0 : mask = si_create_bitmask(cu_per_sh);
2986 :
2987 0 : return ~data & mask;
2988 : }
2989 :
2990 0 : static void si_setup_spi(struct radeon_device *rdev,
2991 : u32 se_num, u32 sh_per_se,
2992 : u32 cu_per_sh)
2993 : {
2994 : int i, j, k;
2995 : u32 data, mask, active_cu;
2996 :
2997 0 : for (i = 0; i < se_num; i++) {
2998 0 : for (j = 0; j < sh_per_se; j++) {
2999 0 : si_select_se_sh(rdev, i, j);
3000 0 : data = RREG32(SPI_STATIC_THREAD_MGMT_3);
3001 0 : active_cu = si_get_cu_enabled(rdev, cu_per_sh);
3002 :
3003 : mask = 1;
3004 0 : for (k = 0; k < 16; k++) {
3005 0 : mask <<= k;
3006 0 : if (active_cu & mask) {
3007 0 : data &= ~mask;
3008 0 : WREG32(SPI_STATIC_THREAD_MGMT_3, data);
3009 0 : break;
3010 : }
3011 : }
3012 : }
3013 : }
3014 0 : si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3015 0 : }
3016 :
3017 0 : static u32 si_get_rb_disabled(struct radeon_device *rdev,
3018 : u32 max_rb_num_per_se,
3019 : u32 sh_per_se)
3020 : {
3021 : u32 data, mask;
3022 :
3023 0 : data = RREG32(CC_RB_BACKEND_DISABLE);
3024 0 : if (data & 1)
3025 0 : data &= BACKEND_DISABLE_MASK;
3026 : else
3027 : data = 0;
3028 0 : data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3029 :
3030 0 : data >>= BACKEND_DISABLE_SHIFT;
3031 :
3032 0 : mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3033 :
3034 0 : return data & mask;
3035 : }
3036 :
3037 0 : static void si_setup_rb(struct radeon_device *rdev,
3038 : u32 se_num, u32 sh_per_se,
3039 : u32 max_rb_num_per_se)
3040 : {
3041 : int i, j;
3042 : u32 data, mask;
3043 : u32 disabled_rbs = 0;
3044 : u32 enabled_rbs = 0;
3045 :
3046 0 : for (i = 0; i < se_num; i++) {
3047 0 : for (j = 0; j < sh_per_se; j++) {
3048 0 : si_select_se_sh(rdev, i, j);
3049 0 : data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3050 0 : disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3051 : }
3052 : }
3053 0 : si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3054 :
3055 : mask = 1;
3056 0 : for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3057 0 : if (!(disabled_rbs & mask))
3058 0 : enabled_rbs |= mask;
3059 0 : mask <<= 1;
3060 : }
3061 :
3062 0 : rdev->config.si.backend_enable_mask = enabled_rbs;
3063 :
3064 0 : for (i = 0; i < se_num; i++) {
3065 0 : si_select_se_sh(rdev, i, 0xffffffff);
3066 : data = 0;
3067 0 : for (j = 0; j < sh_per_se; j++) {
3068 0 : switch (enabled_rbs & 3) {
3069 : case 1:
3070 0 : data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3071 0 : break;
3072 : case 2:
3073 0 : data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3074 0 : break;
3075 : case 3:
3076 : default:
3077 0 : data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3078 0 : break;
3079 : }
3080 0 : enabled_rbs >>= 2;
3081 : }
3082 0 : WREG32(PA_SC_RASTER_CONFIG, data);
3083 : }
3084 0 : si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3085 0 : }
3086 :
3087 0 : static void si_gpu_init(struct radeon_device *rdev)
3088 : {
3089 : u32 gb_addr_config = 0;
3090 : u32 mc_shared_chmap, mc_arb_ramcfg;
3091 : u32 sx_debug_1;
3092 : u32 hdp_host_path_cntl;
3093 : u32 tmp;
3094 : int i, j;
3095 :
3096 0 : switch (rdev->family) {
3097 : case CHIP_TAHITI:
3098 0 : rdev->config.si.max_shader_engines = 2;
3099 0 : rdev->config.si.max_tile_pipes = 12;
3100 0 : rdev->config.si.max_cu_per_sh = 8;
3101 0 : rdev->config.si.max_sh_per_se = 2;
3102 0 : rdev->config.si.max_backends_per_se = 4;
3103 0 : rdev->config.si.max_texture_channel_caches = 12;
3104 0 : rdev->config.si.max_gprs = 256;
3105 0 : rdev->config.si.max_gs_threads = 32;
3106 0 : rdev->config.si.max_hw_contexts = 8;
3107 :
3108 0 : rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3109 0 : rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3110 0 : rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3111 0 : rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3112 : gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3113 0 : break;
3114 : case CHIP_PITCAIRN:
3115 0 : rdev->config.si.max_shader_engines = 2;
3116 0 : rdev->config.si.max_tile_pipes = 8;
3117 0 : rdev->config.si.max_cu_per_sh = 5;
3118 0 : rdev->config.si.max_sh_per_se = 2;
3119 0 : rdev->config.si.max_backends_per_se = 4;
3120 0 : rdev->config.si.max_texture_channel_caches = 8;
3121 0 : rdev->config.si.max_gprs = 256;
3122 0 : rdev->config.si.max_gs_threads = 32;
3123 0 : rdev->config.si.max_hw_contexts = 8;
3124 :
3125 0 : rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3126 0 : rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3127 0 : rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3128 0 : rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3129 : gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3130 0 : break;
3131 : case CHIP_VERDE:
3132 : default:
3133 0 : rdev->config.si.max_shader_engines = 1;
3134 0 : rdev->config.si.max_tile_pipes = 4;
3135 0 : rdev->config.si.max_cu_per_sh = 5;
3136 0 : rdev->config.si.max_sh_per_se = 2;
3137 0 : rdev->config.si.max_backends_per_se = 4;
3138 0 : rdev->config.si.max_texture_channel_caches = 4;
3139 0 : rdev->config.si.max_gprs = 256;
3140 0 : rdev->config.si.max_gs_threads = 32;
3141 0 : rdev->config.si.max_hw_contexts = 8;
3142 :
3143 0 : rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3144 0 : rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3145 0 : rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3146 0 : rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3147 : gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3148 0 : break;
3149 : case CHIP_OLAND:
3150 0 : rdev->config.si.max_shader_engines = 1;
3151 0 : rdev->config.si.max_tile_pipes = 4;
3152 0 : rdev->config.si.max_cu_per_sh = 6;
3153 0 : rdev->config.si.max_sh_per_se = 1;
3154 0 : rdev->config.si.max_backends_per_se = 2;
3155 0 : rdev->config.si.max_texture_channel_caches = 4;
3156 0 : rdev->config.si.max_gprs = 256;
3157 0 : rdev->config.si.max_gs_threads = 16;
3158 0 : rdev->config.si.max_hw_contexts = 8;
3159 :
3160 0 : rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3161 0 : rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3162 0 : rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3163 0 : rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3164 : gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3165 0 : break;
3166 : case CHIP_HAINAN:
3167 0 : rdev->config.si.max_shader_engines = 1;
3168 0 : rdev->config.si.max_tile_pipes = 4;
3169 0 : rdev->config.si.max_cu_per_sh = 5;
3170 0 : rdev->config.si.max_sh_per_se = 1;
3171 0 : rdev->config.si.max_backends_per_se = 1;
3172 0 : rdev->config.si.max_texture_channel_caches = 2;
3173 0 : rdev->config.si.max_gprs = 256;
3174 0 : rdev->config.si.max_gs_threads = 16;
3175 0 : rdev->config.si.max_hw_contexts = 8;
3176 :
3177 0 : rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3178 0 : rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3179 0 : rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3180 0 : rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3181 : gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3182 0 : break;
3183 : }
3184 :
3185 : /* Initialize HDP */
3186 0 : for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3187 0 : WREG32((0x2c14 + j), 0x00000000);
3188 0 : WREG32((0x2c18 + j), 0x00000000);
3189 0 : WREG32((0x2c1c + j), 0x00000000);
3190 0 : WREG32((0x2c20 + j), 0x00000000);
3191 0 : WREG32((0x2c24 + j), 0x00000000);
3192 : }
3193 :
3194 0 : WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3195 0 : WREG32(SRBM_INT_CNTL, 1);
3196 0 : WREG32(SRBM_INT_ACK, 1);
3197 :
3198 0 : evergreen_fix_pci_max_read_req_size(rdev);
3199 :
3200 0 : WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3201 :
3202 0 : mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3203 0 : mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3204 :
3205 0 : rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3206 0 : rdev->config.si.mem_max_burst_length_bytes = 256;
3207 0 : tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3208 0 : rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3209 0 : if (rdev->config.si.mem_row_size_in_kb > 4)
3210 0 : rdev->config.si.mem_row_size_in_kb = 4;
3211 : /* XXX use MC settings? */
3212 0 : rdev->config.si.shader_engine_tile_size = 32;
3213 0 : rdev->config.si.num_gpus = 1;
3214 0 : rdev->config.si.multi_gpu_tile_size = 64;
3215 :
3216 : /* fix up row size */
3217 0 : gb_addr_config &= ~ROW_SIZE_MASK;
3218 0 : switch (rdev->config.si.mem_row_size_in_kb) {
3219 : case 1:
3220 : default:
3221 : gb_addr_config |= ROW_SIZE(0);
3222 0 : break;
3223 : case 2:
3224 0 : gb_addr_config |= ROW_SIZE(1);
3225 0 : break;
3226 : case 4:
3227 0 : gb_addr_config |= ROW_SIZE(2);
3228 0 : break;
3229 : }
3230 :
3231 : /* setup tiling info dword. gb_addr_config is not adequate since it does
3232 : * not have bank info, so create a custom tiling dword.
3233 : * bits 3:0 num_pipes
3234 : * bits 7:4 num_banks
3235 : * bits 11:8 group_size
3236 : * bits 15:12 row_size
3237 : */
3238 0 : rdev->config.si.tile_config = 0;
3239 0 : switch (rdev->config.si.num_tile_pipes) {
3240 : case 1:
3241 0 : rdev->config.si.tile_config |= (0 << 0);
3242 0 : break;
3243 : case 2:
3244 0 : rdev->config.si.tile_config |= (1 << 0);
3245 0 : break;
3246 : case 4:
3247 0 : rdev->config.si.tile_config |= (2 << 0);
3248 0 : break;
3249 : case 8:
3250 : default:
3251 : /* XXX what about 12? */
3252 0 : rdev->config.si.tile_config |= (3 << 0);
3253 0 : break;
3254 : }
3255 0 : switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3256 : case 0: /* four banks */
3257 0 : rdev->config.si.tile_config |= 0 << 4;
3258 0 : break;
3259 : case 1: /* eight banks */
3260 0 : rdev->config.si.tile_config |= 1 << 4;
3261 0 : break;
3262 : case 2: /* sixteen banks */
3263 : default:
3264 0 : rdev->config.si.tile_config |= 2 << 4;
3265 0 : break;
3266 : }
3267 0 : rdev->config.si.tile_config |=
3268 0 : ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3269 0 : rdev->config.si.tile_config |=
3270 0 : ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3271 :
3272 0 : WREG32(GB_ADDR_CONFIG, gb_addr_config);
3273 0 : WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3274 0 : WREG32(DMIF_ADDR_CALC, gb_addr_config);
3275 0 : WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3276 0 : WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3277 0 : WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3278 0 : if (rdev->has_uvd) {
3279 0 : WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3280 0 : WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3281 0 : WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3282 0 : }
3283 :
3284 0 : si_tiling_mode_table_init(rdev);
3285 :
3286 0 : si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3287 0 : rdev->config.si.max_sh_per_se,
3288 0 : rdev->config.si.max_backends_per_se);
3289 :
3290 0 : si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3291 0 : rdev->config.si.max_sh_per_se,
3292 0 : rdev->config.si.max_cu_per_sh);
3293 :
3294 0 : rdev->config.si.active_cus = 0;
3295 0 : for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3296 0 : for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3297 0 : rdev->config.si.active_cus +=
3298 0 : hweight32(si_get_cu_active_bitmap(rdev, i, j));
3299 : }
3300 : }
3301 :
3302 : /* set HW defaults for 3D engine */
3303 0 : WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3304 : ROQ_IB2_START(0x2b)));
3305 0 : WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3306 :
3307 0 : sx_debug_1 = RREG32(SX_DEBUG_1);
3308 0 : WREG32(SX_DEBUG_1, sx_debug_1);
3309 :
3310 0 : WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3311 :
3312 0 : WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3313 : SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3314 : SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3315 : SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3316 :
3317 0 : WREG32(VGT_NUM_INSTANCES, 1);
3318 :
3319 0 : WREG32(CP_PERFMON_CNTL, 0);
3320 :
3321 0 : WREG32(SQ_CONFIG, 0);
3322 :
3323 0 : WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3324 : FORCE_EOV_MAX_REZ_CNT(255)));
3325 :
3326 0 : WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3327 : AUTO_INVLD_EN(ES_AND_GS_AUTO));
3328 :
3329 0 : WREG32(VGT_GS_VERTEX_REUSE, 16);
3330 0 : WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3331 :
3332 0 : WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3333 0 : WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3334 0 : WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3335 0 : WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3336 0 : WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3337 0 : WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3338 0 : WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3339 0 : WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3340 :
3341 0 : tmp = RREG32(HDP_MISC_CNTL);
3342 0 : tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3343 0 : WREG32(HDP_MISC_CNTL, tmp);
3344 :
3345 0 : hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3346 0 : WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3347 :
3348 0 : WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3349 :
3350 0 : udelay(50);
3351 0 : }
3352 :
3353 : /*
3354 : * GPU scratch registers helpers function.
3355 : */
3356 0 : static void si_scratch_init(struct radeon_device *rdev)
3357 : {
3358 : int i;
3359 :
3360 0 : rdev->scratch.num_reg = 7;
3361 0 : rdev->scratch.reg_base = SCRATCH_REG0;
3362 0 : for (i = 0; i < rdev->scratch.num_reg; i++) {
3363 0 : rdev->scratch.free[i] = true;
3364 0 : rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3365 : }
3366 0 : }
3367 :
3368 0 : void si_fence_ring_emit(struct radeon_device *rdev,
3369 : struct radeon_fence *fence)
3370 : {
3371 0 : struct radeon_ring *ring = &rdev->ring[fence->ring];
3372 0 : u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3373 :
3374 : /* flush read cache over gart */
3375 0 : radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3376 0 : radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3377 0 : radeon_ring_write(ring, 0);
3378 0 : radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3379 0 : radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3380 : PACKET3_TC_ACTION_ENA |
3381 : PACKET3_SH_KCACHE_ACTION_ENA |
3382 : PACKET3_SH_ICACHE_ACTION_ENA);
3383 0 : radeon_ring_write(ring, 0xFFFFFFFF);
3384 0 : radeon_ring_write(ring, 0);
3385 0 : radeon_ring_write(ring, 10); /* poll interval */
3386 : /* EVENT_WRITE_EOP - flush caches, send int */
3387 0 : radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3388 0 : radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3389 0 : radeon_ring_write(ring, lower_32_bits(addr));
3390 0 : radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3391 0 : radeon_ring_write(ring, fence->seq);
3392 0 : radeon_ring_write(ring, 0);
3393 0 : }
3394 :
3395 : /*
3396 : * IB stuff
3397 : */
3398 0 : void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3399 : {
3400 0 : struct radeon_ring *ring = &rdev->ring[ib->ring];
3401 0 : unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3402 : u32 header;
3403 :
3404 0 : if (ib->is_const_ib) {
3405 : /* set switch buffer packet before const IB */
3406 0 : radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3407 0 : radeon_ring_write(ring, 0);
3408 :
3409 : header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3410 0 : } else {
3411 : u32 next_rptr;
3412 0 : if (ring->rptr_save_reg) {
3413 0 : next_rptr = ring->wptr + 3 + 4 + 8;
3414 0 : radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3415 0 : radeon_ring_write(ring, ((ring->rptr_save_reg -
3416 0 : PACKET3_SET_CONFIG_REG_START) >> 2));
3417 0 : radeon_ring_write(ring, next_rptr);
3418 0 : } else if (rdev->wb.enabled) {
3419 0 : next_rptr = ring->wptr + 5 + 4 + 8;
3420 0 : radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3421 0 : radeon_ring_write(ring, (1 << 8));
3422 0 : radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3423 0 : radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3424 0 : radeon_ring_write(ring, next_rptr);
3425 0 : }
3426 :
3427 : header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3428 : }
3429 :
3430 0 : radeon_ring_write(ring, header);
3431 0 : radeon_ring_write(ring,
3432 : #ifdef __BIG_ENDIAN
3433 : (2 << 0) |
3434 : #endif
3435 0 : (ib->gpu_addr & 0xFFFFFFFC));
3436 0 : radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3437 0 : radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3438 :
3439 0 : if (!ib->is_const_ib) {
3440 : /* flush read cache over gart for this vmid */
3441 0 : radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3442 0 : radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3443 0 : radeon_ring_write(ring, vm_id);
3444 0 : radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3445 0 : radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3446 : PACKET3_TC_ACTION_ENA |
3447 : PACKET3_SH_KCACHE_ACTION_ENA |
3448 : PACKET3_SH_ICACHE_ACTION_ENA);
3449 0 : radeon_ring_write(ring, 0xFFFFFFFF);
3450 0 : radeon_ring_write(ring, 0);
3451 0 : radeon_ring_write(ring, 10); /* poll interval */
3452 0 : }
3453 0 : }
3454 :
3455 : /*
3456 : * CP.
3457 : */
3458 0 : static void si_cp_enable(struct radeon_device *rdev, bool enable)
3459 : {
3460 0 : if (enable)
3461 0 : WREG32(CP_ME_CNTL, 0);
3462 : else {
3463 0 : if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3464 0 : radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3465 0 : WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3466 0 : WREG32(SCRATCH_UMSK, 0);
3467 0 : rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3468 0 : rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3469 0 : rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3470 : }
3471 0 : udelay(50);
3472 0 : }
3473 :
3474 0 : static int si_cp_load_microcode(struct radeon_device *rdev)
3475 : {
3476 : int i;
3477 :
3478 0 : if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3479 0 : return -EINVAL;
3480 :
3481 0 : si_cp_enable(rdev, false);
3482 :
3483 0 : if (rdev->new_fw) {
3484 : const struct gfx_firmware_header_v1_0 *pfp_hdr =
3485 0 : (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3486 : const struct gfx_firmware_header_v1_0 *ce_hdr =
3487 0 : (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3488 : const struct gfx_firmware_header_v1_0 *me_hdr =
3489 0 : (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3490 : const __le32 *fw_data;
3491 : u32 fw_size;
3492 :
3493 0 : radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3494 0 : radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3495 0 : radeon_ucode_print_gfx_hdr(&me_hdr->header);
3496 :
3497 : /* PFP */
3498 0 : fw_data = (const __le32 *)
3499 0 : (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3500 0 : fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3501 0 : WREG32(CP_PFP_UCODE_ADDR, 0);
3502 0 : for (i = 0; i < fw_size; i++)
3503 0 : WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3504 0 : WREG32(CP_PFP_UCODE_ADDR, 0);
3505 :
3506 : /* CE */
3507 0 : fw_data = (const __le32 *)
3508 0 : (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3509 0 : fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3510 0 : WREG32(CP_CE_UCODE_ADDR, 0);
3511 0 : for (i = 0; i < fw_size; i++)
3512 0 : WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3513 0 : WREG32(CP_CE_UCODE_ADDR, 0);
3514 :
3515 : /* ME */
3516 0 : fw_data = (const __be32 *)
3517 0 : (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3518 0 : fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3519 0 : WREG32(CP_ME_RAM_WADDR, 0);
3520 0 : for (i = 0; i < fw_size; i++)
3521 0 : WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3522 0 : WREG32(CP_ME_RAM_WADDR, 0);
3523 0 : } else {
3524 : const __be32 *fw_data;
3525 :
3526 : /* PFP */
3527 0 : fw_data = (const __be32 *)rdev->pfp_fw->data;
3528 0 : WREG32(CP_PFP_UCODE_ADDR, 0);
3529 0 : for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3530 0 : WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3531 0 : WREG32(CP_PFP_UCODE_ADDR, 0);
3532 :
3533 : /* CE */
3534 0 : fw_data = (const __be32 *)rdev->ce_fw->data;
3535 0 : WREG32(CP_CE_UCODE_ADDR, 0);
3536 0 : for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3537 0 : WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3538 0 : WREG32(CP_CE_UCODE_ADDR, 0);
3539 :
3540 : /* ME */
3541 0 : fw_data = (const __be32 *)rdev->me_fw->data;
3542 0 : WREG32(CP_ME_RAM_WADDR, 0);
3543 0 : for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3544 0 : WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3545 0 : WREG32(CP_ME_RAM_WADDR, 0);
3546 : }
3547 :
3548 0 : WREG32(CP_PFP_UCODE_ADDR, 0);
3549 0 : WREG32(CP_CE_UCODE_ADDR, 0);
3550 0 : WREG32(CP_ME_RAM_WADDR, 0);
3551 0 : WREG32(CP_ME_RAM_RADDR, 0);
3552 0 : return 0;
3553 0 : }
3554 :
3555 0 : static int si_cp_start(struct radeon_device *rdev)
3556 : {
3557 0 : struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3558 : int r, i;
3559 :
3560 0 : r = radeon_ring_lock(rdev, ring, 7 + 4);
3561 0 : if (r) {
3562 0 : DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3563 0 : return r;
3564 : }
3565 : /* init the CP */
3566 0 : radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3567 0 : radeon_ring_write(ring, 0x1);
3568 0 : radeon_ring_write(ring, 0x0);
3569 0 : radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3570 0 : radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3571 0 : radeon_ring_write(ring, 0);
3572 0 : radeon_ring_write(ring, 0);
3573 :
3574 : /* init the CE partitions */
3575 0 : radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3576 0 : radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3577 0 : radeon_ring_write(ring, 0xc000);
3578 0 : radeon_ring_write(ring, 0xe000);
3579 0 : radeon_ring_unlock_commit(rdev, ring, false);
3580 :
3581 0 : si_cp_enable(rdev, true);
3582 :
3583 0 : r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3584 0 : if (r) {
3585 0 : DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3586 0 : return r;
3587 : }
3588 :
3589 : /* setup clear context state */
3590 0 : radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3591 0 : radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3592 :
3593 0 : for (i = 0; i < si_default_size; i++)
3594 0 : radeon_ring_write(ring, si_default_state[i]);
3595 :
3596 0 : radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3597 0 : radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3598 :
3599 : /* set clear context state */
3600 0 : radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3601 0 : radeon_ring_write(ring, 0);
3602 :
3603 0 : radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3604 0 : radeon_ring_write(ring, 0x00000316);
3605 0 : radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3606 0 : radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3607 :
3608 0 : radeon_ring_unlock_commit(rdev, ring, false);
3609 :
3610 0 : for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3611 0 : ring = &rdev->ring[i];
3612 0 : r = radeon_ring_lock(rdev, ring, 2);
3613 :
3614 : /* clear the compute context state */
3615 0 : radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3616 0 : radeon_ring_write(ring, 0);
3617 :
3618 0 : radeon_ring_unlock_commit(rdev, ring, false);
3619 : }
3620 :
3621 0 : return 0;
3622 0 : }
3623 :
3624 0 : static void si_cp_fini(struct radeon_device *rdev)
3625 : {
3626 : struct radeon_ring *ring;
3627 0 : si_cp_enable(rdev, false);
3628 :
3629 0 : ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3630 0 : radeon_ring_fini(rdev, ring);
3631 0 : radeon_scratch_free(rdev, ring->rptr_save_reg);
3632 :
3633 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3634 0 : radeon_ring_fini(rdev, ring);
3635 0 : radeon_scratch_free(rdev, ring->rptr_save_reg);
3636 :
3637 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3638 0 : radeon_ring_fini(rdev, ring);
3639 0 : radeon_scratch_free(rdev, ring->rptr_save_reg);
3640 0 : }
3641 :
3642 0 : static int si_cp_resume(struct radeon_device *rdev)
3643 : {
3644 : struct radeon_ring *ring;
3645 : u32 tmp;
3646 : u32 rb_bufsz;
3647 : int r;
3648 :
3649 0 : si_enable_gui_idle_interrupt(rdev, false);
3650 :
3651 0 : WREG32(CP_SEM_WAIT_TIMER, 0x0);
3652 0 : WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3653 :
3654 : /* Set the write pointer delay */
3655 0 : WREG32(CP_RB_WPTR_DELAY, 0);
3656 :
3657 0 : WREG32(CP_DEBUG, 0);
3658 0 : WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3659 :
3660 : /* ring 0 - compute and gfx */
3661 : /* Set ring buffer size */
3662 0 : ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3663 0 : rb_bufsz = order_base_2(ring->ring_size / 8);
3664 0 : tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3665 : #ifdef __BIG_ENDIAN
3666 : tmp |= BUF_SWAP_32BIT;
3667 : #endif
3668 0 : WREG32(CP_RB0_CNTL, tmp);
3669 :
3670 : /* Initialize the ring buffer's read and write pointers */
3671 0 : WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3672 0 : ring->wptr = 0;
3673 0 : WREG32(CP_RB0_WPTR, ring->wptr);
3674 :
3675 : /* set the wb address whether it's enabled or not */
3676 0 : WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3677 0 : WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3678 :
3679 0 : if (rdev->wb.enabled)
3680 0 : WREG32(SCRATCH_UMSK, 0xff);
3681 : else {
3682 0 : tmp |= RB_NO_UPDATE;
3683 0 : WREG32(SCRATCH_UMSK, 0);
3684 : }
3685 :
3686 0 : mdelay(1);
3687 0 : WREG32(CP_RB0_CNTL, tmp);
3688 :
3689 0 : WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3690 :
3691 : /* ring1 - compute only */
3692 : /* Set ring buffer size */
3693 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3694 0 : rb_bufsz = order_base_2(ring->ring_size / 8);
3695 0 : tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3696 : #ifdef __BIG_ENDIAN
3697 : tmp |= BUF_SWAP_32BIT;
3698 : #endif
3699 0 : WREG32(CP_RB1_CNTL, tmp);
3700 :
3701 : /* Initialize the ring buffer's read and write pointers */
3702 0 : WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3703 0 : ring->wptr = 0;
3704 0 : WREG32(CP_RB1_WPTR, ring->wptr);
3705 :
3706 : /* set the wb address whether it's enabled or not */
3707 0 : WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3708 0 : WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3709 :
3710 0 : mdelay(1);
3711 0 : WREG32(CP_RB1_CNTL, tmp);
3712 :
3713 0 : WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3714 :
3715 : /* ring2 - compute only */
3716 : /* Set ring buffer size */
3717 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3718 0 : rb_bufsz = order_base_2(ring->ring_size / 8);
3719 0 : tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3720 : #ifdef __BIG_ENDIAN
3721 : tmp |= BUF_SWAP_32BIT;
3722 : #endif
3723 0 : WREG32(CP_RB2_CNTL, tmp);
3724 :
3725 : /* Initialize the ring buffer's read and write pointers */
3726 0 : WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3727 0 : ring->wptr = 0;
3728 0 : WREG32(CP_RB2_WPTR, ring->wptr);
3729 :
3730 : /* set the wb address whether it's enabled or not */
3731 0 : WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3732 0 : WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3733 :
3734 0 : mdelay(1);
3735 0 : WREG32(CP_RB2_CNTL, tmp);
3736 :
3737 0 : WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3738 :
3739 : /* start the rings */
3740 0 : si_cp_start(rdev);
3741 0 : rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3742 0 : rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3743 0 : rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3744 0 : r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3745 0 : if (r) {
3746 0 : rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3747 0 : rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3748 0 : rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3749 0 : return r;
3750 : }
3751 0 : r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3752 0 : if (r) {
3753 0 : rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3754 0 : }
3755 0 : r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3756 0 : if (r) {
3757 0 : rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3758 0 : }
3759 :
3760 0 : si_enable_gui_idle_interrupt(rdev, true);
3761 :
3762 0 : if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3763 0 : radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3764 :
3765 0 : return 0;
3766 0 : }
3767 :
3768 0 : u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3769 : {
3770 : u32 reset_mask = 0;
3771 : u32 tmp;
3772 :
3773 : /* GRBM_STATUS */
3774 0 : tmp = RREG32(GRBM_STATUS);
3775 0 : if (tmp & (PA_BUSY | SC_BUSY |
3776 : BCI_BUSY | SX_BUSY |
3777 : TA_BUSY | VGT_BUSY |
3778 : DB_BUSY | CB_BUSY |
3779 : GDS_BUSY | SPI_BUSY |
3780 : IA_BUSY | IA_BUSY_NO_DMA))
3781 0 : reset_mask |= RADEON_RESET_GFX;
3782 :
3783 0 : if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3784 : CP_BUSY | CP_COHERENCY_BUSY))
3785 0 : reset_mask |= RADEON_RESET_CP;
3786 :
3787 0 : if (tmp & GRBM_EE_BUSY)
3788 0 : reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3789 :
3790 : /* GRBM_STATUS2 */
3791 0 : tmp = RREG32(GRBM_STATUS2);
3792 0 : if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3793 0 : reset_mask |= RADEON_RESET_RLC;
3794 :
3795 : /* DMA_STATUS_REG 0 */
3796 0 : tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3797 0 : if (!(tmp & DMA_IDLE))
3798 0 : reset_mask |= RADEON_RESET_DMA;
3799 :
3800 : /* DMA_STATUS_REG 1 */
3801 0 : tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3802 0 : if (!(tmp & DMA_IDLE))
3803 0 : reset_mask |= RADEON_RESET_DMA1;
3804 :
3805 : /* SRBM_STATUS2 */
3806 0 : tmp = RREG32(SRBM_STATUS2);
3807 0 : if (tmp & DMA_BUSY)
3808 0 : reset_mask |= RADEON_RESET_DMA;
3809 :
3810 0 : if (tmp & DMA1_BUSY)
3811 0 : reset_mask |= RADEON_RESET_DMA1;
3812 :
3813 : /* SRBM_STATUS */
3814 0 : tmp = RREG32(SRBM_STATUS);
3815 :
3816 0 : if (tmp & IH_BUSY)
3817 0 : reset_mask |= RADEON_RESET_IH;
3818 :
3819 0 : if (tmp & SEM_BUSY)
3820 0 : reset_mask |= RADEON_RESET_SEM;
3821 :
3822 0 : if (tmp & GRBM_RQ_PENDING)
3823 0 : reset_mask |= RADEON_RESET_GRBM;
3824 :
3825 0 : if (tmp & VMC_BUSY)
3826 0 : reset_mask |= RADEON_RESET_VMC;
3827 :
3828 0 : if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3829 : MCC_BUSY | MCD_BUSY))
3830 0 : reset_mask |= RADEON_RESET_MC;
3831 :
3832 0 : if (evergreen_is_display_hung(rdev))
3833 0 : reset_mask |= RADEON_RESET_DISPLAY;
3834 :
3835 : /* VM_L2_STATUS */
3836 0 : tmp = RREG32(VM_L2_STATUS);
3837 0 : if (tmp & L2_BUSY)
3838 0 : reset_mask |= RADEON_RESET_VMC;
3839 :
3840 : /* Skip MC reset as it's mostly likely not hung, just busy */
3841 0 : if (reset_mask & RADEON_RESET_MC) {
3842 : DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3843 0 : reset_mask &= ~RADEON_RESET_MC;
3844 0 : }
3845 :
3846 0 : return reset_mask;
3847 : }
3848 :
3849 0 : static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3850 : {
3851 0 : struct evergreen_mc_save save;
3852 : u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3853 : u32 tmp;
3854 :
3855 0 : if (reset_mask == 0)
3856 0 : return;
3857 :
3858 : dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3859 :
3860 0 : evergreen_print_gpu_status_regs(rdev);
3861 : dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3862 : RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3863 : dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3864 : RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3865 :
3866 : /* disable PG/CG */
3867 0 : si_fini_pg(rdev);
3868 0 : si_fini_cg(rdev);
3869 :
3870 : /* stop the rlc */
3871 0 : si_rlc_stop(rdev);
3872 :
3873 : /* Disable CP parsing/prefetching */
3874 0 : WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3875 :
3876 0 : if (reset_mask & RADEON_RESET_DMA) {
3877 : /* dma0 */
3878 0 : tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3879 0 : tmp &= ~DMA_RB_ENABLE;
3880 0 : WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3881 0 : }
3882 0 : if (reset_mask & RADEON_RESET_DMA1) {
3883 : /* dma1 */
3884 0 : tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3885 0 : tmp &= ~DMA_RB_ENABLE;
3886 0 : WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3887 0 : }
3888 :
3889 0 : udelay(50);
3890 :
3891 0 : evergreen_mc_stop(rdev, &save);
3892 0 : if (evergreen_mc_wait_for_idle(rdev)) {
3893 0 : dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3894 0 : }
3895 :
3896 0 : if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3897 : grbm_soft_reset = SOFT_RESET_CB |
3898 : SOFT_RESET_DB |
3899 : SOFT_RESET_GDS |
3900 : SOFT_RESET_PA |
3901 : SOFT_RESET_SC |
3902 : SOFT_RESET_BCI |
3903 : SOFT_RESET_SPI |
3904 : SOFT_RESET_SX |
3905 : SOFT_RESET_TC |
3906 : SOFT_RESET_TA |
3907 : SOFT_RESET_VGT |
3908 : SOFT_RESET_IA;
3909 0 : }
3910 :
3911 0 : if (reset_mask & RADEON_RESET_CP) {
3912 0 : grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3913 :
3914 : srbm_soft_reset |= SOFT_RESET_GRBM;
3915 0 : }
3916 :
3917 0 : if (reset_mask & RADEON_RESET_DMA)
3918 0 : srbm_soft_reset |= SOFT_RESET_DMA;
3919 :
3920 0 : if (reset_mask & RADEON_RESET_DMA1)
3921 0 : srbm_soft_reset |= SOFT_RESET_DMA1;
3922 :
3923 0 : if (reset_mask & RADEON_RESET_DISPLAY)
3924 0 : srbm_soft_reset |= SOFT_RESET_DC;
3925 :
3926 0 : if (reset_mask & RADEON_RESET_RLC)
3927 0 : grbm_soft_reset |= SOFT_RESET_RLC;
3928 :
3929 0 : if (reset_mask & RADEON_RESET_SEM)
3930 0 : srbm_soft_reset |= SOFT_RESET_SEM;
3931 :
3932 0 : if (reset_mask & RADEON_RESET_IH)
3933 0 : srbm_soft_reset |= SOFT_RESET_IH;
3934 :
3935 0 : if (reset_mask & RADEON_RESET_GRBM)
3936 0 : srbm_soft_reset |= SOFT_RESET_GRBM;
3937 :
3938 0 : if (reset_mask & RADEON_RESET_VMC)
3939 0 : srbm_soft_reset |= SOFT_RESET_VMC;
3940 :
3941 0 : if (reset_mask & RADEON_RESET_MC)
3942 0 : srbm_soft_reset |= SOFT_RESET_MC;
3943 :
3944 0 : if (grbm_soft_reset) {
3945 0 : tmp = RREG32(GRBM_SOFT_RESET);
3946 0 : tmp |= grbm_soft_reset;
3947 : dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3948 0 : WREG32(GRBM_SOFT_RESET, tmp);
3949 0 : tmp = RREG32(GRBM_SOFT_RESET);
3950 :
3951 0 : udelay(50);
3952 :
3953 0 : tmp &= ~grbm_soft_reset;
3954 0 : WREG32(GRBM_SOFT_RESET, tmp);
3955 0 : tmp = RREG32(GRBM_SOFT_RESET);
3956 0 : }
3957 :
3958 0 : if (srbm_soft_reset) {
3959 0 : tmp = RREG32(SRBM_SOFT_RESET);
3960 0 : tmp |= srbm_soft_reset;
3961 : dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3962 0 : WREG32(SRBM_SOFT_RESET, tmp);
3963 0 : tmp = RREG32(SRBM_SOFT_RESET);
3964 :
3965 0 : udelay(50);
3966 :
3967 0 : tmp &= ~srbm_soft_reset;
3968 0 : WREG32(SRBM_SOFT_RESET, tmp);
3969 0 : tmp = RREG32(SRBM_SOFT_RESET);
3970 0 : }
3971 :
3972 : /* Wait a little for things to settle down */
3973 0 : udelay(50);
3974 :
3975 0 : evergreen_mc_resume(rdev, &save);
3976 0 : udelay(50);
3977 :
3978 0 : evergreen_print_gpu_status_regs(rdev);
3979 0 : }
3980 :
3981 0 : static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3982 : {
3983 : u32 tmp, i;
3984 :
3985 0 : tmp = RREG32(CG_SPLL_FUNC_CNTL);
3986 0 : tmp |= SPLL_BYPASS_EN;
3987 0 : WREG32(CG_SPLL_FUNC_CNTL, tmp);
3988 :
3989 0 : tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3990 0 : tmp |= SPLL_CTLREQ_CHG;
3991 0 : WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3992 :
3993 0 : for (i = 0; i < rdev->usec_timeout; i++) {
3994 0 : if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3995 : break;
3996 0 : udelay(1);
3997 : }
3998 :
3999 0 : tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
4000 0 : tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
4001 0 : WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
4002 :
4003 0 : tmp = RREG32(MPLL_CNTL_MODE);
4004 0 : tmp &= ~MPLL_MCLK_SEL;
4005 0 : WREG32(MPLL_CNTL_MODE, tmp);
4006 0 : }
4007 :
4008 0 : static void si_spll_powerdown(struct radeon_device *rdev)
4009 : {
4010 : u32 tmp;
4011 :
4012 0 : tmp = RREG32(SPLL_CNTL_MODE);
4013 0 : tmp |= SPLL_SW_DIR_CONTROL;
4014 0 : WREG32(SPLL_CNTL_MODE, tmp);
4015 :
4016 0 : tmp = RREG32(CG_SPLL_FUNC_CNTL);
4017 0 : tmp |= SPLL_RESET;
4018 0 : WREG32(CG_SPLL_FUNC_CNTL, tmp);
4019 :
4020 0 : tmp = RREG32(CG_SPLL_FUNC_CNTL);
4021 0 : tmp |= SPLL_SLEEP;
4022 0 : WREG32(CG_SPLL_FUNC_CNTL, tmp);
4023 :
4024 0 : tmp = RREG32(SPLL_CNTL_MODE);
4025 0 : tmp &= ~SPLL_SW_DIR_CONTROL;
4026 0 : WREG32(SPLL_CNTL_MODE, tmp);
4027 0 : }
4028 :
4029 0 : static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4030 : {
4031 0 : struct evergreen_mc_save save;
4032 : u32 tmp, i;
4033 :
4034 : dev_info(rdev->dev, "GPU pci config reset\n");
4035 :
4036 : /* disable dpm? */
4037 :
4038 : /* disable cg/pg */
4039 0 : si_fini_pg(rdev);
4040 0 : si_fini_cg(rdev);
4041 :
4042 : /* Disable CP parsing/prefetching */
4043 0 : WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4044 : /* dma0 */
4045 0 : tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4046 0 : tmp &= ~DMA_RB_ENABLE;
4047 0 : WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4048 : /* dma1 */
4049 0 : tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4050 0 : tmp &= ~DMA_RB_ENABLE;
4051 0 : WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4052 : /* XXX other engines? */
4053 :
4054 : /* halt the rlc, disable cp internal ints */
4055 0 : si_rlc_stop(rdev);
4056 :
4057 0 : udelay(50);
4058 :
4059 : /* disable mem access */
4060 0 : evergreen_mc_stop(rdev, &save);
4061 0 : if (evergreen_mc_wait_for_idle(rdev)) {
4062 0 : dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4063 0 : }
4064 :
4065 : /* set mclk/sclk to bypass */
4066 0 : si_set_clk_bypass_mode(rdev);
4067 : /* powerdown spll */
4068 0 : si_spll_powerdown(rdev);
4069 : /* disable BM */
4070 : pci_clear_master(rdev->pdev);
4071 : /* reset */
4072 0 : radeon_pci_config_reset(rdev);
4073 : /* wait for asic to come out of reset */
4074 0 : for (i = 0; i < rdev->usec_timeout; i++) {
4075 0 : if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4076 : break;
4077 0 : udelay(1);
4078 : }
4079 0 : }
4080 :
4081 0 : int si_asic_reset(struct radeon_device *rdev)
4082 : {
4083 : u32 reset_mask;
4084 :
4085 0 : reset_mask = si_gpu_check_soft_reset(rdev);
4086 :
4087 0 : if (reset_mask)
4088 0 : r600_set_bios_scratch_engine_hung(rdev, true);
4089 :
4090 : /* try soft reset */
4091 0 : si_gpu_soft_reset(rdev, reset_mask);
4092 :
4093 0 : reset_mask = si_gpu_check_soft_reset(rdev);
4094 :
4095 : /* try pci config reset */
4096 0 : if (reset_mask && radeon_hard_reset)
4097 0 : si_gpu_pci_config_reset(rdev);
4098 :
4099 0 : reset_mask = si_gpu_check_soft_reset(rdev);
4100 :
4101 0 : if (!reset_mask)
4102 0 : r600_set_bios_scratch_engine_hung(rdev, false);
4103 :
4104 0 : return 0;
4105 : }
4106 :
4107 : /**
4108 : * si_gfx_is_lockup - Check if the GFX engine is locked up
4109 : *
4110 : * @rdev: radeon_device pointer
4111 : * @ring: radeon_ring structure holding ring information
4112 : *
4113 : * Check if the GFX engine is locked up.
4114 : * Returns true if the engine appears to be locked up, false if not.
4115 : */
4116 0 : bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4117 : {
4118 0 : u32 reset_mask = si_gpu_check_soft_reset(rdev);
4119 :
4120 0 : if (!(reset_mask & (RADEON_RESET_GFX |
4121 : RADEON_RESET_COMPUTE |
4122 : RADEON_RESET_CP))) {
4123 0 : radeon_ring_lockup_update(rdev, ring);
4124 0 : return false;
4125 : }
4126 0 : return radeon_ring_test_lockup(rdev, ring);
4127 0 : }
4128 :
4129 : /* MC */
4130 0 : static void si_mc_program(struct radeon_device *rdev)
4131 : {
4132 0 : struct evergreen_mc_save save;
4133 : u32 tmp;
4134 : int i, j;
4135 :
4136 : /* Initialize HDP */
4137 0 : for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4138 0 : WREG32((0x2c14 + j), 0x00000000);
4139 0 : WREG32((0x2c18 + j), 0x00000000);
4140 0 : WREG32((0x2c1c + j), 0x00000000);
4141 0 : WREG32((0x2c20 + j), 0x00000000);
4142 0 : WREG32((0x2c24 + j), 0x00000000);
4143 : }
4144 0 : WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4145 :
4146 0 : evergreen_mc_stop(rdev, &save);
4147 0 : if (radeon_mc_wait_for_idle(rdev)) {
4148 0 : dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4149 0 : }
4150 0 : if (!ASIC_IS_NODCE(rdev))
4151 : /* Lockout access through VGA aperture*/
4152 0 : WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4153 : /* Update configuration */
4154 0 : WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4155 : rdev->mc.vram_start >> 12);
4156 0 : WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4157 : rdev->mc.vram_end >> 12);
4158 0 : WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4159 : rdev->vram_scratch.gpu_addr >> 12);
4160 0 : tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4161 0 : tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4162 0 : WREG32(MC_VM_FB_LOCATION, tmp);
4163 : /* XXX double check these! */
4164 0 : WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4165 0 : WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4166 0 : WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4167 0 : WREG32(MC_VM_AGP_BASE, 0);
4168 0 : WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4169 0 : WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4170 0 : if (radeon_mc_wait_for_idle(rdev)) {
4171 0 : dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4172 0 : }
4173 0 : evergreen_mc_resume(rdev, &save);
4174 0 : if (!ASIC_IS_NODCE(rdev)) {
4175 : /* we need to own VRAM, so turn off the VGA renderer here
4176 : * to stop it overwriting our objects */
4177 0 : rv515_vga_render_disable(rdev);
4178 0 : }
4179 0 : }
4180 :
4181 0 : void si_vram_gtt_location(struct radeon_device *rdev,
4182 : struct radeon_mc *mc)
4183 : {
4184 0 : if (mc->mc_vram_size > 0xFFC0000000ULL) {
4185 : /* leave room for at least 1024M GTT */
4186 0 : dev_warn(rdev->dev, "limiting VRAM\n");
4187 0 : mc->real_vram_size = 0xFFC0000000ULL;
4188 0 : mc->mc_vram_size = 0xFFC0000000ULL;
4189 0 : }
4190 0 : radeon_vram_location(rdev, &rdev->mc, 0);
4191 0 : rdev->mc.gtt_base_align = 0;
4192 0 : radeon_gtt_location(rdev, mc);
4193 0 : }
4194 :
4195 0 : static int si_mc_init(struct radeon_device *rdev)
4196 : {
4197 : u32 tmp;
4198 : int chansize, numchan;
4199 :
4200 : /* Get VRAM informations */
4201 0 : rdev->mc.vram_is_ddr = true;
4202 0 : tmp = RREG32(MC_ARB_RAMCFG);
4203 0 : if (tmp & CHANSIZE_OVERRIDE) {
4204 : chansize = 16;
4205 0 : } else if (tmp & CHANSIZE_MASK) {
4206 : chansize = 64;
4207 0 : } else {
4208 : chansize = 32;
4209 : }
4210 0 : tmp = RREG32(MC_SHARED_CHMAP);
4211 0 : switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4212 : case 0:
4213 : default:
4214 : numchan = 1;
4215 0 : break;
4216 : case 1:
4217 : numchan = 2;
4218 0 : break;
4219 : case 2:
4220 : numchan = 4;
4221 0 : break;
4222 : case 3:
4223 : numchan = 8;
4224 0 : break;
4225 : case 4:
4226 : numchan = 3;
4227 0 : break;
4228 : case 5:
4229 : numchan = 6;
4230 0 : break;
4231 : case 6:
4232 : numchan = 10;
4233 0 : break;
4234 : case 7:
4235 : numchan = 12;
4236 0 : break;
4237 : case 8:
4238 : numchan = 16;
4239 0 : break;
4240 : }
4241 0 : rdev->mc.vram_width = numchan * chansize;
4242 : /* Could aper size report 0 ? */
4243 0 : rdev->mc.aper_base = rdev->fb_aper_offset;
4244 0 : rdev->mc.aper_size = rdev->fb_aper_size;
4245 : /* size in MB on si */
4246 0 : tmp = RREG32(CONFIG_MEMSIZE);
4247 : /* some boards may have garbage in the upper 16 bits */
4248 0 : if (tmp & 0xffff0000) {
4249 : DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4250 0 : if (tmp & 0xffff)
4251 0 : tmp &= 0xffff;
4252 : }
4253 0 : rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4254 0 : rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4255 0 : rdev->mc.visible_vram_size = rdev->mc.aper_size;
4256 0 : si_vram_gtt_location(rdev, &rdev->mc);
4257 0 : radeon_update_bandwidth_info(rdev);
4258 :
4259 0 : return 0;
4260 : }
4261 :
4262 : /*
4263 : * GART
4264 : */
4265 0 : void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4266 : {
4267 : /* flush hdp cache */
4268 0 : WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4269 :
4270 : /* bits 0-15 are the VM contexts0-15 */
4271 0 : WREG32(VM_INVALIDATE_REQUEST, 1);
4272 0 : }
4273 :
4274 0 : static int si_pcie_gart_enable(struct radeon_device *rdev)
4275 : {
4276 : int r, i;
4277 :
4278 0 : if (rdev->gart.robj == NULL) {
4279 0 : dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4280 0 : return -EINVAL;
4281 : }
4282 0 : r = radeon_gart_table_vram_pin(rdev);
4283 0 : if (r)
4284 0 : return r;
4285 : /* Setup TLB control */
4286 0 : WREG32(MC_VM_MX_L1_TLB_CNTL,
4287 : (0xA << 7) |
4288 : ENABLE_L1_TLB |
4289 : ENABLE_L1_FRAGMENT_PROCESSING |
4290 : SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4291 : ENABLE_ADVANCED_DRIVER_MODEL |
4292 : SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4293 : /* Setup L2 cache */
4294 0 : WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4295 : ENABLE_L2_FRAGMENT_PROCESSING |
4296 : ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4297 : ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4298 : EFFECTIVE_L2_QUEUE_SIZE(7) |
4299 : CONTEXT1_IDENTITY_ACCESS_MODE(1));
4300 0 : WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4301 0 : WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4302 : BANK_SELECT(4) |
4303 : L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4304 : /* setup context0 */
4305 0 : WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4306 0 : WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4307 0 : WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4308 0 : WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4309 : (u32)(rdev->dummy_page.addr >> 12));
4310 0 : WREG32(VM_CONTEXT0_CNTL2, 0);
4311 0 : WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4312 : RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4313 :
4314 0 : WREG32(0x15D4, 0);
4315 0 : WREG32(0x15D8, 0);
4316 0 : WREG32(0x15DC, 0);
4317 :
4318 : /* empty context1-15 */
4319 : /* set vm size, must be a multiple of 4 */
4320 0 : WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4321 0 : WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4322 : /* Assign the pt base to something valid for now; the pts used for
4323 : * the VMs are determined by the application and setup and assigned
4324 : * on the fly in the vm part of radeon_gart.c
4325 : */
4326 0 : for (i = 1; i < 16; i++) {
4327 0 : if (i < 8)
4328 0 : WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4329 : rdev->vm_manager.saved_table_addr[i]);
4330 : else
4331 0 : WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4332 : rdev->vm_manager.saved_table_addr[i]);
4333 : }
4334 :
4335 : /* enable context1-15 */
4336 0 : WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4337 : (u32)(rdev->dummy_page.addr >> 12));
4338 0 : WREG32(VM_CONTEXT1_CNTL2, 4);
4339 0 : WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4340 : PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4341 : RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4342 : RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4343 : DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4344 : DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4345 : PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4346 : PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4347 : VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4348 : VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4349 : READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4350 : READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4351 : WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4352 : WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4353 :
4354 0 : si_pcie_gart_tlb_flush(rdev);
4355 : DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4356 : (unsigned)(rdev->mc.gtt_size >> 20),
4357 : (unsigned long long)rdev->gart.table_addr);
4358 0 : rdev->gart.ready = true;
4359 0 : return 0;
4360 0 : }
4361 :
4362 0 : static void si_pcie_gart_disable(struct radeon_device *rdev)
4363 : {
4364 : unsigned i;
4365 :
4366 0 : for (i = 1; i < 16; ++i) {
4367 : uint32_t reg;
4368 0 : if (i < 8)
4369 0 : reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4370 : else
4371 0 : reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4372 0 : rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4373 : }
4374 :
4375 : /* Disable all tables */
4376 0 : WREG32(VM_CONTEXT0_CNTL, 0);
4377 0 : WREG32(VM_CONTEXT1_CNTL, 0);
4378 : /* Setup TLB control */
4379 0 : WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4380 : SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4381 : /* Setup L2 cache */
4382 0 : WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4383 : ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4384 : EFFECTIVE_L2_QUEUE_SIZE(7) |
4385 : CONTEXT1_IDENTITY_ACCESS_MODE(1));
4386 0 : WREG32(VM_L2_CNTL2, 0);
4387 0 : WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4388 : L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4389 0 : radeon_gart_table_vram_unpin(rdev);
4390 0 : }
4391 :
4392 0 : static void si_pcie_gart_fini(struct radeon_device *rdev)
4393 : {
4394 0 : si_pcie_gart_disable(rdev);
4395 0 : radeon_gart_table_vram_free(rdev);
4396 0 : radeon_gart_fini(rdev);
4397 0 : }
4398 :
4399 : /* vm parser */
4400 0 : static bool si_vm_reg_valid(u32 reg)
4401 : {
4402 : /* context regs are fine */
4403 0 : if (reg >= 0x28000)
4404 0 : return true;
4405 :
4406 : /* check config regs */
4407 0 : switch (reg) {
4408 : case GRBM_GFX_INDEX:
4409 : case CP_STRMOUT_CNTL:
4410 : case VGT_VTX_VECT_EJECT_REG:
4411 : case VGT_CACHE_INVALIDATION:
4412 : case VGT_ESGS_RING_SIZE:
4413 : case VGT_GSVS_RING_SIZE:
4414 : case VGT_GS_VERTEX_REUSE:
4415 : case VGT_PRIMITIVE_TYPE:
4416 : case VGT_INDEX_TYPE:
4417 : case VGT_NUM_INDICES:
4418 : case VGT_NUM_INSTANCES:
4419 : case VGT_TF_RING_SIZE:
4420 : case VGT_HS_OFFCHIP_PARAM:
4421 : case VGT_TF_MEMORY_BASE:
4422 : case PA_CL_ENHANCE:
4423 : case PA_SU_LINE_STIPPLE_VALUE:
4424 : case PA_SC_LINE_STIPPLE_STATE:
4425 : case PA_SC_ENHANCE:
4426 : case SQC_CACHES:
4427 : case SPI_STATIC_THREAD_MGMT_1:
4428 : case SPI_STATIC_THREAD_MGMT_2:
4429 : case SPI_STATIC_THREAD_MGMT_3:
4430 : case SPI_PS_MAX_WAVE_ID:
4431 : case SPI_CONFIG_CNTL:
4432 : case SPI_CONFIG_CNTL_1:
4433 : case TA_CNTL_AUX:
4434 0 : return true;
4435 : default:
4436 0 : DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4437 0 : return false;
4438 : }
4439 0 : }
4440 :
4441 0 : static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4442 : u32 *ib, struct radeon_cs_packet *pkt)
4443 : {
4444 0 : switch (pkt->opcode) {
4445 : case PACKET3_NOP:
4446 : case PACKET3_SET_BASE:
4447 : case PACKET3_SET_CE_DE_COUNTERS:
4448 : case PACKET3_LOAD_CONST_RAM:
4449 : case PACKET3_WRITE_CONST_RAM:
4450 : case PACKET3_WRITE_CONST_RAM_OFFSET:
4451 : case PACKET3_DUMP_CONST_RAM:
4452 : case PACKET3_INCREMENT_CE_COUNTER:
4453 : case PACKET3_WAIT_ON_DE_COUNTER:
4454 : case PACKET3_CE_WRITE:
4455 : break;
4456 : default:
4457 0 : DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4458 0 : return -EINVAL;
4459 : }
4460 0 : return 0;
4461 0 : }
4462 :
4463 0 : static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4464 : {
4465 : u32 start_reg, reg, i;
4466 0 : u32 command = ib[idx + 4];
4467 0 : u32 info = ib[idx + 1];
4468 0 : u32 idx_value = ib[idx];
4469 0 : if (command & PACKET3_CP_DMA_CMD_SAS) {
4470 : /* src address space is register */
4471 0 : if (((info & 0x60000000) >> 29) == 0) {
4472 0 : start_reg = idx_value << 2;
4473 0 : if (command & PACKET3_CP_DMA_CMD_SAIC) {
4474 : reg = start_reg;
4475 0 : if (!si_vm_reg_valid(reg)) {
4476 0 : DRM_ERROR("CP DMA Bad SRC register\n");
4477 0 : return -EINVAL;
4478 : }
4479 : } else {
4480 0 : for (i = 0; i < (command & 0x1fffff); i++) {
4481 0 : reg = start_reg + (4 * i);
4482 0 : if (!si_vm_reg_valid(reg)) {
4483 0 : DRM_ERROR("CP DMA Bad SRC register\n");
4484 0 : return -EINVAL;
4485 : }
4486 : }
4487 : }
4488 : }
4489 : }
4490 0 : if (command & PACKET3_CP_DMA_CMD_DAS) {
4491 : /* dst address space is register */
4492 0 : if (((info & 0x00300000) >> 20) == 0) {
4493 0 : start_reg = ib[idx + 2];
4494 0 : if (command & PACKET3_CP_DMA_CMD_DAIC) {
4495 : reg = start_reg;
4496 0 : if (!si_vm_reg_valid(reg)) {
4497 0 : DRM_ERROR("CP DMA Bad DST register\n");
4498 0 : return -EINVAL;
4499 : }
4500 : } else {
4501 0 : for (i = 0; i < (command & 0x1fffff); i++) {
4502 0 : reg = start_reg + (4 * i);
4503 0 : if (!si_vm_reg_valid(reg)) {
4504 0 : DRM_ERROR("CP DMA Bad DST register\n");
4505 0 : return -EINVAL;
4506 : }
4507 : }
4508 : }
4509 : }
4510 : }
4511 0 : return 0;
4512 0 : }
4513 :
4514 0 : static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4515 : u32 *ib, struct radeon_cs_packet *pkt)
4516 : {
4517 : int r;
4518 0 : u32 idx = pkt->idx + 1;
4519 0 : u32 idx_value = ib[idx];
4520 : u32 start_reg, end_reg, reg, i;
4521 :
4522 0 : switch (pkt->opcode) {
4523 : case PACKET3_NOP:
4524 : case PACKET3_SET_BASE:
4525 : case PACKET3_CLEAR_STATE:
4526 : case PACKET3_INDEX_BUFFER_SIZE:
4527 : case PACKET3_DISPATCH_DIRECT:
4528 : case PACKET3_DISPATCH_INDIRECT:
4529 : case PACKET3_ALLOC_GDS:
4530 : case PACKET3_WRITE_GDS_RAM:
4531 : case PACKET3_ATOMIC_GDS:
4532 : case PACKET3_ATOMIC:
4533 : case PACKET3_OCCLUSION_QUERY:
4534 : case PACKET3_SET_PREDICATION:
4535 : case PACKET3_COND_EXEC:
4536 : case PACKET3_PRED_EXEC:
4537 : case PACKET3_DRAW_INDIRECT:
4538 : case PACKET3_DRAW_INDEX_INDIRECT:
4539 : case PACKET3_INDEX_BASE:
4540 : case PACKET3_DRAW_INDEX_2:
4541 : case PACKET3_CONTEXT_CONTROL:
4542 : case PACKET3_INDEX_TYPE:
4543 : case PACKET3_DRAW_INDIRECT_MULTI:
4544 : case PACKET3_DRAW_INDEX_AUTO:
4545 : case PACKET3_DRAW_INDEX_IMMD:
4546 : case PACKET3_NUM_INSTANCES:
4547 : case PACKET3_DRAW_INDEX_MULTI_AUTO:
4548 : case PACKET3_STRMOUT_BUFFER_UPDATE:
4549 : case PACKET3_DRAW_INDEX_OFFSET_2:
4550 : case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4551 : case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4552 : case PACKET3_MPEG_INDEX:
4553 : case PACKET3_WAIT_REG_MEM:
4554 : case PACKET3_MEM_WRITE:
4555 : case PACKET3_PFP_SYNC_ME:
4556 : case PACKET3_SURFACE_SYNC:
4557 : case PACKET3_EVENT_WRITE:
4558 : case PACKET3_EVENT_WRITE_EOP:
4559 : case PACKET3_EVENT_WRITE_EOS:
4560 : case PACKET3_SET_CONTEXT_REG:
4561 : case PACKET3_SET_CONTEXT_REG_INDIRECT:
4562 : case PACKET3_SET_SH_REG:
4563 : case PACKET3_SET_SH_REG_OFFSET:
4564 : case PACKET3_INCREMENT_DE_COUNTER:
4565 : case PACKET3_WAIT_ON_CE_COUNTER:
4566 : case PACKET3_WAIT_ON_AVAIL_BUFFER:
4567 : case PACKET3_ME_WRITE:
4568 : break;
4569 : case PACKET3_COPY_DATA:
4570 0 : if ((idx_value & 0xf00) == 0) {
4571 0 : reg = ib[idx + 3] * 4;
4572 0 : if (!si_vm_reg_valid(reg))
4573 0 : return -EINVAL;
4574 : }
4575 : break;
4576 : case PACKET3_WRITE_DATA:
4577 0 : if ((idx_value & 0xf00) == 0) {
4578 0 : start_reg = ib[idx + 1] * 4;
4579 0 : if (idx_value & 0x10000) {
4580 0 : if (!si_vm_reg_valid(start_reg))
4581 0 : return -EINVAL;
4582 : } else {
4583 0 : for (i = 0; i < (pkt->count - 2); i++) {
4584 0 : reg = start_reg + (4 * i);
4585 0 : if (!si_vm_reg_valid(reg))
4586 0 : return -EINVAL;
4587 : }
4588 : }
4589 : }
4590 : break;
4591 : case PACKET3_COND_WRITE:
4592 0 : if (idx_value & 0x100) {
4593 0 : reg = ib[idx + 5] * 4;
4594 0 : if (!si_vm_reg_valid(reg))
4595 0 : return -EINVAL;
4596 : }
4597 : break;
4598 : case PACKET3_COPY_DW:
4599 0 : if (idx_value & 0x2) {
4600 0 : reg = ib[idx + 3] * 4;
4601 0 : if (!si_vm_reg_valid(reg))
4602 0 : return -EINVAL;
4603 : }
4604 : break;
4605 : case PACKET3_SET_CONFIG_REG:
4606 0 : start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4607 0 : end_reg = 4 * pkt->count + start_reg - 4;
4608 0 : if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4609 0 : (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4610 0 : (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4611 0 : DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4612 0 : return -EINVAL;
4613 : }
4614 0 : for (i = 0; i < pkt->count; i++) {
4615 0 : reg = start_reg + (4 * i);
4616 0 : if (!si_vm_reg_valid(reg))
4617 0 : return -EINVAL;
4618 : }
4619 : break;
4620 : case PACKET3_CP_DMA:
4621 0 : r = si_vm_packet3_cp_dma_check(ib, idx);
4622 0 : if (r)
4623 0 : return r;
4624 : break;
4625 : default:
4626 0 : DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4627 0 : return -EINVAL;
4628 : }
4629 0 : return 0;
4630 0 : }
4631 :
4632 0 : static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4633 : u32 *ib, struct radeon_cs_packet *pkt)
4634 : {
4635 : int r;
4636 0 : u32 idx = pkt->idx + 1;
4637 0 : u32 idx_value = ib[idx];
4638 : u32 start_reg, reg, i;
4639 :
4640 0 : switch (pkt->opcode) {
4641 : case PACKET3_NOP:
4642 : case PACKET3_SET_BASE:
4643 : case PACKET3_CLEAR_STATE:
4644 : case PACKET3_DISPATCH_DIRECT:
4645 : case PACKET3_DISPATCH_INDIRECT:
4646 : case PACKET3_ALLOC_GDS:
4647 : case PACKET3_WRITE_GDS_RAM:
4648 : case PACKET3_ATOMIC_GDS:
4649 : case PACKET3_ATOMIC:
4650 : case PACKET3_OCCLUSION_QUERY:
4651 : case PACKET3_SET_PREDICATION:
4652 : case PACKET3_COND_EXEC:
4653 : case PACKET3_PRED_EXEC:
4654 : case PACKET3_CONTEXT_CONTROL:
4655 : case PACKET3_STRMOUT_BUFFER_UPDATE:
4656 : case PACKET3_WAIT_REG_MEM:
4657 : case PACKET3_MEM_WRITE:
4658 : case PACKET3_PFP_SYNC_ME:
4659 : case PACKET3_SURFACE_SYNC:
4660 : case PACKET3_EVENT_WRITE:
4661 : case PACKET3_EVENT_WRITE_EOP:
4662 : case PACKET3_EVENT_WRITE_EOS:
4663 : case PACKET3_SET_CONTEXT_REG:
4664 : case PACKET3_SET_CONTEXT_REG_INDIRECT:
4665 : case PACKET3_SET_SH_REG:
4666 : case PACKET3_SET_SH_REG_OFFSET:
4667 : case PACKET3_INCREMENT_DE_COUNTER:
4668 : case PACKET3_WAIT_ON_CE_COUNTER:
4669 : case PACKET3_WAIT_ON_AVAIL_BUFFER:
4670 : case PACKET3_ME_WRITE:
4671 : break;
4672 : case PACKET3_COPY_DATA:
4673 0 : if ((idx_value & 0xf00) == 0) {
4674 0 : reg = ib[idx + 3] * 4;
4675 0 : if (!si_vm_reg_valid(reg))
4676 0 : return -EINVAL;
4677 : }
4678 : break;
4679 : case PACKET3_WRITE_DATA:
4680 0 : if ((idx_value & 0xf00) == 0) {
4681 0 : start_reg = ib[idx + 1] * 4;
4682 0 : if (idx_value & 0x10000) {
4683 0 : if (!si_vm_reg_valid(start_reg))
4684 0 : return -EINVAL;
4685 : } else {
4686 0 : for (i = 0; i < (pkt->count - 2); i++) {
4687 0 : reg = start_reg + (4 * i);
4688 0 : if (!si_vm_reg_valid(reg))
4689 0 : return -EINVAL;
4690 : }
4691 : }
4692 : }
4693 : break;
4694 : case PACKET3_COND_WRITE:
4695 0 : if (idx_value & 0x100) {
4696 0 : reg = ib[idx + 5] * 4;
4697 0 : if (!si_vm_reg_valid(reg))
4698 0 : return -EINVAL;
4699 : }
4700 : break;
4701 : case PACKET3_COPY_DW:
4702 0 : if (idx_value & 0x2) {
4703 0 : reg = ib[idx + 3] * 4;
4704 0 : if (!si_vm_reg_valid(reg))
4705 0 : return -EINVAL;
4706 : }
4707 : break;
4708 : case PACKET3_CP_DMA:
4709 0 : r = si_vm_packet3_cp_dma_check(ib, idx);
4710 0 : if (r)
4711 0 : return r;
4712 : break;
4713 : default:
4714 0 : DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4715 0 : return -EINVAL;
4716 : }
4717 0 : return 0;
4718 0 : }
4719 :
4720 0 : int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4721 : {
4722 : int ret = 0;
4723 : u32 idx = 0, i;
4724 0 : struct radeon_cs_packet pkt;
4725 :
4726 0 : do {
4727 0 : pkt.idx = idx;
4728 0 : pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4729 0 : pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4730 0 : pkt.one_reg_wr = 0;
4731 0 : switch (pkt.type) {
4732 : case RADEON_PACKET_TYPE0:
4733 0 : dev_err(rdev->dev, "Packet0 not allowed!\n");
4734 : ret = -EINVAL;
4735 0 : break;
4736 : case RADEON_PACKET_TYPE2:
4737 0 : idx += 1;
4738 0 : break;
4739 : case RADEON_PACKET_TYPE3:
4740 0 : pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4741 0 : if (ib->is_const_ib)
4742 0 : ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4743 : else {
4744 0 : switch (ib->ring) {
4745 : case RADEON_RING_TYPE_GFX_INDEX:
4746 0 : ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4747 0 : break;
4748 : case CAYMAN_RING_TYPE_CP1_INDEX:
4749 : case CAYMAN_RING_TYPE_CP2_INDEX:
4750 0 : ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4751 0 : break;
4752 : default:
4753 0 : dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4754 : ret = -EINVAL;
4755 0 : break;
4756 : }
4757 : }
4758 0 : idx += pkt.count + 2;
4759 0 : break;
4760 : default:
4761 0 : dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4762 : ret = -EINVAL;
4763 0 : break;
4764 : }
4765 0 : if (ret) {
4766 0 : for (i = 0; i < ib->length_dw; i++) {
4767 0 : if (i == idx)
4768 0 : printk("\t0x%08x <---\n", ib->ptr[i]);
4769 : else
4770 0 : printk("\t0x%08x\n", ib->ptr[i]);
4771 : }
4772 : break;
4773 : }
4774 0 : } while (idx < ib->length_dw);
4775 :
4776 0 : return ret;
4777 0 : }
4778 :
4779 : /*
4780 : * vm
4781 : */
4782 0 : int si_vm_init(struct radeon_device *rdev)
4783 : {
4784 : /* number of VMs */
4785 0 : rdev->vm_manager.nvm = 16;
4786 : /* base offset of vram pages */
4787 0 : rdev->vm_manager.vram_base_offset = 0;
4788 :
4789 0 : return 0;
4790 : }
4791 :
4792 0 : void si_vm_fini(struct radeon_device *rdev)
4793 : {
4794 0 : }
4795 :
4796 : /**
4797 : * si_vm_decode_fault - print human readable fault info
4798 : *
4799 : * @rdev: radeon_device pointer
4800 : * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4801 : * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4802 : *
4803 : * Print human readable fault information (SI).
4804 : */
4805 0 : static void si_vm_decode_fault(struct radeon_device *rdev,
4806 : u32 status, u32 addr)
4807 : {
4808 0 : u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4809 0 : u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4810 0 : u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4811 : char *block;
4812 :
4813 0 : if (rdev->family == CHIP_TAHITI) {
4814 0 : switch (mc_id) {
4815 : case 160:
4816 : case 144:
4817 : case 96:
4818 : case 80:
4819 : case 224:
4820 : case 208:
4821 : case 32:
4822 : case 16:
4823 : block = "CB";
4824 0 : break;
4825 : case 161:
4826 : case 145:
4827 : case 97:
4828 : case 81:
4829 : case 225:
4830 : case 209:
4831 : case 33:
4832 : case 17:
4833 : block = "CB_FMASK";
4834 0 : break;
4835 : case 162:
4836 : case 146:
4837 : case 98:
4838 : case 82:
4839 : case 226:
4840 : case 210:
4841 : case 34:
4842 : case 18:
4843 : block = "CB_CMASK";
4844 0 : break;
4845 : case 163:
4846 : case 147:
4847 : case 99:
4848 : case 83:
4849 : case 227:
4850 : case 211:
4851 : case 35:
4852 : case 19:
4853 : block = "CB_IMMED";
4854 0 : break;
4855 : case 164:
4856 : case 148:
4857 : case 100:
4858 : case 84:
4859 : case 228:
4860 : case 212:
4861 : case 36:
4862 : case 20:
4863 : block = "DB";
4864 0 : break;
4865 : case 165:
4866 : case 149:
4867 : case 101:
4868 : case 85:
4869 : case 229:
4870 : case 213:
4871 : case 37:
4872 : case 21:
4873 : block = "DB_HTILE";
4874 0 : break;
4875 : case 167:
4876 : case 151:
4877 : case 103:
4878 : case 87:
4879 : case 231:
4880 : case 215:
4881 : case 39:
4882 : case 23:
4883 : block = "DB_STEN";
4884 0 : break;
4885 : case 72:
4886 : case 68:
4887 : case 64:
4888 : case 8:
4889 : case 4:
4890 : case 0:
4891 : case 136:
4892 : case 132:
4893 : case 128:
4894 : case 200:
4895 : case 196:
4896 : case 192:
4897 : block = "TC";
4898 0 : break;
4899 : case 112:
4900 : case 48:
4901 : block = "CP";
4902 0 : break;
4903 : case 49:
4904 : case 177:
4905 : case 50:
4906 : case 178:
4907 : block = "SH";
4908 0 : break;
4909 : case 53:
4910 : case 190:
4911 : block = "VGT";
4912 0 : break;
4913 : case 117:
4914 : block = "IH";
4915 0 : break;
4916 : case 51:
4917 : case 115:
4918 : block = "RLC";
4919 0 : break;
4920 : case 119:
4921 : case 183:
4922 : block = "DMA0";
4923 0 : break;
4924 : case 61:
4925 : block = "DMA1";
4926 0 : break;
4927 : case 248:
4928 : case 120:
4929 : block = "HDP";
4930 0 : break;
4931 : default:
4932 : block = "unknown";
4933 0 : break;
4934 : }
4935 : } else {
4936 0 : switch (mc_id) {
4937 : case 32:
4938 : case 16:
4939 : case 96:
4940 : case 80:
4941 : case 160:
4942 : case 144:
4943 : case 224:
4944 : case 208:
4945 : block = "CB";
4946 0 : break;
4947 : case 33:
4948 : case 17:
4949 : case 97:
4950 : case 81:
4951 : case 161:
4952 : case 145:
4953 : case 225:
4954 : case 209:
4955 : block = "CB_FMASK";
4956 0 : break;
4957 : case 34:
4958 : case 18:
4959 : case 98:
4960 : case 82:
4961 : case 162:
4962 : case 146:
4963 : case 226:
4964 : case 210:
4965 : block = "CB_CMASK";
4966 0 : break;
4967 : case 35:
4968 : case 19:
4969 : case 99:
4970 : case 83:
4971 : case 163:
4972 : case 147:
4973 : case 227:
4974 : case 211:
4975 : block = "CB_IMMED";
4976 0 : break;
4977 : case 36:
4978 : case 20:
4979 : case 100:
4980 : case 84:
4981 : case 164:
4982 : case 148:
4983 : case 228:
4984 : case 212:
4985 : block = "DB";
4986 0 : break;
4987 : case 37:
4988 : case 21:
4989 : case 101:
4990 : case 85:
4991 : case 165:
4992 : case 149:
4993 : case 229:
4994 : case 213:
4995 : block = "DB_HTILE";
4996 0 : break;
4997 : case 39:
4998 : case 23:
4999 : case 103:
5000 : case 87:
5001 : case 167:
5002 : case 151:
5003 : case 231:
5004 : case 215:
5005 : block = "DB_STEN";
5006 0 : break;
5007 : case 72:
5008 : case 68:
5009 : case 8:
5010 : case 4:
5011 : case 136:
5012 : case 132:
5013 : case 200:
5014 : case 196:
5015 : block = "TC";
5016 0 : break;
5017 : case 112:
5018 : case 48:
5019 : block = "CP";
5020 0 : break;
5021 : case 49:
5022 : case 177:
5023 : case 50:
5024 : case 178:
5025 : block = "SH";
5026 0 : break;
5027 : case 53:
5028 : block = "VGT";
5029 0 : break;
5030 : case 117:
5031 : block = "IH";
5032 0 : break;
5033 : case 51:
5034 : case 115:
5035 : block = "RLC";
5036 0 : break;
5037 : case 119:
5038 : case 183:
5039 : block = "DMA0";
5040 0 : break;
5041 : case 61:
5042 : block = "DMA1";
5043 0 : break;
5044 : case 248:
5045 : case 120:
5046 : block = "HDP";
5047 0 : break;
5048 : default:
5049 : block = "unknown";
5050 0 : break;
5051 : }
5052 : }
5053 :
5054 0 : printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5055 : protections, vmid, addr,
5056 : (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5057 : block, mc_id);
5058 0 : }
5059 :
5060 0 : void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5061 : unsigned vm_id, uint64_t pd_addr)
5062 : {
5063 : /* write new base address */
5064 0 : radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5065 0 : radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5066 : WRITE_DATA_DST_SEL(0)));
5067 :
5068 0 : if (vm_id < 8) {
5069 0 : radeon_ring_write(ring,
5070 0 : (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5071 0 : } else {
5072 0 : radeon_ring_write(ring,
5073 0 : (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5074 : }
5075 0 : radeon_ring_write(ring, 0);
5076 0 : radeon_ring_write(ring, pd_addr >> 12);
5077 :
5078 : /* flush hdp cache */
5079 0 : radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5080 0 : radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5081 : WRITE_DATA_DST_SEL(0)));
5082 0 : radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5083 0 : radeon_ring_write(ring, 0);
5084 0 : radeon_ring_write(ring, 0x1);
5085 :
5086 : /* bits 0-15 are the VM contexts0-15 */
5087 0 : radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5088 0 : radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5089 : WRITE_DATA_DST_SEL(0)));
5090 0 : radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5091 0 : radeon_ring_write(ring, 0);
5092 0 : radeon_ring_write(ring, 1 << vm_id);
5093 :
5094 : /* wait for the invalidate to complete */
5095 0 : radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5096 0 : radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */
5097 : WAIT_REG_MEM_ENGINE(0))); /* me */
5098 0 : radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5099 0 : radeon_ring_write(ring, 0);
5100 0 : radeon_ring_write(ring, 0); /* ref */
5101 0 : radeon_ring_write(ring, 0); /* mask */
5102 0 : radeon_ring_write(ring, 0x20); /* poll interval */
5103 :
5104 : /* sync PFP to ME, otherwise we might get invalid PFP reads */
5105 0 : radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5106 0 : radeon_ring_write(ring, 0x0);
5107 0 : }
5108 :
5109 : /*
5110 : * Power and clock gating
5111 : */
5112 0 : static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5113 : {
5114 : int i;
5115 :
5116 0 : for (i = 0; i < rdev->usec_timeout; i++) {
5117 0 : if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5118 : break;
5119 0 : udelay(1);
5120 : }
5121 :
5122 0 : for (i = 0; i < rdev->usec_timeout; i++) {
5123 0 : if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5124 : break;
5125 0 : udelay(1);
5126 : }
5127 0 : }
5128 :
5129 0 : static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5130 : bool enable)
5131 : {
5132 0 : u32 tmp = RREG32(CP_INT_CNTL_RING0);
5133 : u32 mask;
5134 : int i;
5135 :
5136 0 : if (enable)
5137 0 : tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5138 : else
5139 0 : tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5140 0 : WREG32(CP_INT_CNTL_RING0, tmp);
5141 :
5142 0 : if (!enable) {
5143 : /* read a gfx register */
5144 0 : tmp = RREG32(DB_DEPTH_INFO);
5145 :
5146 : mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5147 0 : for (i = 0; i < rdev->usec_timeout; i++) {
5148 0 : if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5149 : break;
5150 0 : udelay(1);
5151 : }
5152 : }
5153 0 : }
5154 :
5155 0 : static void si_set_uvd_dcm(struct radeon_device *rdev,
5156 : bool sw_mode)
5157 : {
5158 : u32 tmp, tmp2;
5159 :
5160 0 : tmp = RREG32(UVD_CGC_CTRL);
5161 0 : tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5162 0 : tmp |= DCM | CG_DT(1) | CLK_OD(4);
5163 :
5164 0 : if (sw_mode) {
5165 0 : tmp &= ~0x7ffff800;
5166 : tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5167 0 : } else {
5168 0 : tmp |= 0x7ffff800;
5169 : tmp2 = 0;
5170 : }
5171 :
5172 0 : WREG32(UVD_CGC_CTRL, tmp);
5173 0 : WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5174 0 : }
5175 :
5176 0 : void si_init_uvd_internal_cg(struct radeon_device *rdev)
5177 : {
5178 : bool hw_mode = true;
5179 :
5180 0 : if (hw_mode) {
5181 0 : si_set_uvd_dcm(rdev, false);
5182 0 : } else {
5183 0 : u32 tmp = RREG32(UVD_CGC_CTRL);
5184 0 : tmp &= ~DCM;
5185 0 : WREG32(UVD_CGC_CTRL, tmp);
5186 : }
5187 0 : }
5188 :
5189 0 : static u32 si_halt_rlc(struct radeon_device *rdev)
5190 : {
5191 : u32 data, orig;
5192 :
5193 0 : orig = data = RREG32(RLC_CNTL);
5194 :
5195 0 : if (data & RLC_ENABLE) {
5196 0 : data &= ~RLC_ENABLE;
5197 0 : WREG32(RLC_CNTL, data);
5198 :
5199 0 : si_wait_for_rlc_serdes(rdev);
5200 0 : }
5201 :
5202 0 : return orig;
5203 : }
5204 :
5205 0 : static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5206 : {
5207 : u32 tmp;
5208 :
5209 0 : tmp = RREG32(RLC_CNTL);
5210 0 : if (tmp != rlc)
5211 0 : WREG32(RLC_CNTL, rlc);
5212 0 : }
5213 :
5214 0 : static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5215 : {
5216 : u32 data, orig;
5217 :
5218 0 : orig = data = RREG32(DMA_PG);
5219 0 : if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5220 0 : data |= PG_CNTL_ENABLE;
5221 : else
5222 0 : data &= ~PG_CNTL_ENABLE;
5223 0 : if (orig != data)
5224 0 : WREG32(DMA_PG, data);
5225 0 : }
5226 :
5227 0 : static void si_init_dma_pg(struct radeon_device *rdev)
5228 : {
5229 : u32 tmp;
5230 :
5231 0 : WREG32(DMA_PGFSM_WRITE, 0x00002000);
5232 0 : WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5233 :
5234 0 : for (tmp = 0; tmp < 5; tmp++)
5235 0 : WREG32(DMA_PGFSM_WRITE, 0);
5236 0 : }
5237 :
5238 0 : static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5239 : bool enable)
5240 : {
5241 : u32 tmp;
5242 :
5243 0 : if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5244 : tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5245 0 : WREG32(RLC_TTOP_D, tmp);
5246 :
5247 0 : tmp = RREG32(RLC_PG_CNTL);
5248 0 : tmp |= GFX_PG_ENABLE;
5249 0 : WREG32(RLC_PG_CNTL, tmp);
5250 :
5251 0 : tmp = RREG32(RLC_AUTO_PG_CTRL);
5252 0 : tmp |= AUTO_PG_EN;
5253 0 : WREG32(RLC_AUTO_PG_CTRL, tmp);
5254 0 : } else {
5255 0 : tmp = RREG32(RLC_AUTO_PG_CTRL);
5256 0 : tmp &= ~AUTO_PG_EN;
5257 0 : WREG32(RLC_AUTO_PG_CTRL, tmp);
5258 :
5259 0 : tmp = RREG32(DB_RENDER_CONTROL);
5260 : }
5261 0 : }
5262 :
5263 0 : static void si_init_gfx_cgpg(struct radeon_device *rdev)
5264 : {
5265 : u32 tmp;
5266 :
5267 0 : WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5268 :
5269 0 : tmp = RREG32(RLC_PG_CNTL);
5270 0 : tmp |= GFX_PG_SRC;
5271 0 : WREG32(RLC_PG_CNTL, tmp);
5272 :
5273 0 : WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5274 :
5275 0 : tmp = RREG32(RLC_AUTO_PG_CTRL);
5276 :
5277 0 : tmp &= ~GRBM_REG_SGIT_MASK;
5278 0 : tmp |= GRBM_REG_SGIT(0x700);
5279 0 : tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5280 0 : WREG32(RLC_AUTO_PG_CTRL, tmp);
5281 0 : }
5282 :
5283 0 : static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5284 : {
5285 : u32 mask = 0, tmp, tmp1;
5286 : int i;
5287 :
5288 0 : si_select_se_sh(rdev, se, sh);
5289 0 : tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5290 0 : tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5291 0 : si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5292 :
5293 0 : tmp &= 0xffff0000;
5294 :
5295 0 : tmp |= tmp1;
5296 0 : tmp >>= 16;
5297 :
5298 0 : for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5299 0 : mask <<= 1;
5300 0 : mask |= 1;
5301 : }
5302 :
5303 0 : return (~tmp) & mask;
5304 : }
5305 :
5306 0 : static void si_init_ao_cu_mask(struct radeon_device *rdev)
5307 : {
5308 : u32 i, j, k, active_cu_number = 0;
5309 : u32 mask, counter, cu_bitmap;
5310 : u32 tmp = 0;
5311 :
5312 0 : for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5313 0 : for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5314 : mask = 1;
5315 : cu_bitmap = 0;
5316 : counter = 0;
5317 0 : for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5318 0 : if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5319 0 : if (counter < 2)
5320 0 : cu_bitmap |= mask;
5321 0 : counter++;
5322 0 : }
5323 0 : mask <<= 1;
5324 : }
5325 :
5326 0 : active_cu_number += counter;
5327 0 : tmp |= (cu_bitmap << (i * 16 + j * 8));
5328 : }
5329 : }
5330 :
5331 0 : WREG32(RLC_PG_AO_CU_MASK, tmp);
5332 :
5333 0 : tmp = RREG32(RLC_MAX_PG_CU);
5334 0 : tmp &= ~MAX_PU_CU_MASK;
5335 0 : tmp |= MAX_PU_CU(active_cu_number);
5336 0 : WREG32(RLC_MAX_PG_CU, tmp);
5337 0 : }
5338 :
5339 0 : static void si_enable_cgcg(struct radeon_device *rdev,
5340 : bool enable)
5341 : {
5342 : u32 data, orig, tmp;
5343 :
5344 0 : orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5345 :
5346 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5347 0 : si_enable_gui_idle_interrupt(rdev, true);
5348 :
5349 0 : WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5350 :
5351 0 : tmp = si_halt_rlc(rdev);
5352 :
5353 0 : WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5354 0 : WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5355 0 : WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5356 :
5357 0 : si_wait_for_rlc_serdes(rdev);
5358 :
5359 0 : si_update_rlc(rdev, tmp);
5360 :
5361 0 : WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5362 :
5363 0 : data |= CGCG_EN | CGLS_EN;
5364 0 : } else {
5365 0 : si_enable_gui_idle_interrupt(rdev, false);
5366 :
5367 0 : RREG32(CB_CGTT_SCLK_CTRL);
5368 0 : RREG32(CB_CGTT_SCLK_CTRL);
5369 0 : RREG32(CB_CGTT_SCLK_CTRL);
5370 0 : RREG32(CB_CGTT_SCLK_CTRL);
5371 :
5372 0 : data &= ~(CGCG_EN | CGLS_EN);
5373 : }
5374 :
5375 0 : if (orig != data)
5376 0 : WREG32(RLC_CGCG_CGLS_CTRL, data);
5377 0 : }
5378 :
5379 0 : static void si_enable_mgcg(struct radeon_device *rdev,
5380 : bool enable)
5381 : {
5382 : u32 data, orig, tmp = 0;
5383 :
5384 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5385 0 : orig = data = RREG32(CGTS_SM_CTRL_REG);
5386 : data = 0x96940200;
5387 0 : if (orig != data)
5388 0 : WREG32(CGTS_SM_CTRL_REG, data);
5389 :
5390 0 : if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5391 0 : orig = data = RREG32(CP_MEM_SLP_CNTL);
5392 0 : data |= CP_MEM_LS_EN;
5393 0 : if (orig != data)
5394 0 : WREG32(CP_MEM_SLP_CNTL, data);
5395 : }
5396 :
5397 0 : orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5398 0 : data &= 0xffffffc0;
5399 0 : if (orig != data)
5400 0 : WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5401 :
5402 0 : tmp = si_halt_rlc(rdev);
5403 :
5404 0 : WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5405 0 : WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5406 0 : WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5407 :
5408 0 : si_update_rlc(rdev, tmp);
5409 0 : } else {
5410 0 : orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5411 0 : data |= 0x00000003;
5412 0 : if (orig != data)
5413 0 : WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5414 :
5415 0 : data = RREG32(CP_MEM_SLP_CNTL);
5416 0 : if (data & CP_MEM_LS_EN) {
5417 0 : data &= ~CP_MEM_LS_EN;
5418 0 : WREG32(CP_MEM_SLP_CNTL, data);
5419 0 : }
5420 0 : orig = data = RREG32(CGTS_SM_CTRL_REG);
5421 0 : data |= LS_OVERRIDE | OVERRIDE;
5422 0 : if (orig != data)
5423 0 : WREG32(CGTS_SM_CTRL_REG, data);
5424 :
5425 0 : tmp = si_halt_rlc(rdev);
5426 :
5427 0 : WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5428 0 : WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5429 0 : WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5430 :
5431 0 : si_update_rlc(rdev, tmp);
5432 : }
5433 0 : }
5434 :
5435 0 : static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5436 : bool enable)
5437 : {
5438 : u32 orig, data, tmp;
5439 :
5440 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5441 0 : tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5442 0 : tmp |= 0x3fff;
5443 0 : WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5444 :
5445 0 : orig = data = RREG32(UVD_CGC_CTRL);
5446 0 : data |= DCM;
5447 0 : if (orig != data)
5448 0 : WREG32(UVD_CGC_CTRL, data);
5449 :
5450 0 : WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5451 0 : WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5452 0 : } else {
5453 0 : tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5454 0 : tmp &= ~0x3fff;
5455 0 : WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5456 :
5457 0 : orig = data = RREG32(UVD_CGC_CTRL);
5458 0 : data &= ~DCM;
5459 0 : if (orig != data)
5460 0 : WREG32(UVD_CGC_CTRL, data);
5461 :
5462 0 : WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5463 0 : WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5464 : }
5465 0 : }
5466 :
5467 : static const u32 mc_cg_registers[] =
5468 : {
5469 : MC_HUB_MISC_HUB_CG,
5470 : MC_HUB_MISC_SIP_CG,
5471 : MC_HUB_MISC_VM_CG,
5472 : MC_XPB_CLK_GAT,
5473 : ATC_MISC_CG,
5474 : MC_CITF_MISC_WR_CG,
5475 : MC_CITF_MISC_RD_CG,
5476 : MC_CITF_MISC_VM_CG,
5477 : VM_L2_CG,
5478 : };
5479 :
5480 0 : static void si_enable_mc_ls(struct radeon_device *rdev,
5481 : bool enable)
5482 : {
5483 : int i;
5484 : u32 orig, data;
5485 :
5486 0 : for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5487 0 : orig = data = RREG32(mc_cg_registers[i]);
5488 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5489 0 : data |= MC_LS_ENABLE;
5490 : else
5491 0 : data &= ~MC_LS_ENABLE;
5492 0 : if (data != orig)
5493 0 : WREG32(mc_cg_registers[i], data);
5494 : }
5495 0 : }
5496 :
5497 0 : static void si_enable_mc_mgcg(struct radeon_device *rdev,
5498 : bool enable)
5499 : {
5500 : int i;
5501 : u32 orig, data;
5502 :
5503 0 : for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5504 0 : orig = data = RREG32(mc_cg_registers[i]);
5505 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5506 0 : data |= MC_CG_ENABLE;
5507 : else
5508 0 : data &= ~MC_CG_ENABLE;
5509 0 : if (data != orig)
5510 0 : WREG32(mc_cg_registers[i], data);
5511 : }
5512 0 : }
5513 :
5514 0 : static void si_enable_dma_mgcg(struct radeon_device *rdev,
5515 : bool enable)
5516 : {
5517 : u32 orig, data, offset;
5518 : int i;
5519 :
5520 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5521 0 : for (i = 0; i < 2; i++) {
5522 0 : if (i == 0)
5523 0 : offset = DMA0_REGISTER_OFFSET;
5524 : else
5525 : offset = DMA1_REGISTER_OFFSET;
5526 0 : orig = data = RREG32(DMA_POWER_CNTL + offset);
5527 0 : data &= ~MEM_POWER_OVERRIDE;
5528 0 : if (data != orig)
5529 0 : WREG32(DMA_POWER_CNTL + offset, data);
5530 0 : WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5531 : }
5532 : } else {
5533 0 : for (i = 0; i < 2; i++) {
5534 0 : if (i == 0)
5535 0 : offset = DMA0_REGISTER_OFFSET;
5536 : else
5537 : offset = DMA1_REGISTER_OFFSET;
5538 0 : orig = data = RREG32(DMA_POWER_CNTL + offset);
5539 0 : data |= MEM_POWER_OVERRIDE;
5540 0 : if (data != orig)
5541 0 : WREG32(DMA_POWER_CNTL + offset, data);
5542 :
5543 0 : orig = data = RREG32(DMA_CLK_CTRL + offset);
5544 : data = 0xff000000;
5545 0 : if (data != orig)
5546 0 : WREG32(DMA_CLK_CTRL + offset, data);
5547 : }
5548 : }
5549 0 : }
5550 :
5551 0 : static void si_enable_bif_mgls(struct radeon_device *rdev,
5552 : bool enable)
5553 : {
5554 : u32 orig, data;
5555 :
5556 0 : orig = data = RREG32_PCIE(PCIE_CNTL2);
5557 :
5558 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5559 0 : data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5560 : REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5561 : else
5562 0 : data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5563 : REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5564 :
5565 0 : if (orig != data)
5566 0 : WREG32_PCIE(PCIE_CNTL2, data);
5567 0 : }
5568 :
5569 0 : static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5570 : bool enable)
5571 : {
5572 : u32 orig, data;
5573 :
5574 0 : orig = data = RREG32(HDP_HOST_PATH_CNTL);
5575 :
5576 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5577 0 : data &= ~CLOCK_GATING_DIS;
5578 : else
5579 0 : data |= CLOCK_GATING_DIS;
5580 :
5581 0 : if (orig != data)
5582 0 : WREG32(HDP_HOST_PATH_CNTL, data);
5583 0 : }
5584 :
5585 0 : static void si_enable_hdp_ls(struct radeon_device *rdev,
5586 : bool enable)
5587 : {
5588 : u32 orig, data;
5589 :
5590 0 : orig = data = RREG32(HDP_MEM_POWER_LS);
5591 :
5592 0 : if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5593 0 : data |= HDP_LS_ENABLE;
5594 : else
5595 0 : data &= ~HDP_LS_ENABLE;
5596 :
5597 0 : if (orig != data)
5598 0 : WREG32(HDP_MEM_POWER_LS, data);
5599 0 : }
5600 :
5601 0 : static void si_update_cg(struct radeon_device *rdev,
5602 : u32 block, bool enable)
5603 : {
5604 0 : if (block & RADEON_CG_BLOCK_GFX) {
5605 0 : si_enable_gui_idle_interrupt(rdev, false);
5606 : /* order matters! */
5607 0 : if (enable) {
5608 0 : si_enable_mgcg(rdev, true);
5609 0 : si_enable_cgcg(rdev, true);
5610 0 : } else {
5611 0 : si_enable_cgcg(rdev, false);
5612 0 : si_enable_mgcg(rdev, false);
5613 : }
5614 0 : si_enable_gui_idle_interrupt(rdev, true);
5615 0 : }
5616 :
5617 0 : if (block & RADEON_CG_BLOCK_MC) {
5618 0 : si_enable_mc_mgcg(rdev, enable);
5619 0 : si_enable_mc_ls(rdev, enable);
5620 0 : }
5621 :
5622 0 : if (block & RADEON_CG_BLOCK_SDMA) {
5623 0 : si_enable_dma_mgcg(rdev, enable);
5624 0 : }
5625 :
5626 0 : if (block & RADEON_CG_BLOCK_BIF) {
5627 0 : si_enable_bif_mgls(rdev, enable);
5628 0 : }
5629 :
5630 0 : if (block & RADEON_CG_BLOCK_UVD) {
5631 0 : if (rdev->has_uvd) {
5632 0 : si_enable_uvd_mgcg(rdev, enable);
5633 0 : }
5634 : }
5635 :
5636 0 : if (block & RADEON_CG_BLOCK_HDP) {
5637 0 : si_enable_hdp_mgcg(rdev, enable);
5638 0 : si_enable_hdp_ls(rdev, enable);
5639 0 : }
5640 0 : }
5641 :
5642 0 : static void si_init_cg(struct radeon_device *rdev)
5643 : {
5644 0 : si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5645 : RADEON_CG_BLOCK_MC |
5646 : RADEON_CG_BLOCK_SDMA |
5647 : RADEON_CG_BLOCK_BIF |
5648 : RADEON_CG_BLOCK_HDP), true);
5649 0 : if (rdev->has_uvd) {
5650 0 : si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5651 0 : si_init_uvd_internal_cg(rdev);
5652 0 : }
5653 0 : }
5654 :
5655 0 : static void si_fini_cg(struct radeon_device *rdev)
5656 : {
5657 0 : if (rdev->has_uvd) {
5658 0 : si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5659 0 : }
5660 0 : si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5661 : RADEON_CG_BLOCK_MC |
5662 : RADEON_CG_BLOCK_SDMA |
5663 : RADEON_CG_BLOCK_BIF |
5664 : RADEON_CG_BLOCK_HDP), false);
5665 0 : }
5666 :
5667 0 : u32 si_get_csb_size(struct radeon_device *rdev)
5668 : {
5669 : u32 count = 0;
5670 : const struct cs_section_def *sect = NULL;
5671 : const struct cs_extent_def *ext = NULL;
5672 :
5673 0 : if (rdev->rlc.cs_data == NULL)
5674 0 : return 0;
5675 :
5676 : /* begin clear state */
5677 : count += 2;
5678 : /* context control state */
5679 : count += 3;
5680 :
5681 0 : for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5682 0 : for (ext = sect->section; ext->extent != NULL; ++ext) {
5683 0 : if (sect->id == SECT_CONTEXT)
5684 0 : count += 2 + ext->reg_count;
5685 : else
5686 0 : return 0;
5687 : }
5688 : }
5689 : /* pa_sc_raster_config */
5690 0 : count += 3;
5691 : /* end clear state */
5692 0 : count += 2;
5693 : /* clear state */
5694 0 : count += 2;
5695 :
5696 0 : return count;
5697 0 : }
5698 :
5699 0 : void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5700 : {
5701 : u32 count = 0, i;
5702 : const struct cs_section_def *sect = NULL;
5703 : const struct cs_extent_def *ext = NULL;
5704 :
5705 0 : if (rdev->rlc.cs_data == NULL)
5706 0 : return;
5707 0 : if (buffer == NULL)
5708 0 : return;
5709 :
5710 0 : buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5711 0 : buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5712 :
5713 0 : buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5714 0 : buffer[count++] = cpu_to_le32(0x80000000);
5715 0 : buffer[count++] = cpu_to_le32(0x80000000);
5716 :
5717 0 : for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5718 0 : for (ext = sect->section; ext->extent != NULL; ++ext) {
5719 0 : if (sect->id == SECT_CONTEXT) {
5720 0 : buffer[count++] =
5721 0 : cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5722 0 : buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5723 0 : for (i = 0; i < ext->reg_count; i++)
5724 0 : buffer[count++] = cpu_to_le32(ext->extent[i]);
5725 : } else {
5726 0 : return;
5727 : }
5728 : }
5729 : }
5730 :
5731 0 : buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5732 0 : buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5733 0 : switch (rdev->family) {
5734 : case CHIP_TAHITI:
5735 : case CHIP_PITCAIRN:
5736 0 : buffer[count++] = cpu_to_le32(0x2a00126a);
5737 0 : break;
5738 : case CHIP_VERDE:
5739 0 : buffer[count++] = cpu_to_le32(0x0000124a);
5740 0 : break;
5741 : case CHIP_OLAND:
5742 0 : buffer[count++] = cpu_to_le32(0x00000082);
5743 0 : break;
5744 : case CHIP_HAINAN:
5745 0 : buffer[count++] = cpu_to_le32(0x00000000);
5746 0 : break;
5747 : default:
5748 0 : buffer[count++] = cpu_to_le32(0x00000000);
5749 0 : break;
5750 : }
5751 :
5752 0 : buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5753 0 : buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5754 :
5755 0 : buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5756 0 : buffer[count++] = cpu_to_le32(0);
5757 0 : }
5758 :
5759 0 : static void si_init_pg(struct radeon_device *rdev)
5760 : {
5761 0 : if (rdev->pg_flags) {
5762 0 : if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5763 0 : si_init_dma_pg(rdev);
5764 0 : }
5765 0 : si_init_ao_cu_mask(rdev);
5766 0 : if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5767 0 : si_init_gfx_cgpg(rdev);
5768 0 : } else {
5769 0 : WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5770 0 : WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5771 : }
5772 0 : si_enable_dma_pg(rdev, true);
5773 0 : si_enable_gfx_cgpg(rdev, true);
5774 0 : } else {
5775 0 : WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5776 0 : WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5777 : }
5778 0 : }
5779 :
5780 0 : static void si_fini_pg(struct radeon_device *rdev)
5781 : {
5782 0 : if (rdev->pg_flags) {
5783 0 : si_enable_dma_pg(rdev, false);
5784 0 : si_enable_gfx_cgpg(rdev, false);
5785 0 : }
5786 0 : }
5787 :
5788 : /*
5789 : * RLC
5790 : */
5791 0 : void si_rlc_reset(struct radeon_device *rdev)
5792 : {
5793 0 : u32 tmp = RREG32(GRBM_SOFT_RESET);
5794 :
5795 0 : tmp |= SOFT_RESET_RLC;
5796 0 : WREG32(GRBM_SOFT_RESET, tmp);
5797 0 : udelay(50);
5798 0 : tmp &= ~SOFT_RESET_RLC;
5799 0 : WREG32(GRBM_SOFT_RESET, tmp);
5800 0 : udelay(50);
5801 0 : }
5802 :
5803 0 : static void si_rlc_stop(struct radeon_device *rdev)
5804 : {
5805 0 : WREG32(RLC_CNTL, 0);
5806 :
5807 0 : si_enable_gui_idle_interrupt(rdev, false);
5808 :
5809 0 : si_wait_for_rlc_serdes(rdev);
5810 0 : }
5811 :
5812 0 : static void si_rlc_start(struct radeon_device *rdev)
5813 : {
5814 0 : WREG32(RLC_CNTL, RLC_ENABLE);
5815 :
5816 0 : si_enable_gui_idle_interrupt(rdev, true);
5817 :
5818 0 : udelay(50);
5819 0 : }
5820 :
5821 0 : static bool si_lbpw_supported(struct radeon_device *rdev)
5822 : {
5823 : u32 tmp;
5824 :
5825 : /* Enable LBPW only for DDR3 */
5826 0 : tmp = RREG32(MC_SEQ_MISC0);
5827 0 : if ((tmp & 0xF0000000) == 0xB0000000)
5828 0 : return true;
5829 0 : return false;
5830 0 : }
5831 :
5832 0 : static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5833 : {
5834 : u32 tmp;
5835 :
5836 0 : tmp = RREG32(RLC_LB_CNTL);
5837 0 : if (enable)
5838 0 : tmp |= LOAD_BALANCE_ENABLE;
5839 : else
5840 0 : tmp &= ~LOAD_BALANCE_ENABLE;
5841 0 : WREG32(RLC_LB_CNTL, tmp);
5842 :
5843 0 : if (!enable) {
5844 0 : si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5845 0 : WREG32(SPI_LB_CU_MASK, 0x00ff);
5846 0 : }
5847 0 : }
5848 :
5849 0 : static int si_rlc_resume(struct radeon_device *rdev)
5850 : {
5851 : u32 i;
5852 :
5853 0 : if (!rdev->rlc_fw)
5854 0 : return -EINVAL;
5855 :
5856 0 : si_rlc_stop(rdev);
5857 :
5858 0 : si_rlc_reset(rdev);
5859 :
5860 0 : si_init_pg(rdev);
5861 :
5862 0 : si_init_cg(rdev);
5863 :
5864 0 : WREG32(RLC_RL_BASE, 0);
5865 0 : WREG32(RLC_RL_SIZE, 0);
5866 0 : WREG32(RLC_LB_CNTL, 0);
5867 0 : WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5868 0 : WREG32(RLC_LB_CNTR_INIT, 0);
5869 0 : WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5870 :
5871 0 : WREG32(RLC_MC_CNTL, 0);
5872 0 : WREG32(RLC_UCODE_CNTL, 0);
5873 :
5874 0 : if (rdev->new_fw) {
5875 : const struct rlc_firmware_header_v1_0 *hdr =
5876 0 : (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5877 0 : u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5878 0 : const __le32 *fw_data = (const __le32 *)
5879 0 : (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5880 :
5881 0 : radeon_ucode_print_rlc_hdr(&hdr->header);
5882 :
5883 0 : for (i = 0; i < fw_size; i++) {
5884 0 : WREG32(RLC_UCODE_ADDR, i);
5885 0 : WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5886 : }
5887 0 : } else {
5888 : const __be32 *fw_data =
5889 0 : (const __be32 *)rdev->rlc_fw->data;
5890 0 : for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5891 0 : WREG32(RLC_UCODE_ADDR, i);
5892 0 : WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5893 : }
5894 : }
5895 0 : WREG32(RLC_UCODE_ADDR, 0);
5896 :
5897 0 : si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5898 :
5899 0 : si_rlc_start(rdev);
5900 :
5901 0 : return 0;
5902 0 : }
5903 :
5904 0 : static void si_enable_interrupts(struct radeon_device *rdev)
5905 : {
5906 0 : u32 ih_cntl = RREG32(IH_CNTL);
5907 0 : u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5908 :
5909 0 : ih_cntl |= ENABLE_INTR;
5910 0 : ih_rb_cntl |= IH_RB_ENABLE;
5911 0 : WREG32(IH_CNTL, ih_cntl);
5912 0 : WREG32(IH_RB_CNTL, ih_rb_cntl);
5913 0 : rdev->ih.enabled = true;
5914 0 : }
5915 :
5916 0 : static void si_disable_interrupts(struct radeon_device *rdev)
5917 : {
5918 0 : u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5919 0 : u32 ih_cntl = RREG32(IH_CNTL);
5920 :
5921 0 : ih_rb_cntl &= ~IH_RB_ENABLE;
5922 0 : ih_cntl &= ~ENABLE_INTR;
5923 0 : WREG32(IH_RB_CNTL, ih_rb_cntl);
5924 0 : WREG32(IH_CNTL, ih_cntl);
5925 : /* set rptr, wptr to 0 */
5926 0 : WREG32(IH_RB_RPTR, 0);
5927 0 : WREG32(IH_RB_WPTR, 0);
5928 0 : rdev->ih.enabled = false;
5929 0 : rdev->ih.rptr = 0;
5930 0 : }
5931 :
5932 0 : static void si_disable_interrupt_state(struct radeon_device *rdev)
5933 : {
5934 : u32 tmp;
5935 :
5936 0 : tmp = RREG32(CP_INT_CNTL_RING0) &
5937 : (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5938 0 : WREG32(CP_INT_CNTL_RING0, tmp);
5939 0 : WREG32(CP_INT_CNTL_RING1, 0);
5940 0 : WREG32(CP_INT_CNTL_RING2, 0);
5941 0 : tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5942 0 : WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5943 0 : tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5944 0 : WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5945 0 : WREG32(GRBM_INT_CNTL, 0);
5946 0 : WREG32(SRBM_INT_CNTL, 0);
5947 0 : if (rdev->num_crtc >= 2) {
5948 0 : WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5949 0 : WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5950 0 : }
5951 0 : if (rdev->num_crtc >= 4) {
5952 0 : WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5953 0 : WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5954 0 : }
5955 0 : if (rdev->num_crtc >= 6) {
5956 0 : WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5957 0 : WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5958 0 : }
5959 :
5960 0 : if (rdev->num_crtc >= 2) {
5961 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5962 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5963 0 : }
5964 0 : if (rdev->num_crtc >= 4) {
5965 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5966 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5967 0 : }
5968 0 : if (rdev->num_crtc >= 6) {
5969 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5970 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5971 0 : }
5972 :
5973 0 : if (!ASIC_IS_NODCE(rdev)) {
5974 0 : WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5975 :
5976 0 : tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5977 0 : WREG32(DC_HPD1_INT_CONTROL, tmp);
5978 0 : tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5979 0 : WREG32(DC_HPD2_INT_CONTROL, tmp);
5980 0 : tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5981 0 : WREG32(DC_HPD3_INT_CONTROL, tmp);
5982 0 : tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5983 0 : WREG32(DC_HPD4_INT_CONTROL, tmp);
5984 0 : tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5985 0 : WREG32(DC_HPD5_INT_CONTROL, tmp);
5986 0 : tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5987 0 : WREG32(DC_HPD6_INT_CONTROL, tmp);
5988 0 : }
5989 0 : }
5990 :
5991 0 : static int si_irq_init(struct radeon_device *rdev)
5992 : {
5993 : int ret = 0;
5994 : int rb_bufsz;
5995 : u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5996 :
5997 : /* allocate ring */
5998 0 : ret = r600_ih_ring_alloc(rdev);
5999 0 : if (ret)
6000 0 : return ret;
6001 :
6002 : /* disable irqs */
6003 0 : si_disable_interrupts(rdev);
6004 :
6005 : /* init rlc */
6006 0 : ret = si_rlc_resume(rdev);
6007 0 : if (ret) {
6008 0 : r600_ih_ring_fini(rdev);
6009 0 : return ret;
6010 : }
6011 :
6012 : /* setup interrupt control */
6013 : /* set dummy read address to ring address */
6014 0 : WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6015 0 : interrupt_cntl = RREG32(INTERRUPT_CNTL);
6016 : /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6017 : * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6018 : */
6019 0 : interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6020 : /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6021 0 : interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6022 0 : WREG32(INTERRUPT_CNTL, interrupt_cntl);
6023 :
6024 0 : WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6025 0 : rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6026 :
6027 : ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6028 0 : IH_WPTR_OVERFLOW_CLEAR |
6029 0 : (rb_bufsz << 1));
6030 :
6031 0 : if (rdev->wb.enabled)
6032 0 : ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6033 :
6034 : /* set the writeback address whether it's enabled or not */
6035 0 : WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6036 0 : WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6037 :
6038 0 : WREG32(IH_RB_CNTL, ih_rb_cntl);
6039 :
6040 : /* set rptr, wptr to 0 */
6041 0 : WREG32(IH_RB_RPTR, 0);
6042 0 : WREG32(IH_RB_WPTR, 0);
6043 :
6044 : /* Default settings for IH_CNTL (disabled at first) */
6045 : ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6046 : /* RPTR_REARM only works if msi's are enabled */
6047 0 : if (rdev->msi_enabled)
6048 0 : ih_cntl |= RPTR_REARM;
6049 0 : WREG32(IH_CNTL, ih_cntl);
6050 :
6051 : /* force the active interrupt state to all disabled */
6052 0 : si_disable_interrupt_state(rdev);
6053 :
6054 : pci_set_master(rdev->pdev);
6055 :
6056 : /* enable irqs */
6057 0 : si_enable_interrupts(rdev);
6058 :
6059 0 : return ret;
6060 0 : }
6061 :
6062 0 : int si_irq_set(struct radeon_device *rdev)
6063 : {
6064 : u32 cp_int_cntl;
6065 : u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6066 : u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6067 : u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
6068 : u32 grbm_int_cntl = 0;
6069 : u32 dma_cntl, dma_cntl1;
6070 : u32 thermal_int = 0;
6071 :
6072 0 : if (!rdev->irq.installed) {
6073 0 : WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6074 0 : return -EINVAL;
6075 : }
6076 : /* don't enable anything if the ih is disabled */
6077 0 : if (!rdev->ih.enabled) {
6078 0 : si_disable_interrupts(rdev);
6079 : /* force the active interrupt state to all disabled */
6080 0 : si_disable_interrupt_state(rdev);
6081 0 : return 0;
6082 : }
6083 :
6084 0 : cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6085 : (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6086 :
6087 0 : if (!ASIC_IS_NODCE(rdev)) {
6088 0 : hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6089 0 : hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6090 0 : hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6091 0 : hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6092 0 : hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6093 0 : hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6094 0 : }
6095 :
6096 0 : dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6097 0 : dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6098 :
6099 0 : thermal_int = RREG32(CG_THERMAL_INT) &
6100 : ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6101 :
6102 : /* enable CP interrupts on all rings */
6103 0 : if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6104 : DRM_DEBUG("si_irq_set: sw int gfx\n");
6105 0 : cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6106 0 : }
6107 0 : if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6108 : DRM_DEBUG("si_irq_set: sw int cp1\n");
6109 : cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6110 0 : }
6111 0 : if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6112 : DRM_DEBUG("si_irq_set: sw int cp2\n");
6113 : cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6114 0 : }
6115 0 : if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6116 : DRM_DEBUG("si_irq_set: sw int dma\n");
6117 0 : dma_cntl |= TRAP_ENABLE;
6118 0 : }
6119 :
6120 0 : if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6121 : DRM_DEBUG("si_irq_set: sw int dma1\n");
6122 0 : dma_cntl1 |= TRAP_ENABLE;
6123 0 : }
6124 0 : if (rdev->irq.crtc_vblank_int[0] ||
6125 0 : atomic_read(&rdev->irq.pflip[0])) {
6126 : DRM_DEBUG("si_irq_set: vblank 0\n");
6127 : crtc1 |= VBLANK_INT_MASK;
6128 0 : }
6129 0 : if (rdev->irq.crtc_vblank_int[1] ||
6130 0 : atomic_read(&rdev->irq.pflip[1])) {
6131 : DRM_DEBUG("si_irq_set: vblank 1\n");
6132 : crtc2 |= VBLANK_INT_MASK;
6133 0 : }
6134 0 : if (rdev->irq.crtc_vblank_int[2] ||
6135 0 : atomic_read(&rdev->irq.pflip[2])) {
6136 : DRM_DEBUG("si_irq_set: vblank 2\n");
6137 : crtc3 |= VBLANK_INT_MASK;
6138 0 : }
6139 0 : if (rdev->irq.crtc_vblank_int[3] ||
6140 0 : atomic_read(&rdev->irq.pflip[3])) {
6141 : DRM_DEBUG("si_irq_set: vblank 3\n");
6142 : crtc4 |= VBLANK_INT_MASK;
6143 0 : }
6144 0 : if (rdev->irq.crtc_vblank_int[4] ||
6145 0 : atomic_read(&rdev->irq.pflip[4])) {
6146 : DRM_DEBUG("si_irq_set: vblank 4\n");
6147 : crtc5 |= VBLANK_INT_MASK;
6148 0 : }
6149 0 : if (rdev->irq.crtc_vblank_int[5] ||
6150 0 : atomic_read(&rdev->irq.pflip[5])) {
6151 : DRM_DEBUG("si_irq_set: vblank 5\n");
6152 : crtc6 |= VBLANK_INT_MASK;
6153 0 : }
6154 0 : if (rdev->irq.hpd[0]) {
6155 : DRM_DEBUG("si_irq_set: hpd 1\n");
6156 0 : hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6157 0 : }
6158 0 : if (rdev->irq.hpd[1]) {
6159 : DRM_DEBUG("si_irq_set: hpd 2\n");
6160 0 : hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6161 0 : }
6162 0 : if (rdev->irq.hpd[2]) {
6163 : DRM_DEBUG("si_irq_set: hpd 3\n");
6164 0 : hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6165 0 : }
6166 0 : if (rdev->irq.hpd[3]) {
6167 : DRM_DEBUG("si_irq_set: hpd 4\n");
6168 0 : hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6169 0 : }
6170 0 : if (rdev->irq.hpd[4]) {
6171 : DRM_DEBUG("si_irq_set: hpd 5\n");
6172 0 : hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6173 0 : }
6174 0 : if (rdev->irq.hpd[5]) {
6175 : DRM_DEBUG("si_irq_set: hpd 6\n");
6176 0 : hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6177 0 : }
6178 :
6179 0 : WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6180 0 : WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6181 0 : WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6182 :
6183 0 : WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6184 0 : WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6185 :
6186 0 : WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6187 :
6188 0 : if (rdev->irq.dpm_thermal) {
6189 : DRM_DEBUG("dpm thermal\n");
6190 0 : thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6191 0 : }
6192 :
6193 0 : if (rdev->num_crtc >= 2) {
6194 0 : WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6195 0 : WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6196 0 : }
6197 0 : if (rdev->num_crtc >= 4) {
6198 0 : WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6199 0 : WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6200 0 : }
6201 0 : if (rdev->num_crtc >= 6) {
6202 0 : WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6203 0 : WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6204 0 : }
6205 :
6206 0 : if (rdev->num_crtc >= 2) {
6207 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6208 : GRPH_PFLIP_INT_MASK);
6209 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6210 : GRPH_PFLIP_INT_MASK);
6211 0 : }
6212 0 : if (rdev->num_crtc >= 4) {
6213 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6214 : GRPH_PFLIP_INT_MASK);
6215 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6216 : GRPH_PFLIP_INT_MASK);
6217 0 : }
6218 0 : if (rdev->num_crtc >= 6) {
6219 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6220 : GRPH_PFLIP_INT_MASK);
6221 0 : WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6222 : GRPH_PFLIP_INT_MASK);
6223 0 : }
6224 :
6225 0 : if (!ASIC_IS_NODCE(rdev)) {
6226 0 : WREG32(DC_HPD1_INT_CONTROL, hpd1);
6227 0 : WREG32(DC_HPD2_INT_CONTROL, hpd2);
6228 0 : WREG32(DC_HPD3_INT_CONTROL, hpd3);
6229 0 : WREG32(DC_HPD4_INT_CONTROL, hpd4);
6230 0 : WREG32(DC_HPD5_INT_CONTROL, hpd5);
6231 0 : WREG32(DC_HPD6_INT_CONTROL, hpd6);
6232 0 : }
6233 :
6234 0 : WREG32(CG_THERMAL_INT, thermal_int);
6235 :
6236 : /* posting read */
6237 0 : RREG32(SRBM_STATUS);
6238 :
6239 0 : return 0;
6240 0 : }
6241 :
6242 0 : static inline void si_irq_ack(struct radeon_device *rdev)
6243 : {
6244 : u32 tmp;
6245 :
6246 0 : if (ASIC_IS_NODCE(rdev))
6247 0 : return;
6248 :
6249 0 : rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6250 0 : rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6251 0 : rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6252 0 : rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6253 0 : rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6254 0 : rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6255 0 : rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6256 0 : rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6257 0 : if (rdev->num_crtc >= 4) {
6258 0 : rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6259 0 : rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6260 0 : }
6261 0 : if (rdev->num_crtc >= 6) {
6262 0 : rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6263 0 : rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6264 0 : }
6265 :
6266 0 : if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6267 0 : WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6268 0 : if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6269 0 : WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6270 0 : if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6271 0 : WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6272 0 : if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6273 0 : WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6274 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6275 0 : WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6276 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6277 0 : WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6278 :
6279 0 : if (rdev->num_crtc >= 4) {
6280 0 : if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6281 0 : WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6282 0 : if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6283 0 : WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6284 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6285 0 : WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6286 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6287 0 : WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6288 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6289 0 : WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6290 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6291 0 : WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6292 : }
6293 :
6294 0 : if (rdev->num_crtc >= 6) {
6295 0 : if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6296 0 : WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6297 0 : if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6298 0 : WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6299 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6300 0 : WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6301 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6302 0 : WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6303 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6304 0 : WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6305 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6306 0 : WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6307 : }
6308 :
6309 0 : if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6310 0 : tmp = RREG32(DC_HPD1_INT_CONTROL);
6311 0 : tmp |= DC_HPDx_INT_ACK;
6312 0 : WREG32(DC_HPD1_INT_CONTROL, tmp);
6313 0 : }
6314 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6315 0 : tmp = RREG32(DC_HPD2_INT_CONTROL);
6316 0 : tmp |= DC_HPDx_INT_ACK;
6317 0 : WREG32(DC_HPD2_INT_CONTROL, tmp);
6318 0 : }
6319 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6320 0 : tmp = RREG32(DC_HPD3_INT_CONTROL);
6321 0 : tmp |= DC_HPDx_INT_ACK;
6322 0 : WREG32(DC_HPD3_INT_CONTROL, tmp);
6323 0 : }
6324 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6325 0 : tmp = RREG32(DC_HPD4_INT_CONTROL);
6326 0 : tmp |= DC_HPDx_INT_ACK;
6327 0 : WREG32(DC_HPD4_INT_CONTROL, tmp);
6328 0 : }
6329 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6330 0 : tmp = RREG32(DC_HPD5_INT_CONTROL);
6331 0 : tmp |= DC_HPDx_INT_ACK;
6332 0 : WREG32(DC_HPD5_INT_CONTROL, tmp);
6333 0 : }
6334 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6335 0 : tmp = RREG32(DC_HPD6_INT_CONTROL);
6336 0 : tmp |= DC_HPDx_INT_ACK;
6337 0 : WREG32(DC_HPD6_INT_CONTROL, tmp);
6338 0 : }
6339 :
6340 0 : if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
6341 0 : tmp = RREG32(DC_HPD1_INT_CONTROL);
6342 0 : tmp |= DC_HPDx_RX_INT_ACK;
6343 0 : WREG32(DC_HPD1_INT_CONTROL, tmp);
6344 0 : }
6345 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
6346 0 : tmp = RREG32(DC_HPD2_INT_CONTROL);
6347 0 : tmp |= DC_HPDx_RX_INT_ACK;
6348 0 : WREG32(DC_HPD2_INT_CONTROL, tmp);
6349 0 : }
6350 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
6351 0 : tmp = RREG32(DC_HPD3_INT_CONTROL);
6352 0 : tmp |= DC_HPDx_RX_INT_ACK;
6353 0 : WREG32(DC_HPD3_INT_CONTROL, tmp);
6354 0 : }
6355 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
6356 0 : tmp = RREG32(DC_HPD4_INT_CONTROL);
6357 0 : tmp |= DC_HPDx_RX_INT_ACK;
6358 0 : WREG32(DC_HPD4_INT_CONTROL, tmp);
6359 0 : }
6360 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
6361 0 : tmp = RREG32(DC_HPD5_INT_CONTROL);
6362 0 : tmp |= DC_HPDx_RX_INT_ACK;
6363 0 : WREG32(DC_HPD5_INT_CONTROL, tmp);
6364 0 : }
6365 0 : if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
6366 0 : tmp = RREG32(DC_HPD6_INT_CONTROL);
6367 0 : tmp |= DC_HPDx_RX_INT_ACK;
6368 0 : WREG32(DC_HPD6_INT_CONTROL, tmp);
6369 0 : }
6370 0 : }
6371 :
6372 0 : static void si_irq_disable(struct radeon_device *rdev)
6373 : {
6374 0 : si_disable_interrupts(rdev);
6375 : /* Wait and acknowledge irq */
6376 0 : mdelay(1);
6377 0 : si_irq_ack(rdev);
6378 0 : si_disable_interrupt_state(rdev);
6379 0 : }
6380 :
6381 0 : static void si_irq_suspend(struct radeon_device *rdev)
6382 : {
6383 0 : si_irq_disable(rdev);
6384 0 : si_rlc_stop(rdev);
6385 0 : }
6386 :
6387 0 : static void si_irq_fini(struct radeon_device *rdev)
6388 : {
6389 0 : si_irq_suspend(rdev);
6390 0 : r600_ih_ring_fini(rdev);
6391 0 : }
6392 :
6393 0 : static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6394 : {
6395 : u32 wptr, tmp;
6396 :
6397 0 : if (rdev->wb.enabled)
6398 0 : wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6399 : else
6400 0 : wptr = RREG32(IH_RB_WPTR);
6401 :
6402 0 : if (wptr & RB_OVERFLOW) {
6403 0 : wptr &= ~RB_OVERFLOW;
6404 : /* When a ring buffer overflow happen start parsing interrupt
6405 : * from the last not overwritten vector (wptr + 16). Hopefully
6406 : * this should allow us to catchup.
6407 : */
6408 0 : dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6409 : wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6410 0 : rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6411 0 : tmp = RREG32(IH_RB_CNTL);
6412 0 : tmp |= IH_WPTR_OVERFLOW_CLEAR;
6413 0 : WREG32(IH_RB_CNTL, tmp);
6414 0 : }
6415 0 : return (wptr & rdev->ih.ptr_mask);
6416 : }
6417 :
6418 : /* SI IV Ring
6419 : * Each IV ring entry is 128 bits:
6420 : * [7:0] - interrupt source id
6421 : * [31:8] - reserved
6422 : * [59:32] - interrupt source data
6423 : * [63:60] - reserved
6424 : * [71:64] - RINGID
6425 : * [79:72] - VMID
6426 : * [127:80] - reserved
6427 : */
6428 0 : int si_irq_process(struct radeon_device *rdev)
6429 : {
6430 : u32 wptr;
6431 : u32 rptr;
6432 : u32 src_id, src_data, ring_id;
6433 : u32 ring_index;
6434 : bool queue_hotplug = false;
6435 : bool queue_dp = false;
6436 : bool queue_thermal = false;
6437 : u32 status, addr;
6438 :
6439 0 : if (!rdev->ih.enabled || rdev->shutdown)
6440 0 : return IRQ_NONE;
6441 :
6442 0 : wptr = si_get_ih_wptr(rdev);
6443 :
6444 0 : if (wptr == rdev->ih.rptr)
6445 0 : return IRQ_NONE;
6446 : restart_ih:
6447 : /* is somebody else already processing irqs? */
6448 0 : if (atomic_xchg(&rdev->ih.lock, 1))
6449 0 : return IRQ_NONE;
6450 :
6451 0 : rptr = rdev->ih.rptr;
6452 : DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6453 :
6454 : /* Order reading of wptr vs. reading of IH ring data */
6455 0 : rmb();
6456 :
6457 : /* display interrupts */
6458 0 : si_irq_ack(rdev);
6459 :
6460 0 : while (rptr != wptr) {
6461 : /* wptr/rptr are in bytes! */
6462 0 : ring_index = rptr / 4;
6463 0 : src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6464 0 : src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6465 0 : ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6466 :
6467 0 : switch (src_id) {
6468 : case 1: /* D1 vblank/vline */
6469 0 : switch (src_data) {
6470 : case 0: /* D1 vblank */
6471 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
6472 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6473 :
6474 0 : if (rdev->irq.crtc_vblank_int[0]) {
6475 0 : drm_handle_vblank(rdev->ddev, 0);
6476 0 : rdev->pm.vblank_sync = true;
6477 0 : wake_up(&rdev->irq.vblank_queue);
6478 0 : }
6479 0 : if (atomic_read(&rdev->irq.pflip[0]))
6480 0 : radeon_crtc_handle_vblank(rdev, 0);
6481 0 : rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6482 : DRM_DEBUG("IH: D1 vblank\n");
6483 :
6484 0 : break;
6485 : case 1: /* D1 vline */
6486 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
6487 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6488 :
6489 0 : rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6490 : DRM_DEBUG("IH: D1 vline\n");
6491 :
6492 0 : break;
6493 : default:
6494 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6495 : break;
6496 : }
6497 : break;
6498 : case 2: /* D2 vblank/vline */
6499 0 : switch (src_data) {
6500 : case 0: /* D2 vblank */
6501 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
6502 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6503 :
6504 0 : if (rdev->irq.crtc_vblank_int[1]) {
6505 0 : drm_handle_vblank(rdev->ddev, 1);
6506 0 : rdev->pm.vblank_sync = true;
6507 0 : wake_up(&rdev->irq.vblank_queue);
6508 0 : }
6509 0 : if (atomic_read(&rdev->irq.pflip[1]))
6510 0 : radeon_crtc_handle_vblank(rdev, 1);
6511 0 : rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6512 : DRM_DEBUG("IH: D2 vblank\n");
6513 :
6514 0 : break;
6515 : case 1: /* D2 vline */
6516 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
6517 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6518 :
6519 0 : rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6520 : DRM_DEBUG("IH: D2 vline\n");
6521 :
6522 0 : break;
6523 : default:
6524 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6525 : break;
6526 : }
6527 : break;
6528 : case 3: /* D3 vblank/vline */
6529 0 : switch (src_data) {
6530 : case 0: /* D3 vblank */
6531 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
6532 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6533 :
6534 0 : if (rdev->irq.crtc_vblank_int[2]) {
6535 0 : drm_handle_vblank(rdev->ddev, 2);
6536 0 : rdev->pm.vblank_sync = true;
6537 0 : wake_up(&rdev->irq.vblank_queue);
6538 0 : }
6539 0 : if (atomic_read(&rdev->irq.pflip[2]))
6540 0 : radeon_crtc_handle_vblank(rdev, 2);
6541 0 : rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6542 : DRM_DEBUG("IH: D3 vblank\n");
6543 :
6544 0 : break;
6545 : case 1: /* D3 vline */
6546 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
6547 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6548 :
6549 0 : rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6550 : DRM_DEBUG("IH: D3 vline\n");
6551 :
6552 0 : break;
6553 : default:
6554 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6555 : break;
6556 : }
6557 : break;
6558 : case 4: /* D4 vblank/vline */
6559 0 : switch (src_data) {
6560 : case 0: /* D4 vblank */
6561 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
6562 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6563 :
6564 0 : if (rdev->irq.crtc_vblank_int[3]) {
6565 0 : drm_handle_vblank(rdev->ddev, 3);
6566 0 : rdev->pm.vblank_sync = true;
6567 0 : wake_up(&rdev->irq.vblank_queue);
6568 0 : }
6569 0 : if (atomic_read(&rdev->irq.pflip[3]))
6570 0 : radeon_crtc_handle_vblank(rdev, 3);
6571 0 : rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6572 : DRM_DEBUG("IH: D4 vblank\n");
6573 :
6574 0 : break;
6575 : case 1: /* D4 vline */
6576 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
6577 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6578 :
6579 0 : rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6580 : DRM_DEBUG("IH: D4 vline\n");
6581 :
6582 0 : break;
6583 : default:
6584 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6585 : break;
6586 : }
6587 : break;
6588 : case 5: /* D5 vblank/vline */
6589 0 : switch (src_data) {
6590 : case 0: /* D5 vblank */
6591 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
6592 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6593 :
6594 0 : if (rdev->irq.crtc_vblank_int[4]) {
6595 0 : drm_handle_vblank(rdev->ddev, 4);
6596 0 : rdev->pm.vblank_sync = true;
6597 0 : wake_up(&rdev->irq.vblank_queue);
6598 0 : }
6599 0 : if (atomic_read(&rdev->irq.pflip[4]))
6600 0 : radeon_crtc_handle_vblank(rdev, 4);
6601 0 : rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6602 : DRM_DEBUG("IH: D5 vblank\n");
6603 :
6604 0 : break;
6605 : case 1: /* D5 vline */
6606 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
6607 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6608 :
6609 0 : rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6610 : DRM_DEBUG("IH: D5 vline\n");
6611 :
6612 0 : break;
6613 : default:
6614 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6615 : break;
6616 : }
6617 : break;
6618 : case 6: /* D6 vblank/vline */
6619 0 : switch (src_data) {
6620 : case 0: /* D6 vblank */
6621 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
6622 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6623 :
6624 0 : if (rdev->irq.crtc_vblank_int[5]) {
6625 0 : drm_handle_vblank(rdev->ddev, 5);
6626 0 : rdev->pm.vblank_sync = true;
6627 0 : wake_up(&rdev->irq.vblank_queue);
6628 0 : }
6629 0 : if (atomic_read(&rdev->irq.pflip[5]))
6630 0 : radeon_crtc_handle_vblank(rdev, 5);
6631 0 : rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6632 : DRM_DEBUG("IH: D6 vblank\n");
6633 :
6634 0 : break;
6635 : case 1: /* D6 vline */
6636 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
6637 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6638 :
6639 0 : rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6640 : DRM_DEBUG("IH: D6 vline\n");
6641 :
6642 0 : break;
6643 : default:
6644 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6645 : break;
6646 : }
6647 : break;
6648 : case 8: /* D1 page flip */
6649 : case 10: /* D2 page flip */
6650 : case 12: /* D3 page flip */
6651 : case 14: /* D4 page flip */
6652 : case 16: /* D5 page flip */
6653 : case 18: /* D6 page flip */
6654 : DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6655 0 : if (radeon_use_pflipirq > 0)
6656 0 : radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6657 : break;
6658 : case 42: /* HPD hotplug */
6659 0 : switch (src_data) {
6660 : case 0:
6661 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
6662 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6663 :
6664 0 : rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6665 : queue_hotplug = true;
6666 : DRM_DEBUG("IH: HPD1\n");
6667 :
6668 0 : break;
6669 : case 1:
6670 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
6671 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6672 :
6673 0 : rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6674 : queue_hotplug = true;
6675 : DRM_DEBUG("IH: HPD2\n");
6676 :
6677 0 : break;
6678 : case 2:
6679 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
6680 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6681 :
6682 0 : rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6683 : queue_hotplug = true;
6684 : DRM_DEBUG("IH: HPD3\n");
6685 :
6686 0 : break;
6687 : case 3:
6688 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
6689 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6690 :
6691 0 : rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6692 : queue_hotplug = true;
6693 : DRM_DEBUG("IH: HPD4\n");
6694 :
6695 0 : break;
6696 : case 4:
6697 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
6698 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6699 :
6700 0 : rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6701 : queue_hotplug = true;
6702 : DRM_DEBUG("IH: HPD5\n");
6703 :
6704 0 : break;
6705 : case 5:
6706 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
6707 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6708 :
6709 0 : rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6710 : queue_hotplug = true;
6711 : DRM_DEBUG("IH: HPD6\n");
6712 :
6713 0 : break;
6714 : case 6:
6715 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
6716 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6717 :
6718 0 : rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
6719 : queue_dp = true;
6720 : DRM_DEBUG("IH: HPD_RX 1\n");
6721 :
6722 0 : break;
6723 : case 7:
6724 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
6725 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6726 :
6727 0 : rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
6728 : queue_dp = true;
6729 : DRM_DEBUG("IH: HPD_RX 2\n");
6730 :
6731 0 : break;
6732 : case 8:
6733 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
6734 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6735 :
6736 0 : rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
6737 : queue_dp = true;
6738 : DRM_DEBUG("IH: HPD_RX 3\n");
6739 :
6740 0 : break;
6741 : case 9:
6742 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
6743 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6744 :
6745 0 : rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
6746 : queue_dp = true;
6747 : DRM_DEBUG("IH: HPD_RX 4\n");
6748 :
6749 0 : break;
6750 : case 10:
6751 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
6752 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6753 :
6754 0 : rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
6755 : queue_dp = true;
6756 : DRM_DEBUG("IH: HPD_RX 5\n");
6757 :
6758 0 : break;
6759 : case 11:
6760 0 : if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
6761 : DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6762 :
6763 0 : rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
6764 : queue_dp = true;
6765 : DRM_DEBUG("IH: HPD_RX 6\n");
6766 :
6767 0 : break;
6768 : default:
6769 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6770 : break;
6771 : }
6772 : break;
6773 : case 96:
6774 0 : DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6775 0 : WREG32(SRBM_INT_ACK, 0x1);
6776 0 : break;
6777 : case 124: /* UVD */
6778 : DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6779 0 : radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6780 0 : break;
6781 : case 146:
6782 : case 147:
6783 0 : addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6784 0 : status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6785 : /* reset addr and status */
6786 0 : WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6787 0 : if (addr == 0x0 && status == 0x0)
6788 : break;
6789 0 : dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6790 0 : dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6791 : addr);
6792 0 : dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6793 : status);
6794 0 : si_vm_decode_fault(rdev, status, addr);
6795 0 : break;
6796 : case 176: /* RINGID0 CP_INT */
6797 0 : radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6798 0 : break;
6799 : case 177: /* RINGID1 CP_INT */
6800 0 : radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6801 0 : break;
6802 : case 178: /* RINGID2 CP_INT */
6803 0 : radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6804 0 : break;
6805 : case 181: /* CP EOP event */
6806 : DRM_DEBUG("IH: CP EOP\n");
6807 0 : switch (ring_id) {
6808 : case 0:
6809 0 : radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6810 0 : break;
6811 : case 1:
6812 0 : radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6813 0 : break;
6814 : case 2:
6815 0 : radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6816 0 : break;
6817 : }
6818 : break;
6819 : case 224: /* DMA trap event */
6820 : DRM_DEBUG("IH: DMA trap\n");
6821 0 : radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6822 0 : break;
6823 : case 230: /* thermal low to high */
6824 : DRM_DEBUG("IH: thermal low to high\n");
6825 0 : rdev->pm.dpm.thermal.high_to_low = false;
6826 : queue_thermal = true;
6827 0 : break;
6828 : case 231: /* thermal high to low */
6829 : DRM_DEBUG("IH: thermal high to low\n");
6830 0 : rdev->pm.dpm.thermal.high_to_low = true;
6831 : queue_thermal = true;
6832 0 : break;
6833 : case 233: /* GUI IDLE */
6834 : DRM_DEBUG("IH: GUI idle\n");
6835 : break;
6836 : case 244: /* DMA trap event */
6837 : DRM_DEBUG("IH: DMA1 trap\n");
6838 0 : radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6839 0 : break;
6840 : default:
6841 : DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6842 : break;
6843 : }
6844 :
6845 : /* wptr/rptr are in bytes! */
6846 0 : rptr += 16;
6847 0 : rptr &= rdev->ih.ptr_mask;
6848 0 : WREG32(IH_RB_RPTR, rptr);
6849 : }
6850 0 : if (queue_dp)
6851 0 : schedule_work(&rdev->dp_work);
6852 0 : if (queue_hotplug)
6853 0 : schedule_delayed_work(&rdev->hotplug_work, 0);
6854 0 : if (queue_thermal && rdev->pm.dpm_enabled)
6855 0 : schedule_work(&rdev->pm.dpm.thermal.work);
6856 0 : rdev->ih.rptr = rptr;
6857 0 : atomic_set(&rdev->ih.lock, 0);
6858 :
6859 : /* make sure wptr hasn't changed while processing */
6860 0 : wptr = si_get_ih_wptr(rdev);
6861 0 : if (wptr != rptr)
6862 0 : goto restart_ih;
6863 :
6864 0 : return IRQ_HANDLED;
6865 0 : }
6866 :
6867 : /*
6868 : * startup/shutdown callbacks
6869 : */
6870 0 : static int si_startup(struct radeon_device *rdev)
6871 : {
6872 : struct radeon_ring *ring;
6873 : int r;
6874 :
6875 : /* enable pcie gen2/3 link */
6876 0 : si_pcie_gen3_enable(rdev);
6877 : /* enable aspm */
6878 0 : si_program_aspm(rdev);
6879 :
6880 : /* scratch needs to be initialized before MC */
6881 0 : r = r600_vram_scratch_init(rdev);
6882 0 : if (r)
6883 0 : return r;
6884 :
6885 0 : si_mc_program(rdev);
6886 :
6887 0 : if (!rdev->pm.dpm_enabled) {
6888 0 : r = si_mc_load_microcode(rdev);
6889 0 : if (r) {
6890 0 : DRM_ERROR("Failed to load MC firmware!\n");
6891 0 : return r;
6892 : }
6893 : }
6894 :
6895 0 : r = si_pcie_gart_enable(rdev);
6896 0 : if (r)
6897 0 : return r;
6898 0 : si_gpu_init(rdev);
6899 :
6900 : /* allocate rlc buffers */
6901 0 : if (rdev->family == CHIP_VERDE) {
6902 0 : rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6903 0 : rdev->rlc.reg_list_size =
6904 : (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6905 0 : }
6906 0 : rdev->rlc.cs_data = si_cs_data;
6907 0 : r = sumo_rlc_init(rdev);
6908 0 : if (r) {
6909 0 : DRM_ERROR("Failed to init rlc BOs!\n");
6910 0 : return r;
6911 : }
6912 :
6913 : /* allocate wb buffer */
6914 0 : r = radeon_wb_init(rdev);
6915 0 : if (r)
6916 0 : return r;
6917 :
6918 0 : r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6919 0 : if (r) {
6920 0 : dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6921 0 : return r;
6922 : }
6923 :
6924 0 : r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6925 0 : if (r) {
6926 0 : dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6927 0 : return r;
6928 : }
6929 :
6930 0 : r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6931 0 : if (r) {
6932 0 : dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6933 0 : return r;
6934 : }
6935 :
6936 0 : r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6937 0 : if (r) {
6938 0 : dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6939 0 : return r;
6940 : }
6941 :
6942 0 : r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6943 0 : if (r) {
6944 0 : dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6945 0 : return r;
6946 : }
6947 :
6948 0 : if (rdev->has_uvd) {
6949 0 : r = uvd_v2_2_resume(rdev);
6950 0 : if (!r) {
6951 0 : r = radeon_fence_driver_start_ring(rdev,
6952 : R600_RING_TYPE_UVD_INDEX);
6953 0 : if (r)
6954 0 : dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6955 : }
6956 0 : if (r)
6957 0 : rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6958 : }
6959 :
6960 0 : r = radeon_vce_resume(rdev);
6961 0 : if (!r) {
6962 0 : r = vce_v1_0_resume(rdev);
6963 0 : if (!r)
6964 0 : r = radeon_fence_driver_start_ring(rdev,
6965 : TN_RING_TYPE_VCE1_INDEX);
6966 0 : if (!r)
6967 0 : r = radeon_fence_driver_start_ring(rdev,
6968 : TN_RING_TYPE_VCE2_INDEX);
6969 : }
6970 0 : if (r) {
6971 0 : dev_err(rdev->dev, "VCE init error (%d).\n", r);
6972 0 : rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6973 0 : rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6974 0 : }
6975 :
6976 : /* Enable IRQ */
6977 0 : if (!rdev->irq.installed) {
6978 0 : r = radeon_irq_kms_init(rdev);
6979 0 : if (r)
6980 0 : return r;
6981 : }
6982 :
6983 0 : r = si_irq_init(rdev);
6984 0 : if (r) {
6985 0 : DRM_ERROR("radeon: IH init failed (%d).\n", r);
6986 0 : radeon_irq_kms_fini(rdev);
6987 0 : return r;
6988 : }
6989 0 : si_irq_set(rdev);
6990 :
6991 0 : ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6992 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6993 : RADEON_CP_PACKET2);
6994 0 : if (r)
6995 0 : return r;
6996 :
6997 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6998 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6999 : RADEON_CP_PACKET2);
7000 0 : if (r)
7001 0 : return r;
7002 :
7003 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7004 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7005 : RADEON_CP_PACKET2);
7006 0 : if (r)
7007 0 : return r;
7008 :
7009 0 : ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7010 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7011 : DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7012 0 : if (r)
7013 0 : return r;
7014 :
7015 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7016 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7017 : DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7018 0 : if (r)
7019 0 : return r;
7020 :
7021 0 : r = si_cp_load_microcode(rdev);
7022 0 : if (r)
7023 0 : return r;
7024 0 : r = si_cp_resume(rdev);
7025 0 : if (r)
7026 0 : return r;
7027 :
7028 0 : r = cayman_dma_resume(rdev);
7029 0 : if (r)
7030 0 : return r;
7031 :
7032 0 : if (rdev->has_uvd) {
7033 0 : ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7034 0 : if (ring->ring_size) {
7035 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7036 : RADEON_CP_PACKET2);
7037 0 : if (!r)
7038 0 : r = uvd_v1_0_init(rdev);
7039 0 : if (r)
7040 0 : DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7041 : }
7042 : }
7043 :
7044 : r = -ENOENT;
7045 :
7046 0 : ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7047 0 : if (ring->ring_size)
7048 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7049 : VCE_CMD_NO_OP);
7050 :
7051 0 : ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7052 0 : if (ring->ring_size)
7053 0 : r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7054 : VCE_CMD_NO_OP);
7055 :
7056 0 : if (!r)
7057 0 : r = vce_v1_0_init(rdev);
7058 0 : else if (r != -ENOENT)
7059 0 : DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7060 :
7061 0 : r = radeon_ib_pool_init(rdev);
7062 0 : if (r) {
7063 0 : dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7064 0 : return r;
7065 : }
7066 :
7067 0 : r = radeon_vm_manager_init(rdev);
7068 0 : if (r) {
7069 0 : dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7070 0 : return r;
7071 : }
7072 :
7073 0 : r = radeon_audio_init(rdev);
7074 0 : if (r)
7075 0 : return r;
7076 :
7077 0 : return 0;
7078 0 : }
7079 :
7080 0 : int si_resume(struct radeon_device *rdev)
7081 : {
7082 : int r;
7083 :
7084 : /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
7085 : * posting will perform necessary task to bring back GPU into good
7086 : * shape.
7087 : */
7088 : /* post card */
7089 0 : atom_asic_init(rdev->mode_info.atom_context);
7090 :
7091 : /* init golden registers */
7092 0 : si_init_golden_registers(rdev);
7093 :
7094 0 : if (rdev->pm.pm_method == PM_METHOD_DPM)
7095 0 : radeon_pm_resume(rdev);
7096 :
7097 0 : rdev->accel_working = true;
7098 0 : r = si_startup(rdev);
7099 0 : if (r) {
7100 0 : DRM_ERROR("si startup failed on resume\n");
7101 0 : rdev->accel_working = false;
7102 0 : return r;
7103 : }
7104 :
7105 0 : return r;
7106 :
7107 0 : }
7108 :
7109 0 : int si_suspend(struct radeon_device *rdev)
7110 : {
7111 0 : radeon_pm_suspend(rdev);
7112 0 : radeon_audio_fini(rdev);
7113 0 : radeon_vm_manager_fini(rdev);
7114 0 : si_cp_enable(rdev, false);
7115 0 : cayman_dma_stop(rdev);
7116 0 : if (rdev->has_uvd) {
7117 0 : uvd_v1_0_fini(rdev);
7118 0 : radeon_uvd_suspend(rdev);
7119 0 : radeon_vce_suspend(rdev);
7120 0 : }
7121 0 : si_fini_pg(rdev);
7122 0 : si_fini_cg(rdev);
7123 0 : si_irq_suspend(rdev);
7124 0 : radeon_wb_disable(rdev);
7125 0 : si_pcie_gart_disable(rdev);
7126 0 : return 0;
7127 : }
7128 :
7129 : /* Plan is to move initialization in that function and use
7130 : * helper function so that radeon_device_init pretty much
7131 : * do nothing more than calling asic specific function. This
7132 : * should also allow to remove a bunch of callback function
7133 : * like vram_info.
7134 : */
7135 0 : int si_init(struct radeon_device *rdev)
7136 : {
7137 0 : struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7138 : int r;
7139 :
7140 : /* Read BIOS */
7141 0 : if (!radeon_get_bios(rdev)) {
7142 0 : if (ASIC_IS_AVIVO(rdev))
7143 0 : return -EINVAL;
7144 : }
7145 : /* Must be an ATOMBIOS */
7146 0 : if (!rdev->is_atom_bios) {
7147 0 : dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7148 0 : return -EINVAL;
7149 : }
7150 0 : r = radeon_atombios_init(rdev);
7151 0 : if (r)
7152 0 : return r;
7153 :
7154 : /* Post card if necessary */
7155 0 : if (!radeon_card_posted(rdev)) {
7156 0 : if (!rdev->bios) {
7157 0 : dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7158 0 : return -EINVAL;
7159 : }
7160 : DRM_INFO("GPU not posted. posting now...\n");
7161 0 : atom_asic_init(rdev->mode_info.atom_context);
7162 0 : }
7163 : /* init golden registers */
7164 0 : si_init_golden_registers(rdev);
7165 : /* Initialize scratch registers */
7166 0 : si_scratch_init(rdev);
7167 : /* Initialize surface registers */
7168 0 : radeon_surface_init(rdev);
7169 : /* Initialize clocks */
7170 0 : radeon_get_clock_info(rdev->ddev);
7171 :
7172 : /* Fence driver */
7173 0 : r = radeon_fence_driver_init(rdev);
7174 0 : if (r)
7175 0 : return r;
7176 :
7177 : /* initialize memory controller */
7178 0 : r = si_mc_init(rdev);
7179 0 : if (r)
7180 0 : return r;
7181 : /* Memory manager */
7182 0 : r = radeon_bo_init(rdev);
7183 0 : if (r)
7184 0 : return r;
7185 :
7186 0 : if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7187 0 : !rdev->rlc_fw || !rdev->mc_fw) {
7188 0 : r = si_init_microcode(rdev);
7189 0 : if (r) {
7190 0 : DRM_ERROR("Failed to load firmware!\n");
7191 0 : return r;
7192 : }
7193 : }
7194 :
7195 : /* Initialize power management */
7196 0 : radeon_pm_init(rdev);
7197 :
7198 0 : ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7199 0 : ring->ring_obj = NULL;
7200 0 : r600_ring_init(rdev, ring, 1024 * 1024);
7201 :
7202 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7203 0 : ring->ring_obj = NULL;
7204 0 : r600_ring_init(rdev, ring, 1024 * 1024);
7205 :
7206 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7207 0 : ring->ring_obj = NULL;
7208 0 : r600_ring_init(rdev, ring, 1024 * 1024);
7209 :
7210 0 : ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7211 0 : ring->ring_obj = NULL;
7212 0 : r600_ring_init(rdev, ring, 64 * 1024);
7213 :
7214 0 : ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7215 0 : ring->ring_obj = NULL;
7216 0 : r600_ring_init(rdev, ring, 64 * 1024);
7217 :
7218 0 : if (rdev->has_uvd) {
7219 0 : r = radeon_uvd_init(rdev);
7220 0 : if (!r) {
7221 0 : ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7222 0 : ring->ring_obj = NULL;
7223 0 : r600_ring_init(rdev, ring, 4096);
7224 0 : }
7225 : }
7226 :
7227 0 : r = radeon_vce_init(rdev);
7228 0 : if (!r) {
7229 0 : ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7230 0 : ring->ring_obj = NULL;
7231 0 : r600_ring_init(rdev, ring, 4096);
7232 :
7233 0 : ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7234 0 : ring->ring_obj = NULL;
7235 0 : r600_ring_init(rdev, ring, 4096);
7236 0 : }
7237 :
7238 0 : rdev->ih.ring_obj = NULL;
7239 0 : r600_ih_ring_init(rdev, 64 * 1024);
7240 :
7241 0 : r = r600_pcie_gart_init(rdev);
7242 0 : if (r)
7243 0 : return r;
7244 :
7245 0 : rdev->accel_working = true;
7246 0 : r = si_startup(rdev);
7247 0 : if (r) {
7248 0 : dev_err(rdev->dev, "disabling GPU acceleration\n");
7249 0 : si_cp_fini(rdev);
7250 0 : cayman_dma_fini(rdev);
7251 0 : si_irq_fini(rdev);
7252 0 : sumo_rlc_fini(rdev);
7253 0 : radeon_wb_fini(rdev);
7254 0 : radeon_ib_pool_fini(rdev);
7255 0 : radeon_vm_manager_fini(rdev);
7256 0 : radeon_irq_kms_fini(rdev);
7257 0 : si_pcie_gart_fini(rdev);
7258 0 : rdev->accel_working = false;
7259 0 : }
7260 :
7261 : /* Don't start up if the MC ucode is missing.
7262 : * The default clocks and voltages before the MC ucode
7263 : * is loaded are not suffient for advanced operations.
7264 : */
7265 0 : if (!rdev->mc_fw) {
7266 0 : DRM_ERROR("radeon: MC ucode required for NI+.\n");
7267 0 : return -EINVAL;
7268 : }
7269 :
7270 0 : return 0;
7271 0 : }
7272 :
7273 0 : void si_fini(struct radeon_device *rdev)
7274 : {
7275 0 : radeon_pm_fini(rdev);
7276 0 : si_cp_fini(rdev);
7277 0 : cayman_dma_fini(rdev);
7278 0 : si_fini_pg(rdev);
7279 0 : si_fini_cg(rdev);
7280 0 : si_irq_fini(rdev);
7281 0 : sumo_rlc_fini(rdev);
7282 0 : radeon_wb_fini(rdev);
7283 0 : radeon_vm_manager_fini(rdev);
7284 0 : radeon_ib_pool_fini(rdev);
7285 0 : radeon_irq_kms_fini(rdev);
7286 0 : if (rdev->has_uvd) {
7287 0 : uvd_v1_0_fini(rdev);
7288 0 : radeon_uvd_fini(rdev);
7289 0 : radeon_vce_fini(rdev);
7290 0 : }
7291 0 : si_pcie_gart_fini(rdev);
7292 0 : r600_vram_scratch_fini(rdev);
7293 0 : radeon_gem_fini(rdev);
7294 0 : radeon_fence_driver_fini(rdev);
7295 0 : radeon_bo_fini(rdev);
7296 0 : radeon_atombios_fini(rdev);
7297 0 : kfree(rdev->bios);
7298 0 : rdev->bios = NULL;
7299 0 : }
7300 :
7301 : /**
7302 : * si_get_gpu_clock_counter - return GPU clock counter snapshot
7303 : *
7304 : * @rdev: radeon_device pointer
7305 : *
7306 : * Fetches a GPU clock counter snapshot (SI).
7307 : * Returns the 64 bit clock counter snapshot.
7308 : */
7309 0 : uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7310 : {
7311 : uint64_t clock;
7312 :
7313 0 : mutex_lock(&rdev->gpu_clock_mutex);
7314 0 : WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7315 0 : clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7316 0 : ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7317 0 : mutex_unlock(&rdev->gpu_clock_mutex);
7318 0 : return clock;
7319 : }
7320 :
7321 0 : int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7322 : {
7323 0 : unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7324 : int r;
7325 :
7326 : /* bypass vclk and dclk with bclk */
7327 0 : WREG32_P(CG_UPLL_FUNC_CNTL_2,
7328 : VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7329 : ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7330 :
7331 : /* put PLL in bypass mode */
7332 0 : WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7333 :
7334 0 : if (!vclk || !dclk) {
7335 : /* keep the Bypass mode */
7336 0 : return 0;
7337 : }
7338 :
7339 0 : r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7340 : 16384, 0x03FFFFFF, 0, 128, 5,
7341 : &fb_div, &vclk_div, &dclk_div);
7342 0 : if (r)
7343 0 : return r;
7344 :
7345 : /* set RESET_ANTI_MUX to 0 */
7346 0 : WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7347 :
7348 : /* set VCO_MODE to 1 */
7349 0 : WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7350 :
7351 : /* disable sleep mode */
7352 0 : WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7353 :
7354 : /* deassert UPLL_RESET */
7355 0 : WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7356 :
7357 0 : mdelay(1);
7358 :
7359 0 : r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7360 0 : if (r)
7361 0 : return r;
7362 :
7363 : /* assert UPLL_RESET again */
7364 0 : WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7365 :
7366 : /* disable spread spectrum. */
7367 0 : WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7368 :
7369 : /* set feedback divider */
7370 0 : WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7371 :
7372 : /* set ref divider to 0 */
7373 0 : WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7374 :
7375 0 : if (fb_div < 307200)
7376 0 : WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7377 : else
7378 0 : WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7379 :
7380 : /* set PDIV_A and PDIV_B */
7381 0 : WREG32_P(CG_UPLL_FUNC_CNTL_2,
7382 : UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7383 : ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7384 :
7385 : /* give the PLL some time to settle */
7386 0 : mdelay(15);
7387 :
7388 : /* deassert PLL_RESET */
7389 0 : WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7390 :
7391 0 : mdelay(15);
7392 :
7393 : /* switch from bypass mode to normal mode */
7394 0 : WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7395 :
7396 0 : r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7397 0 : if (r)
7398 0 : return r;
7399 :
7400 : /* switch VCLK and DCLK selection */
7401 0 : WREG32_P(CG_UPLL_FUNC_CNTL_2,
7402 : VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7403 : ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7404 :
7405 0 : mdelay(100);
7406 :
7407 0 : return 0;
7408 0 : }
7409 :
7410 0 : static void si_pcie_gen3_enable(struct radeon_device *rdev)
7411 : {
7412 0 : struct pci_dev _root;
7413 : struct pci_dev *root;
7414 : int bridge_pos, gpu_pos;
7415 0 : u32 speed_cntl, mask, current_data_rate;
7416 : int ret, i;
7417 0 : u16 tmp16;
7418 :
7419 : root = &_root;
7420 0 : root->pc = rdev->pdev->pc;
7421 0 : root->tag = *rdev->ddev->bridgetag;
7422 :
7423 0 : if (pci_is_root_bus(rdev->pdev->bus))
7424 0 : return;
7425 :
7426 0 : if (radeon_pcie_gen2 == 0)
7427 0 : return;
7428 :
7429 0 : if (rdev->flags & RADEON_IS_IGP)
7430 0 : return;
7431 :
7432 0 : if (!(rdev->flags & RADEON_IS_PCIE))
7433 0 : return;
7434 :
7435 0 : ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7436 0 : if (ret != 0)
7437 0 : return;
7438 :
7439 0 : if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7440 0 : return;
7441 :
7442 0 : speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7443 0 : current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7444 : LC_CURRENT_DATA_RATE_SHIFT;
7445 0 : if (mask & DRM_PCIE_SPEED_80) {
7446 0 : if (current_data_rate == 2) {
7447 : DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7448 0 : return;
7449 : }
7450 : DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7451 0 : } else if (mask & DRM_PCIE_SPEED_50) {
7452 0 : if (current_data_rate == 1) {
7453 : DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7454 0 : return;
7455 : }
7456 : DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7457 : }
7458 :
7459 0 : bridge_pos = pci_pcie_cap(root);
7460 0 : if (!bridge_pos)
7461 0 : return;
7462 :
7463 0 : gpu_pos = pci_pcie_cap(rdev->pdev);
7464 0 : if (!gpu_pos)
7465 0 : return;
7466 :
7467 0 : if (mask & DRM_PCIE_SPEED_80) {
7468 : /* re-try equalization if gen3 is not already enabled */
7469 0 : if (current_data_rate != 2) {
7470 0 : u16 bridge_cfg, gpu_cfg;
7471 0 : u16 bridge_cfg2, gpu_cfg2;
7472 : u32 max_lw, current_lw, tmp;
7473 :
7474 0 : pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7475 0 : pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7476 :
7477 0 : tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7478 0 : pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7479 :
7480 0 : tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7481 0 : pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7482 :
7483 0 : tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7484 0 : max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7485 0 : current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7486 :
7487 0 : if (current_lw < max_lw) {
7488 0 : tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7489 0 : if (tmp & LC_RENEGOTIATION_SUPPORT) {
7490 0 : tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7491 0 : tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7492 0 : tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7493 0 : WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7494 0 : }
7495 : }
7496 :
7497 0 : for (i = 0; i < 10; i++) {
7498 : /* check status */
7499 0 : pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7500 0 : if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7501 : break;
7502 :
7503 0 : pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7504 0 : pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7505 :
7506 0 : pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7507 0 : pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7508 :
7509 0 : tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7510 0 : tmp |= LC_SET_QUIESCE;
7511 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7512 :
7513 0 : tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7514 0 : tmp |= LC_REDO_EQ;
7515 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7516 :
7517 0 : mdelay(100);
7518 :
7519 : /* linkctl */
7520 0 : pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7521 0 : tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7522 0 : tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7523 0 : pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7524 :
7525 0 : pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7526 0 : tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7527 0 : tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7528 0 : pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7529 :
7530 : /* linkctl2 */
7531 0 : pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7532 0 : tmp16 &= ~((1 << 4) | (7 << 9));
7533 0 : tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7534 0 : pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7535 :
7536 0 : pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7537 0 : tmp16 &= ~((1 << 4) | (7 << 9));
7538 0 : tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7539 0 : pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7540 :
7541 0 : tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7542 0 : tmp &= ~LC_SET_QUIESCE;
7543 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7544 : }
7545 0 : }
7546 : }
7547 :
7548 : /* set the link speed */
7549 0 : speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7550 0 : speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7551 0 : WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7552 :
7553 0 : pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7554 0 : tmp16 &= ~0xf;
7555 0 : if (mask & DRM_PCIE_SPEED_80)
7556 0 : tmp16 |= 3; /* gen3 */
7557 0 : else if (mask & DRM_PCIE_SPEED_50)
7558 0 : tmp16 |= 2; /* gen2 */
7559 : else
7560 0 : tmp16 |= 1; /* gen1 */
7561 0 : pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7562 :
7563 0 : speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7564 0 : speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7565 0 : WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7566 :
7567 0 : for (i = 0; i < rdev->usec_timeout; i++) {
7568 0 : speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7569 0 : if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7570 : break;
7571 0 : udelay(1);
7572 : }
7573 0 : }
7574 :
7575 0 : static void si_program_aspm(struct radeon_device *rdev)
7576 : {
7577 : u32 data, orig;
7578 : bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7579 : bool disable_clkreq = false;
7580 :
7581 0 : if (radeon_aspm == 0)
7582 0 : return;
7583 :
7584 0 : if (!(rdev->flags & RADEON_IS_PCIE))
7585 0 : return;
7586 :
7587 0 : orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7588 0 : data &= ~LC_XMIT_N_FTS_MASK;
7589 0 : data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7590 0 : if (orig != data)
7591 0 : WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7592 :
7593 0 : orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7594 0 : data |= LC_GO_TO_RECOVERY;
7595 0 : if (orig != data)
7596 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7597 :
7598 0 : orig = data = RREG32_PCIE(PCIE_P_CNTL);
7599 0 : data |= P_IGNORE_EDB_ERR;
7600 0 : if (orig != data)
7601 0 : WREG32_PCIE(PCIE_P_CNTL, data);
7602 :
7603 0 : orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7604 0 : data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7605 0 : data |= LC_PMI_TO_L1_DIS;
7606 0 : if (!disable_l0s)
7607 0 : data |= LC_L0S_INACTIVITY(7);
7608 :
7609 0 : if (!disable_l1) {
7610 0 : data |= LC_L1_INACTIVITY(7);
7611 0 : data &= ~LC_PMI_TO_L1_DIS;
7612 0 : if (orig != data)
7613 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7614 :
7615 0 : if (!disable_plloff_in_l1) {
7616 : bool clk_req_support;
7617 :
7618 0 : orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7619 0 : data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7620 0 : data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7621 0 : if (orig != data)
7622 0 : WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7623 :
7624 0 : orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7625 0 : data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7626 0 : data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7627 0 : if (orig != data)
7628 0 : WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7629 :
7630 0 : orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7631 0 : data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7632 0 : data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7633 0 : if (orig != data)
7634 0 : WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7635 :
7636 0 : orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7637 0 : data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7638 0 : data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7639 0 : if (orig != data)
7640 0 : WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7641 :
7642 0 : if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7643 0 : orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7644 0 : data &= ~PLL_RAMP_UP_TIME_0_MASK;
7645 0 : if (orig != data)
7646 0 : WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7647 :
7648 0 : orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7649 0 : data &= ~PLL_RAMP_UP_TIME_1_MASK;
7650 0 : if (orig != data)
7651 0 : WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7652 :
7653 0 : orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7654 0 : data &= ~PLL_RAMP_UP_TIME_2_MASK;
7655 0 : if (orig != data)
7656 0 : WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7657 :
7658 0 : orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7659 0 : data &= ~PLL_RAMP_UP_TIME_3_MASK;
7660 0 : if (orig != data)
7661 0 : WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7662 :
7663 0 : orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7664 0 : data &= ~PLL_RAMP_UP_TIME_0_MASK;
7665 0 : if (orig != data)
7666 0 : WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7667 :
7668 0 : orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7669 0 : data &= ~PLL_RAMP_UP_TIME_1_MASK;
7670 0 : if (orig != data)
7671 0 : WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7672 :
7673 0 : orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7674 0 : data &= ~PLL_RAMP_UP_TIME_2_MASK;
7675 0 : if (orig != data)
7676 0 : WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7677 :
7678 0 : orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7679 0 : data &= ~PLL_RAMP_UP_TIME_3_MASK;
7680 0 : if (orig != data)
7681 0 : WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7682 : }
7683 0 : orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7684 0 : data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7685 0 : data |= LC_DYN_LANES_PWR_STATE(3);
7686 0 : if (orig != data)
7687 0 : WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7688 :
7689 0 : orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7690 0 : data &= ~LS2_EXIT_TIME_MASK;
7691 0 : if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7692 0 : data |= LS2_EXIT_TIME(5);
7693 0 : if (orig != data)
7694 0 : WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7695 :
7696 0 : orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7697 0 : data &= ~LS2_EXIT_TIME_MASK;
7698 0 : if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7699 0 : data |= LS2_EXIT_TIME(5);
7700 0 : if (orig != data)
7701 0 : WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7702 :
7703 0 : if (!disable_clkreq &&
7704 0 : !pci_is_root_bus(rdev->pdev->bus)) {
7705 0 : u32 lnkcap;
7706 0 : struct pci_dev _root;
7707 : struct pci_dev *root;
7708 :
7709 : root = &_root;
7710 0 : root->pc = rdev->pdev->pc;
7711 0 : root->tag = *rdev->ddev->bridgetag;
7712 :
7713 : clk_req_support = false;
7714 0 : pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7715 0 : if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7716 0 : clk_req_support = true;
7717 0 : } else {
7718 : clk_req_support = false;
7719 : }
7720 :
7721 0 : if (clk_req_support) {
7722 0 : orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7723 0 : data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7724 0 : if (orig != data)
7725 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7726 :
7727 0 : orig = data = RREG32(THM_CLK_CNTL);
7728 0 : data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7729 0 : data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7730 0 : if (orig != data)
7731 0 : WREG32(THM_CLK_CNTL, data);
7732 :
7733 0 : orig = data = RREG32(MISC_CLK_CNTL);
7734 0 : data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7735 0 : data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7736 0 : if (orig != data)
7737 0 : WREG32(MISC_CLK_CNTL, data);
7738 :
7739 0 : orig = data = RREG32(CG_CLKPIN_CNTL);
7740 0 : data &= ~BCLK_AS_XCLK;
7741 0 : if (orig != data)
7742 0 : WREG32(CG_CLKPIN_CNTL, data);
7743 :
7744 0 : orig = data = RREG32(CG_CLKPIN_CNTL_2);
7745 0 : data &= ~FORCE_BIF_REFCLK_EN;
7746 0 : if (orig != data)
7747 0 : WREG32(CG_CLKPIN_CNTL_2, data);
7748 :
7749 0 : orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7750 0 : data &= ~MPLL_CLKOUT_SEL_MASK;
7751 0 : data |= MPLL_CLKOUT_SEL(4);
7752 0 : if (orig != data)
7753 0 : WREG32(MPLL_BYPASSCLK_SEL, data);
7754 :
7755 0 : orig = data = RREG32(SPLL_CNTL_MODE);
7756 0 : data &= ~SPLL_REFCLK_SEL_MASK;
7757 0 : if (orig != data)
7758 0 : WREG32(SPLL_CNTL_MODE, data);
7759 : }
7760 0 : }
7761 : } else {
7762 0 : if (orig != data)
7763 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7764 : }
7765 :
7766 0 : orig = data = RREG32_PCIE(PCIE_CNTL2);
7767 0 : data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7768 0 : if (orig != data)
7769 0 : WREG32_PCIE(PCIE_CNTL2, data);
7770 :
7771 0 : if (!disable_l0s) {
7772 0 : data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7773 0 : if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7774 0 : data = RREG32_PCIE(PCIE_LC_STATUS1);
7775 0 : if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7776 0 : orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7777 0 : data &= ~LC_L0S_INACTIVITY_MASK;
7778 0 : if (orig != data)
7779 0 : WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7780 : }
7781 : }
7782 : }
7783 0 : }
7784 :
7785 0 : int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7786 : {
7787 : unsigned i;
7788 :
7789 : /* make sure VCEPLL_CTLREQ is deasserted */
7790 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7791 :
7792 0 : mdelay(10);
7793 :
7794 : /* assert UPLL_CTLREQ */
7795 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7796 :
7797 : /* wait for CTLACK and CTLACK2 to get asserted */
7798 0 : for (i = 0; i < 100; ++i) {
7799 : uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7800 0 : if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7801 0 : break;
7802 0 : mdelay(10);
7803 0 : }
7804 :
7805 : /* deassert UPLL_CTLREQ */
7806 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7807 :
7808 0 : if (i == 100) {
7809 0 : DRM_ERROR("Timeout setting UVD clocks!\n");
7810 0 : return -ETIMEDOUT;
7811 : }
7812 :
7813 0 : return 0;
7814 0 : }
7815 :
7816 0 : int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7817 : {
7818 0 : unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7819 : int r;
7820 :
7821 : /* bypass evclk and ecclk with bclk */
7822 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7823 : EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7824 : ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7825 :
7826 : /* put PLL in bypass mode */
7827 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7828 : ~VCEPLL_BYPASS_EN_MASK);
7829 :
7830 0 : if (!evclk || !ecclk) {
7831 : /* keep the Bypass mode, put PLL to sleep */
7832 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7833 : ~VCEPLL_SLEEP_MASK);
7834 0 : return 0;
7835 : }
7836 :
7837 0 : r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7838 : 16384, 0x03FFFFFF, 0, 128, 5,
7839 : &fb_div, &evclk_div, &ecclk_div);
7840 0 : if (r)
7841 0 : return r;
7842 :
7843 : /* set RESET_ANTI_MUX to 0 */
7844 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7845 :
7846 : /* set VCO_MODE to 1 */
7847 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7848 : ~VCEPLL_VCO_MODE_MASK);
7849 :
7850 : /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7851 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7852 : ~VCEPLL_SLEEP_MASK);
7853 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7854 :
7855 : /* deassert VCEPLL_RESET */
7856 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7857 :
7858 0 : mdelay(1);
7859 :
7860 0 : r = si_vce_send_vcepll_ctlreq(rdev);
7861 0 : if (r)
7862 0 : return r;
7863 :
7864 : /* assert VCEPLL_RESET again */
7865 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7866 :
7867 : /* disable spread spectrum. */
7868 0 : WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7869 :
7870 : /* set feedback divider */
7871 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7872 :
7873 : /* set ref divider to 0 */
7874 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7875 :
7876 : /* set PDIV_A and PDIV_B */
7877 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7878 : VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7879 : ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7880 :
7881 : /* give the PLL some time to settle */
7882 0 : mdelay(15);
7883 :
7884 : /* deassert PLL_RESET */
7885 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7886 :
7887 0 : mdelay(15);
7888 :
7889 : /* switch from bypass mode to normal mode */
7890 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7891 :
7892 0 : r = si_vce_send_vcepll_ctlreq(rdev);
7893 0 : if (r)
7894 0 : return r;
7895 :
7896 : /* switch VCLK and DCLK selection */
7897 0 : WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7898 : EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7899 : ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7900 :
7901 0 : mdelay(100);
7902 :
7903 0 : return 0;
7904 0 : }
|
