Line data Source code
1 : /* $OpenBSD: ahci.c,v 1.32 2017/08/21 21:43:46 jmatthew Exp $ */
2 :
3 : /*
4 : * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
5 : * Copyright (c) 2010 Conformal Systems LLC <info@conformal.com>
6 : * Copyright (c) 2010 Jonathan Matthew <jonathan@d14n.org>
7 : *
8 : * Permission to use, copy, modify, and distribute this software for any
9 : * purpose with or without fee is hereby granted, provided that the above
10 : * copyright notice and this permission notice appear in all copies.
11 : *
12 : * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 : * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 : * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 : * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 : * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 : * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 : * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 : */
20 :
21 : #include <sys/param.h>
22 : #include <sys/systm.h>
23 : #include <sys/buf.h>
24 : #include <sys/kernel.h>
25 : #include <sys/malloc.h>
26 : #include <sys/device.h>
27 : #include <sys/queue.h>
28 : #include <sys/mutex.h>
29 : #include <sys/pool.h>
30 :
31 : #include <machine/bus.h>
32 :
33 : #include <dev/ic/ahcireg.h>
34 : #include <dev/ic/ahcivar.h>
35 :
36 : #ifdef AHCI_DEBUG
37 : #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
38 : while (0)
39 : #define AHCI_D_TIMEOUT 0x00
40 : #define AHCI_D_VERBOSE 0x01
41 : #define AHCI_D_INTR 0x02
42 : #define AHCI_D_XFER 0x08
43 : int ahcidebug = AHCI_D_VERBOSE;
44 : #else
45 : #define DPRINTF(m, f...)
46 : #endif
47 :
48 : #ifdef HIBERNATE
49 : #include <uvm/uvm_extern.h>
50 : #include <sys/hibernate.h>
51 : #include <sys/disk.h>
52 : #include <sys/disklabel.h>
53 :
54 : #include <scsi/scsi_all.h>
55 : #include <scsi/scsiconf.h>
56 :
57 : void ahci_hibernate_io_start(struct ahci_port *,
58 : struct ahci_ccb *);
59 : int ahci_hibernate_io_poll(struct ahci_port *,
60 : struct ahci_ccb *);
61 : void ahci_hibernate_load_prdt(struct ahci_ccb *);
62 :
63 : int ahci_hibernate_io(dev_t dev, daddr_t blkno,
64 : vaddr_t addr, size_t size, int wr, void *page);
65 : #endif
66 :
67 : struct cfdriver ahci_cd = {
68 : NULL, "ahci", DV_DULL
69 : };
70 :
71 : void ahci_enable_interrupts(struct ahci_port *);
72 :
73 : int ahci_init(struct ahci_softc *);
74 : int ahci_port_alloc(struct ahci_softc *, u_int);
75 : void ahci_port_detect(struct ahci_softc *, u_int);
76 : void ahci_port_free(struct ahci_softc *, u_int);
77 : int ahci_port_init(struct ahci_softc *, u_int);
78 :
79 : int ahci_default_port_start(struct ahci_port *, int);
80 : int ahci_port_stop(struct ahci_port *, int);
81 : int ahci_port_clo(struct ahci_port *);
82 : int ahci_port_softreset(struct ahci_port *);
83 : void ahci_port_comreset(struct ahci_port *);
84 : int ahci_port_portreset(struct ahci_port *, int);
85 : void ahci_port_portreset_start(struct ahci_port *);
86 : int ahci_port_portreset_poll(struct ahci_port *);
87 : void ahci_port_portreset_wait(struct ahci_port *);
88 : int ahci_port_portreset_finish(struct ahci_port *, int);
89 : int ahci_port_signature(struct ahci_port *);
90 : int ahci_pmp_port_softreset(struct ahci_port *, int);
91 : int ahci_pmp_port_portreset(struct ahci_port *, int);
92 : int ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port);
93 :
94 : int ahci_load_prdt(struct ahci_ccb *);
95 : void ahci_load_prdt_seg(struct ahci_prdt *, u_int64_t,
96 : u_int32_t, u_int32_t);
97 : void ahci_unload_prdt(struct ahci_ccb *);
98 :
99 : int ahci_poll(struct ahci_ccb *, int, void (*)(void *));
100 : void ahci_start(struct ahci_ccb *);
101 :
102 : void ahci_issue_pending_ncq_commands(struct ahci_port *);
103 : void ahci_issue_pending_commands(struct ahci_port *, int);
104 :
105 : int ahci_intr(void *);
106 : u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
107 :
108 : struct ahci_ccb *ahci_get_ccb(struct ahci_port *);
109 : void ahci_put_ccb(struct ahci_ccb *);
110 :
111 : struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
112 : void ahci_put_err_ccb(struct ahci_ccb *);
113 :
114 : struct ahci_ccb *ahci_get_pmp_ccb(struct ahci_port *);
115 : void ahci_put_pmp_ccb(struct ahci_ccb *);
116 :
117 : int ahci_port_read_ncq_error(struct ahci_port *, int *, int);
118 :
119 : struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t);
120 : void ahci_dmamem_free(struct ahci_softc *,
121 : struct ahci_dmamem *);
122 :
123 : u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
124 : void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
125 : int ahci_wait_ne(struct ahci_softc *, bus_size_t,
126 : u_int32_t, u_int32_t);
127 :
128 : u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
129 : void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
130 : int ahci_pwait_eq(struct ahci_port *, bus_size_t,
131 : u_int32_t, u_int32_t, int);
132 : void ahci_flush_tfd(struct ahci_port *ap);
133 : u_int32_t ahci_active_mask(struct ahci_port *);
134 : int ahci_port_detect_pmp(struct ahci_port *);
135 : void ahci_pmp_probe_timeout(void *);
136 :
137 : /* pmp operations */
138 : int ahci_pmp_read(struct ahci_port *, int, int,
139 : u_int32_t *);
140 : int ahci_pmp_write(struct ahci_port *, int, int, u_int32_t);
141 : int ahci_pmp_phy_status(struct ahci_port *, int,
142 : u_int32_t *);
143 : int ahci_pmp_identify(struct ahci_port *, int *);
144 :
145 :
146 : /* Wait for all bits in _b to be cleared */
147 : #define ahci_pwait_clr(_ap, _r, _b, _n) \
148 : ahci_pwait_eq((_ap), (_r), (_b), 0, (_n))
149 :
150 : /* Wait for all bits in _b to be set */
151 : #define ahci_pwait_set(_ap, _r, _b, _n) \
152 : ahci_pwait_eq((_ap), (_r), (_b), (_b), (_n))
153 :
154 :
155 :
156 : /* provide methods for atascsi to call */
157 : int ahci_ata_probe(void *, int, int);
158 : void ahci_ata_free(void *, int, int);
159 : struct ata_xfer * ahci_ata_get_xfer(void *, int);
160 : void ahci_ata_put_xfer(struct ata_xfer *);
161 : void ahci_ata_cmd(struct ata_xfer *);
162 :
163 : struct atascsi_methods ahci_atascsi_methods = {
164 : ahci_ata_probe,
165 : ahci_ata_free,
166 : ahci_ata_get_xfer,
167 : ahci_ata_put_xfer,
168 : ahci_ata_cmd
169 : };
170 :
171 : /* ccb completions */
172 : void ahci_ata_cmd_done(struct ahci_ccb *);
173 : void ahci_pmp_cmd_done(struct ahci_ccb *);
174 : void ahci_ata_cmd_timeout(void *);
175 : void ahci_empty_done(struct ahci_ccb *);
176 :
177 : int
178 0 : ahci_attach(struct ahci_softc *sc)
179 : {
180 0 : struct atascsi_attach_args aaa;
181 : u_int32_t pi;
182 : int i, j, done;
183 :
184 0 : if (sc->sc_port_start == NULL)
185 0 : sc->sc_port_start = ahci_default_port_start;
186 :
187 0 : if (ahci_init(sc) != 0) {
188 : /* error already printed by ahci_init */
189 : goto unmap;
190 : }
191 :
192 0 : printf("\n");
193 :
194 0 : sc->sc_cap = ahci_read(sc, AHCI_REG_CAP);
195 0 : sc->sc_ncmds = AHCI_REG_CAP_NCS(sc->sc_cap);
196 : #ifdef AHCI_DEBUG
197 : if (ahcidebug & AHCI_D_VERBOSE) {
198 : const char *gen;
199 :
200 : switch (sc->sc_cap & AHCI_REG_CAP_ISS) {
201 : case AHCI_REG_CAP_ISS_G1:
202 : gen = "1 (1.5Gbps)";
203 : break;
204 : case AHCI_REG_CAP_ISS_G2:
205 : gen = "2 (3.0Gb/s)";
206 : break;
207 : case AHCI_REG_CAP_ISS_G3:
208 : gen = "3 (6.0Gb/s)";
209 : break;
210 : default:
211 : gen = "unknown";
212 : break;
213 : }
214 :
215 : printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
216 : DEVNAME(sc), sc->sc_cap, AHCI_FMT_CAP,
217 : AHCI_REG_CAP_NP(sc->sc_cap), sc->sc_ncmds, gen);
218 : printf("%s: extended capabilities 0x%b\n", DEVNAME(sc),
219 : ahci_read(sc, AHCI_REG_CAP2), AHCI_FMT_CAP2);
220 : }
221 : #endif
222 :
223 0 : pi = ahci_read(sc, AHCI_REG_PI);
224 : DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
225 : DEVNAME(sc), pi);
226 :
227 : #ifdef AHCI_COALESCE
228 : /* Naive coalescing support - enable for all ports. */
229 : if (sc->sc_cap & AHCI_REG_CAP_CCCS) {
230 : u_int16_t ccc_timeout = 20;
231 : u_int8_t ccc_numcomplete = 12;
232 : u_int32_t ccc_ctl;
233 :
234 : /* disable coalescing during reconfiguration. */
235 : ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
236 : ccc_ctl &= ~0x00000001;
237 : ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
238 :
239 : sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
240 : if (pi & sc->sc_ccc_mask) {
241 : /* A conflict with the implemented port list? */
242 : printf("%s: coalescing interrupt/implemented port list "
243 : "conflict, PI: %08x, ccc_mask: %08x\n",
244 : DEVNAME(sc), pi, sc->sc_ccc_mask);
245 : sc->sc_ccc_mask = 0;
246 : goto noccc;
247 : }
248 :
249 : /* ahci_port_start will enable each port when it starts. */
250 : sc->sc_ccc_ports = pi;
251 : sc->sc_ccc_ports_cur = 0;
252 :
253 : /* program thresholds and enable overall coalescing. */
254 : ccc_ctl &= ~0xffffff00;
255 : ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
256 : ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
257 : ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
258 : ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
259 : }
260 : noccc:
261 : #endif
262 : /*
263 : * Given that ahci_port_alloc() will grab one CCB for error recovery
264 : * in the NCQ case from the pool of CCBs sized based on sc->sc_ncmds
265 : * pretend at least 2 command slots for devices without NCQ support.
266 : * That way, also at least 1 slot is made available for atascsi(4).
267 : */
268 0 : sc->sc_ncmds = max(2, sc->sc_ncmds);
269 0 : for (i = 0; i < AHCI_MAX_PORTS; i++) {
270 0 : if (!ISSET(pi, 1 << i)) {
271 : /* dont allocate stuff if the port isnt implemented */
272 : continue;
273 : }
274 :
275 0 : if (ahci_port_alloc(sc, i) == ENOMEM)
276 : goto freeports;
277 :
278 0 : if (sc->sc_ports[i] != NULL)
279 0 : ahci_port_portreset_start(sc->sc_ports[i]);
280 : }
281 :
282 : /*
283 : * Poll for device detection until all ports report a device, or one
284 : * second has elapsed.
285 : */
286 0 : for (i = 0; i < 1000; i++) {
287 : done = 1;
288 0 : for (j = 0; j < AHCI_MAX_PORTS; j++) {
289 0 : if (sc->sc_ports[j] == NULL)
290 : continue;
291 :
292 0 : if (ahci_port_portreset_poll(sc->sc_ports[j]))
293 0 : done = 0;
294 : }
295 :
296 0 : if (done)
297 : break;
298 :
299 0 : delay(1000);
300 : }
301 :
302 : /*
303 : * Finish device detection on all ports that initialized.
304 : */
305 0 : for (i = 0; i < AHCI_MAX_PORTS; i++) {
306 0 : if (sc->sc_ports[i] != NULL)
307 0 : ahci_port_detect(sc, i);
308 : }
309 :
310 0 : memset(&aaa, 0, sizeof(aaa));
311 0 : aaa.aaa_cookie = sc;
312 0 : aaa.aaa_methods = &ahci_atascsi_methods;
313 0 : aaa.aaa_minphys = NULL;
314 0 : aaa.aaa_nports = AHCI_MAX_PORTS;
315 0 : aaa.aaa_ncmds = sc->sc_ncmds - 1;
316 0 : if (!(sc->sc_flags & AHCI_F_NO_NCQ) &&
317 0 : sc->sc_ncmds > 2 &&
318 0 : (sc->sc_cap & AHCI_REG_CAP_SNCQ)) {
319 0 : aaa.aaa_capability |= ASAA_CAP_NCQ | ASAA_CAP_PMP_NCQ;
320 0 : }
321 :
322 0 : sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
323 :
324 : /* Enable interrupts */
325 0 : ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
326 :
327 0 : return 0;
328 :
329 : freeports:
330 0 : for (i = 0; i < AHCI_MAX_PORTS; i++) {
331 0 : if (sc->sc_ports[i] != NULL)
332 0 : ahci_port_free(sc, i);
333 : }
334 : unmap:
335 : /* Disable controller */
336 0 : ahci_write(sc, AHCI_REG_GHC, 0);
337 0 : return 1;
338 0 : }
339 :
340 : int
341 0 : ahci_detach(struct ahci_softc *sc, int flags)
342 : {
343 : int rv, i;
344 :
345 0 : if (sc->sc_atascsi != NULL) {
346 0 : rv = atascsi_detach(sc->sc_atascsi, flags);
347 0 : if (rv != 0)
348 0 : return (rv);
349 : }
350 :
351 0 : for (i = 0; i < AHCI_MAX_PORTS; i++) {
352 0 : if (sc->sc_ports[i] != NULL)
353 0 : ahci_port_free(sc, i);
354 : }
355 :
356 0 : return (0);
357 0 : }
358 :
359 : int
360 0 : ahci_activate(struct device *self, int act)
361 : {
362 0 : struct ahci_softc *sc = (struct ahci_softc *)self;
363 : int i, rv = 0;
364 :
365 0 : switch (act) {
366 : case DVACT_RESUME:
367 : /* enable ahci (global interrupts disabled) */
368 0 : ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
369 :
370 : /* restore BIOS initialised parameters */
371 0 : ahci_write(sc, AHCI_REG_CAP, sc->sc_cap);
372 :
373 0 : for (i = 0; i < AHCI_MAX_PORTS; i++) {
374 0 : if (sc->sc_ports[i] != NULL)
375 0 : ahci_port_init(sc, i);
376 : }
377 :
378 : /* Enable interrupts */
379 0 : ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
380 :
381 0 : rv = config_activate_children(self, act);
382 0 : break;
383 : case DVACT_POWERDOWN:
384 0 : rv = config_activate_children(self, act);
385 0 : for (i = 0; i < AHCI_MAX_PORTS; i++) {
386 0 : if (sc->sc_ports[i] != NULL)
387 0 : ahci_port_stop(sc->sc_ports[i], 1);
388 : }
389 : break;
390 : default:
391 0 : rv = config_activate_children(self, act);
392 0 : break;
393 : }
394 0 : return (rv);
395 : }
396 :
397 : int
398 0 : ahci_init(struct ahci_softc *sc)
399 : {
400 : u_int32_t reg, cap, pi;
401 : const char *revision;
402 :
403 : DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
404 : AHCI_FMT_GHC);
405 :
406 : /* save BIOS initialised parameters, enable staggered spin up */
407 0 : cap = ahci_read(sc, AHCI_REG_CAP);
408 0 : cap &= AHCI_REG_CAP_SMPS;
409 0 : cap |= AHCI_REG_CAP_SSS;
410 0 : pi = ahci_read(sc, AHCI_REG_PI);
411 :
412 0 : if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
413 : /* reset the controller */
414 0 : ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
415 0 : if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
416 0 : AHCI_REG_GHC_HR) != 0) {
417 0 : printf(" unable to reset controller\n");
418 0 : return (1);
419 : }
420 : }
421 :
422 : /* enable ahci (global interrupts disabled) */
423 0 : ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
424 :
425 : /* restore parameters */
426 0 : ahci_write(sc, AHCI_REG_CAP, cap);
427 0 : ahci_write(sc, AHCI_REG_PI, pi);
428 :
429 : /* check the revision */
430 0 : reg = ahci_read(sc, AHCI_REG_VS);
431 0 : switch (reg) {
432 : case AHCI_REG_VS_0_95:
433 : revision = "0.95";
434 0 : break;
435 : case AHCI_REG_VS_1_0:
436 : revision = "1.0";
437 0 : break;
438 : case AHCI_REG_VS_1_1:
439 : revision = "1.1";
440 0 : break;
441 : case AHCI_REG_VS_1_2:
442 : revision = "1.2";
443 0 : break;
444 : case AHCI_REG_VS_1_3:
445 : revision = "1.3";
446 0 : break;
447 : case AHCI_REG_VS_1_3_1:
448 : revision = "1.3.1";
449 0 : break;
450 :
451 : default:
452 0 : printf(" unsupported AHCI revision 0x%08x\n", reg);
453 0 : return (1);
454 : }
455 :
456 0 : printf(" AHCI %s", revision);
457 :
458 0 : return (0);
459 0 : }
460 :
461 : void
462 0 : ahci_enable_interrupts(struct ahci_port *ap)
463 : {
464 0 : ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
465 : AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
466 0 : AHCI_PREG_IE_UFE |
467 0 : ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) ? AHCI_PREG_IE_IPME : 0) |
468 : #ifdef AHCI_COALESCE
469 : ((ap->ap_sc->sc_ccc_ports & (1 << ap->ap_port)) ? 0 :
470 : (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE))
471 : #else
472 0 : AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
473 : #endif
474 : );
475 0 : }
476 :
477 : int
478 0 : ahci_port_alloc(struct ahci_softc *sc, u_int port)
479 : {
480 : struct ahci_port *ap;
481 : struct ahci_ccb *ccb;
482 : u_int64_t dva;
483 : u_int32_t cmd;
484 : struct ahci_cmd_hdr *hdr;
485 : struct ahci_cmd_table *table;
486 : int i, rc = ENOMEM;
487 :
488 0 : ap = malloc(sizeof(*ap), M_DEVBUF, M_NOWAIT | M_ZERO);
489 0 : if (ap == NULL) {
490 0 : printf("%s: unable to allocate memory for port %d\n",
491 0 : DEVNAME(sc), port);
492 0 : goto reterr;
493 : }
494 0 : ap->ap_err_scratch = dma_alloc(DEV_BSIZE, PR_NOWAIT | PR_ZERO);
495 0 : if (ap->ap_err_scratch == NULL) {
496 0 : printf("%s: unable to allocate DMA scratch buf for port %d\n",
497 0 : DEVNAME(sc), port);
498 0 : free(ap, M_DEVBUF, sizeof(*ap));
499 0 : goto reterr;
500 : }
501 :
502 : #ifdef AHCI_DEBUG
503 : snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
504 : DEVNAME(sc), port);
505 : #endif
506 0 : ap->ap_port = port;
507 0 : sc->sc_ports[port] = ap;
508 :
509 0 : if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
510 0 : AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
511 0 : printf("%s: unable to create register window for port %d\n",
512 0 : DEVNAME(sc), port);
513 0 : goto freeport;
514 : }
515 :
516 0 : ap->ap_sc = sc;
517 : #ifdef AHCI_COALESCE
518 : ap->ap_num = port;
519 : #endif
520 0 : TAILQ_INIT(&ap->ap_ccb_free);
521 0 : TAILQ_INIT(&ap->ap_ccb_pending);
522 0 : mtx_init(&ap->ap_ccb_mtx, IPL_BIO);
523 :
524 : /* Disable port interrupts */
525 0 : ahci_pwrite(ap, AHCI_PREG_IE, 0);
526 :
527 : /* Sec 10.1.2 - deinitialise port if it is already running */
528 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD);
529 0 : if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
530 0 : AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
531 0 : ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
532 : int r;
533 :
534 0 : r = ahci_port_stop(ap, 1);
535 0 : if (r) {
536 0 : printf("%s: unable to disable %s, ignoring port %d\n",
537 0 : DEVNAME(sc), r == 2 ? "CR" : "FR", port);
538 : rc = ENXIO;
539 0 : goto freeport;
540 : }
541 :
542 : /* Write DET to zero */
543 0 : ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
544 0 : }
545 :
546 : /* Allocate RFIS */
547 0 : ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
548 0 : if (ap->ap_dmamem_rfis == NULL)
549 : goto nomem;
550 :
551 : /* Setup RFIS base address */
552 0 : ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
553 0 : dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
554 0 : ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
555 0 : ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
556 :
557 : /* Enable FIS reception and activate port. */
558 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
559 0 : cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
560 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
561 :
562 : /* Check whether port activated. Skip it if not. */
563 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
564 0 : if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
565 : rc = ENXIO;
566 0 : goto freeport;
567 : }
568 :
569 : /* Allocate a CCB for each command slot */
570 0 : ap->ap_ccbs = mallocarray(sc->sc_ncmds, sizeof(struct ahci_ccb),
571 : M_DEVBUF, M_NOWAIT | M_ZERO);
572 0 : if (ap->ap_ccbs == NULL) {
573 0 : printf("%s: unable to allocate command list for port %d\n",
574 0 : DEVNAME(sc), port);
575 0 : goto freeport;
576 : }
577 :
578 : /* Command List Structures and Command Tables */
579 0 : ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
580 0 : sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
581 0 : ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
582 0 : sc->sc_ncmds * sizeof(struct ahci_cmd_table));
583 0 : if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
584 : nomem:
585 0 : printf("%s: unable to allocate DMA memory for port %d\n",
586 0 : DEVNAME(sc), port);
587 0 : goto freeport;
588 : }
589 :
590 : /* Setup command list base address */
591 0 : dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
592 0 : ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
593 0 : ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
594 :
595 : /* Split CCB allocation into CCBs and assign to command header/table */
596 0 : hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
597 0 : table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
598 0 : for (i = 0; i < sc->sc_ncmds; i++) {
599 0 : ccb = &ap->ap_ccbs[i];
600 :
601 0 : if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
602 : (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
603 0 : &ccb->ccb_dmamap) != 0) {
604 0 : printf("%s: unable to create dmamap for port %d "
605 0 : "ccb %d\n", DEVNAME(sc), port, i);
606 0 : goto freeport;
607 : }
608 :
609 0 : ccb->ccb_slot = i;
610 0 : ccb->ccb_port = ap;
611 0 : ccb->ccb_cmd_hdr = &hdr[i];
612 0 : ccb->ccb_cmd_table = &table[i];
613 0 : htolem64(&ccb->ccb_cmd_hdr->ctba,
614 : AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
615 : ccb->ccb_slot * sizeof(struct ahci_cmd_table));
616 :
617 0 : ccb->ccb_xa.fis =
618 0 : (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
619 0 : ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
620 0 : ccb->ccb_xa.tag = i;
621 :
622 0 : ccb->ccb_xa.state = ATA_S_COMPLETE;
623 0 : ahci_put_ccb(ccb);
624 : }
625 :
626 : /* grab a ccb for use during error recovery */
627 0 : ap->ap_ccb_err = &ap->ap_ccbs[sc->sc_ncmds - 1];
628 0 : TAILQ_REMOVE(&ap->ap_ccb_free, ap->ap_ccb_err, ccb_entry);
629 0 : ap->ap_ccb_err->ccb_xa.state = ATA_S_COMPLETE;
630 :
631 : /* Wait for ICC change to complete */
632 0 : ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
633 0 : rc = 0;
634 :
635 : freeport:
636 0 : if (rc != 0)
637 0 : ahci_port_free(sc, port);
638 : reterr:
639 0 : return (rc);
640 0 : }
641 :
642 : void
643 0 : ahci_port_detect(struct ahci_softc *sc, u_int port)
644 : {
645 : struct ahci_port *ap;
646 : const char *speed;
647 : int rc;
648 :
649 0 : ap = sc->sc_ports[port];
650 :
651 0 : rc = ahci_port_portreset_finish(ap, 1);
652 0 : switch (rc) {
653 : case ENODEV:
654 0 : switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
655 : case AHCI_PREG_SSTS_DET_DEV_NE:
656 0 : printf("%s: device not communicating on port %d\n",
657 0 : DEVNAME(sc), port);
658 0 : break;
659 : case AHCI_PREG_SSTS_DET_PHYOFFLINE:
660 0 : printf("%s: PHY offline on port %d\n", DEVNAME(sc),
661 : port);
662 0 : break;
663 : default:
664 : DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
665 : "on port %d\n", DEVNAME(sc), port);
666 : break;
667 : }
668 : goto freeport;
669 :
670 : case EBUSY:
671 0 : printf("%s: device on port %d didn't come ready, "
672 0 : "TFD: 0x%b\n", DEVNAME(sc), port,
673 0 : ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
674 :
675 : /* Try a soft reset to clear busy */
676 0 : rc = ahci_port_softreset(ap);
677 0 : if (rc) {
678 0 : printf("%s: unable to communicate "
679 : "with device on port %d\n", DEVNAME(sc), port);
680 0 : goto freeport;
681 : }
682 : break;
683 :
684 : default:
685 : break;
686 : }
687 :
688 : DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d; %d\n",
689 : DEVNAME(sc), port, rc);
690 :
691 : /* Read current link speed */
692 0 : switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) {
693 : case AHCI_PREG_SSTS_SPD_GEN1:
694 : speed = "1.5Gb/s";
695 0 : break;
696 : case AHCI_PREG_SSTS_SPD_GEN2:
697 : speed = "3.0Gb/s";
698 0 : break;
699 : case AHCI_PREG_SSTS_SPD_GEN3:
700 : speed = "6.0Gb/s";
701 0 : break;
702 : default:
703 : speed = NULL;
704 0 : break;
705 : }
706 0 : if (speed != NULL)
707 0 : printf("%s: port %d: %s\n", PORTNAME(ap), port, speed);
708 :
709 : /* Enable command transfers on port */
710 0 : if (ahci_port_start(ap, 0)) {
711 0 : printf("%s: failed to start command DMA on port %d, "
712 0 : "disabling\n", DEVNAME(sc), port);
713 : rc = ENXIO; /* couldn't start port */
714 0 : }
715 :
716 : /* Flush interrupts for port */
717 0 : ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
718 0 : ahci_write(sc, AHCI_REG_IS, 1 << port);
719 :
720 0 : ahci_enable_interrupts(ap);
721 : freeport:
722 0 : if (rc != 0)
723 0 : ahci_port_free(sc, port);
724 0 : }
725 :
726 : void
727 0 : ahci_port_free(struct ahci_softc *sc, u_int port)
728 : {
729 0 : struct ahci_port *ap = sc->sc_ports[port];
730 : struct ahci_ccb *ccb;
731 :
732 : /* Ensure port is disabled and its interrupts are flushed */
733 0 : if (ap->ap_sc) {
734 0 : ahci_pwrite(ap, AHCI_PREG_CMD, 0);
735 0 : ahci_pwrite(ap, AHCI_PREG_IE, 0);
736 0 : ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
737 0 : ahci_write(sc, AHCI_REG_IS, 1 << port);
738 0 : }
739 :
740 0 : if (ap->ap_ccb_err)
741 0 : ahci_put_ccb(ap->ap_ccb_err);
742 :
743 0 : if (ap->ap_ccbs) {
744 0 : while ((ccb = ahci_get_ccb(ap)) != NULL)
745 0 : bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
746 0 : free(ap->ap_ccbs, M_DEVBUF, sc->sc_ncmds * sizeof(*ccb));
747 0 : }
748 :
749 0 : if (ap->ap_dmamem_cmd_list)
750 0 : ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
751 0 : if (ap->ap_dmamem_rfis)
752 0 : ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
753 0 : if (ap->ap_dmamem_cmd_table)
754 0 : ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
755 0 : if (ap->ap_err_scratch)
756 0 : dma_free(ap->ap_err_scratch, DEV_BSIZE);
757 :
758 : /* bus_space(9) says we dont free the subregions handle */
759 :
760 0 : free(ap, M_DEVBUF, sizeof(*ap));
761 0 : sc->sc_ports[port] = NULL;
762 0 : }
763 :
764 : int
765 0 : ahci_port_init(struct ahci_softc *sc, u_int port)
766 : {
767 : struct ahci_port *ap;
768 : u_int64_t dva;
769 : u_int32_t cmd;
770 : int rc = ENOMEM;
771 :
772 0 : ap = sc->sc_ports[port];
773 : #ifdef AHCI_DEBUG
774 : snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
775 : DEVNAME(sc), port);
776 : #endif
777 :
778 : /* Disable port interrupts */
779 0 : ahci_pwrite(ap, AHCI_PREG_IE, 0);
780 :
781 : /* Sec 10.1.2 - deinitialise port if it is already running */
782 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD);
783 0 : if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
784 0 : AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
785 0 : ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
786 : int r;
787 :
788 0 : r = ahci_port_stop(ap, 1);
789 0 : if (r) {
790 0 : printf("%s: unable to disable %s, ignoring port %d\n",
791 0 : DEVNAME(sc), r == 2 ? "CR" : "FR", port);
792 : rc = ENXIO;
793 0 : goto reterr;
794 : }
795 :
796 : /* Write DET to zero */
797 0 : ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
798 0 : }
799 :
800 : /* Setup RFIS base address */
801 0 : ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
802 0 : dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
803 0 : ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
804 0 : ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
805 :
806 : /* Enable FIS reception and activate port. */
807 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
808 0 : cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
809 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
810 :
811 : /* Check whether port activated. Skip it if not. */
812 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
813 0 : if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
814 : rc = ENXIO;
815 0 : goto reterr;
816 : }
817 :
818 : /* Setup command list base address */
819 0 : dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
820 0 : ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
821 0 : ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
822 :
823 : /* Wait for ICC change to complete */
824 0 : ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
825 :
826 : /* Reset port */
827 0 : rc = ahci_port_portreset(ap, 1);
828 0 : switch (rc) {
829 : case ENODEV:
830 0 : switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
831 : case AHCI_PREG_SSTS_DET_DEV_NE:
832 0 : printf("%s: device not communicating on port %d\n",
833 0 : DEVNAME(sc), port);
834 0 : break;
835 : case AHCI_PREG_SSTS_DET_PHYOFFLINE:
836 0 : printf("%s: PHY offline on port %d\n", DEVNAME(sc),
837 : port);
838 0 : break;
839 : default:
840 : DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
841 : "on port %d\n", DEVNAME(sc), port);
842 : break;
843 : }
844 : goto reterr;
845 :
846 : case EBUSY:
847 0 : printf("%s: device on port %d didn't come ready, "
848 0 : "TFD: 0x%b\n", DEVNAME(sc), port,
849 0 : ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
850 :
851 : /* Try a soft reset to clear busy */
852 0 : rc = ahci_port_softreset(ap);
853 0 : if (rc) {
854 0 : printf("%s: unable to communicate "
855 : "with device on port %d\n", DEVNAME(sc), port);
856 0 : goto reterr;
857 : }
858 : break;
859 :
860 : default:
861 : break;
862 : }
863 : DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
864 : DEVNAME(sc), port);
865 :
866 0 : if (ap->ap_pmp_ports > 0) {
867 : int p;
868 :
869 0 : for (p = 0; p < ap->ap_pmp_ports; p++) {
870 : int sig;
871 :
872 : /* might need to do a portreset first here? */
873 :
874 : /* softreset the port */
875 0 : if (ahci_pmp_port_softreset(ap, p)) {
876 0 : printf("%s.%d: unable to probe PMP port due to"
877 0 : " softreset failure\n", PORTNAME(ap), p);
878 0 : continue;
879 : }
880 :
881 0 : sig = ahci_port_signature(ap);
882 0 : printf("%s.%d: port signature returned %d\n",
883 0 : PORTNAME(ap), p, sig);
884 0 : }
885 0 : }
886 :
887 : /* Enable command transfers on port */
888 0 : if (ahci_port_start(ap, 0)) {
889 0 : printf("%s: failed to start command DMA on port %d, "
890 0 : "disabling\n", DEVNAME(sc), port);
891 : rc = ENXIO; /* couldn't start port */
892 0 : }
893 :
894 : /* Flush interrupts for port */
895 0 : ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
896 0 : ahci_write(sc, AHCI_REG_IS, 1 << port);
897 :
898 0 : ahci_enable_interrupts(ap);
899 :
900 : reterr:
901 0 : return (rc);
902 0 : }
903 :
904 : int
905 0 : ahci_default_port_start(struct ahci_port *ap, int fre_only)
906 : {
907 : u_int32_t r;
908 :
909 : /* Turn on FRE (and ST) */
910 0 : r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
911 0 : r |= AHCI_PREG_CMD_FRE;
912 0 : if (!fre_only)
913 0 : r |= AHCI_PREG_CMD_ST;
914 0 : ahci_pwrite(ap, AHCI_PREG_CMD, r);
915 :
916 : #ifdef AHCI_COALESCE
917 : /* (Re-)enable coalescing on the port. */
918 : if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
919 : ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
920 : ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
921 : ap->ap_sc->sc_ccc_ports_cur);
922 : }
923 : #endif
924 :
925 : /* Wait for CR to come on */
926 0 : if (!fre_only &&
927 0 : ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1))
928 0 : return (1);
929 :
930 0 : return (0);
931 0 : }
932 :
933 : int
934 0 : ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
935 : {
936 : u_int32_t r;
937 :
938 : #ifdef AHCI_COALESCE
939 : /* Disable coalescing on the port while it is stopped. */
940 : if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
941 : ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
942 : ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
943 : ap->ap_sc->sc_ccc_ports_cur);
944 : }
945 : #endif
946 :
947 : /* Turn off ST (and FRE) */
948 0 : r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
949 0 : r &= ~AHCI_PREG_CMD_ST;
950 0 : if (stop_fis_rx)
951 0 : r &= ~AHCI_PREG_CMD_FRE;
952 0 : ahci_pwrite(ap, AHCI_PREG_CMD, r);
953 :
954 : /* Wait for CR to go off */
955 0 : if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR, 1))
956 0 : return (1);
957 :
958 : /* Wait for FR to go off */
959 0 : if (stop_fis_rx &&
960 0 : ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR, 1))
961 0 : return (2);
962 :
963 0 : return (0);
964 0 : }
965 :
966 : /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
967 : int
968 0 : ahci_port_clo(struct ahci_port *ap)
969 : {
970 0 : struct ahci_softc *sc = ap->ap_sc;
971 : u_int32_t cmd;
972 :
973 : /* Only attempt CLO if supported by controller */
974 0 : if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
975 0 : return (1);
976 :
977 : /* Issue CLO */
978 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
979 : #ifdef DIAGNOSTIC
980 0 : if (ISSET(cmd, AHCI_PREG_CMD_ST))
981 0 : printf("%s: CLO requested while port running\n", PORTNAME(ap));
982 : #endif
983 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
984 :
985 : /* Wait for completion */
986 0 : if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO, 1)) {
987 0 : printf("%s: CLO did not complete\n", PORTNAME(ap));
988 0 : return (1);
989 : }
990 :
991 0 : return (0);
992 0 : }
993 :
994 : /* AHCI soft reset, Section 10.4.1 */
995 : int
996 0 : ahci_port_softreset(struct ahci_port *ap)
997 : {
998 : struct ahci_ccb *ccb = NULL;
999 : struct ahci_cmd_hdr *cmd_slot;
1000 : u_int8_t *fis;
1001 : int s, rc = EIO, oldstate;
1002 : u_int32_t cmd;
1003 :
1004 : DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
1005 :
1006 0 : s = splbio();
1007 0 : oldstate = ap->ap_state;
1008 0 : ap->ap_state = AP_S_ERROR_RECOVERY;
1009 :
1010 : /* Save previous command register state */
1011 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1012 :
1013 : /* Idle port */
1014 0 : if (ahci_port_stop(ap, 0)) {
1015 0 : printf("%s: failed to stop port, cannot softreset\n",
1016 0 : PORTNAME(ap));
1017 0 : goto err;
1018 : }
1019 :
1020 : /* Request CLO if device appears hung */
1021 0 : if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
1022 : AHCI_PREG_TFD_STS_DRQ))
1023 0 : ahci_port_clo(ap);
1024 :
1025 : /* Clear port errors to permit TFD transfer */
1026 0 : ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1027 :
1028 : /* Restart port */
1029 0 : if (ahci_port_start(ap, 0)) {
1030 0 : printf("%s: failed to start port, cannot softreset\n",
1031 0 : PORTNAME(ap));
1032 0 : goto err;
1033 : }
1034 :
1035 : /* Check whether CLO worked */
1036 0 : if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1037 : AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1038 0 : printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
1039 0 : ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
1040 : ? "failed" : "unsupported");
1041 : rc = EBUSY;
1042 0 : goto err;
1043 : }
1044 :
1045 : /* Prep first D2H command with SRST feature & clear busy/reset flags */
1046 0 : ccb = ahci_get_err_ccb(ap);
1047 0 : cmd_slot = ccb->ccb_cmd_hdr;
1048 0 : memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1049 :
1050 0 : fis = ccb->ccb_cmd_table->cfis;
1051 0 : fis[0] = ATA_FIS_TYPE_H2D;
1052 0 : fis[15] = ATA_FIS_CONTROL_SRST;
1053 :
1054 0 : cmd_slot->prdtl = 0;
1055 0 : htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1056 : AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1057 : AHCI_CMD_LIST_FLAG_W);
1058 :
1059 0 : ccb->ccb_xa.state = ATA_S_PENDING;
1060 0 : if (ahci_poll(ccb, 1000, NULL) != 0)
1061 : goto err;
1062 :
1063 : /* Prep second D2H command to read status and complete reset sequence */
1064 0 : fis[0] = ATA_FIS_TYPE_H2D;
1065 0 : fis[15] = 0;
1066 :
1067 0 : cmd_slot->prdtl = 0;
1068 0 : htolem16(&cmd_slot->flags, 5 | AHCI_CMD_LIST_FLAG_W);
1069 :
1070 0 : ccb->ccb_xa.state = ATA_S_PENDING;
1071 0 : if (ahci_poll(ccb, 1000, NULL) != 0)
1072 : goto err;
1073 :
1074 0 : if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1075 : AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 1)) {
1076 0 : printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1077 0 : PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
1078 : AHCI_PFMT_TFD_STS);
1079 : rc = EBUSY;
1080 0 : goto err;
1081 : }
1082 :
1083 0 : rc = 0;
1084 : err:
1085 0 : if (ccb != NULL) {
1086 : /* Abort our command, if it failed, by stopping command DMA. */
1087 0 : if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
1088 0 : printf("%s: stopping the port, softreset slot %d was "
1089 0 : "still active.\n", PORTNAME(ap), ccb->ccb_slot);
1090 0 : ahci_port_stop(ap, 0);
1091 0 : }
1092 0 : ccb->ccb_xa.state = ATA_S_ERROR;
1093 0 : ahci_put_err_ccb(ccb);
1094 0 : }
1095 :
1096 : /* Restore saved CMD register state */
1097 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1098 0 : ap->ap_state = oldstate;
1099 :
1100 0 : splx(s);
1101 :
1102 0 : return (rc);
1103 : }
1104 :
1105 : int
1106 0 : ahci_pmp_port_softreset(struct ahci_port *ap, int pmp_port)
1107 : {
1108 : struct ahci_ccb *ccb = NULL;
1109 0 : u_int32_t data;
1110 : int count;
1111 : int rc;
1112 : int s;
1113 : struct ahci_cmd_hdr *cmd_slot;
1114 : u_int8_t *fis;
1115 :
1116 0 : s = splbio();
1117 : /* ignore spurious IFS errors while resetting */
1118 : DPRINTF(AHCI_D_VERBOSE, "%s: now ignoring IFS\n", PORTNAME(ap));
1119 0 : ap->ap_pmp_ignore_ifs = 1;
1120 :
1121 : count = 2;
1122 : rc = 0;
1123 0 : do {
1124 0 : if (ccb != NULL) {
1125 0 : ahci_put_pmp_ccb(ccb);
1126 : ccb = NULL;
1127 0 : }
1128 :
1129 0 : if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1130 0 : printf("%s.%d: unable to clear PHY status\n",
1131 0 : PORTNAME(ap), pmp_port);
1132 0 : }
1133 0 : ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1134 : /* maybe don't do this on the first loop: */
1135 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1136 0 : ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1137 :
1138 : /* send first softreset FIS */
1139 0 : ccb = ahci_get_pmp_ccb(ap);
1140 0 : cmd_slot = ccb->ccb_cmd_hdr;
1141 0 : memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1142 :
1143 0 : fis = ccb->ccb_cmd_table->cfis;
1144 0 : fis[0] = ATA_FIS_TYPE_H2D;
1145 0 : fis[1] = pmp_port;
1146 0 : fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1147 :
1148 0 : cmd_slot->prdtl = 0;
1149 0 : htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1150 : AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1151 : (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
1152 :
1153 0 : ccb->ccb_xa.state = ATA_S_PENDING;
1154 :
1155 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP softreset cmd\n",
1156 : PORTNAME(ap), pmp_port);
1157 0 : if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1158 0 : printf("%s.%d: PMP port softreset cmd failed\n",
1159 0 : PORTNAME(ap), pmp_port);
1160 : rc = EBUSY;
1161 : if (count > 0) {
1162 : /* probably delay a while to allow
1163 : * it to settle down?
1164 : */
1165 : }
1166 0 : continue;
1167 : }
1168 :
1169 : /* send signature FIS */
1170 0 : memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
1171 0 : fis[0] = ATA_FIS_TYPE_H2D;
1172 0 : fis[1] = pmp_port;
1173 0 : fis[15] = ATA_FIS_CONTROL_4BIT;
1174 :
1175 0 : cmd_slot->prdtl = 0;
1176 0 : htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1177 : (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
1178 :
1179 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: sending PMP probe status cmd\n",
1180 : PORTNAME(ap), pmp_port);
1181 0 : ccb->ccb_xa.state = ATA_S_PENDING;
1182 0 : if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1183 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP probe status cmd "
1184 : "failed\n", PORTNAME(ap), pmp_port);
1185 : rc = EBUSY;
1186 : if (count > 0) {
1187 : /* sleep a while? */
1188 : }
1189 0 : continue;
1190 : }
1191 :
1192 0 : fis[15] = 0;
1193 0 : break;
1194 0 : } while (count--);
1195 :
1196 0 : if (ccb != NULL) {
1197 0 : ahci_put_pmp_ccb(ccb);
1198 : ccb = NULL;
1199 0 : }
1200 :
1201 : /* clean up a bit */
1202 0 : ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1203 0 : ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1204 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1205 0 : ap->ap_pmp_ignore_ifs = 0;
1206 : DPRINTF(AHCI_D_VERBOSE, "%s: no longer ignoring IFS\n", PORTNAME(ap));
1207 0 : splx(s);
1208 :
1209 0 : return (rc);
1210 0 : }
1211 :
1212 : int
1213 0 : ahci_pmp_port_probe(struct ahci_port *ap, int pmp_port)
1214 : {
1215 : int sig;
1216 :
1217 0 : ap->ap_state = AP_S_PMP_PORT_PROBE;
1218 :
1219 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: probing pmp port\n", PORTNAME(ap),
1220 : pmp_port);
1221 0 : if (ahci_pmp_port_portreset(ap, pmp_port)) {
1222 0 : printf("%s.%d: unable to probe PMP port; portreset failed\n",
1223 0 : PORTNAME(ap), pmp_port);
1224 0 : ap->ap_state = AP_S_NORMAL;
1225 0 : return (ATA_PORT_T_NONE);
1226 : }
1227 :
1228 0 : if (ahci_pmp_port_softreset(ap, pmp_port)) {
1229 0 : printf("%s.%d: unable to probe PMP port due to softreset "
1230 0 : "failure\n", PORTNAME(ap), pmp_port);
1231 0 : ap->ap_state = AP_S_NORMAL;
1232 0 : return (ATA_PORT_T_NONE);
1233 : }
1234 :
1235 0 : sig = ahci_port_signature(ap);
1236 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: port signature returned %d\n",
1237 : PORTNAME(ap), pmp_port, sig);
1238 0 : ap->ap_state = AP_S_NORMAL;
1239 0 : return (sig);
1240 0 : }
1241 :
1242 :
1243 : void
1244 0 : ahci_flush_tfd(struct ahci_port *ap)
1245 : {
1246 : u_int32_t r;
1247 :
1248 0 : r = ahci_pread(ap, AHCI_PREG_SERR);
1249 0 : if (r & AHCI_PREG_SERR_DIAG_X)
1250 0 : ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X);
1251 0 : }
1252 :
1253 : u_int32_t
1254 0 : ahci_active_mask(struct ahci_port *ap)
1255 : {
1256 : u_int32_t mask;
1257 :
1258 0 : mask = ahci_pread(ap, AHCI_PREG_CI);
1259 0 : if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ)
1260 0 : mask |= ahci_pread(ap, AHCI_PREG_SACT);
1261 0 : return mask;
1262 : }
1263 :
1264 : void
1265 0 : ahci_pmp_probe_timeout(void *cookie)
1266 : {
1267 0 : struct ahci_ccb *ccb = cookie;
1268 0 : struct ahci_port *ap = ccb->ccb_port;
1269 : u_int32_t mask;
1270 :
1271 : DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe cmd timed out\n", PORTNAME(ap));
1272 0 : switch (ccb->ccb_xa.state) {
1273 : case ATA_S_PENDING:
1274 0 : TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
1275 0 : ccb->ccb_xa.state = ATA_S_TIMEOUT;
1276 0 : break;
1277 :
1278 : case ATA_S_ONCHIP:
1279 : case ATA_S_ERROR: /* currently mostly here for the ATI SBx00 quirk */
1280 : /* clear the command on-chip */
1281 0 : KASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&
1282 : ap->ap_sactive == 0);
1283 0 : ahci_port_stop(ap, 0);
1284 0 : ahci_port_start(ap, 0);
1285 :
1286 0 : if (ahci_active_mask(ap) != 0) {
1287 0 : ahci_port_stop(ap, 0);
1288 0 : ahci_port_start(ap, 0);
1289 0 : mask = ahci_active_mask(ap);
1290 0 : if (mask != 0) {
1291 0 : printf("%s: ahci_pmp_probe_timeout: failed to "
1292 0 : "clear active cmds: %08x\n", PORTNAME(ap),
1293 : mask);
1294 0 : }
1295 : }
1296 :
1297 0 : ccb->ccb_xa.state = ATA_S_TIMEOUT;
1298 0 : ap->ap_active &= ~(1 << ccb->ccb_slot);
1299 0 : KASSERT(ap->ap_active_cnt > 0);
1300 0 : --ap->ap_active_cnt;
1301 : DPRINTF(AHCI_D_VERBOSE, "%s: timed out %d, active %x, count %d\n",
1302 : PORTNAME(ap), ccb->ccb_slot, ap->ap_active, ap->ap_active_cnt);
1303 0 : break;
1304 :
1305 : default:
1306 0 : panic("%s: ahci_pmp_probe_timeout: ccb in bad state %d",
1307 0 : PORTNAME(ap), ccb->ccb_xa.state);
1308 : }
1309 0 : }
1310 :
1311 : int
1312 0 : ahci_port_signature(struct ahci_port *ap)
1313 : {
1314 : u_int32_t sig;
1315 :
1316 0 : sig = ahci_pread(ap, AHCI_PREG_SIG);
1317 0 : if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
1318 0 : return (ATA_PORT_T_ATAPI);
1319 0 : else if ((sig & 0xffff0000) == (SATA_SIGNATURE_PORT_MULTIPLIER &
1320 : 0xffff0000))
1321 0 : return (ATA_PORT_T_PM);
1322 : else
1323 0 : return (ATA_PORT_T_DISK);
1324 0 : }
1325 :
1326 : int
1327 0 : ahci_pmp_port_portreset(struct ahci_port *ap, int pmp_port)
1328 : {
1329 0 : u_int32_t cmd, data;
1330 : int loop;
1331 : int rc = 1;
1332 : int s;
1333 :
1334 0 : s = splbio();
1335 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: PMP port reset\n", PORTNAME(ap),
1336 : pmp_port);
1337 :
1338 : /* Save previous command register state */
1339 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1340 :
1341 : /* turn off power management and disable the PHY */
1342 0 : data = AHCI_PREG_SCTL_IPM_DISABLED;
1343 : /* maybe add AHCI_PREG_SCTL_DET_DISABLE */
1344 0 : if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1))
1345 : goto err;
1346 0 : if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1347 : goto err;
1348 0 : delay(10000);
1349 :
1350 : /* start COMRESET */
1351 0 : data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1352 0 : if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1353 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: forcing GEN1\n", PORTNAME(ap),
1354 : pmp_port);
1355 0 : data |= AHCI_PREG_SCTL_SPD_GEN1;
1356 0 : } else
1357 0 : data |= AHCI_PREG_SCTL_SPD_ANY;
1358 :
1359 0 : if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1360 : goto err;
1361 :
1362 : /* give it a while to settle down */
1363 0 : delay(100000);
1364 :
1365 0 : if (ahci_pmp_phy_status(ap, pmp_port, &data)) {
1366 0 : printf("%s.%d: cannot clear PHY status\n", PORTNAME(ap),
1367 : pmp_port);
1368 0 : }
1369 :
1370 : /* start trying to negotiate */
1371 0 : ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1372 0 : data = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_NONE;
1373 0 : if (ahci_pmp_write(ap, pmp_port, SATA_PMREG_SCTL, data))
1374 : goto err;
1375 :
1376 : /* give it a while to detect */
1377 0 : for (loop = 3; loop; --loop) {
1378 0 : if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
1379 : goto err;
1380 0 : if (data & AHCI_PREG_SSTS_DET)
1381 : break;
1382 0 : delay(100000);
1383 : }
1384 0 : if (loop == 0) {
1385 0 : printf("%s.%d: port is unplugged\n", PORTNAME(ap), pmp_port);
1386 0 : goto err;
1387 : }
1388 :
1389 : /* give it even longer to fully negotiate */
1390 0 : for (loop = 30; loop; --loop) {
1391 0 : if (ahci_pmp_read(ap, pmp_port, SATA_PMREG_SSTS, &data))
1392 : goto err;
1393 0 : if ((data & AHCI_PREG_SSTS_DET) == AHCI_PREG_SSTS_DET_DEV)
1394 : break;
1395 0 : delay(100000);
1396 : }
1397 :
1398 0 : if (loop == 0) {
1399 0 : printf("%s.%d: device is not negotiating\n", PORTNAME(ap),
1400 : pmp_port);
1401 0 : goto err;
1402 : }
1403 :
1404 : /* device detected */
1405 : DPRINTF(AHCI_D_VERBOSE, "%s.%d: device detected\n", PORTNAME(ap),
1406 : pmp_port);
1407 :
1408 : /* clean up a bit */
1409 0 : delay(100000);
1410 0 : ahci_pmp_write(ap, pmp_port, SATA_PMREG_SERR, -1);
1411 0 : ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1412 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
1413 :
1414 0 : rc = 0;
1415 : err:
1416 : /* Restore preserved port state */
1417 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1418 0 : splx(s);
1419 0 : return (rc);
1420 0 : }
1421 :
1422 : /* AHCI port reset, Section 10.4.2 */
1423 :
1424 : void
1425 0 : ahci_port_comreset(struct ahci_port *ap)
1426 : {
1427 : u_int32_t r;
1428 :
1429 : r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1430 0 : if ((ap->ap_sc->sc_dev.dv_cfdata->cf_flags & 0x01) != 0) {
1431 : DPRINTF(AHCI_D_VERBOSE, "%s: forcing GEN1\n", PORTNAME(ap));
1432 : r |= AHCI_PREG_SCTL_SPD_GEN1;
1433 0 : } else
1434 : r |= AHCI_PREG_SCTL_SPD_ANY;
1435 0 : ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1436 0 : delay(10000); /* wait at least 1ms for COMRESET to be sent */
1437 0 : r &= ~AHCI_PREG_SCTL_DET_INIT;
1438 : r |= AHCI_PREG_SCTL_DET_NONE;
1439 0 : ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1440 0 : delay(10000);
1441 0 : }
1442 :
1443 : void
1444 0 : ahci_port_portreset_start(struct ahci_port *ap)
1445 : {
1446 : int s;
1447 :
1448 0 : s = splbio();
1449 : DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1450 :
1451 : /* Save previous command register state */
1452 0 : ap->ap_saved_cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1453 :
1454 : /* Clear ST, ignoring failure */
1455 0 : ahci_port_stop(ap, 0);
1456 :
1457 : /* Perform device detection */
1458 0 : ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
1459 0 : delay(10000);
1460 0 : ahci_port_comreset(ap);
1461 0 : splx(s);
1462 0 : }
1463 :
1464 : int
1465 0 : ahci_port_portreset_poll(struct ahci_port *ap)
1466 : {
1467 0 : if ((ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) !=
1468 : AHCI_PREG_SSTS_DET_DEV)
1469 0 : return (EAGAIN);
1470 0 : return (0);
1471 0 : }
1472 :
1473 : void
1474 0 : ahci_port_portreset_wait(struct ahci_port *ap)
1475 : {
1476 : int i;
1477 :
1478 0 : for (i = 0; i < 1000; i++) {
1479 0 : if (ahci_port_portreset_poll(ap) == 0)
1480 : break;
1481 0 : delay(1000);
1482 : }
1483 0 : }
1484 :
1485 : int
1486 0 : ahci_port_portreset_finish(struct ahci_port *ap, int pmp)
1487 : {
1488 : int rc, s, retries = 0;
1489 :
1490 0 : s = splbio();
1491 : retry:
1492 0 : if (ahci_port_portreset_poll(ap)) {
1493 : rc = ENODEV;
1494 0 : if (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
1495 : /* this may be a port multiplier with no device
1496 : * on port 0, so still do the pmp check if requested.
1497 : */
1498 : } else {
1499 : goto err;
1500 : }
1501 : } else {
1502 : /* Clear SERR (incl X bit), so TFD can update */
1503 0 : ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1504 :
1505 : /* Wait for device to become ready */
1506 0 : if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1507 : AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR, 3)) {
1508 : /* even if the device doesn't wake up, check if there's
1509 : * a port multiplier there
1510 : */
1511 0 : if (retries == 0) {
1512 : retries = 1;
1513 0 : ahci_port_comreset(ap);
1514 0 : ahci_port_portreset_wait(ap);
1515 0 : goto retry;
1516 : }
1517 : rc = EBUSY;
1518 0 : } else {
1519 : rc = 0;
1520 : }
1521 : }
1522 :
1523 0 : if (pmp != 0) {
1524 0 : if (ahci_port_detect_pmp(ap)) {
1525 : /* reset again without pmp support */
1526 : pmp = 0;
1527 : retries = 0;
1528 0 : ahci_port_comreset(ap);
1529 0 : ahci_port_portreset_wait(ap);
1530 0 : goto retry;
1531 : }
1532 : }
1533 :
1534 : err:
1535 : /* Restore preserved port state */
1536 0 : ahci_pwrite(ap, AHCI_PREG_CMD, ap->ap_saved_cmd);
1537 0 : ap->ap_saved_cmd = 0;
1538 0 : splx(s);
1539 :
1540 0 : return (rc);
1541 : }
1542 :
1543 : int
1544 0 : ahci_port_portreset(struct ahci_port *ap, int pmp)
1545 : {
1546 0 : ahci_port_portreset_start(ap);
1547 0 : ahci_port_portreset_wait(ap);
1548 0 : return (ahci_port_portreset_finish(ap, pmp));
1549 : }
1550 :
1551 : int
1552 0 : ahci_port_detect_pmp(struct ahci_port *ap)
1553 : {
1554 : int count, pmp_rc, rc;
1555 : u_int32_t r, cmd;
1556 : struct ahci_cmd_hdr *cmd_slot;
1557 : struct ahci_ccb *ccb = NULL;
1558 : u_int8_t *fis = NULL;
1559 :
1560 0 : if ((ap->ap_sc->sc_flags & AHCI_F_NO_PMP) ||
1561 0 : !ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SPM)) {
1562 0 : return 0;
1563 : }
1564 :
1565 : rc = 0;
1566 : pmp_rc = 0;
1567 : count = 2;
1568 0 : do {
1569 : DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe %d\n", PORTNAME(ap),
1570 : count);
1571 0 : if (ccb != NULL) {
1572 0 : ahci_put_pmp_ccb(ccb);
1573 : ccb = NULL;
1574 0 : }
1575 0 : ahci_port_stop(ap, 0);
1576 0 : ap->ap_state = AP_S_PMP_PROBE;
1577 :
1578 : /* set PMA in cmd reg */
1579 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1580 0 : if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
1581 0 : cmd |= AHCI_PREG_CMD_PMA;
1582 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1583 0 : }
1584 :
1585 : /* Flush errors and request CLO unconditionally,
1586 : * then start the port
1587 : */
1588 0 : r = ahci_pread(ap, AHCI_PREG_SERR);
1589 0 : if (r & AHCI_PREG_SERR_DIAG_X)
1590 0 : ahci_pwrite(ap, AHCI_PREG_SERR,
1591 : AHCI_PREG_SERR_DIAG_X);
1592 :
1593 : /* Request CLO */
1594 0 : ahci_port_clo(ap);
1595 :
1596 : /* Clear port errors to permit TFD transfer */
1597 0 : r = ahci_pread(ap, AHCI_PREG_SERR);
1598 0 : ahci_pwrite(ap, AHCI_PREG_SERR, r);
1599 :
1600 : /* Restart port */
1601 0 : if (ahci_port_start(ap, 0)) {
1602 : rc = EBUSY;
1603 0 : printf("%s: failed to start port, cannot probe PMP\n",
1604 0 : PORTNAME(ap));
1605 0 : break;
1606 : }
1607 :
1608 : /* Check whether CLO worked */
1609 0 : if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1610 : AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1611 : u_int32_t cap;
1612 :
1613 0 : cap = ahci_read(ap->ap_sc, AHCI_REG_CAP);
1614 0 : printf("%s: CLO %s, need port reset\n",
1615 0 : PORTNAME(ap),
1616 0 : ISSET(cap, AHCI_REG_CAP_SCLO)
1617 : ? "failed" : "unsupported");
1618 : pmp_rc = EBUSY;
1619 : break;
1620 : }
1621 :
1622 : /* Prep first command with SRST feature &
1623 : * clear busy/reset flags
1624 : */
1625 0 : ccb = ahci_get_pmp_ccb(ap);
1626 0 : cmd_slot = ccb->ccb_cmd_hdr;
1627 0 : memset(ccb->ccb_cmd_table, 0,
1628 : sizeof(struct ahci_cmd_table));
1629 :
1630 0 : fis = ccb->ccb_cmd_table->cfis;
1631 0 : fis[0] = ATA_FIS_TYPE_H2D;
1632 0 : fis[1] = SATA_PMP_CONTROL_PORT;
1633 0 : fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1634 :
1635 0 : cmd_slot->prdtl = 0;
1636 0 : htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1637 : AHCI_CMD_LIST_FLAG_C | AHCI_CMD_LIST_FLAG_R |
1638 : AHCI_CMD_LIST_FLAG_PMP);
1639 :
1640 : DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP reset cmd\n",
1641 : PORTNAME(ap));
1642 0 : ccb->ccb_xa.state = ATA_S_PENDING;
1643 0 : if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
1644 : DPRINTF(AHCI_D_VERBOSE, "%s: PMP reset cmd failed\n",
1645 : PORTNAME(ap));
1646 : pmp_rc = EBUSY;
1647 0 : continue;
1648 : }
1649 :
1650 0 : if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1651 : AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ, 1)) {
1652 0 : printf("%s: port busy after first PMP probe FIS\n",
1653 0 : PORTNAME(ap));
1654 0 : }
1655 :
1656 : /* clear errors in case the device
1657 : * didn't reset cleanly
1658 : */
1659 0 : ahci_flush_tfd(ap);
1660 0 : r = ahci_pread(ap, AHCI_PREG_SERR);
1661 0 : ahci_pwrite(ap, AHCI_PREG_SERR, r);
1662 :
1663 : /* Prep second command to read status and
1664 : * complete reset sequence
1665 : */
1666 0 : memset(ccb->ccb_cmd_table, 0,
1667 : sizeof(struct ahci_cmd_table));
1668 0 : fis[0] = ATA_FIS_TYPE_H2D;
1669 0 : fis[1] = SATA_PMP_CONTROL_PORT;
1670 0 : fis[15] = ATA_FIS_CONTROL_4BIT;
1671 :
1672 0 : cmd_slot->prdtl = 0;
1673 0 : htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
1674 : AHCI_CMD_LIST_FLAG_PMP);
1675 :
1676 : DPRINTF(AHCI_D_VERBOSE, "%s: sending PMP probe status cmd\n",
1677 : PORTNAME(ap));
1678 0 : ccb->ccb_xa.state = ATA_S_PENDING;
1679 0 : if (ahci_poll(ccb, 5000, ahci_pmp_probe_timeout) != 0) {
1680 : DPRINTF(AHCI_D_VERBOSE, "%s: PMP probe status "
1681 : "cmd failed\n", PORTNAME(ap));
1682 : pmp_rc = EBUSY;
1683 0 : continue;
1684 : }
1685 :
1686 : /* apparently we need to retry at least once
1687 : * to get the right signature
1688 : */
1689 0 : fis[15] = 0;
1690 : pmp_rc = 0;
1691 0 : } while (--count);
1692 :
1693 0 : if (ccb != NULL) {
1694 0 : ahci_put_pmp_ccb(ccb);
1695 : ccb = NULL;
1696 0 : }
1697 :
1698 0 : if (ap->ap_state == AP_S_PMP_PROBE) {
1699 0 : ap->ap_state = AP_S_NORMAL;
1700 0 : }
1701 :
1702 0 : if (pmp_rc == 0) {
1703 0 : if (ahci_port_signature(ap) != ATA_PORT_T_PM) {
1704 : DPRINTF(AHCI_D_VERBOSE, "%s: device is not a PMP\n",
1705 : PORTNAME(ap));
1706 : pmp_rc = EBUSY;
1707 0 : } else {
1708 : DPRINTF(AHCI_D_VERBOSE, "%s: PMP found\n",
1709 : PORTNAME(ap));
1710 : }
1711 : }
1712 :
1713 0 : if (pmp_rc == 0) {
1714 0 : if (ahci_pmp_identify(ap, &ap->ap_pmp_ports)) {
1715 : pmp_rc = EBUSY;
1716 0 : } else {
1717 : rc = 0;
1718 : }
1719 : }
1720 :
1721 : /* if PMP detection failed, so turn off the PMA bit and
1722 : * reset the port again
1723 : */
1724 0 : if (pmp_rc != 0) {
1725 : DPRINTF(AHCI_D_VERBOSE, "%s: no PMP found, resetting "
1726 : "the port\n", PORTNAME(ap));
1727 0 : ahci_port_stop(ap, 0);
1728 0 : ahci_port_clo(ap);
1729 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1730 0 : cmd &= ~AHCI_PREG_CMD_PMA;
1731 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1732 :
1733 0 : ahci_pwrite(ap, AHCI_PREG_IE, 0);
1734 0 : ahci_port_stop(ap, 0);
1735 0 : if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
1736 0 : ahci_pwrite(ap, AHCI_PREG_SNTF, -1);
1737 0 : ahci_flush_tfd(ap);
1738 0 : ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1739 :
1740 0 : ahci_pwrite(ap, AHCI_PREG_IS, -1);
1741 :
1742 0 : ahci_enable_interrupts(ap);
1743 :
1744 : rc = pmp_rc;
1745 0 : }
1746 :
1747 0 : return (rc);
1748 0 : }
1749 :
1750 : void
1751 0 : ahci_load_prdt_seg(struct ahci_prdt *prd, u_int64_t addr, u_int32_t len,
1752 : u_int32_t flags)
1753 : {
1754 0 : flags |= len - 1;
1755 :
1756 0 : htolem64(&prd->dba, addr);
1757 0 : htolem32(&prd->flags, flags);
1758 0 : }
1759 :
1760 : int
1761 0 : ahci_load_prdt(struct ahci_ccb *ccb)
1762 : {
1763 0 : struct ahci_port *ap = ccb->ccb_port;
1764 0 : struct ahci_softc *sc = ap->ap_sc;
1765 0 : struct ata_xfer *xa = &ccb->ccb_xa;
1766 0 : struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
1767 0 : bus_dmamap_t dmap = ccb->ccb_dmamap;
1768 0 : struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1769 : int i, error;
1770 :
1771 0 : if (xa->datalen == 0) {
1772 0 : ccb->ccb_cmd_hdr->prdtl = 0;
1773 0 : return (0);
1774 : }
1775 :
1776 0 : error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
1777 : (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1778 0 : if (error != 0) {
1779 0 : printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
1780 0 : return (1);
1781 : }
1782 :
1783 0 : for (i = 0; i < dmap->dm_nsegs - 1; i++) {
1784 0 : ahci_load_prdt_seg(&prdt[i], dmap->dm_segs[i].ds_addr,
1785 : dmap->dm_segs[i].ds_len, 0);
1786 : }
1787 :
1788 0 : ahci_load_prdt_seg(&prdt[i],
1789 : dmap->dm_segs[i].ds_addr, dmap->dm_segs[i].ds_len,
1790 0 : ISSET(xa->flags, ATA_F_PIO) ? AHCI_PRDT_FLAG_INTR : 0);
1791 :
1792 0 : htolem16(&cmd_slot->prdtl, dmap->dm_nsegs);
1793 :
1794 0 : bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1795 : (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
1796 : BUS_DMASYNC_PREWRITE);
1797 :
1798 0 : return (0);
1799 0 : }
1800 :
1801 : void
1802 0 : ahci_unload_prdt(struct ahci_ccb *ccb)
1803 : {
1804 0 : struct ahci_port *ap = ccb->ccb_port;
1805 0 : struct ahci_softc *sc = ap->ap_sc;
1806 0 : struct ata_xfer *xa = &ccb->ccb_xa;
1807 0 : bus_dmamap_t dmap = ccb->ccb_dmamap;
1808 :
1809 0 : if (xa->datalen != 0) {
1810 0 : bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1811 : (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
1812 : BUS_DMASYNC_POSTWRITE);
1813 :
1814 0 : bus_dmamap_unload(sc->sc_dmat, dmap);
1815 :
1816 0 : if (ccb->ccb_xa.flags & ATA_F_NCQ)
1817 0 : xa->resid = 0;
1818 : else
1819 0 : xa->resid = xa->datalen -
1820 0 : lemtoh32(&ccb->ccb_cmd_hdr->prdbc);
1821 : }
1822 0 : }
1823 :
1824 : int
1825 0 : ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
1826 : {
1827 0 : struct ahci_port *ap = ccb->ccb_port;
1828 : int s;
1829 :
1830 0 : s = splbio();
1831 0 : ahci_start(ccb);
1832 0 : do {
1833 0 : if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
1834 : 1 << ccb->ccb_slot)) {
1835 0 : splx(s);
1836 0 : return (0);
1837 : }
1838 0 : if (ccb->ccb_xa.state == ATA_S_ERROR) {
1839 : DPRINTF(AHCI_D_VERBOSE, "%s: ccb in slot %d errored\n",
1840 : PORTNAME(ap), ccb->ccb_slot);
1841 : /* pretend it timed out? */
1842 0 : if (timeout_fn != NULL) {
1843 0 : timeout_fn(ccb);
1844 0 : }
1845 0 : splx(s);
1846 0 : return (1);
1847 : }
1848 :
1849 0 : delay(1000);
1850 0 : } while (--timeout > 0);
1851 :
1852 : /* Run timeout while at splbio, otherwise ahci_intr could interfere. */
1853 0 : if (timeout_fn != NULL)
1854 0 : timeout_fn(ccb);
1855 :
1856 0 : splx(s);
1857 :
1858 0 : return (1);
1859 0 : }
1860 :
1861 : void
1862 0 : ahci_start(struct ahci_ccb *ccb)
1863 : {
1864 0 : struct ahci_port *ap = ccb->ccb_port;
1865 0 : struct ahci_softc *sc = ap->ap_sc;
1866 :
1867 : /* Zero transferred byte count before transfer */
1868 0 : ccb->ccb_cmd_hdr->prdbc = 0;
1869 :
1870 : /* Sync command list entry and corresponding command table entry */
1871 0 : bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
1872 : ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
1873 : sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
1874 0 : bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
1875 : ccb->ccb_slot * sizeof(struct ahci_cmd_table),
1876 : sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
1877 :
1878 : /* Prepare RFIS area for write by controller */
1879 0 : bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
1880 : sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
1881 :
1882 0 : if (ccb->ccb_xa.flags & ATA_F_NCQ) {
1883 : /* Issue NCQ commands only when there are no outstanding
1884 : * standard commands. */
1885 0 : if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending) ||
1886 0 : (ap->ap_sactive != 0 &&
1887 0 : ap->ap_pmp_ncq_port != ccb->ccb_xa.pmp_port)) {
1888 0 : TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1889 0 : } else {
1890 0 : KASSERT(ap->ap_active_cnt == 0);
1891 0 : ap->ap_sactive |= (1 << ccb->ccb_slot);
1892 0 : ccb->ccb_xa.state = ATA_S_ONCHIP;
1893 0 : ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
1894 0 : ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1895 0 : ap->ap_pmp_ncq_port = ccb->ccb_xa.pmp_port;
1896 : }
1897 : } else {
1898 : /* Wait for all NCQ commands to finish before issuing standard
1899 : * command. */
1900 0 : if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
1901 0 : TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1902 0 : else if (ap->ap_active_cnt < 2) {
1903 0 : ap->ap_active |= 1 << ccb->ccb_slot;
1904 0 : ccb->ccb_xa.state = ATA_S_ONCHIP;
1905 0 : ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1906 0 : ap->ap_active_cnt++;
1907 0 : }
1908 : }
1909 0 : }
1910 :
1911 : void
1912 0 : ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1913 : {
1914 : struct ahci_ccb *nextccb;
1915 : u_int32_t sact_change = 0;
1916 :
1917 0 : KASSERT(ap->ap_active_cnt == 0);
1918 :
1919 0 : nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1920 0 : if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
1921 0 : return;
1922 :
1923 : /* Start all the NCQ commands at the head of the pending list.
1924 : * If a port multiplier is attached to the port, we can only
1925 : * issue commands for one of its ports at a time.
1926 : */
1927 0 : if (ap->ap_sactive != 0 &&
1928 0 : ap->ap_pmp_ncq_port != nextccb->ccb_xa.pmp_port) {
1929 0 : return;
1930 : }
1931 :
1932 0 : ap->ap_pmp_ncq_port = nextccb->ccb_xa.pmp_port;
1933 0 : do {
1934 0 : TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1935 0 : sact_change |= 1 << nextccb->ccb_slot;
1936 0 : nextccb->ccb_xa.state = ATA_S_ONCHIP;
1937 0 : nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1938 0 : } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) &&
1939 0 : (nextccb->ccb_xa.pmp_port == ap->ap_pmp_ncq_port));
1940 :
1941 0 : ap->ap_sactive |= sact_change;
1942 0 : ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
1943 0 : ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
1944 0 : }
1945 :
1946 : void
1947 0 : ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
1948 : {
1949 : struct ahci_ccb *nextccb;
1950 :
1951 0 : nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1952 0 : if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
1953 0 : if (last_was_ncq) {
1954 0 : KASSERT(nextccb->ccb_xa.pmp_port !=
1955 : ap->ap_pmp_ncq_port);
1956 : /* otherwise it should have been started already */
1957 : } else {
1958 0 : ap->ap_active_cnt--;
1959 : }
1960 :
1961 : /* Issue NCQ commands only when there are no outstanding
1962 : * standard commands, and previous NCQ commands for other
1963 : * PMP ports have finished.
1964 : */
1965 0 : if (ap->ap_active == 0)
1966 0 : ahci_issue_pending_ncq_commands(ap);
1967 : else
1968 0 : KASSERT(ap->ap_active_cnt == 1);
1969 0 : } else if (nextccb) {
1970 0 : if (ap->ap_sactive != 0 || last_was_ncq)
1971 0 : KASSERT(ap->ap_active_cnt == 0);
1972 :
1973 : /* Wait for all NCQ commands to finish before issuing standard
1974 : * command. */
1975 0 : if (ap->ap_sactive != 0)
1976 0 : return;
1977 :
1978 : /* Keep up to 2 standard commands on-chip at a time. */
1979 0 : do {
1980 0 : TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1981 0 : ap->ap_active |= 1 << nextccb->ccb_slot;
1982 0 : nextccb->ccb_xa.state = ATA_S_ONCHIP;
1983 0 : ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
1984 0 : if (last_was_ncq)
1985 0 : ap->ap_active_cnt++;
1986 0 : if (ap->ap_active_cnt == 2)
1987 : break;
1988 0 : KASSERT(ap->ap_active_cnt == 1);
1989 0 : nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1990 0 : } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
1991 0 : } else if (!last_was_ncq) {
1992 0 : KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
1993 :
1994 : /* Standard command finished, none waiting to start. */
1995 0 : ap->ap_active_cnt--;
1996 0 : } else {
1997 0 : KASSERT(ap->ap_active_cnt == 0);
1998 :
1999 : /* NCQ command finished. */
2000 : }
2001 0 : }
2002 :
2003 : int
2004 0 : ahci_intr(void *arg)
2005 : {
2006 0 : struct ahci_softc *sc = arg;
2007 : u_int32_t is, ack = 0;
2008 : int port;
2009 :
2010 : /* Read global interrupt status */
2011 0 : is = ahci_read(sc, AHCI_REG_IS);
2012 0 : if (is == 0 || is == 0xffffffff)
2013 0 : return (0);
2014 : ack = is;
2015 :
2016 : #ifdef AHCI_COALESCE
2017 : /* Check coalescing interrupt first */
2018 : if (is & sc->sc_ccc_mask) {
2019 : DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2020 : DEVNAME(sc));
2021 : is &= ~sc->sc_ccc_mask;
2022 : is |= sc->sc_ccc_ports_cur;
2023 : }
2024 : #endif
2025 :
2026 : /* Process interrupts for each port */
2027 0 : while (is) {
2028 0 : port = ffs(is) - 1;
2029 0 : if (sc->sc_ports[port])
2030 0 : ahci_port_intr(sc->sc_ports[port],
2031 : AHCI_PREG_CI_ALL_SLOTS);
2032 0 : is &= ~(1 << port);
2033 : }
2034 :
2035 : /* Finally, acknowledge global interrupt */
2036 0 : ahci_write(sc, AHCI_REG_IS, ack);
2037 :
2038 0 : return (1);
2039 0 : }
2040 :
2041 : u_int32_t
2042 0 : ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
2043 : {
2044 0 : struct ahci_softc *sc = ap->ap_sc;
2045 : u_int32_t is, ci_saved, ci_masked, processed = 0;
2046 : int slot, need_restart = 0;
2047 : int process_error = 0;
2048 : struct ahci_ccb *ccb;
2049 : volatile u_int32_t *active;
2050 : #ifdef DIAGNOSTIC
2051 : u_int32_t tmp;
2052 : #endif
2053 :
2054 0 : is = ahci_pread(ap, AHCI_PREG_IS);
2055 :
2056 : /* Ack port interrupt only if checking all command slots. */
2057 0 : if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
2058 0 : ahci_pwrite(ap, AHCI_PREG_IS, is);
2059 :
2060 : if (is)
2061 : DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
2062 : is, AHCI_PFMT_IS);
2063 :
2064 0 : if (ap->ap_sactive) {
2065 : /* Active NCQ commands - use SActive instead of CI */
2066 0 : KASSERT(ap->ap_active == 0);
2067 0 : KASSERT(ap->ap_active_cnt == 0);
2068 0 : ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
2069 : active = &ap->ap_sactive;
2070 0 : } else {
2071 : /* Save CI */
2072 0 : ci_saved = ahci_pread(ap, AHCI_PREG_CI);
2073 0 : active = &ap->ap_active;
2074 : }
2075 :
2076 0 : if (is & AHCI_PREG_IS_TFES) {
2077 : process_error = 1;
2078 0 : } else if (is & AHCI_PREG_IS_DHRS) {
2079 : u_int32_t tfd;
2080 : u_int32_t cmd;
2081 : u_int32_t serr;
2082 :
2083 0 : tfd = ahci_pread(ap, AHCI_PREG_TFD);
2084 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD);
2085 0 : serr = ahci_pread(ap, AHCI_PREG_SERR);
2086 0 : if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
2087 0 : (cmd & AHCI_PREG_CMD_CR) == 0) {
2088 : DPRINTF(AHCI_D_VERBOSE, "%s: DHRS error, TFD: %b, SERR:"
2089 : " %b, DIAG: %b\n", PORTNAME(ap), tfd,
2090 : AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2091 : AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2092 : AHCI_PFMT_SERR_DIAG);
2093 : process_error = 1;
2094 0 : } else {
2095 : /* rfis copy back is in the normal execution path */
2096 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
2097 : }
2098 0 : }
2099 :
2100 : /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
2101 0 : if (process_error) {
2102 : u_int32_t tfd, serr;
2103 0 : int err_slot;
2104 :
2105 0 : tfd = ahci_pread(ap, AHCI_PREG_TFD);
2106 0 : serr = ahci_pread(ap, AHCI_PREG_SERR);
2107 :
2108 0 : if (ap->ap_sactive == 0) {
2109 : /* Errored slot is easy to determine from CMD. */
2110 0 : err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
2111 : AHCI_PREG_CMD));
2112 :
2113 0 : if ((ci_saved & (1 << err_slot)) == 0) {
2114 : /*
2115 : * Hardware doesn't seem to report correct
2116 : * slot number. If there's only one
2117 : * outstanding command we can cope,
2118 : * otherwise fail all active commands.
2119 : */
2120 0 : if (ap->ap_active_cnt == 1)
2121 0 : err_slot = ffs(ap->ap_active) - 1;
2122 : else
2123 0 : goto failall;
2124 0 : }
2125 :
2126 0 : ccb = &ap->ap_ccbs[err_slot];
2127 :
2128 : /* Preserve received taskfile data from the RFIS. */
2129 0 : memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
2130 : sizeof(struct ata_fis_d2h));
2131 0 : } else
2132 0 : err_slot = -1; /* Must extract error from log page */
2133 :
2134 : DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
2135 : " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
2136 : AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
2137 : AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
2138 : AHCI_PFMT_SERR_DIAG);
2139 :
2140 : /* Turn off ST to clear CI and SACT. */
2141 0 : ahci_port_stop(ap, 0);
2142 : need_restart = 1;
2143 :
2144 : /* Clear SERR to enable capturing new errors. */
2145 0 : ahci_pwrite(ap, AHCI_PREG_SERR, serr);
2146 :
2147 : /* Acknowledge the interrupts we can recover from. */
2148 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
2149 : AHCI_PREG_IS_IFS);
2150 0 : is = ahci_pread(ap, AHCI_PREG_IS);
2151 :
2152 : /* If device hasn't cleared its busy status, try to idle it. */
2153 0 : if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
2154 :
2155 0 : if ((ap->ap_state == AP_S_PMP_PORT_PROBE) ||
2156 0 : (ap->ap_state == AP_S_ERROR_RECOVERY)) {
2157 : /* can't reset the port here, just make sure
2158 : * the operation fails and the port still works.
2159 : */
2160 0 : } else if (ap->ap_pmp_ports != 0 && err_slot != -1) {
2161 0 : printf("%s: error on PMP port %d, idling "
2162 : "device\n", PORTNAME(ap),
2163 0 : ccb->ccb_xa.pmp_port);
2164 0 : if (ahci_pmp_port_softreset(ap,
2165 0 : ccb->ccb_xa.pmp_port) == 0) {
2166 0 : printf("%s: unable to softreset port "
2167 0 : "%d\n", PORTNAME(ap),
2168 0 : ccb->ccb_xa.pmp_port);
2169 0 : if (ahci_pmp_port_portreset(ap,
2170 0 : ccb->ccb_xa.pmp_port)) {
2171 0 : printf("%s: failed to port "
2172 : " reset %d, giving up on "
2173 0 : "it\n", PORTNAME(ap),
2174 0 : ccb->ccb_xa.pmp_port);
2175 0 : goto fatal;
2176 : }
2177 : }
2178 : } else {
2179 0 : printf("%s: attempting to idle device\n",
2180 : PORTNAME(ap));
2181 0 : if (ahci_port_softreset(ap)) {
2182 0 : printf("%s: failed to soft reset "
2183 0 : "device\n", PORTNAME(ap));
2184 0 : if (ahci_port_portreset(ap, 0)) {
2185 0 : printf("%s: failed to port "
2186 : "reset device, give up on "
2187 0 : "it\n", PORTNAME(ap));
2188 0 : goto fatal;
2189 : }
2190 : }
2191 : }
2192 :
2193 : /* Had to reset device, can't gather extended info. */
2194 0 : } else if (ap->ap_sactive) {
2195 : /* Recover the NCQ error from log page 10h.
2196 : * We can only have queued commands active for one port
2197 : * at a time, so we know which device errored.
2198 : */
2199 0 : ahci_port_read_ncq_error(ap, &err_slot,
2200 0 : ap->ap_pmp_ncq_port);
2201 0 : if (err_slot < 0)
2202 0 : goto failall;
2203 :
2204 : DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2205 : PORTNAME(ap), err_slot);
2206 :
2207 0 : ccb = &ap->ap_ccbs[err_slot];
2208 0 : if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
2209 0 : printf("%s: NCQ errored slot %d is idle"
2210 0 : " (%08x active)\n", PORTNAME(ap), err_slot,
2211 : ci_saved);
2212 0 : goto failall;
2213 : }
2214 : } else {
2215 : /* Didn't reset, could gather extended info from log. */
2216 : }
2217 :
2218 : /*
2219 : * If we couldn't determine the errored slot, reset the port
2220 : * and fail all the active slots.
2221 : */
2222 0 : if (err_slot == -1) {
2223 0 : if (ahci_port_softreset(ap) != 0 &&
2224 0 : ahci_port_portreset(ap, 0) != 0) {
2225 0 : printf("%s: couldn't reset after NCQ error, "
2226 0 : "disabling device.\n", PORTNAME(ap));
2227 0 : goto fatal;
2228 : }
2229 0 : printf("%s: couldn't recover NCQ error, failing "
2230 0 : "all outstanding commands.\n", PORTNAME(ap));
2231 0 : goto failall;
2232 : }
2233 :
2234 : /* Clear the failed command in saved CI so completion runs. */
2235 0 : ci_saved &= ~(1 << err_slot);
2236 :
2237 : /* Note the error in the ata_xfer. */
2238 0 : KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2239 0 : ccb->ccb_xa.state = ATA_S_ERROR;
2240 :
2241 : #ifdef DIAGNOSTIC
2242 : /* There may only be one outstanding standard command now. */
2243 0 : if (ap->ap_sactive == 0) {
2244 : tmp = ci_saved;
2245 0 : if (tmp) {
2246 0 : slot = ffs(tmp) - 1;
2247 0 : tmp &= ~(1 << slot);
2248 0 : KASSERT(tmp == 0);
2249 : }
2250 : }
2251 : #endif
2252 0 : }
2253 :
2254 : /* ATI SBx00 AHCI controllers respond to PMP probes with IPMS interrupts
2255 : * when there's a normal SATA device attached.
2256 : */
2257 0 : if ((ap->ap_state == AP_S_PMP_PROBE) &&
2258 0 : (ap->ap_sc->sc_flags & AHCI_F_IPMS_PROBE) &&
2259 0 : (is & AHCI_PREG_IS_IPMS)) {
2260 0 : slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, AHCI_PREG_CMD));
2261 : DPRINTF(AHCI_D_INTR, "%s: slot %d received IPMS\n",
2262 : PORTNAME(ap), slot);
2263 :
2264 0 : ccb = &ap->ap_ccbs[slot];
2265 0 : ccb->ccb_xa.state = ATA_S_ERROR;
2266 :
2267 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IPMS);
2268 0 : is &= ~AHCI_PREG_IS_IPMS;
2269 0 : }
2270 :
2271 : /* ignore IFS errors while resetting a PMP port */
2272 0 : if ((is & AHCI_PREG_IS_IFS) /*&& ap->ap_pmp_ignore_ifs*/) {
2273 : DPRINTF(AHCI_D_INTR, "%s: ignoring IFS while resetting PMP "
2274 : "port\n", PORTNAME(ap));
2275 :
2276 : need_restart = 1;
2277 0 : ahci_pwrite(ap, AHCI_PREG_SERR, -1);
2278 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
2279 : is &= ~AHCI_PREG_IS_IFS;
2280 0 : goto failall;
2281 : }
2282 :
2283 : /* Check for remaining errors - they are fatal. */
2284 0 : if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
2285 : AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
2286 0 : printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
2287 0 : PORTNAME(ap), is, AHCI_PFMT_IS);
2288 :
2289 : /* XXX try recovery first */
2290 0 : goto fatal;
2291 : }
2292 :
2293 : /* Fail all outstanding commands if we know the port won't recover. */
2294 0 : if (ap->ap_state == AP_S_FATAL_ERROR) {
2295 : fatal:
2296 0 : ap->ap_state = AP_S_FATAL_ERROR;
2297 : failall:
2298 :
2299 : /* Ensure port is shut down. */
2300 0 : ahci_port_stop(ap, 1);
2301 :
2302 : /* Error all the active slots. */
2303 0 : ci_masked = ci_saved & *active;
2304 0 : while (ci_masked) {
2305 0 : slot = ffs(ci_masked) - 1;
2306 0 : ccb = &ap->ap_ccbs[slot];
2307 0 : ci_masked &= ~(1 << slot);
2308 0 : ccb->ccb_xa.state = ATA_S_ERROR;
2309 : }
2310 :
2311 : /* Run completion for all active slots. */
2312 0 : ci_saved &= ~*active;
2313 :
2314 : /* Don't restart the port if our problems were deemed fatal. */
2315 0 : if (ap->ap_state == AP_S_FATAL_ERROR)
2316 0 : need_restart = 0;
2317 : }
2318 :
2319 : /*
2320 : * CCB completion is detected by noticing its slot's bit in CI has
2321 : * changed to zero some time after we activated it.
2322 : * If we are polling, we may only be interested in particular slot(s).
2323 : */
2324 0 : ci_masked = ~ci_saved & *active & ci_mask;
2325 0 : while (ci_masked) {
2326 0 : slot = ffs(ci_masked) - 1;
2327 0 : ccb = &ap->ap_ccbs[slot];
2328 0 : ci_masked &= ~(1 << slot);
2329 :
2330 : DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2331 : PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2332 : " (error)" : "");
2333 :
2334 0 : bus_dmamap_sync(sc->sc_dmat,
2335 : AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
2336 : ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
2337 : sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
2338 :
2339 0 : bus_dmamap_sync(sc->sc_dmat,
2340 : AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
2341 : ccb->ccb_slot * sizeof(struct ahci_cmd_table),
2342 : sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
2343 :
2344 0 : bus_dmamap_sync(sc->sc_dmat,
2345 : AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
2346 : sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
2347 :
2348 0 : *active &= ~(1 << ccb->ccb_slot);
2349 : /* Copy the rfis into the ccb if we were asked for it */
2350 0 : if (ccb->ccb_xa.state == ATA_S_ONCHIP &&
2351 0 : ccb->ccb_xa.flags & ATA_F_GET_RFIS) {
2352 0 : memcpy(&ccb->ccb_xa.rfis,
2353 : ap->ap_rfis->rfis,
2354 : sizeof(struct ata_fis_d2h));
2355 0 : }
2356 :
2357 0 : ccb->ccb_done(ccb);
2358 :
2359 0 : processed |= 1 << ccb->ccb_slot;
2360 : }
2361 :
2362 0 : if (need_restart) {
2363 : /* Restart command DMA on the port */
2364 0 : ahci_port_start(ap, 0);
2365 :
2366 : /* Re-enable outstanding commands on port. */
2367 0 : if (ci_saved) {
2368 : #ifdef DIAGNOSTIC
2369 : tmp = ci_saved;
2370 0 : while (tmp) {
2371 0 : slot = ffs(tmp) - 1;
2372 0 : tmp &= ~(1 << slot);
2373 0 : ccb = &ap->ap_ccbs[slot];
2374 0 : KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2375 0 : KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
2376 : (!!ap->ap_sactive));
2377 : }
2378 : #endif
2379 : DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2380 : "re-enabling%s slots %08x\n", PORTNAME(ap),
2381 : ap->ap_sactive ? " NCQ" : "", ci_saved);
2382 :
2383 0 : if (ap->ap_sactive)
2384 0 : ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
2385 0 : ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
2386 0 : }
2387 : }
2388 :
2389 0 : return (processed);
2390 0 : }
2391 :
2392 : struct ahci_ccb *
2393 0 : ahci_get_ccb(struct ahci_port *ap)
2394 : {
2395 : struct ahci_ccb *ccb;
2396 :
2397 0 : mtx_enter(&ap->ap_ccb_mtx);
2398 0 : ccb = TAILQ_FIRST(&ap->ap_ccb_free);
2399 0 : if (ccb != NULL) {
2400 0 : KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2401 0 : TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2402 0 : ccb->ccb_xa.state = ATA_S_SETUP;
2403 0 : }
2404 0 : mtx_leave(&ap->ap_ccb_mtx);
2405 :
2406 0 : return (ccb);
2407 : }
2408 :
2409 : void
2410 0 : ahci_put_ccb(struct ahci_ccb *ccb)
2411 : {
2412 0 : struct ahci_port *ap = ccb->ccb_port;
2413 :
2414 : #ifdef DIAGNOSTIC
2415 0 : if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
2416 0 : ccb->ccb_xa.state != ATA_S_TIMEOUT &&
2417 0 : ccb->ccb_xa.state != ATA_S_ERROR) {
2418 0 : printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2419 0 : "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
2420 0 : ccb->ccb_slot);
2421 0 : }
2422 : #endif
2423 :
2424 0 : ccb->ccb_xa.state = ATA_S_PUT;
2425 0 : mtx_enter(&ap->ap_ccb_mtx);
2426 0 : TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2427 0 : mtx_leave(&ap->ap_ccb_mtx);
2428 0 : }
2429 :
2430 : struct ahci_ccb *
2431 0 : ahci_get_err_ccb(struct ahci_port *ap)
2432 : {
2433 : struct ahci_ccb *err_ccb;
2434 : u_int32_t sact;
2435 :
2436 0 : splassert(IPL_BIO);
2437 :
2438 : /* No commands may be active on the chip. */
2439 0 : sact = ahci_pread(ap, AHCI_PREG_SACT);
2440 0 : if (sact != 0)
2441 0 : printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2442 0 : KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2443 :
2444 : #ifdef DIAGNOSTIC
2445 0 : KASSERT(ap->ap_err_busy == 0);
2446 0 : ap->ap_err_busy = 1;
2447 : #endif
2448 : /* Save outstanding command state. */
2449 0 : ap->ap_err_saved_active = ap->ap_active;
2450 0 : ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2451 0 : ap->ap_err_saved_sactive = ap->ap_sactive;
2452 :
2453 : /*
2454 : * Pretend we have no commands outstanding, so that completions won't
2455 : * run prematurely.
2456 : */
2457 0 : ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2458 :
2459 : /*
2460 : * Grab a CCB to use for error recovery. This should never fail, as
2461 : * we ask atascsi to reserve one for us at init time.
2462 : */
2463 0 : err_ccb = ap->ap_ccb_err;
2464 0 : err_ccb->ccb_xa.flags = 0;
2465 0 : err_ccb->ccb_xa.state = ATA_S_SETUP;
2466 0 : err_ccb->ccb_done = ahci_empty_done;
2467 :
2468 0 : return (err_ccb);
2469 : }
2470 :
2471 : void
2472 0 : ahci_put_err_ccb(struct ahci_ccb *ccb)
2473 : {
2474 0 : struct ahci_port *ap = ccb->ccb_port;
2475 : u_int32_t sact;
2476 :
2477 0 : splassert(IPL_BIO);
2478 :
2479 : #ifdef DIAGNOSTIC
2480 0 : KASSERT(ap->ap_err_busy);
2481 : #endif
2482 : /* No commands may be active on the chip */
2483 0 : sact = ahci_pread(ap, AHCI_PREG_SACT);
2484 0 : if (sact != 0)
2485 0 : printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact);
2486 0 : KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2487 :
2488 : #ifdef DIAGNOSTIC
2489 : /* Done with the CCB */
2490 0 : KASSERT(ccb == ap->ap_ccb_err);
2491 : #endif
2492 :
2493 : /* Restore outstanding command state */
2494 0 : ap->ap_sactive = ap->ap_err_saved_sactive;
2495 0 : ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2496 0 : ap->ap_active = ap->ap_err_saved_active;
2497 :
2498 : #ifdef DIAGNOSTIC
2499 0 : ap->ap_err_busy = 0;
2500 : #endif
2501 0 : }
2502 :
2503 : struct ahci_ccb *
2504 0 : ahci_get_pmp_ccb(struct ahci_port *ap)
2505 : {
2506 : struct ahci_ccb *ccb;
2507 : u_int32_t sact;
2508 :
2509 : /* some PMP commands need to be issued on slot 1,
2510 : * particularly the command that clears SRST and
2511 : * fetches the device signature.
2512 : *
2513 : * ensure the chip is idle and ccb 1 is available.
2514 : */
2515 0 : splassert(IPL_BIO);
2516 :
2517 0 : sact = ahci_pread(ap, AHCI_PREG_SACT);
2518 0 : if (sact != 0)
2519 0 : printf("ahci_get_pmp_ccb; SACT %08x != 0\n", sact);
2520 0 : KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2521 :
2522 0 : ccb = &ap->ap_ccbs[1];
2523 0 : KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2524 0 : ccb->ccb_xa.flags = 0;
2525 0 : ccb->ccb_done = ahci_pmp_cmd_done;
2526 :
2527 0 : mtx_enter(&ap->ap_ccb_mtx);
2528 0 : TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2529 0 : mtx_leave(&ap->ap_ccb_mtx);
2530 :
2531 0 : return ccb;
2532 : }
2533 :
2534 : void
2535 0 : ahci_put_pmp_ccb(struct ahci_ccb *ccb)
2536 : {
2537 0 : struct ahci_port *ap = ccb->ccb_port;
2538 : u_int32_t sact;
2539 :
2540 : /* make sure this is the right ccb */
2541 0 : KASSERT(ccb == &ap->ap_ccbs[1]);
2542 :
2543 : /* No commands may be active on the chip */
2544 0 : sact = ahci_pread(ap, AHCI_PREG_SACT);
2545 0 : if (sact != 0)
2546 0 : printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact);
2547 0 : KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2548 :
2549 0 : ccb->ccb_xa.state = ATA_S_PUT;
2550 0 : mtx_enter(&ap->ap_ccb_mtx);
2551 0 : TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2552 0 : mtx_leave(&ap->ap_ccb_mtx);
2553 0 : }
2554 :
2555 : int
2556 0 : ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp, int pmp_port)
2557 : {
2558 : struct ahci_ccb *ccb;
2559 : struct ahci_cmd_hdr *cmd_slot;
2560 : u_int32_t cmd;
2561 : struct ata_fis_h2d *fis;
2562 : int rc = EIO, oldstate;
2563 :
2564 : DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2565 0 : oldstate = ap->ap_state;
2566 0 : ap->ap_state = AP_S_ERROR_RECOVERY;
2567 :
2568 : /* Save command register state. */
2569 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
2570 :
2571 : /* Port should have been idled already. Start it. */
2572 0 : KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
2573 0 : ahci_port_start(ap, 0);
2574 :
2575 : /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2576 0 : ccb = ahci_get_err_ccb(ap);
2577 0 : ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
2578 0 : ccb->ccb_xa.data = ap->ap_err_scratch;
2579 0 : ccb->ccb_xa.datalen = 512;
2580 0 : cmd_slot = ccb->ccb_cmd_hdr;
2581 0 : memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
2582 :
2583 0 : fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2584 0 : fis->type = ATA_FIS_TYPE_H2D;
2585 0 : fis->flags = ATA_H2D_FLAGS_CMD | pmp_port;
2586 0 : fis->command = ATA_C_READ_LOG_EXT;
2587 0 : fis->lba_low = 0x10; /* queued error log page (10h) */
2588 0 : fis->sector_count = 1; /* number of sectors (1) */
2589 0 : fis->sector_count_exp = 0;
2590 0 : fis->lba_mid = 0; /* starting offset */
2591 0 : fis->lba_mid_exp = 0;
2592 0 : fis->device = 0;
2593 :
2594 0 : htolem16(&cmd_slot->flags, 5 /* FIS length: 5 DWORDS */ |
2595 : (pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT));
2596 :
2597 0 : if (ahci_load_prdt(ccb) != 0) {
2598 : rc = ENOMEM; /* XXX caller must abort all commands */
2599 0 : goto err;
2600 : }
2601 :
2602 0 : ccb->ccb_xa.state = ATA_S_PENDING;
2603 0 : if (ahci_poll(ccb, 1000, NULL) != 0 ||
2604 0 : ccb->ccb_xa.state == ATA_S_ERROR)
2605 : goto err;
2606 :
2607 0 : rc = 0;
2608 : err:
2609 : /* Abort our command, if it failed, by stopping command DMA. */
2610 0 : if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
2611 0 : printf("%s: log page read failed, slot %d was still active.\n",
2612 0 : PORTNAME(ap), ccb->ccb_slot);
2613 0 : ahci_port_stop(ap, 0);
2614 0 : }
2615 :
2616 : /* Done with the error CCB now. */
2617 0 : ahci_unload_prdt(ccb);
2618 0 : ahci_put_err_ccb(ccb);
2619 :
2620 : /* Extract failed register set and tags from the scratch space. */
2621 0 : if (rc == 0) {
2622 : struct ata_log_page_10h *log;
2623 : int err_slot;
2624 :
2625 0 : log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2626 0 : if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
2627 : /* Not queued bit was set - wasn't an NCQ error? */
2628 0 : printf("%s: read NCQ error page, but not an NCQ "
2629 0 : "error?\n", PORTNAME(ap));
2630 : rc = ESRCH;
2631 0 : } else {
2632 : /* Copy back the log record as a D2H register FIS. */
2633 0 : *err_slotp = err_slot = log->err_regs.type &
2634 : ATA_LOG_10H_TYPE_TAG_MASK;
2635 :
2636 0 : ccb = &ap->ap_ccbs[err_slot];
2637 0 : memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
2638 : sizeof(struct ata_fis_d2h));
2639 0 : ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
2640 0 : ccb->ccb_xa.rfis.flags = 0;
2641 : }
2642 0 : }
2643 :
2644 : /* Restore saved CMD register state */
2645 0 : ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
2646 0 : ap->ap_state = oldstate;
2647 :
2648 0 : return (rc);
2649 : }
2650 :
2651 : struct ahci_dmamem *
2652 0 : ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
2653 : {
2654 : struct ahci_dmamem *adm;
2655 0 : int nsegs;
2656 :
2657 0 : adm = malloc(sizeof(*adm), M_DEVBUF, M_NOWAIT | M_ZERO);
2658 0 : if (adm == NULL)
2659 0 : return (NULL);
2660 :
2661 0 : adm->adm_size = size;
2662 :
2663 0 : if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
2664 0 : BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
2665 : goto admfree;
2666 :
2667 0 : if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
2668 0 : 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0)
2669 : goto destroy;
2670 :
2671 0 : if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
2672 0 : &adm->adm_kva, BUS_DMA_NOWAIT | BUS_DMA_COHERENT) != 0)
2673 : goto free;
2674 :
2675 0 : if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
2676 0 : NULL, BUS_DMA_NOWAIT) != 0)
2677 : goto unmap;
2678 :
2679 0 : return (adm);
2680 :
2681 : unmap:
2682 0 : bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
2683 : free:
2684 0 : bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2685 : destroy:
2686 0 : bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2687 : admfree:
2688 0 : free(adm, M_DEVBUF, sizeof(*adm));
2689 :
2690 0 : return (NULL);
2691 0 : }
2692 :
2693 : void
2694 0 : ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
2695 : {
2696 0 : bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
2697 0 : bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
2698 0 : bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2699 0 : bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2700 0 : free(adm, M_DEVBUF, sizeof(*adm));
2701 0 : }
2702 :
2703 : u_int32_t
2704 0 : ahci_read(struct ahci_softc *sc, bus_size_t r)
2705 : {
2706 0 : bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2707 : BUS_SPACE_BARRIER_READ);
2708 0 : return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
2709 : }
2710 :
2711 : void
2712 0 : ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2713 : {
2714 0 : bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
2715 0 : bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2716 : BUS_SPACE_BARRIER_WRITE);
2717 0 : }
2718 :
2719 : int
2720 0 : ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
2721 : u_int32_t target)
2722 : {
2723 : int i;
2724 :
2725 0 : for (i = 0; i < 1000; i++) {
2726 0 : if ((ahci_read(sc, r) & mask) != target)
2727 0 : return (0);
2728 0 : delay(1000);
2729 : }
2730 :
2731 0 : return (1);
2732 0 : }
2733 :
2734 : u_int32_t
2735 0 : ahci_pread(struct ahci_port *ap, bus_size_t r)
2736 : {
2737 0 : bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2738 : BUS_SPACE_BARRIER_READ);
2739 0 : return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
2740 : }
2741 :
2742 : void
2743 0 : ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2744 : {
2745 0 : bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
2746 0 : bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2747 : BUS_SPACE_BARRIER_WRITE);
2748 0 : }
2749 :
2750 : int
2751 0 : ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
2752 : u_int32_t target, int n)
2753 : {
2754 : int i;
2755 :
2756 0 : for (i = 0; i < n * 1000; i++) {
2757 0 : if ((ahci_pread(ap, r) & mask) == target)
2758 0 : return (0);
2759 0 : delay(1000);
2760 : }
2761 :
2762 0 : return (1);
2763 0 : }
2764 :
2765 : int
2766 0 : ahci_ata_probe(void *xsc, int port, int lun)
2767 : {
2768 0 : struct ahci_softc *sc = xsc;
2769 0 : struct ahci_port *ap = sc->sc_ports[port];
2770 :
2771 0 : if (ap == NULL)
2772 0 : return (ATA_PORT_T_NONE);
2773 :
2774 0 : if (lun != 0) {
2775 0 : int pmp_port = lun - 1;
2776 0 : if (pmp_port >= ap->ap_pmp_ports) {
2777 0 : return (ATA_PORT_T_NONE);
2778 : }
2779 0 : return (ahci_pmp_port_probe(ap, pmp_port));
2780 : } else {
2781 0 : return (ahci_port_signature(ap));
2782 : }
2783 0 : }
2784 :
2785 : void
2786 0 : ahci_ata_free(void *xsc, int port, int lun)
2787 : {
2788 :
2789 0 : }
2790 :
2791 : struct ata_xfer *
2792 0 : ahci_ata_get_xfer(void *aaa_cookie, int port)
2793 : {
2794 0 : struct ahci_softc *sc = aaa_cookie;
2795 0 : struct ahci_port *ap = sc->sc_ports[port];
2796 : struct ahci_ccb *ccb;
2797 :
2798 0 : ccb = ahci_get_ccb(ap);
2799 0 : if (ccb == NULL) {
2800 : DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
2801 : PORTNAME(ap));
2802 0 : return (NULL);
2803 : }
2804 :
2805 : DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
2806 : PORTNAME(ap), ccb->ccb_slot);
2807 :
2808 0 : return ((struct ata_xfer *)ccb);
2809 0 : }
2810 :
2811 : void
2812 0 : ahci_ata_put_xfer(struct ata_xfer *xa)
2813 : {
2814 0 : struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2815 :
2816 : DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
2817 :
2818 0 : ahci_put_ccb(ccb);
2819 0 : }
2820 :
2821 : void
2822 0 : ahci_ata_cmd(struct ata_xfer *xa)
2823 : {
2824 0 : struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2825 : struct ahci_cmd_hdr *cmd_slot;
2826 : int s;
2827 : u_int16_t flags;
2828 :
2829 0 : if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
2830 : goto failcmd;
2831 :
2832 0 : ccb->ccb_done = ahci_ata_cmd_done;
2833 :
2834 0 : cmd_slot = ccb->ccb_cmd_hdr;
2835 : flags = 5 /* FIS length (in DWORDs) */;
2836 0 : flags |= xa->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
2837 :
2838 0 : if (xa->flags & ATA_F_WRITE)
2839 0 : flags |= AHCI_CMD_LIST_FLAG_W;
2840 :
2841 0 : if (xa->flags & ATA_F_PACKET)
2842 0 : flags |= AHCI_CMD_LIST_FLAG_A;
2843 :
2844 0 : htolem16(&cmd_slot->flags, flags);
2845 :
2846 0 : if (ahci_load_prdt(ccb) != 0)
2847 : goto failcmd;
2848 :
2849 0 : timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
2850 :
2851 0 : xa->state = ATA_S_PENDING;
2852 :
2853 0 : if (xa->flags & ATA_F_POLL)
2854 0 : ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
2855 : else {
2856 0 : s = splbio();
2857 0 : timeout_add_msec(&xa->stimeout, xa->timeout);
2858 0 : ahci_start(ccb);
2859 0 : splx(s);
2860 : }
2861 :
2862 0 : return;
2863 :
2864 : failcmd:
2865 0 : s = splbio();
2866 0 : xa->state = ATA_S_ERROR;
2867 0 : ata_complete(xa);
2868 0 : splx(s);
2869 0 : }
2870 :
2871 : void
2872 0 : ahci_pmp_cmd_done(struct ahci_ccb *ccb)
2873 : {
2874 0 : struct ata_xfer *xa = &ccb->ccb_xa;
2875 :
2876 0 : if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
2877 0 : ahci_issue_pending_commands(ccb->ccb_port,
2878 0 : xa->flags & ATA_F_NCQ);
2879 :
2880 0 : xa->state = ATA_S_COMPLETE;
2881 0 : }
2882 :
2883 :
2884 : void
2885 0 : ahci_ata_cmd_done(struct ahci_ccb *ccb)
2886 : {
2887 0 : struct ata_xfer *xa = &ccb->ccb_xa;
2888 :
2889 0 : timeout_del(&xa->stimeout);
2890 :
2891 0 : if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
2892 0 : ahci_issue_pending_commands(ccb->ccb_port,
2893 0 : xa->flags & ATA_F_NCQ);
2894 :
2895 0 : ahci_unload_prdt(ccb);
2896 :
2897 0 : if (xa->state == ATA_S_ONCHIP)
2898 0 : xa->state = ATA_S_COMPLETE;
2899 : #ifdef DIAGNOSTIC
2900 0 : else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
2901 0 : printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
2902 0 : "slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
2903 0 : ccb->ccb_slot);
2904 : #endif
2905 0 : if (xa->state != ATA_S_TIMEOUT)
2906 0 : ata_complete(xa);
2907 0 : }
2908 :
2909 : void
2910 0 : ahci_ata_cmd_timeout(void *arg)
2911 : {
2912 0 : struct ahci_ccb *ccb = arg;
2913 0 : struct ata_xfer *xa = &ccb->ccb_xa;
2914 0 : struct ahci_port *ap = ccb->ccb_port;
2915 : int s, ccb_was_started, ncq_cmd;
2916 : volatile u_int32_t *active;
2917 :
2918 0 : s = splbio();
2919 :
2920 0 : ncq_cmd = (xa->flags & ATA_F_NCQ);
2921 0 : active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
2922 :
2923 0 : if (ccb->ccb_xa.state == ATA_S_PENDING) {
2924 : DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
2925 : "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
2926 0 : TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
2927 : ccb_was_started = 0;
2928 0 : } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
2929 0 : 1 << ccb->ccb_slot)) {
2930 : DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
2931 : "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
2932 : goto ret;
2933 0 : } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
2934 : DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
2935 : "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
2936 : ISSET(*active, 1 << ccb->ccb_slot) ?
2937 : " but slot is still active?" : ".");
2938 : goto ret;
2939 0 : } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
2940 0 : AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
2941 : 1 << ccb->ccb_slot)) {
2942 : DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
2943 : "IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
2944 : ccb->ccb_slot);
2945 0 : *active &= ~(1 << ccb->ccb_slot);
2946 0 : ccb->ccb_done(ccb);
2947 0 : goto ret;
2948 : } else {
2949 : ccb_was_started = 1;
2950 : }
2951 :
2952 : /* Complete the slot with a timeout error. */
2953 0 : ccb->ccb_xa.state = ATA_S_TIMEOUT;
2954 0 : *active &= ~(1 << ccb->ccb_slot);
2955 : DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
2956 0 : ccb->ccb_done(ccb); /* This won't issue pending commands or run the
2957 : atascsi completion. */
2958 :
2959 : /* Reset port to abort running command. */
2960 0 : if (ccb_was_started) {
2961 : DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
2962 : "in slot %d, pmp port %d, active %08x\n", PORTNAME(ap),
2963 : ncq_cmd ? " NCQ" : "", ccb->ccb_slot, xa->pmp_port, *active);
2964 0 : if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap, 0)
2965 0 : != 0) {
2966 0 : printf("%s: failed to reset port during timeout "
2967 0 : "handling, disabling it\n", PORTNAME(ap));
2968 0 : ap->ap_state = AP_S_FATAL_ERROR;
2969 0 : }
2970 :
2971 : /* Restart any other commands that were aborted by the reset. */
2972 0 : if (*active) {
2973 : DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
2974 : "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
2975 : *active);
2976 0 : if (ncq_cmd)
2977 0 : ahci_pwrite(ap, AHCI_PREG_SACT, *active);
2978 0 : ahci_pwrite(ap, AHCI_PREG_CI, *active);
2979 0 : }
2980 : }
2981 :
2982 : /* Issue any pending commands now. */
2983 : DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
2984 0 : if (ccb_was_started)
2985 0 : ahci_issue_pending_commands(ap, ncq_cmd);
2986 0 : else if (ap->ap_active == 0)
2987 0 : ahci_issue_pending_ncq_commands(ap);
2988 :
2989 : /* Complete the timed out ata_xfer I/O (may generate new I/O). */
2990 : DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
2991 0 : ata_complete(xa);
2992 :
2993 : DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
2994 : ret:
2995 0 : splx(s);
2996 0 : }
2997 :
2998 : void
2999 0 : ahci_empty_done(struct ahci_ccb *ccb)
3000 : {
3001 0 : if (ccb->ccb_xa.state != ATA_S_ERROR)
3002 0 : ccb->ccb_xa.state = ATA_S_COMPLETE;
3003 0 : }
3004 :
3005 : int
3006 0 : ahci_pmp_read(struct ahci_port *ap, int target, int which, u_int32_t *datap)
3007 : {
3008 : struct ahci_ccb *ccb;
3009 : struct ata_fis_h2d *fis;
3010 : int error;
3011 :
3012 0 : ccb = ahci_get_pmp_ccb(ap);
3013 0 : if (ccb == NULL) {
3014 0 : printf("%s: NULL ccb!\n", PORTNAME(ap));
3015 0 : return (1);
3016 : }
3017 0 : ccb->ccb_xa.flags = ATA_F_POLL | ATA_F_GET_RFIS;
3018 0 : ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
3019 0 : ccb->ccb_xa.state = ATA_S_PENDING;
3020 :
3021 0 : memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
3022 0 : fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3023 0 : fis->type = ATA_FIS_TYPE_H2D;
3024 0 : fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
3025 0 : fis->command = ATA_C_READ_PM;
3026 0 : fis->features = which;
3027 0 : fis->device = target | ATA_H2D_DEVICE_LBA;
3028 0 : fis->control = ATA_FIS_CONTROL_4BIT;
3029 :
3030 0 : if (ahci_poll(ccb, 1000, ahci_pmp_probe_timeout) != 0) {
3031 : error = 1;
3032 0 : } else {
3033 0 : *datap = ccb->ccb_xa.rfis.sector_count |
3034 0 : (ccb->ccb_xa.rfis.lba_low << 8) |
3035 0 : (ccb->ccb_xa.rfis.lba_mid << 16) |
3036 0 : (ccb->ccb_xa.rfis.lba_high << 24);
3037 : error = 0;
3038 : }
3039 0 : ahci_put_pmp_ccb(ccb);
3040 0 : return (error);
3041 0 : }
3042 :
3043 : int
3044 0 : ahci_pmp_write(struct ahci_port *ap, int target, int which, u_int32_t data)
3045 : {
3046 : struct ahci_ccb *ccb;
3047 : struct ata_fis_h2d *fis;
3048 : int error;
3049 :
3050 0 : ccb = ahci_get_pmp_ccb(ap);
3051 0 : if (ccb == NULL) {
3052 0 : printf("%s: NULL ccb!\n", PORTNAME(ap));
3053 0 : return (1);
3054 : }
3055 0 : ccb->ccb_xa.flags = ATA_F_POLL;
3056 0 : ccb->ccb_xa.pmp_port = SATA_PMP_CONTROL_PORT;
3057 0 : ccb->ccb_xa.state = ATA_S_PENDING;
3058 :
3059 0 : memset(ccb->ccb_cmd_table, 0, sizeof(struct ahci_cmd_table));
3060 0 : fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
3061 0 : fis->type = ATA_FIS_TYPE_H2D;
3062 0 : fis->flags = ATA_H2D_FLAGS_CMD | SATA_PMP_CONTROL_PORT;
3063 0 : fis->command = ATA_C_WRITE_PM;
3064 0 : fis->features = which;
3065 0 : fis->device = target | ATA_H2D_DEVICE_LBA;
3066 0 : fis->sector_count = (u_int8_t)data;
3067 0 : fis->lba_low = (u_int8_t)(data >> 8);
3068 0 : fis->lba_mid = (u_int8_t)(data >> 16);
3069 0 : fis->lba_high = (u_int8_t)(data >> 24);
3070 0 : fis->control = ATA_FIS_CONTROL_4BIT;
3071 :
3072 0 : error = ahci_poll(ccb, 1000, ahci_pmp_probe_timeout);
3073 0 : ahci_put_pmp_ccb(ccb);
3074 0 : return (error);
3075 0 : }
3076 :
3077 : int
3078 0 : ahci_pmp_phy_status(struct ahci_port *ap, int target, u_int32_t *datap)
3079 : {
3080 : int error;
3081 :
3082 0 : error = ahci_pmp_read(ap, target, SATA_PMREG_SSTS, datap);
3083 0 : if (error == 0)
3084 0 : error = ahci_pmp_write(ap, target, SATA_PMREG_SERR, -1);
3085 0 : if (error)
3086 0 : *datap = 0;
3087 :
3088 0 : return (error);
3089 : }
3090 :
3091 : int
3092 0 : ahci_pmp_identify(struct ahci_port *ap, int *ret_nports)
3093 : {
3094 0 : u_int32_t chipid;
3095 0 : u_int32_t rev;
3096 0 : u_int32_t nports;
3097 0 : u_int32_t features;
3098 0 : u_int32_t enabled;
3099 : int s;
3100 :
3101 0 : s = splbio();
3102 :
3103 0 : if (ahci_pmp_read(ap, 15, 0, &chipid) ||
3104 0 : ahci_pmp_read(ap, 15, 1, &rev) ||
3105 0 : ahci_pmp_read(ap, 15, 2, &nports) ||
3106 0 : ahci_pmp_read(ap, 15, SATA_PMREG_FEA, &features) ||
3107 0 : ahci_pmp_read(ap, 15, SATA_PMREG_FEAEN, &enabled)) {
3108 0 : printf("%s: port multiplier identification failed\n",
3109 0 : PORTNAME(ap));
3110 0 : splx(s);
3111 0 : return (1);
3112 : }
3113 0 : splx(s);
3114 :
3115 0 : nports &= 0x0F;
3116 :
3117 : /* ignore SEMB port on SiI3726 port multiplier chips */
3118 0 : if (chipid == 0x37261095) {
3119 0 : nports--;
3120 0 : }
3121 :
3122 0 : printf("%s: port multiplier found: chip=%08x rev=0x%b nports=%d, "
3123 0 : "features: 0x%b, enabled: 0x%b\n", PORTNAME(ap), chipid, rev,
3124 0 : SATA_PFMT_PM_REV, nports, features, SATA_PFMT_PM_FEA, enabled,
3125 : SATA_PFMT_PM_FEA);
3126 :
3127 0 : *ret_nports = nports;
3128 0 : return (0);
3129 0 : }
3130 :
3131 :
3132 : #ifdef HIBERNATE
3133 : void
3134 0 : ahci_hibernate_io_start(struct ahci_port *ap, struct ahci_ccb *ccb)
3135 : {
3136 0 : ccb->ccb_cmd_hdr->prdbc = 0;
3137 0 : ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
3138 0 : }
3139 :
3140 : int
3141 0 : ahci_hibernate_io_poll(struct ahci_port *ap, struct ahci_ccb *ccb)
3142 : {
3143 : u_int32_t is, ci_saved;
3144 : int process_error = 0;
3145 :
3146 0 : is = ahci_pread(ap, AHCI_PREG_IS);
3147 :
3148 0 : ci_saved = ahci_pread(ap, AHCI_PREG_CI);
3149 :
3150 0 : if (is & AHCI_PREG_IS_DHRS) {
3151 : u_int32_t tfd;
3152 : u_int32_t cmd;
3153 : u_int32_t serr;
3154 :
3155 0 : tfd = ahci_pread(ap, AHCI_PREG_TFD);
3156 0 : cmd = ahci_pread(ap, AHCI_PREG_CMD);
3157 0 : serr = ahci_pread(ap, AHCI_PREG_SERR);
3158 0 : if ((tfd & AHCI_PREG_TFD_STS_ERR) &&
3159 0 : (cmd & AHCI_PREG_CMD_CR) == 0) {
3160 : process_error = 1;
3161 0 : } else {
3162 0 : ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
3163 : }
3164 0 : } else if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
3165 : AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
3166 : process_error = 1;
3167 0 : }
3168 :
3169 : /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
3170 0 : if (process_error) {
3171 :
3172 : /* Turn off ST to clear CI and SACT. */
3173 0 : ahci_port_stop(ap, 0);
3174 :
3175 : /* just return an error indicator? we can't meaningfully
3176 : * recover, and on the way back out we'll DVACT_RESUME which
3177 : * resets and reinits the port.
3178 : */
3179 0 : return (EIO);
3180 : }
3181 :
3182 : /* command is finished when the bit in CI for the slot goes to 0 */
3183 0 : if (ci_saved & (1 << ccb->ccb_slot)) {
3184 0 : return (EAGAIN);
3185 : }
3186 :
3187 0 : return (0);
3188 0 : }
3189 :
3190 : void
3191 0 : ahci_hibernate_load_prdt(struct ahci_ccb *ccb)
3192 : {
3193 0 : struct ata_xfer *xa = &ccb->ccb_xa;
3194 0 : struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt, *prd;
3195 0 : struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
3196 : int i;
3197 0 : paddr_t data_phys;
3198 : u_int64_t data_bus_phys;
3199 : vaddr_t data_addr;
3200 : size_t seglen;
3201 : size_t buflen;
3202 :
3203 0 : if (xa->datalen == 0) {
3204 0 : ccb->ccb_cmd_hdr->prdtl = 0;
3205 0 : return;
3206 : }
3207 :
3208 : /* derived from i386/amd64 _bus_dma_load_buffer;
3209 : * for amd64 the buffer will always be dma safe.
3210 : */
3211 :
3212 : buflen = xa->datalen;
3213 0 : data_addr = (vaddr_t)xa->data;
3214 0 : for (i = 0; buflen > 0; i++) {
3215 0 : prd = &prdt[i];
3216 :
3217 0 : pmap_extract(pmap_kernel(), data_addr, &data_phys);
3218 0 : data_bus_phys = data_phys;
3219 :
3220 0 : seglen = PAGE_SIZE - ((u_long)data_addr & PGOFSET);
3221 0 : if (buflen < seglen)
3222 0 : seglen = buflen;
3223 :
3224 0 : ahci_load_prdt_seg(&prdt[i], data_bus_phys, seglen, 0);
3225 :
3226 0 : data_addr += seglen;
3227 0 : buflen -= seglen;
3228 : }
3229 :
3230 0 : htolem16(&cmd_slot->prdtl, i);
3231 0 : }
3232 :
3233 : int
3234 0 : ahci_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size,
3235 : int op, void *page)
3236 : {
3237 : /* we use the 'real' ahci_port and ahci_softc here, but
3238 : * never write to them
3239 : */
3240 : struct {
3241 : struct ahci_cmd_hdr cmd_hdr[32]; /* page aligned, 1024 bytes */
3242 : struct ahci_rfis rfis; /* 1k aligned, 256 bytes */
3243 : /* cmd table isn't actually used because of mysteries */
3244 : struct ahci_cmd_table cmd_table; /* 256 aligned, 512 bytes */
3245 : struct ahci_port *ap;
3246 : struct ahci_ccb ccb_buf;
3247 : struct ahci_ccb *ccb;
3248 : struct ahci_cmd_hdr *hdr_buf;
3249 : int pmp_port;
3250 : daddr_t poffset;
3251 : size_t psize;
3252 0 : } *my = page;
3253 : struct ata_fis_h2d *fis;
3254 : u_int32_t sector_count;
3255 : struct ahci_cmd_hdr *cmd_slot;
3256 : int rc;
3257 : int timeout;
3258 : u_int16_t flags;
3259 :
3260 0 : if (op == HIB_INIT) {
3261 : struct device *disk;
3262 : struct device *scsibus;
3263 : struct ahci_softc *sc;
3264 : extern struct cfdriver sd_cd;
3265 : struct scsi_link *link;
3266 : struct scsibus_softc *bus_sc;
3267 : int port;
3268 0 : paddr_t page_phys;
3269 : u_int64_t item_phys;
3270 : u_int32_t cmd;
3271 :
3272 0 : my->poffset = blkno;
3273 0 : my->psize = size;
3274 :
3275 : /* map dev to an ahci port */
3276 0 : disk = disk_lookup(&sd_cd, DISKUNIT(dev));
3277 0 : scsibus = disk->dv_parent;
3278 0 : sc = (struct ahci_softc *)disk->dv_parent->dv_parent;
3279 :
3280 : /* find the scsi_link for the device, which has the port */
3281 : port = -1;
3282 0 : bus_sc = (struct scsibus_softc *)scsibus;
3283 0 : SLIST_FOREACH(link, &bus_sc->sc_link_list, bus_list) {
3284 0 : if (link->device_softc == disk) {
3285 : /* link->adapter_softc == sc->sc_atascsi */
3286 0 : port = link->target;
3287 0 : if (link->lun > 0)
3288 0 : my->pmp_port = link->lun - 1;
3289 : else
3290 0 : my->pmp_port = 0;
3291 :
3292 : break;
3293 : }
3294 : }
3295 0 : if (port == -1) {
3296 : /* don't know where the disk is */
3297 0 : return (EIO);
3298 : }
3299 :
3300 0 : my->ap = sc->sc_ports[port];
3301 :
3302 : /* we're going to use the first command slot,
3303 : * so ensure it's not already in use
3304 : */
3305 0 : if (my->ap->ap_ccbs[0].ccb_xa.state != ATA_S_PUT) {
3306 : /* this shouldn't happen, we should be idle */
3307 0 : return (EIO);
3308 : }
3309 :
3310 : /* stop the port so we can relocate to the hibernate page */
3311 0 : if (ahci_port_stop(my->ap, 1)) {
3312 0 : return (EIO);
3313 : }
3314 0 : ahci_pwrite(my->ap, AHCI_PREG_SCTL, 0);
3315 :
3316 0 : pmap_extract(pmap_kernel(), (vaddr_t)page, &page_phys);
3317 :
3318 : /* Setup RFIS base address */
3319 0 : item_phys = page_phys + ((void *)&my->rfis - page);
3320 0 : ahci_pwrite(my->ap, AHCI_PREG_FBU,
3321 0 : (u_int32_t)(item_phys >> 32));
3322 0 : ahci_pwrite(my->ap, AHCI_PREG_FB, (u_int32_t)item_phys);
3323 :
3324 : /* Enable FIS reception and activate port. */
3325 0 : cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
3326 0 : cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD |
3327 : AHCI_PREG_CMD_SUD;
3328 0 : ahci_pwrite(my->ap, AHCI_PREG_CMD, cmd |
3329 : AHCI_PREG_CMD_ICC_ACTIVE);
3330 :
3331 : /* Check whether port activated. */
3332 0 : cmd = ahci_pread(my->ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
3333 0 : if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
3334 0 : return (EIO);
3335 : }
3336 :
3337 : /* Set up the single CCB */
3338 0 : my->ccb = &my->ccb_buf;
3339 0 : my->ccb->ccb_slot = 0;
3340 0 : my->ccb->ccb_port = my->ap;
3341 :
3342 : /* Setup command list base address */
3343 0 : item_phys = page_phys + ((void *)&my->cmd_hdr - page);
3344 0 : ahci_pwrite(my->ap, AHCI_PREG_CLBU,
3345 0 : (u_int32_t)(item_phys >> 32));
3346 0 : ahci_pwrite(my->ap, AHCI_PREG_CLB, (u_int32_t)item_phys);
3347 :
3348 0 : my->ccb->ccb_cmd_hdr = &my->cmd_hdr[0];
3349 :
3350 : /* use existing cmd table - don't know why moving to a new one fails */
3351 0 : my->ccb->ccb_cmd_table = my->ap->ap_ccbs[0].ccb_cmd_table;
3352 0 : pmap_extract(pmap_kernel(),
3353 0 : (vaddr_t)AHCI_DMA_KVA(my->ap->ap_dmamem_cmd_table),
3354 : &page_phys);
3355 0 : item_phys = page_phys;
3356 : #if 0
3357 : /* use cmd table in hibernate page (doesn't work) */
3358 : my->ccb->ccb_cmd_table = &my->cmd_table;
3359 : item_phys = page_phys + ((void *)&my->cmd_table - page);
3360 : #endif
3361 0 : htolem64(&my->ccb->ccb_cmd_hdr->ctba, item_phys);
3362 :
3363 0 : my->ccb->ccb_xa.fis =
3364 0 : (struct ata_fis_h2d *)my->ccb->ccb_cmd_table->cfis;
3365 0 : my->ccb->ccb_xa.packetcmd = my->ccb->ccb_cmd_table->acmd;
3366 0 : my->ccb->ccb_xa.tag = 0;
3367 :
3368 : /* Wait for ICC change to complete */
3369 0 : ahci_pwait_clr(my->ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC, 1);
3370 :
3371 0 : if (ahci_port_start(my->ap, 0)) {
3372 0 : return (EIO);
3373 : }
3374 :
3375 : /* Flush interrupts for port */
3376 0 : ahci_pwrite(my->ap, AHCI_PREG_IS, ahci_pread(my->ap,
3377 : AHCI_PREG_IS));
3378 0 : ahci_write(sc, AHCI_REG_IS, 1 << port);
3379 :
3380 0 : ahci_enable_interrupts(my->ap);
3381 0 : return (0);
3382 0 : } else if (op == HIB_DONE) {
3383 0 : ahci_activate(&my->ap->ap_sc->sc_dev, DVACT_RESUME);
3384 0 : return (0);
3385 : }
3386 :
3387 0 : if (blkno > my->psize)
3388 0 : return (E2BIG);
3389 0 : blkno += my->poffset;
3390 :
3391 : /* build fis */
3392 0 : sector_count = size / 512; /* dlg promises this is okay */
3393 0 : my->ccb->ccb_xa.flags = op == HIB_W ? ATA_F_WRITE : ATA_F_READ;
3394 0 : fis = my->ccb->ccb_xa.fis;
3395 0 : fis->flags = ATA_H2D_FLAGS_CMD | my->pmp_port;
3396 0 : fis->lba_low = blkno & 0xff;
3397 0 : fis->lba_mid = (blkno >> 8) & 0xff;
3398 0 : fis->lba_high = (blkno >> 16) & 0xff;
3399 :
3400 0 : if (sector_count > 0x100 || blkno > 0xfffffff) {
3401 : /* Use LBA48 */
3402 0 : fis->command = op == HIB_W ? ATA_C_WRITEDMA_EXT :
3403 : ATA_C_READDMA_EXT;
3404 0 : fis->device = ATA_H2D_DEVICE_LBA;
3405 0 : fis->lba_low_exp = (blkno >> 24) & 0xff;
3406 0 : fis->lba_mid_exp = (blkno >> 32) & 0xff;
3407 0 : fis->lba_high_exp = (blkno >> 40) & 0xff;
3408 0 : fis->sector_count = sector_count & 0xff;
3409 0 : fis->sector_count_exp = (sector_count >> 8) & 0xff;
3410 0 : } else {
3411 : /* Use LBA */
3412 0 : fis->command = op == HIB_W ? ATA_C_WRITEDMA : ATA_C_READDMA;
3413 0 : fis->device = ATA_H2D_DEVICE_LBA | ((blkno >> 24) & 0x0f);
3414 0 : fis->sector_count = sector_count & 0xff;
3415 : }
3416 :
3417 0 : my->ccb->ccb_xa.data = (void *)addr;
3418 0 : my->ccb->ccb_xa.datalen = size;
3419 0 : my->ccb->ccb_xa.pmp_port = my->pmp_port;
3420 0 : my->ccb->ccb_xa.flags |= ATA_F_POLL;
3421 :
3422 0 : cmd_slot = my->ccb->ccb_cmd_hdr;
3423 : flags = 5; /* FIS length (in DWORDs) */
3424 0 : flags |= my->pmp_port << AHCI_CMD_LIST_FLAG_PMP_SHIFT;
3425 :
3426 0 : if (op == HIB_W)
3427 0 : flags |= AHCI_CMD_LIST_FLAG_W;
3428 :
3429 0 : htolem16(&cmd_slot->flags, flags);
3430 :
3431 0 : ahci_hibernate_load_prdt(my->ccb);
3432 :
3433 0 : ahci_hibernate_io_start(my->ap, my->ccb);
3434 : timeout = 1000000;
3435 0 : while ((rc = ahci_hibernate_io_poll(my->ap, my->ccb)) == EAGAIN) {
3436 0 : delay(1);
3437 0 : timeout--;
3438 0 : if (timeout == 0) {
3439 0 : return (EIO);
3440 : }
3441 : }
3442 :
3443 0 : return (0);
3444 0 : }
3445 :
3446 : #endif
|
