Line data Source code
1 : /* $OpenBSD: safe.c,v 1.43 2018/04/28 15:44:59 jasper Exp $ */
2 :
3 : /*-
4 : * Copyright (c) 2003 Sam Leffler, Errno Consulting
5 : * Copyright (c) 2003 Global Technology Associates, Inc.
6 : * All rights reserved.
7 : *
8 : * Redistribution and use in source and binary forms, with or without
9 : * modification, are permitted provided that the following conditions
10 : * are met:
11 : * 1. Redistributions of source code must retain the above copyright
12 : * notice, this list of conditions and the following disclaimer.
13 : * 2. Redistributions in binary form must reproduce the above copyright
14 : * notice, this list of conditions and the following disclaimer in the
15 : * documentation and/or other materials provided with the distribution.
16 : *
17 : * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 : * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 : * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 : * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 : * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 : * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 : * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 : * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 : * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 : * SUCH DAMAGE.
28 : *
29 : * $FreeBSD: /repoman/r/ncvs/src/sys/dev/safe/safe.c,v 1.1 2003/07/21 21:46:07 sam Exp $
30 : */
31 :
32 : /*
33 : * SafeNet SafeXcel-1141 hardware crypto accelerator
34 : */
35 : #include <sys/param.h>
36 : #include <sys/systm.h>
37 : #include <sys/errno.h>
38 : #include <sys/malloc.h>
39 : #include <sys/mbuf.h>
40 : #include <sys/device.h>
41 : #include <sys/timeout.h>
42 :
43 : #include <machine/bus.h>
44 :
45 : #include <crypto/md5.h>
46 : #include <crypto/sha1.h>
47 : #include <crypto/cryptodev.h>
48 : #include <crypto/cryptosoft.h>
49 : #include <dev/rndvar.h>
50 :
51 : #include <dev/pci/pcivar.h>
52 : #include <dev/pci/pcireg.h>
53 : #include <dev/pci/pcidevs.h>
54 :
55 : #include <dev/pci/safereg.h>
56 : #include <dev/pci/safevar.h>
57 :
58 : #ifndef bswap32
59 : #define bswap32(x) (x) = ntohl((u_int32_t)(x))
60 : #endif
61 :
62 : #define KASSERT_X(x,y)
63 :
64 : /*
65 : * Prototypes and count for the pci_device structure
66 : */
67 : int safe_probe(struct device *, void *, void *);
68 : void safe_attach(struct device *, struct device *, void *);
69 :
70 : struct cfattach safe_ca = {
71 : sizeof(struct safe_softc), safe_probe, safe_attach
72 : };
73 :
74 : struct cfdriver safe_cd = {
75 : 0, "safe", DV_DULL
76 : };
77 :
78 : int safe_intr(void *);
79 : int safe_newsession(u_int32_t *, struct cryptoini *);
80 : int safe_freesession(u_int64_t);
81 : int safe_process(struct cryptop *);
82 : void safe_callback(struct safe_softc *, struct safe_ringentry *);
83 : void safe_feed(struct safe_softc *, struct safe_ringentry *);
84 : void safe_mcopy(struct mbuf *, struct mbuf *, u_int);
85 : void safe_rng_init(struct safe_softc *);
86 : void safe_rng(void *);
87 : int safe_dma_malloc(struct safe_softc *, bus_size_t,
88 : struct safe_dma_alloc *, int);
89 : #define safe_dma_sync(_sc, _dma, _flags) \
90 : bus_dmamap_sync((_sc)->sc_dmat, (_dma)->dma_map, 0, \
91 : (_dma)->dma_map->dm_mapsize, (_flags))
92 : void safe_dma_free(struct safe_softc *, struct safe_dma_alloc *);
93 : int safe_dmamap_aligned(const struct safe_operand *);
94 : int safe_dmamap_uniform(const struct safe_operand *);
95 :
96 : void safe_reset_board(struct safe_softc *);
97 : void safe_init_board(struct safe_softc *);
98 : void safe_init_pciregs(struct safe_softc *);
99 : void safe_cleanchip(struct safe_softc *);
100 : static __inline u_int32_t safe_rng_read(struct safe_softc *);
101 :
102 : int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
103 :
104 : #ifdef SAFE_DEBUG
105 : int safe_debug;
106 : #define DPRINTF(_x) if (safe_debug) printf _x
107 :
108 : void safe_dump_dmastatus(struct safe_softc *, const char *);
109 : void safe_dump_intrstate(struct safe_softc *, const char *);
110 : void safe_dump_ringstate(struct safe_softc *, const char *);
111 : void safe_dump_request(struct safe_softc *, const char *,
112 : struct safe_ringentry *);
113 : void safe_dump_ring(struct safe_softc *sc, const char *tag);
114 : #else
115 : #define DPRINTF(_x)
116 : #endif
117 :
118 : #define READ_REG(sc,r) \
119 : bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
120 :
121 : #define WRITE_REG(sc,reg,val) \
122 : bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
123 :
124 : struct safe_stats safestats;
125 :
126 : int safe_rnginterval = 1; /* poll once a second */
127 : int safe_rngbufsize = 16; /* 64 bytes each poll */
128 : int safe_rngmaxalarm = 8; /* max alarms before reset */
129 :
130 : int
131 0 : safe_probe(struct device *parent, void *match, void *aux)
132 : {
133 0 : struct pci_attach_args *pa = aux;
134 :
135 0 : if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_SAFENET &&
136 0 : PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_SAFENET_SAFEXCEL)
137 0 : return (1);
138 0 : return (0);
139 0 : }
140 :
141 : void
142 0 : safe_attach(struct device *parent, struct device *self, void *aux)
143 : {
144 0 : struct safe_softc *sc = (struct safe_softc *)self;
145 0 : struct pci_attach_args *pa = aux;
146 0 : pci_intr_handle_t ih;
147 : const char *intrstr = NULL;
148 0 : bus_size_t iosize;
149 : bus_addr_t raddr;
150 : u_int32_t devinfo;
151 0 : int algs[CRYPTO_ALGORITHM_MAX + 1], i;
152 :
153 : /* XXX handle power management */
154 :
155 0 : sc->sc_dmat = pa->pa_dmat;
156 :
157 : /*
158 : * Setup memory-mapping of PCI registers.
159 : */
160 0 : if (pci_mapreg_map(pa, SAFE_BAR, PCI_MAPREG_TYPE_MEM, 0,
161 0 : &sc->sc_st, &sc->sc_sh, NULL, &iosize, 0)) {
162 0 : printf(": can't map register space\n");
163 0 : goto bad;
164 : }
165 :
166 0 : if (pci_intr_map(pa, &ih)) {
167 0 : printf(": couldn't map interrupt\n");
168 0 : goto bad1;
169 : }
170 0 : intrstr = pci_intr_string(pa->pa_pc, ih);
171 0 : sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_NET, safe_intr, sc,
172 0 : self->dv_xname);
173 0 : if (sc->sc_ih == NULL) {
174 0 : printf(": couldn't establish interrupt");
175 0 : if (intrstr != NULL)
176 0 : printf(" at %s", intrstr);
177 0 : printf("\n");
178 0 : goto bad2;
179 : }
180 :
181 0 : sc->sc_cid = crypto_get_driverid(0);
182 0 : if (sc->sc_cid < 0) {
183 0 : printf(": could not get crypto driver id\n");
184 0 : goto bad3;
185 : }
186 :
187 0 : sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
188 : (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
189 :
190 : /*
191 : * Allocate packet engine descriptors.
192 : */
193 0 : if (safe_dma_malloc(sc,
194 : SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
195 0 : &sc->sc_ringalloc, 0)) {
196 0 : printf(": cannot allocate PE descriptor ring\n");
197 0 : goto bad4;
198 : }
199 : /*
200 : * Hookup the static portion of all our data structures.
201 : */
202 0 : sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
203 0 : sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
204 0 : sc->sc_front = sc->sc_ring;
205 0 : sc->sc_back = sc->sc_ring;
206 0 : raddr = sc->sc_ringalloc.dma_paddr;
207 0 : bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
208 0 : for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
209 0 : struct safe_ringentry *re = &sc->sc_ring[i];
210 :
211 0 : re->re_desc.d_sa = raddr +
212 : offsetof(struct safe_ringentry, re_sa);
213 0 : re->re_sa.sa_staterec = raddr +
214 : offsetof(struct safe_ringentry, re_sastate);
215 :
216 0 : raddr += sizeof (struct safe_ringentry);
217 : }
218 :
219 : /*
220 : * Allocate scatter and gather particle descriptors.
221 : */
222 0 : if (safe_dma_malloc(sc, SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
223 0 : &sc->sc_spalloc, 0)) {
224 0 : printf(": cannot allocate source particle descriptor ring\n");
225 0 : safe_dma_free(sc, &sc->sc_ringalloc);
226 0 : goto bad4;
227 : }
228 0 : sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
229 0 : sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
230 0 : sc->sc_spfree = sc->sc_spring;
231 0 : bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
232 :
233 0 : if (safe_dma_malloc(sc, SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
234 0 : &sc->sc_dpalloc, 0)) {
235 0 : printf(": cannot allocate destination particle "
236 : "descriptor ring\n");
237 0 : safe_dma_free(sc, &sc->sc_spalloc);
238 0 : safe_dma_free(sc, &sc->sc_ringalloc);
239 0 : goto bad4;
240 : }
241 0 : sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
242 0 : sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
243 0 : sc->sc_dpfree = sc->sc_dpring;
244 0 : bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
245 :
246 0 : printf(":");
247 :
248 0 : devinfo = READ_REG(sc, SAFE_DEVINFO);
249 0 : if (devinfo & SAFE_DEVINFO_RNG)
250 0 : printf(" RNG");
251 :
252 0 : bzero(algs, sizeof(algs));
253 0 : if (devinfo & SAFE_DEVINFO_DES) {
254 0 : printf(" 3DES");
255 0 : algs[CRYPTO_3DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
256 0 : }
257 0 : if (devinfo & SAFE_DEVINFO_AES) {
258 0 : printf(" AES");
259 0 : algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
260 0 : }
261 0 : if (devinfo & SAFE_DEVINFO_MD5) {
262 0 : printf(" MD5");
263 0 : algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
264 0 : }
265 0 : if (devinfo & SAFE_DEVINFO_SHA1) {
266 0 : printf(" SHA1");
267 0 : algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
268 0 : }
269 0 : crypto_register(sc->sc_cid, algs, safe_newsession,
270 : safe_freesession, safe_process);
271 : /* XXX other supported algorithms? */
272 :
273 0 : printf(", %s\n", intrstr);
274 :
275 0 : safe_reset_board(sc); /* reset h/w */
276 0 : safe_init_pciregs(sc); /* init pci settings */
277 0 : safe_init_board(sc); /* init h/w */
278 :
279 0 : if (devinfo & SAFE_DEVINFO_RNG) {
280 0 : safe_rng_init(sc);
281 :
282 0 : timeout_set(&sc->sc_rngto, safe_rng, sc);
283 0 : timeout_add_sec(&sc->sc_rngto, safe_rnginterval);
284 0 : }
285 0 : return;
286 :
287 : bad4:
288 : /* XXX crypto_unregister_all(sc->sc_cid); */
289 : bad3:
290 0 : pci_intr_disestablish(pa->pa_pc, sc->sc_ih);
291 : bad2:
292 : /* pci_intr_unmap? */;
293 : bad1:
294 0 : bus_space_unmap(sc->sc_st, sc->sc_sh, iosize);
295 : bad:
296 0 : return;
297 0 : }
298 :
299 : int
300 0 : safe_process(struct cryptop *crp)
301 : {
302 : int err = 0, i, nicealign, uniform, s;
303 : struct safe_softc *sc;
304 : struct cryptodesc *crd1, *crd2 = NULL, *maccrd, *enccrd;
305 : int bypass, oplen, ivsize, card;
306 : int16_t coffset;
307 : struct safe_session *ses;
308 : struct safe_ringentry *re;
309 : struct safe_sarec *sa;
310 : struct safe_pdesc *pd;
311 0 : u_int32_t cmd0, cmd1, staterec, iv[4];
312 :
313 0 : s = splnet();
314 0 : if (crp == NULL || crp->crp_callback == NULL) {
315 0 : safestats.st_invalid++;
316 0 : splx(s);
317 0 : return (EINVAL);
318 : }
319 0 : card = SAFE_CARD(crp->crp_sid);
320 0 : if (card >= safe_cd.cd_ndevs || safe_cd.cd_devs[card] == NULL) {
321 0 : safestats.st_invalid++;
322 0 : splx(s);
323 0 : return (EINVAL);
324 : }
325 0 : sc = safe_cd.cd_devs[card];
326 :
327 0 : if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
328 0 : safestats.st_badsession++;
329 0 : splx(s);
330 0 : return (EINVAL);
331 : }
332 :
333 0 : if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
334 0 : safestats.st_ringfull++;
335 0 : splx(s);
336 0 : return (ERESTART);
337 : }
338 0 : re = sc->sc_front;
339 :
340 0 : staterec = re->re_sa.sa_staterec; /* save */
341 : /* NB: zero everything but the PE descriptor */
342 0 : bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
343 0 : re->re_sa.sa_staterec = staterec; /* restore */
344 :
345 0 : re->re_crp = crp;
346 0 : re->re_sesn = SAFE_SESSION(crp->crp_sid);
347 :
348 0 : if (crp->crp_flags & CRYPTO_F_IMBUF) {
349 0 : re->re_src_m = (struct mbuf *)crp->crp_buf;
350 0 : re->re_dst_m = (struct mbuf *)crp->crp_buf;
351 0 : } else if (crp->crp_flags & CRYPTO_F_IOV) {
352 0 : re->re_src_io = (struct uio *)crp->crp_buf;
353 0 : re->re_dst_io = (struct uio *)crp->crp_buf;
354 : } else {
355 0 : safestats.st_badflags++;
356 : err = EINVAL;
357 0 : goto errout; /* XXX we don't handle contiguous blocks! */
358 : }
359 :
360 : sa = &re->re_sa;
361 0 : ses = &sc->sc_sessions[re->re_sesn];
362 :
363 0 : if (crp->crp_ndesc < 1) {
364 0 : safestats.st_nodesc++;
365 : err = EINVAL;
366 0 : goto errout;
367 : }
368 0 : crd1 = &crp->crp_desc[0];
369 0 : if (crp->crp_ndesc >= 2)
370 0 : crd2 = &crp->crp_desc[1];
371 :
372 : cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
373 : cmd1 = 0;
374 0 : if (crd2 == NULL) {
375 0 : if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
376 0 : crd1->crd_alg == CRYPTO_SHA1_HMAC) {
377 : maccrd = crd1;
378 : enccrd = NULL;
379 : cmd0 |= SAFE_SA_CMD0_OP_HASH;
380 0 : } else if (crd1->crd_alg == CRYPTO_3DES_CBC ||
381 0 : crd1->crd_alg == CRYPTO_AES_CBC) {
382 : maccrd = NULL;
383 : enccrd = crd1;
384 : cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
385 : } else {
386 0 : safestats.st_badalg++;
387 : err = EINVAL;
388 0 : goto errout;
389 : }
390 : } else {
391 0 : if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
392 0 : crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
393 0 : (crd2->crd_alg == CRYPTO_3DES_CBC ||
394 0 : crd2->crd_alg == CRYPTO_AES_CBC) &&
395 0 : ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
396 : maccrd = crd1;
397 : enccrd = crd2;
398 0 : } else if ((crd1->crd_alg == CRYPTO_3DES_CBC ||
399 0 : crd1->crd_alg == CRYPTO_AES_CBC) &&
400 0 : (crd2->crd_alg == CRYPTO_MD5_HMAC ||
401 0 : crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
402 0 : (crd1->crd_flags & CRD_F_ENCRYPT)) {
403 : enccrd = crd1;
404 : maccrd = crd2;
405 : } else {
406 0 : safestats.st_badalg++;
407 : err = EINVAL;
408 0 : goto errout;
409 : }
410 : cmd0 |= SAFE_SA_CMD0_OP_BOTH;
411 : }
412 :
413 0 : if (enccrd) {
414 0 : if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
415 0 : cmd0 |= SAFE_SA_CMD0_3DES;
416 : cmd1 |= SAFE_SA_CMD1_CBC;
417 : ivsize = 2*sizeof(u_int32_t);
418 0 : } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
419 0 : cmd0 |= SAFE_SA_CMD0_AES;
420 : cmd1 |= SAFE_SA_CMD1_CBC;
421 0 : if (ses->ses_klen == 128)
422 0 : cmd1 |= SAFE_SA_CMD1_AES128;
423 0 : else if (ses->ses_klen == 192)
424 0 : cmd1 |= SAFE_SA_CMD1_AES192;
425 : else
426 : cmd1 |= SAFE_SA_CMD1_AES256;
427 : ivsize = 4*sizeof(u_int32_t);
428 0 : } else {
429 0 : cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
430 : ivsize = 0;
431 : }
432 :
433 : /*
434 : * Setup encrypt/decrypt state. When using basic ops
435 : * we can't use an inline IV because hash/crypt offset
436 : * must be from the end of the IV to the start of the
437 : * crypt data and this leaves out the preceding header
438 : * from the hash calculation. Instead we place the IV
439 : * in the state record and set the hash/crypt offset to
440 : * copy both the header+IV.
441 : */
442 0 : if (enccrd->crd_flags & CRD_F_ENCRYPT) {
443 : cmd0 |= SAFE_SA_CMD0_OUTBOUND;
444 :
445 0 : if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
446 0 : bcopy(enccrd->crd_iv, iv, ivsize);
447 : else
448 0 : arc4random_buf(iv, ivsize);
449 :
450 0 : if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
451 0 : if (crp->crp_flags & CRYPTO_F_IMBUF)
452 0 : err = m_copyback(re->re_src_m,
453 0 : enccrd->crd_inject, ivsize, iv,
454 : M_NOWAIT);
455 0 : else if (crp->crp_flags & CRYPTO_F_IOV)
456 0 : cuio_copyback(re->re_src_io,
457 0 : enccrd->crd_inject, ivsize, iv);
458 0 : if (err)
459 : goto errout;
460 : }
461 0 : for (i = 0; i < ivsize / sizeof(iv[0]); i++)
462 0 : re->re_sastate.sa_saved_iv[i] = htole32(iv[i]);
463 0 : cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
464 0 : } else {
465 0 : cmd0 |= SAFE_SA_CMD0_INBOUND;
466 :
467 0 : if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
468 0 : bcopy(enccrd->crd_iv, iv, ivsize);
469 0 : else if (crp->crp_flags & CRYPTO_F_IMBUF)
470 0 : m_copydata(re->re_src_m, enccrd->crd_inject,
471 0 : ivsize, (caddr_t)iv);
472 0 : else if (crp->crp_flags & CRYPTO_F_IOV)
473 0 : cuio_copydata(re->re_src_io, enccrd->crd_inject,
474 0 : ivsize, (caddr_t)iv);
475 0 : for (i = 0; i < ivsize / sizeof(iv[0]); i++)
476 0 : re->re_sastate.sa_saved_iv[i] = htole32(iv[i]);
477 0 : cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
478 : }
479 : /*
480 : * For basic encryption use the zero pad algorithm.
481 : * This pads results to an 8-byte boundary and
482 : * suppresses padding verification for inbound (i.e.
483 : * decrypt) operations.
484 : *
485 : * NB: Not sure if the 8-byte pad boundary is a problem.
486 : */
487 0 : cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
488 :
489 : /* XXX assert key bufs have the same size */
490 0 : for (i = 0; i < nitems(sa->sa_key); i++)
491 0 : sa->sa_key[i] = ses->ses_key[i];
492 : }
493 :
494 0 : if (maccrd) {
495 0 : if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
496 : cmd0 |= SAFE_SA_CMD0_MD5;
497 0 : cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
498 0 : } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
499 0 : cmd0 |= SAFE_SA_CMD0_SHA1;
500 0 : cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
501 0 : } else {
502 0 : cmd0 |= SAFE_SA_CMD0_HASH_NULL;
503 : }
504 : /*
505 : * Digest data is loaded from the SA and the hash
506 : * result is saved to the state block where we
507 : * retrieve it for return to the caller.
508 : */
509 : /* XXX assert digest bufs have the same size */
510 0 : for (i = 0;
511 0 : i < nitems(sa->sa_outdigest);
512 0 : i++) {
513 0 : sa->sa_indigest[i] = ses->ses_hminner[i];
514 0 : sa->sa_outdigest[i] = ses->ses_hmouter[i];
515 : }
516 :
517 0 : cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
518 0 : re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
519 0 : }
520 :
521 0 : if (enccrd && maccrd) {
522 : /*
523 : * The offset from hash data to the start of
524 : * crypt data is the difference in the skips.
525 : */
526 0 : bypass = maccrd->crd_skip;
527 0 : coffset = enccrd->crd_skip - maccrd->crd_skip;
528 0 : if (coffset < 0) {
529 : DPRINTF(("%s: hash does not precede crypt; "
530 : "mac skip %u enc skip %u\n",
531 : __func__, maccrd->crd_skip, enccrd->crd_skip));
532 0 : safestats.st_skipmismatch++;
533 : err = EINVAL;
534 0 : goto errout;
535 : }
536 0 : oplen = enccrd->crd_skip + enccrd->crd_len;
537 0 : if (maccrd->crd_skip + maccrd->crd_len != oplen) {
538 : DPRINTF(("%s: hash amount %u != crypt amount %u\n",
539 : __func__, maccrd->crd_skip + maccrd->crd_len,
540 : oplen));
541 0 : safestats.st_lenmismatch++;
542 : err = EINVAL;
543 0 : goto errout;
544 : }
545 : #ifdef SAFE_DEBUG
546 : if (safe_debug) {
547 : printf("mac: skip %d, len %d, inject %d\n",
548 : maccrd->crd_skip, maccrd->crd_len,
549 : maccrd->crd_inject);
550 : printf("enc: skip %d, len %d, inject %d\n",
551 : enccrd->crd_skip, enccrd->crd_len,
552 : enccrd->crd_inject);
553 : printf("bypass %d coffset %d oplen %d\n",
554 : bypass, coffset, oplen);
555 : }
556 : #endif
557 0 : if (coffset & 3) { /* offset must be 32-bit aligned */
558 : DPRINTF(("%s: coffset %u misaligned\n",
559 : __func__, coffset));
560 0 : safestats.st_coffmisaligned++;
561 : err = EINVAL;
562 0 : goto errout;
563 : }
564 0 : coffset >>= 2;
565 0 : if (coffset > 255) { /* offset must be <256 dwords */
566 : DPRINTF(("%s: coffset %u too big\n",
567 : __func__, coffset));
568 0 : safestats.st_cofftoobig++;
569 : err = EINVAL;
570 0 : goto errout;
571 : }
572 : /*
573 : * Tell the hardware to copy the header to the output.
574 : * The header is defined as the data from the end of
575 : * the bypass to the start of data to be encrypted.
576 : * Typically this is the inline IV. Note that you need
577 : * to do this even if src+dst are the same; it appears
578 : * that w/o this bit the crypted data is written
579 : * immediately after the bypass data.
580 : */
581 0 : cmd1 |= SAFE_SA_CMD1_HDRCOPY;
582 : /*
583 : * Disable IP header mutable bit handling. This is
584 : * needed to get correct HMAC calculations.
585 : */
586 0 : cmd1 |= SAFE_SA_CMD1_MUTABLE;
587 0 : } else {
588 0 : if (enccrd) {
589 0 : bypass = enccrd->crd_skip;
590 0 : oplen = bypass + enccrd->crd_len;
591 0 : } else {
592 0 : bypass = maccrd->crd_skip;
593 0 : oplen = bypass + maccrd->crd_len;
594 : }
595 : coffset = 0;
596 : }
597 : /* XXX verify multiple of 4 when using s/g */
598 0 : if (bypass > 96) { /* bypass offset must be <= 96 bytes */
599 : DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
600 0 : safestats.st_bypasstoobig++;
601 : err = EINVAL;
602 0 : goto errout;
603 : }
604 :
605 0 : if (bus_dmamap_create(sc->sc_dmat, SAFE_MAX_DMA, SAFE_MAX_PART,
606 : SAFE_MAX_DSIZE, SAFE_MAX_DSIZE, BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
607 : &re->re_src_map)) {
608 0 : safestats.st_nomap++;
609 : err = ENOMEM;
610 0 : goto errout;
611 : }
612 0 : if (crp->crp_flags & CRYPTO_F_IMBUF) {
613 0 : if (bus_dmamap_load_mbuf(sc->sc_dmat, re->re_src_map,
614 : re->re_src_m, BUS_DMA_NOWAIT)) {
615 0 : bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
616 0 : re->re_src_map = NULL;
617 0 : safestats.st_noload++;
618 : err = ENOMEM;
619 0 : goto errout;
620 : }
621 0 : } else if (crp->crp_flags & CRYPTO_F_IOV) {
622 0 : if (bus_dmamap_load_uio(sc->sc_dmat, re->re_src_map,
623 0 : re->re_src_io, BUS_DMA_NOWAIT) != 0) {
624 0 : bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
625 0 : re->re_src_map = NULL;
626 0 : safestats.st_noload++;
627 : err = ENOMEM;
628 0 : goto errout;
629 : }
630 : }
631 0 : nicealign = safe_dmamap_aligned(&re->re_src);
632 0 : uniform = safe_dmamap_uniform(&re->re_src);
633 :
634 : DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
635 : nicealign, uniform, re->re_src_nsegs));
636 0 : if (re->re_src_nsegs > 1) {
637 0 : re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
638 0 : ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
639 0 : for (i = 0; i < re->re_src_nsegs; i++) {
640 : /* NB: no need to check if there's space */
641 0 : pd = sc->sc_spfree;
642 0 : if (++(sc->sc_spfree) == sc->sc_springtop)
643 0 : sc->sc_spfree = sc->sc_spring;
644 :
645 : KASSERT_X((pd->pd_flags&3) == 0 ||
646 : (pd->pd_flags&3) == SAFE_PD_DONE,
647 : ("bogus source particle descriptor; flags %x",
648 : pd->pd_flags));
649 0 : pd->pd_addr = re->re_src_segs[i].ds_addr;
650 0 : pd->pd_ctrl = SAFE_PD_READY |
651 0 : ((re->re_src_segs[i].ds_len << SAFE_PD_LEN_S)
652 0 : & SAFE_PD_LEN_M);
653 : }
654 0 : cmd0 |= SAFE_SA_CMD0_IGATHER;
655 0 : } else {
656 : /*
657 : * No need for gather, reference the operand directly.
658 : */
659 0 : re->re_desc.d_src = re->re_src_segs[0].ds_addr;
660 : }
661 :
662 0 : if (enccrd == NULL && maccrd != NULL) {
663 : /*
664 : * Hash op; no destination needed.
665 : */
666 : } else {
667 0 : if (crp->crp_flags & CRYPTO_F_IOV) {
668 0 : if (!nicealign) {
669 0 : safestats.st_iovmisaligned++;
670 : err = EINVAL;
671 0 : goto errout;
672 : }
673 0 : if (uniform != 1) {
674 : /*
675 : * Source is not suitable for direct use as
676 : * the destination. Create a new scatter/gather
677 : * list based on the destination requirements
678 : * and check if that's ok.
679 : */
680 0 : if (bus_dmamap_create(sc->sc_dmat,
681 : SAFE_MAX_DMA, SAFE_MAX_PART,
682 : SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
683 : BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
684 : &re->re_dst_map)) {
685 0 : safestats.st_nomap++;
686 : err = ENOMEM;
687 0 : goto errout;
688 : }
689 0 : if (bus_dmamap_load_uio(sc->sc_dmat,
690 : re->re_dst_map, re->re_dst_io,
691 0 : BUS_DMA_NOWAIT) != 0) {
692 0 : bus_dmamap_destroy(sc->sc_dmat,
693 : re->re_dst_map);
694 0 : re->re_dst_map = NULL;
695 0 : safestats.st_noload++;
696 : err = ENOMEM;
697 0 : goto errout;
698 : }
699 0 : uniform = safe_dmamap_uniform(&re->re_dst);
700 0 : if (!uniform) {
701 : /*
702 : * There's no way to handle the DMA
703 : * requirements with this uio. We
704 : * could create a separate DMA area for
705 : * the result and then copy it back,
706 : * but for now we just bail and return
707 : * an error. Note that uio requests
708 : * > SAFE_MAX_DSIZE are handled because
709 : * the DMA map and segment list for the
710 : * destination will result in a
711 : * destination particle list that does
712 : * the necessary scatter DMA.
713 : */
714 0 : safestats.st_iovnotuniform++;
715 : err = EINVAL;
716 0 : goto errout;
717 : }
718 : } else
719 0 : re->re_dst = re->re_src;
720 0 : } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
721 0 : if (nicealign && uniform == 1) {
722 : /*
723 : * Source layout is suitable for direct
724 : * sharing of the DMA map and segment list.
725 : */
726 0 : re->re_dst = re->re_src;
727 0 : } else if (nicealign && uniform == 2) {
728 : /*
729 : * The source is properly aligned but requires a
730 : * different particle list to handle DMA of the
731 : * result. Create a new map and do the load to
732 : * create the segment list. The particle
733 : * descriptor setup code below will handle the
734 : * rest.
735 : */
736 0 : if (bus_dmamap_create(sc->sc_dmat,
737 : SAFE_MAX_DMA, SAFE_MAX_PART,
738 : SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
739 : BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
740 : &re->re_dst_map)) {
741 0 : safestats.st_nomap++;
742 : err = ENOMEM;
743 0 : goto errout;
744 : }
745 0 : if (bus_dmamap_load_mbuf(sc->sc_dmat,
746 : re->re_dst_map, re->re_dst_m,
747 0 : BUS_DMA_NOWAIT) != 0) {
748 0 : bus_dmamap_destroy(sc->sc_dmat,
749 : re->re_dst_map);
750 0 : re->re_dst_map = NULL;
751 0 : safestats.st_noload++;
752 : err = ENOMEM;
753 0 : goto errout;
754 : }
755 : } else { /* !(aligned and/or uniform) */
756 : int totlen, len;
757 0 : struct mbuf *m, *top, **mp;
758 :
759 : /*
760 : * DMA constraints require that we allocate a
761 : * new mbuf chain for the destination. We
762 : * allocate an entire new set of mbufs of
763 : * optimal/required size and then tell the
764 : * hardware to copy any bits that are not
765 : * created as a byproduct of the operation.
766 : */
767 0 : if (!nicealign)
768 0 : safestats.st_unaligned++;
769 0 : if (!uniform)
770 0 : safestats.st_notuniform++;
771 0 : totlen = re->re_src_mapsize;
772 0 : if (re->re_src_m->m_flags & M_PKTHDR) {
773 : len = MHLEN;
774 0 : MGETHDR(m, M_DONTWAIT, MT_DATA);
775 0 : } else {
776 : len = MLEN;
777 0 : MGET(m, M_DONTWAIT, MT_DATA);
778 : }
779 0 : if (m == NULL) {
780 0 : safestats.st_nombuf++;
781 0 : err = sc->sc_nqchip ? ERESTART : ENOMEM;
782 0 : goto errout;
783 : }
784 0 : if (len == MHLEN) {
785 0 : err = m_dup_pkthdr(m, re->re_src_m,
786 : M_DONTWAIT);
787 0 : if (err) {
788 0 : m_free(m);
789 0 : goto errout;
790 : }
791 : }
792 0 : if (totlen >= MINCLSIZE) {
793 0 : MCLGET(m, M_DONTWAIT);
794 0 : if ((m->m_flags & M_EXT) == 0) {
795 0 : m_free(m);
796 0 : safestats.st_nomcl++;
797 0 : err = sc->sc_nqchip ?
798 : ERESTART : ENOMEM;
799 0 : goto errout;
800 : }
801 : len = MCLBYTES;
802 0 : }
803 0 : m->m_len = len;
804 0 : top = NULL;
805 : mp = ⊤
806 :
807 0 : while (totlen > 0) {
808 0 : if (top) {
809 0 : MGET(m, M_DONTWAIT, MT_DATA);
810 0 : if (m == NULL) {
811 0 : m_freem(top);
812 0 : safestats.st_nombuf++;
813 0 : err = sc->sc_nqchip ?
814 : ERESTART : ENOMEM;
815 0 : goto errout;
816 : }
817 : len = MLEN;
818 0 : }
819 0 : if (top && totlen >= MINCLSIZE) {
820 0 : MCLGET(m, M_DONTWAIT);
821 0 : if ((m->m_flags & M_EXT) == 0) {
822 0 : *mp = m;
823 0 : m_freem(top);
824 0 : safestats.st_nomcl++;
825 0 : err = sc->sc_nqchip ?
826 : ERESTART : ENOMEM;
827 0 : goto errout;
828 : }
829 : len = MCLBYTES;
830 0 : }
831 0 : m->m_len = len = min(totlen, len);
832 0 : totlen -= len;
833 0 : *mp = m;
834 0 : mp = &m->m_next;
835 : }
836 0 : re->re_dst_m = top;
837 0 : if (bus_dmamap_create(sc->sc_dmat,
838 : SAFE_MAX_DMA, SAFE_MAX_PART,
839 : SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
840 : BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
841 0 : &re->re_dst_map) != 0) {
842 0 : safestats.st_nomap++;
843 : err = ENOMEM;
844 0 : goto errout;
845 : }
846 0 : if (bus_dmamap_load_mbuf(sc->sc_dmat,
847 : re->re_dst_map, re->re_dst_m,
848 0 : BUS_DMA_NOWAIT) != 0) {
849 0 : bus_dmamap_destroy(sc->sc_dmat,
850 : re->re_dst_map);
851 0 : re->re_dst_map = NULL;
852 0 : safestats.st_noload++;
853 : err = ENOMEM;
854 0 : goto errout;
855 : }
856 0 : if (re->re_src_mapsize > oplen) {
857 : /*
858 : * There's data following what the
859 : * hardware will copy for us. If this
860 : * isn't just the ICV (that's going to
861 : * be written on completion), copy it
862 : * to the new mbufs
863 : */
864 0 : if (!(maccrd &&
865 0 : (re->re_src_mapsize-oplen) == 12 &&
866 0 : maccrd->crd_inject == oplen))
867 0 : safe_mcopy(re->re_src_m,
868 0 : re->re_dst_m,
869 : oplen);
870 : else
871 0 : safestats.st_noicvcopy++;
872 : }
873 0 : }
874 : } else {
875 0 : safestats.st_badflags++;
876 : err = EINVAL;
877 0 : goto errout;
878 : }
879 :
880 0 : if (re->re_dst_nsegs > 1) {
881 0 : re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
882 0 : ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
883 0 : for (i = 0; i < re->re_dst_nsegs; i++) {
884 0 : pd = sc->sc_dpfree;
885 : KASSERT_X((pd->pd_flags&3) == 0 ||
886 : (pd->pd_flags&3) == SAFE_PD_DONE,
887 : ("bogus dest particle descriptor; flags %x",
888 : pd->pd_flags));
889 0 : if (++(sc->sc_dpfree) == sc->sc_dpringtop)
890 0 : sc->sc_dpfree = sc->sc_dpring;
891 0 : pd->pd_addr = re->re_dst_segs[i].ds_addr;
892 0 : pd->pd_ctrl = SAFE_PD_READY;
893 : }
894 0 : cmd0 |= SAFE_SA_CMD0_OSCATTER;
895 0 : } else {
896 : /*
897 : * No need for scatter, reference the operand directly.
898 : */
899 0 : re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
900 : }
901 : }
902 :
903 : /*
904 : * All done with setup; fillin the SA command words
905 : * and the packet engine descriptor. The operation
906 : * is now ready for submission to the hardware.
907 : */
908 0 : sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
909 0 : sa->sa_cmd1 = cmd1
910 0 : | (coffset << SAFE_SA_CMD1_OFFSET_S)
911 0 : | SAFE_SA_CMD1_SAREV1 /* Rev 1 SA data structure */
912 0 : | SAFE_SA_CMD1_SRPCI;
913 :
914 : /*
915 : * NB: the order of writes is important here. In case the
916 : * chip is scanning the ring because of an outstanding request
917 : * it might nab this one too. In that case we need to make
918 : * sure the setup is complete before we write the length
919 : * field of the descriptor as it signals the descriptor is
920 : * ready for processing.
921 : */
922 0 : re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
923 0 : if (maccrd)
924 0 : re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
925 0 : re->re_desc.d_len = oplen
926 0 : | SAFE_PE_LEN_READY
927 0 : | (bypass << SAFE_PE_LEN_BYPASS_S)
928 : ;
929 :
930 0 : safestats.st_ipackets++;
931 0 : safestats.st_ibytes += oplen;
932 :
933 0 : if (++(sc->sc_front) == sc->sc_ringtop)
934 0 : sc->sc_front = sc->sc_ring;
935 :
936 : /* XXX honor batching */
937 0 : safe_feed(sc, re);
938 0 : splx(s);
939 0 : return (0);
940 :
941 : errout:
942 0 : if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
943 0 : m_freem(re->re_dst_m);
944 :
945 0 : if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
946 0 : bus_dmamap_unload(sc->sc_dmat, re->re_dst_map);
947 0 : bus_dmamap_destroy(sc->sc_dmat, re->re_dst_map);
948 0 : }
949 0 : if (re->re_src_map != NULL) {
950 0 : bus_dmamap_unload(sc->sc_dmat, re->re_src_map);
951 0 : bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
952 0 : }
953 0 : crp->crp_etype = err;
954 0 : crypto_done(crp);
955 0 : splx(s);
956 0 : return (err);
957 0 : }
958 :
959 : /*
960 : * Resets the board. Values in the regesters are left as is
961 : * from the reset (i.e. initial values are assigned elsewhere).
962 : */
963 : void
964 0 : safe_reset_board(struct safe_softc *sc)
965 : {
966 : u_int32_t v;
967 :
968 : /*
969 : * Reset the device. The manual says no delay
970 : * is needed between marking and clearing reset.
971 : */
972 0 : v = READ_REG(sc, SAFE_PE_DMACFG) &
973 : ~(SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
974 : SAFE_PE_DMACFG_SGRESET);
975 0 : WRITE_REG(sc, SAFE_PE_DMACFG, v
976 : | SAFE_PE_DMACFG_PERESET
977 : | SAFE_PE_DMACFG_PDRRESET
978 : | SAFE_PE_DMACFG_SGRESET);
979 0 : WRITE_REG(sc, SAFE_PE_DMACFG, v);
980 0 : }
981 :
982 : /*
983 : * Initialize registers we need to touch only once.
984 : */
985 : void
986 0 : safe_init_board(struct safe_softc *sc)
987 : {
988 : u_int32_t v, dwords;
989 :
990 0 : v = READ_REG(sc, SAFE_PE_DMACFG);
991 0 : v &= ~(SAFE_PE_DMACFG_PEMODE | SAFE_PE_DMACFG_ESPACKET);
992 0 : v |= SAFE_PE_DMACFG_FSENA /* failsafe enable */
993 : | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
994 : | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
995 : | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
996 : | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
997 : | SAFE_PE_DMACFG_ESSA /* endian-swap SA data */
998 : ;
999 0 : WRITE_REG(sc, SAFE_PE_DMACFG, v);
1000 :
1001 0 : WRITE_REG(sc, SAFE_CRYPTO_CTRL, SAFE_CRYPTO_CTRL_PKEY |
1002 : SAFE_CRYPTO_CTRL_3DES | SAFE_CRYPTO_CTRL_RNG);
1003 :
1004 : #if BYTE_ORDER == LITTLE_ENDIAN
1005 0 : WRITE_REG(sc, SAFE_ENDIAN, SAFE_ENDIAN_TGT_PASS|SAFE_ENDIAN_DMA_PASS);
1006 : #elif BYTE_ORDER == BIG_ENDIAN
1007 : WRITE_REG(sc, SAFE_ENDIAN, SAFE_ENDIAN_TGT_PASS|SAFE_ENDIAN_DMA_SWAB);
1008 : #endif
1009 :
1010 0 : if (sc->sc_chiprev == SAFE_REV(1,0)) {
1011 : /*
1012 : * Avoid large PCI DMA transfers. Rev 1.0 has a bug where
1013 : * "target mode transfers" done while the chip is DMA'ing
1014 : * >1020 bytes cause the hardware to lockup. To avoid this
1015 : * we reduce the max PCI transfer size and use small source
1016 : * particle descriptors (<= 256 bytes).
1017 : */
1018 0 : WRITE_REG(sc, SAFE_DMA_CFG, 256);
1019 0 : printf("%s: Reduce max DMA size to %u words for rev %u.%u WAR\n",
1020 0 : sc->sc_dev.dv_xname,
1021 0 : (READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff,
1022 0 : SAFE_REV_MAJ(sc->sc_chiprev),
1023 0 : SAFE_REV_MIN(sc->sc_chiprev));
1024 0 : }
1025 :
1026 : /* NB: operands+results are overlaid */
1027 0 : WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1028 0 : WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1029 : /*
1030 : * Configure ring entry size and number of items in the ring.
1031 : */
1032 : KASSERT_X((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1033 : ("PE ring entry not 32-bit aligned!"));
1034 : dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1035 0 : WRITE_REG(sc, SAFE_PE_RINGCFG,
1036 : (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1037 0 : WRITE_REG(sc, SAFE_PE_RINGPOLL, 0); /* disable polling */
1038 :
1039 0 : WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1040 0 : WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1041 0 : WRITE_REG(sc, SAFE_PE_PARTSIZE,
1042 : (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1043 : /*
1044 : * NB: destination particles are fixed size. We use
1045 : * an mbuf cluster and require all results go to
1046 : * clusters or smaller.
1047 : */
1048 0 : WRITE_REG(sc, SAFE_PE_PARTCFG, SAFE_MAX_DSIZE);
1049 :
1050 0 : WRITE_REG(sc, SAFE_HI_CLR, SAFE_INT_PE_CDONE | SAFE_INT_PE_DDONE |
1051 : SAFE_INT_PE_ERROR | SAFE_INT_PE_ODONE);
1052 :
1053 : /* it's now safe to enable PE mode, do it */
1054 0 : WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1055 :
1056 : /*
1057 : * Configure hardware to use level-triggered interrupts and
1058 : * to interrupt after each descriptor is processed.
1059 : */
1060 0 : DELAY(1000);
1061 0 : WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1062 0 : DELAY(1000);
1063 0 : WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1064 0 : DELAY(1000);
1065 0 : WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1066 0 : DELAY(1000);
1067 0 : }
1068 :
1069 : /*
1070 : * Init PCI registers
1071 : */
1072 : void
1073 0 : safe_init_pciregs(struct safe_softc *sc)
1074 : {
1075 0 : }
1076 :
1077 : int
1078 0 : safe_dma_malloc(struct safe_softc *sc, bus_size_t size,
1079 : struct safe_dma_alloc *dma, int mapflags)
1080 : {
1081 : int r;
1082 :
1083 0 : if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0,
1084 0 : &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0)
1085 : goto fail_0;
1086 :
1087 0 : if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
1088 0 : size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0)
1089 : goto fail_1;
1090 :
1091 0 : if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
1092 0 : BUS_DMA_NOWAIT, &dma->dma_map)) != 0)
1093 : goto fail_2;
1094 :
1095 0 : if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
1096 0 : size, NULL, BUS_DMA_NOWAIT)) != 0)
1097 : goto fail_3;
1098 :
1099 0 : dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
1100 0 : dma->dma_size = size;
1101 0 : return (0);
1102 :
1103 : fail_3:
1104 0 : bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
1105 : fail_2:
1106 0 : bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
1107 : fail_1:
1108 0 : bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
1109 : fail_0:
1110 0 : dma->dma_map = NULL;
1111 0 : return (r);
1112 0 : }
1113 :
1114 : void
1115 0 : safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma)
1116 : {
1117 0 : bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
1118 0 : bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
1119 0 : bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
1120 0 : bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
1121 0 : }
1122 :
1123 :
1124 : #define SAFE_RNG_MAXWAIT 1000
1125 :
1126 : void
1127 0 : safe_rng_init(struct safe_softc *sc)
1128 : {
1129 : u_int32_t w, v;
1130 : int i;
1131 :
1132 0 : WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1133 : /* use default value according to the manual */
1134 0 : WRITE_REG(sc, SAFE_RNG_CNFG, 0x834); /* magic from SafeNet */
1135 0 : WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1136 :
1137 : /*
1138 : * There is a bug in rev 1.0 of the 1140 that when the RNG
1139 : * is brought out of reset the ready status flag does not
1140 : * work until the RNG has finished its internal initialization.
1141 : *
1142 : * So in order to determine the device is through its
1143 : * initialization we must read the data register, using the
1144 : * status reg in the read in case it is initialized. Then read
1145 : * the data register until it changes from the first read.
1146 : * Once it changes read the data register until it changes
1147 : * again. At this time the RNG is considered initialized.
1148 : * This could take between 750ms - 1000ms in time.
1149 : */
1150 : i = 0;
1151 0 : w = READ_REG(sc, SAFE_RNG_OUT);
1152 0 : do {
1153 0 : v = READ_REG(sc, SAFE_RNG_OUT);
1154 0 : if (v != w) {
1155 : w = v;
1156 0 : break;
1157 : }
1158 0 : DELAY(10);
1159 0 : } while (++i < SAFE_RNG_MAXWAIT);
1160 :
1161 : /* Wait Until data changes again */
1162 : i = 0;
1163 0 : do {
1164 0 : v = READ_REG(sc, SAFE_RNG_OUT);
1165 0 : if (v != w)
1166 : break;
1167 0 : DELAY(10);
1168 0 : } while (++i < SAFE_RNG_MAXWAIT);
1169 0 : }
1170 :
1171 : static __inline u_int32_t
1172 0 : safe_rng_read(struct safe_softc *sc)
1173 : {
1174 : int i;
1175 :
1176 : i = 0;
1177 0 : while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1178 : ;
1179 0 : return (READ_REG(sc, SAFE_RNG_OUT));
1180 : }
1181 :
1182 : void
1183 0 : safe_rng(void *arg)
1184 : {
1185 0 : struct safe_softc *sc = arg;
1186 0 : u_int32_t buf[SAFE_RNG_MAXBUFSIZ]; /* NB: maybe move to softc */
1187 : u_int maxwords;
1188 : int i;
1189 :
1190 0 : safestats.st_rng++;
1191 : /*
1192 : * Fetch the next block of data.
1193 : */
1194 0 : maxwords = safe_rngbufsize;
1195 0 : if (maxwords > SAFE_RNG_MAXBUFSIZ)
1196 : maxwords = SAFE_RNG_MAXBUFSIZ;
1197 : retry:
1198 0 : for (i = 0; i < maxwords; i++)
1199 0 : buf[i] = safe_rng_read(sc);
1200 : /*
1201 : * Check the comparator alarm count and reset the h/w if
1202 : * it exceeds our threshold. This guards against the
1203 : * hardware oscillators resonating with external signals.
1204 : */
1205 0 : if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1206 : u_int32_t freq_inc, w;
1207 :
1208 : DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1209 : READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1210 0 : safestats.st_rngalarm++;
1211 0 : WRITE_REG(sc, SAFE_RNG_CTRL,
1212 : READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1213 : freq_inc = 18;
1214 0 : for (i = 0; i < 64; i++) {
1215 0 : w = READ_REG(sc, SAFE_RNG_CNFG);
1216 0 : freq_inc = ((w + freq_inc) & 0x3fL);
1217 0 : w = ((w & ~0x3fL) | freq_inc);
1218 0 : WRITE_REG(sc, SAFE_RNG_CNFG, w);
1219 :
1220 0 : WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1221 :
1222 0 : (void) safe_rng_read(sc);
1223 0 : DELAY(25);
1224 :
1225 0 : if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1226 0 : WRITE_REG(sc, SAFE_RNG_CTRL,
1227 : READ_REG(sc, SAFE_RNG_CTRL) &
1228 : ~SAFE_RNG_CTRL_SHORTEN);
1229 0 : goto retry;
1230 : }
1231 : freq_inc = 1;
1232 : }
1233 0 : WRITE_REG(sc, SAFE_RNG_CTRL,
1234 : READ_REG(sc, SAFE_RNG_CTRL) & ~SAFE_RNG_CTRL_SHORTEN);
1235 0 : } else
1236 0 : WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1237 :
1238 0 : for (i = 0; i < maxwords; i++)
1239 0 : enqueue_randomness(buf[i]);
1240 :
1241 0 : timeout_add_sec(&sc->sc_rngto, safe_rnginterval);
1242 0 : }
1243 :
1244 : /*
1245 : * Allocate a new 'session' and return an encoded session id. 'sidp'
1246 : * contains our registration id, and should contain an encoded session
1247 : * id on successful allocation.
1248 : */
1249 : int
1250 0 : safe_newsession(u_int32_t *sidp, struct cryptoini *cri)
1251 : {
1252 : struct cryptoini *c, *encini = NULL, *macini = NULL;
1253 : struct safe_softc *sc = NULL;
1254 : struct safe_session *ses = NULL;
1255 0 : MD5_CTX md5ctx;
1256 0 : SHA1_CTX sha1ctx;
1257 : int i, sesn;
1258 :
1259 0 : if (sidp == NULL || cri == NULL)
1260 0 : return (EINVAL);
1261 0 : for (i = 0; i < safe_cd.cd_ndevs; i++) {
1262 0 : sc = safe_cd.cd_devs[i];
1263 0 : if (sc == NULL || sc->sc_cid == (*sidp))
1264 : break;
1265 : }
1266 0 : if (sc == NULL)
1267 0 : return (EINVAL);
1268 :
1269 0 : for (c = cri; c != NULL; c = c->cri_next) {
1270 0 : if (c->cri_alg == CRYPTO_MD5_HMAC ||
1271 0 : c->cri_alg == CRYPTO_SHA1_HMAC) {
1272 0 : if (macini)
1273 0 : return (EINVAL);
1274 : macini = c;
1275 0 : } else if (c->cri_alg == CRYPTO_3DES_CBC ||
1276 0 : c->cri_alg == CRYPTO_AES_CBC) {
1277 0 : if (encini)
1278 0 : return (EINVAL);
1279 : encini = c;
1280 : } else
1281 0 : return (EINVAL);
1282 : }
1283 0 : if (encini == NULL && macini == NULL)
1284 0 : return (EINVAL);
1285 0 : if (encini) { /* validate key length */
1286 0 : switch (encini->cri_alg) {
1287 : case CRYPTO_3DES_CBC:
1288 0 : if (encini->cri_klen != 192)
1289 0 : return (EINVAL);
1290 : break;
1291 : case CRYPTO_AES_CBC:
1292 0 : if (encini->cri_klen != 128 &&
1293 0 : encini->cri_klen != 192 &&
1294 0 : encini->cri_klen != 256)
1295 0 : return (EINVAL);
1296 : break;
1297 : }
1298 : }
1299 :
1300 0 : if (sc->sc_sessions == NULL) {
1301 0 : ses = sc->sc_sessions = (struct safe_session *)malloc(
1302 : sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
1303 0 : if (ses == NULL)
1304 0 : return (ENOMEM);
1305 : sesn = 0;
1306 0 : sc->sc_nsessions = 1;
1307 0 : } else {
1308 0 : for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
1309 0 : if (sc->sc_sessions[sesn].ses_used == 0) {
1310 : ses = &sc->sc_sessions[sesn];
1311 0 : break;
1312 : }
1313 : }
1314 :
1315 0 : if (ses == NULL) {
1316 0 : sesn = sc->sc_nsessions;
1317 0 : ses = mallocarray((sesn + 1),
1318 : sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
1319 0 : if (ses == NULL)
1320 0 : return (ENOMEM);
1321 0 : bcopy(sc->sc_sessions, ses, sesn *
1322 : sizeof(struct safe_session));
1323 0 : explicit_bzero(sc->sc_sessions, sesn *
1324 : sizeof(struct safe_session));
1325 0 : free(sc->sc_sessions, M_DEVBUF, 0);
1326 0 : sc->sc_sessions = ses;
1327 0 : ses = &sc->sc_sessions[sesn];
1328 0 : sc->sc_nsessions++;
1329 0 : }
1330 : }
1331 :
1332 0 : bzero(ses, sizeof(struct safe_session));
1333 0 : ses->ses_used = 1;
1334 :
1335 0 : if (encini) {
1336 0 : ses->ses_klen = encini->cri_klen;
1337 0 : bcopy(encini->cri_key, ses->ses_key, ses->ses_klen / 8);
1338 :
1339 0 : for (i = 0; i < nitems(ses->ses_key); i++)
1340 0 : ses->ses_key[i] = htole32(ses->ses_key[i]);
1341 : }
1342 :
1343 0 : if (macini) {
1344 0 : for (i = 0; i < macini->cri_klen / 8; i++)
1345 0 : macini->cri_key[i] ^= HMAC_IPAD_VAL;
1346 :
1347 0 : if (macini->cri_alg == CRYPTO_MD5_HMAC) {
1348 0 : MD5Init(&md5ctx);
1349 0 : MD5Update(&md5ctx, macini->cri_key,
1350 0 : macini->cri_klen / 8);
1351 0 : MD5Update(&md5ctx, hmac_ipad_buffer,
1352 0 : HMAC_MD5_BLOCK_LEN - (macini->cri_klen / 8));
1353 0 : bcopy(md5ctx.state, ses->ses_hminner,
1354 : sizeof(md5ctx.state));
1355 0 : } else {
1356 0 : SHA1Init(&sha1ctx);
1357 0 : SHA1Update(&sha1ctx, macini->cri_key,
1358 0 : macini->cri_klen / 8);
1359 0 : SHA1Update(&sha1ctx, hmac_ipad_buffer,
1360 0 : HMAC_SHA1_BLOCK_LEN - (macini->cri_klen / 8));
1361 0 : bcopy(sha1ctx.state, ses->ses_hminner,
1362 : sizeof(sha1ctx.state));
1363 : }
1364 :
1365 0 : for (i = 0; i < macini->cri_klen / 8; i++)
1366 0 : macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
1367 :
1368 0 : if (macini->cri_alg == CRYPTO_MD5_HMAC) {
1369 0 : MD5Init(&md5ctx);
1370 0 : MD5Update(&md5ctx, macini->cri_key,
1371 0 : macini->cri_klen / 8);
1372 0 : MD5Update(&md5ctx, hmac_opad_buffer,
1373 0 : HMAC_MD5_BLOCK_LEN - (macini->cri_klen / 8));
1374 0 : bcopy(md5ctx.state, ses->ses_hmouter,
1375 : sizeof(md5ctx.state));
1376 0 : } else {
1377 0 : SHA1Init(&sha1ctx);
1378 0 : SHA1Update(&sha1ctx, macini->cri_key,
1379 0 : macini->cri_klen / 8);
1380 0 : SHA1Update(&sha1ctx, hmac_opad_buffer,
1381 0 : HMAC_SHA1_BLOCK_LEN - (macini->cri_klen / 8));
1382 0 : bcopy(sha1ctx.state, ses->ses_hmouter,
1383 : sizeof(sha1ctx.state));
1384 : }
1385 :
1386 0 : for (i = 0; i < macini->cri_klen / 8; i++)
1387 0 : macini->cri_key[i] ^= HMAC_OPAD_VAL;
1388 :
1389 : /* PE is little-endian, insure proper byte order */
1390 0 : for (i = 0;
1391 0 : i < sizeof(ses->ses_hminner)/sizeof(ses->ses_hminner[0]);
1392 0 : i++) {
1393 0 : ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
1394 0 : ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
1395 : }
1396 : }
1397 :
1398 0 : *sidp = SAFE_SID(sc->sc_dev.dv_unit, sesn);
1399 0 : return (0);
1400 0 : }
1401 :
1402 : /*
1403 : * Deallocate a session.
1404 : */
1405 : int
1406 0 : safe_freesession(u_int64_t tid)
1407 : {
1408 : struct safe_softc *sc;
1409 : int session, ret, card;
1410 0 : u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
1411 :
1412 0 : card = SAFE_CARD(sid);
1413 0 : if (card >= safe_cd.cd_ndevs || safe_cd.cd_devs[card] == NULL)
1414 0 : return (EINVAL);
1415 0 : sc = safe_cd.cd_devs[card];
1416 :
1417 0 : if (sc == NULL)
1418 0 : return (EINVAL);
1419 :
1420 0 : session = SAFE_SESSION(sid);
1421 0 : if (session < sc->sc_nsessions) {
1422 0 : explicit_bzero(&sc->sc_sessions[session],
1423 : sizeof(sc->sc_sessions[session]));
1424 : ret = 0;
1425 0 : } else
1426 : ret = EINVAL;
1427 0 : return (ret);
1428 0 : }
1429 :
1430 : /*
1431 : * Is the operand suitable aligned for direct DMA. Each
1432 : * segment must be aligned on a 32-bit boundary and all
1433 : * but the last segment must be a multiple of 4 bytes.
1434 : */
1435 : int
1436 0 : safe_dmamap_aligned(const struct safe_operand *op)
1437 : {
1438 : int i;
1439 :
1440 0 : for (i = 0; i < op->map->dm_nsegs; i++) {
1441 0 : if (op->map->dm_segs[i].ds_addr & 3)
1442 0 : return (0);
1443 0 : if (i != (op->map->dm_nsegs - 1) &&
1444 0 : (op->map->dm_segs[i].ds_len & 3))
1445 0 : return (0);
1446 : }
1447 0 : return (1);
1448 0 : }
1449 :
1450 : /*
1451 : * Clean up after a chip crash.
1452 : * It is assumed that the caller in splnet()
1453 : */
1454 : void
1455 0 : safe_cleanchip(struct safe_softc *sc)
1456 : {
1457 :
1458 0 : if (sc->sc_nqchip != 0) {
1459 0 : struct safe_ringentry *re = sc->sc_back;
1460 :
1461 0 : while (re != sc->sc_front) {
1462 0 : if (re->re_desc.d_csr != 0)
1463 0 : safe_free_entry(sc, re);
1464 0 : if (++re == sc->sc_ringtop)
1465 0 : re = sc->sc_ring;
1466 : }
1467 0 : sc->sc_back = re;
1468 0 : sc->sc_nqchip = 0;
1469 0 : }
1470 0 : }
1471 :
1472 : /*
1473 : * free a safe_q
1474 : * It is assumed that the caller is within splnet().
1475 : */
1476 : int
1477 0 : safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1478 : {
1479 : struct cryptop *crp;
1480 :
1481 : /*
1482 : * Free header MCR
1483 : */
1484 0 : if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
1485 0 : m_freem(re->re_dst_m);
1486 :
1487 0 : crp = (struct cryptop *)re->re_crp;
1488 :
1489 0 : re->re_desc.d_csr = 0;
1490 :
1491 0 : crp->crp_etype = EFAULT;
1492 0 : crypto_done(crp);
1493 0 : return (0);
1494 : }
1495 :
1496 : /*
1497 : * safe_feed() - post a request to chip
1498 : */
1499 : void
1500 0 : safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
1501 : {
1502 0 : bus_dmamap_sync(sc->sc_dmat, re->re_src_map,
1503 : 0, re->re_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1504 0 : if (re->re_dst_map != NULL)
1505 0 : bus_dmamap_sync(sc->sc_dmat, re->re_dst_map, 0,
1506 : re->re_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
1507 : /* XXX have no smaller granularity */
1508 0 : safe_dma_sync(sc, &sc->sc_ringalloc,
1509 : BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1510 0 : safe_dma_sync(sc, &sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
1511 0 : safe_dma_sync(sc, &sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
1512 :
1513 : #ifdef SAFE_DEBUG
1514 : if (safe_debug) {
1515 : safe_dump_ringstate(sc, __func__);
1516 : safe_dump_request(sc, __func__, re);
1517 : }
1518 : #endif
1519 0 : sc->sc_nqchip++;
1520 0 : if (sc->sc_nqchip > safestats.st_maxqchip)
1521 0 : safestats.st_maxqchip = sc->sc_nqchip;
1522 : /* poke h/w to check descriptor ring, any value can be written */
1523 0 : WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
1524 0 : }
1525 :
1526 : /*
1527 : * Is the operand suitable for direct DMA as the destination
1528 : * of an operation. The hardware requires that each ``particle''
1529 : * but the last in an operation result have the same size. We
1530 : * fix that size at SAFE_MAX_DSIZE bytes. This routine returns
1531 : * 0 if some segment is not a multiple of this size, 1 if all
1532 : * segments are exactly this size, or 2 if segments are at worst
1533 : * a multple of this size.
1534 : */
1535 : int
1536 0 : safe_dmamap_uniform(const struct safe_operand *op)
1537 : {
1538 : int result = 1, i;
1539 :
1540 0 : if (op->map->dm_nsegs <= 0)
1541 0 : return (result);
1542 :
1543 0 : for (i = 0; i < op->map->dm_nsegs-1; i++) {
1544 0 : if (op->map->dm_segs[i].ds_len % SAFE_MAX_DSIZE)
1545 0 : return (0);
1546 0 : if (op->map->dm_segs[i].ds_len != SAFE_MAX_DSIZE)
1547 0 : result = 2;
1548 : }
1549 0 : return (result);
1550 0 : }
1551 :
1552 : /*
1553 : * Copy all data past offset from srcm to dstm.
1554 : */
1555 : void
1556 0 : safe_mcopy(struct mbuf *srcm, struct mbuf *dstm, u_int offset)
1557 : {
1558 : u_int j, dlen, slen;
1559 : caddr_t dptr, sptr;
1560 :
1561 : /*
1562 : * Advance src and dst to offset.
1563 : */
1564 0 : for (j = offset; srcm->m_len <= j;) {
1565 0 : j -= srcm->m_len;
1566 0 : srcm = srcm->m_next;
1567 0 : if (srcm == NULL)
1568 0 : return;
1569 : }
1570 0 : sptr = mtod(srcm, caddr_t) + j;
1571 0 : slen = srcm->m_len - j;
1572 :
1573 0 : for (j = offset; dstm->m_len <= j;) {
1574 0 : j -= dstm->m_len;
1575 0 : dstm = dstm->m_next;
1576 0 : if (dstm == NULL)
1577 0 : return;
1578 : }
1579 0 : dptr = mtod(dstm, caddr_t) + j;
1580 0 : dlen = dstm->m_len - j;
1581 :
1582 : /*
1583 : * Copy everything that remains.
1584 : */
1585 0 : for (;;) {
1586 0 : j = min(slen, dlen);
1587 0 : bcopy(sptr, dptr, j);
1588 0 : if (slen == j) {
1589 0 : srcm = srcm->m_next;
1590 0 : if (srcm == NULL)
1591 0 : return;
1592 0 : sptr = srcm->m_data;
1593 0 : slen = srcm->m_len;
1594 0 : } else
1595 0 : sptr += j, slen -= j;
1596 0 : if (dlen == j) {
1597 0 : dstm = dstm->m_next;
1598 0 : if (dstm == NULL)
1599 0 : return;
1600 0 : dptr = dstm->m_data;
1601 0 : dlen = dstm->m_len;
1602 0 : } else
1603 0 : dptr += j, dlen -= j;
1604 : }
1605 0 : }
1606 :
1607 : void
1608 0 : safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1609 : {
1610 0 : struct cryptop *crp = (struct cryptop *)re->re_crp;
1611 : struct cryptodesc *crd;
1612 : int i;
1613 :
1614 0 : safestats.st_opackets++;
1615 0 : safestats.st_obytes += (re->re_dst_map == NULL) ?
1616 0 : re->re_src_mapsize : re->re_dst_mapsize;
1617 :
1618 0 : safe_dma_sync(sc, &sc->sc_ringalloc,
1619 : BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1620 0 : if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1621 0 : printf("%s: csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1622 0 : sc->sc_dev.dv_xname, re->re_desc.d_csr,
1623 0 : re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1624 0 : safestats.st_peoperr++;
1625 0 : crp->crp_etype = EIO; /* something more meaningful? */
1626 0 : }
1627 0 : if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
1628 0 : bus_dmamap_sync(sc->sc_dmat, re->re_dst_map, 0,
1629 : re->re_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1630 0 : bus_dmamap_unload(sc->sc_dmat, re->re_dst_map);
1631 0 : bus_dmamap_destroy(sc->sc_dmat, re->re_dst_map);
1632 0 : }
1633 0 : bus_dmamap_sync(sc->sc_dmat, re->re_src_map, 0,
1634 : re->re_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1635 0 : bus_dmamap_unload(sc->sc_dmat, re->re_src_map);
1636 0 : bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
1637 :
1638 : /*
1639 : * If result was written to a different mbuf chain, swap
1640 : * it in as the return value and reclaim the original.
1641 : */
1642 0 : if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
1643 0 : m_freem(re->re_src_m);
1644 0 : crp->crp_buf = (caddr_t)re->re_dst_m;
1645 0 : }
1646 :
1647 0 : if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1648 : /* copy out ICV result */
1649 0 : for (i = 0; i < crp->crp_ndesc; i++) {
1650 0 : crd = &crp->crp_desc[i];
1651 0 : if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
1652 0 : crd->crd_alg == CRYPTO_SHA1_HMAC))
1653 : continue;
1654 0 : if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
1655 : /*
1656 : * SHA-1 ICV's are byte-swapped; fix 'em up
1657 : * before copy them to their destination.
1658 : */
1659 0 : bswap32(re->re_sastate.sa_saved_indigest[0]);
1660 0 : bswap32(re->re_sastate.sa_saved_indigest[1]);
1661 0 : bswap32(re->re_sastate.sa_saved_indigest[2]);
1662 0 : }
1663 0 : if (crp->crp_flags & CRYPTO_F_IMBUF) {
1664 0 : crp->crp_etype =
1665 0 : m_copyback((struct mbuf *)crp->crp_buf,
1666 0 : crd->crd_inject, 12,
1667 0 : (caddr_t)re->re_sastate.sa_saved_indigest,
1668 : M_NOWAIT);
1669 0 : } else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) {
1670 0 : bcopy((caddr_t)re->re_sastate.sa_saved_indigest,
1671 : crp->crp_mac, 12);
1672 0 : }
1673 : break;
1674 : }
1675 : }
1676 :
1677 0 : crypto_done(crp);
1678 0 : }
1679 :
1680 : /*
1681 : * SafeXcel Interrupt routine
1682 : */
1683 : int
1684 0 : safe_intr(void *arg)
1685 : {
1686 0 : struct safe_softc *sc = arg;
1687 0 : volatile u_int32_t stat;
1688 :
1689 0 : stat = READ_REG(sc, SAFE_HM_STAT);
1690 0 : if (stat == 0) /* shared irq, not for us */
1691 0 : return (0);
1692 :
1693 0 : WRITE_REG(sc, SAFE_HI_CLR, stat); /* IACK */
1694 :
1695 0 : if ((stat & SAFE_INT_PE_DDONE)) {
1696 : /*
1697 : * Descriptor(s) done; scan the ring and
1698 : * process completed operations.
1699 : */
1700 0 : while (sc->sc_back != sc->sc_front) {
1701 : struct safe_ringentry *re = sc->sc_back;
1702 : #ifdef SAFE_DEBUG
1703 : if (safe_debug) {
1704 : safe_dump_ringstate(sc, __func__);
1705 : safe_dump_request(sc, __func__, re);
1706 : }
1707 : #endif
1708 : /*
1709 : * safe_process marks ring entries that were allocated
1710 : * but not used with a csr of zero. This insures the
1711 : * ring front pointer never needs to be set backwards
1712 : * in the event that an entry is allocated but not used
1713 : * because of a setup error.
1714 : */
1715 0 : if (re->re_desc.d_csr != 0) {
1716 0 : if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
1717 0 : break;
1718 0 : if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
1719 0 : break;
1720 0 : sc->sc_nqchip--;
1721 0 : safe_callback(sc, re);
1722 0 : }
1723 0 : if (++(sc->sc_back) == sc->sc_ringtop)
1724 0 : sc->sc_back = sc->sc_ring;
1725 0 : }
1726 : }
1727 :
1728 0 : return (1);
1729 0 : }
1730 :
1731 : #ifdef SAFE_DEBUG
1732 :
1733 : void
1734 : safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
1735 : {
1736 : printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n", tag,
1737 : READ_REG(sc, SAFE_DMA_ENDIAN), READ_REG(sc, SAFE_DMA_SRCADDR),
1738 : READ_REG(sc, SAFE_DMA_DSTADDR), READ_REG(sc, SAFE_DMA_STAT));
1739 : }
1740 :
1741 : void
1742 : safe_dump_intrstate(struct safe_softc *sc, const char *tag)
1743 : {
1744 : printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n",
1745 : tag, READ_REG(sc, SAFE_HI_CFG), READ_REG(sc, SAFE_HI_MASK),
1746 : READ_REG(sc, SAFE_HI_DESC_CNT), READ_REG(sc, SAFE_HU_STAT),
1747 : READ_REG(sc, SAFE_HM_STAT));
1748 : }
1749 :
1750 : void
1751 : safe_dump_ringstate(struct safe_softc *sc, const char *tag)
1752 : {
1753 : u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
1754 :
1755 : /* NB: assume caller has lock on ring */
1756 : printf("%s: ERNGSTAT %x (next %u) back %u front %u\n",
1757 : tag, estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
1758 : sc->sc_back - sc->sc_ring, sc->sc_front - sc->sc_ring);
1759 : }
1760 :
1761 : void
1762 : safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
1763 : {
1764 : int ix, nsegs;
1765 :
1766 : ix = re - sc->sc_ring;
1767 : printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n", tag,
1768 : re, ix, re->re_desc.d_csr, re->re_desc.d_src, re->re_desc.d_dst,
1769 : re->re_desc.d_sa, re->re_desc.d_len);
1770 : if (re->re_src_nsegs > 1) {
1771 : ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1772 : sizeof(struct safe_pdesc);
1773 : for (nsegs = re->re_src_nsegs; nsegs; nsegs--) {
1774 : printf(" spd[%u] %p: %p", ix,
1775 : &sc->sc_spring[ix],
1776 : (caddr_t)sc->sc_spring[ix].pd_addr);
1777 : printf("\n");
1778 : if (++ix == SAFE_TOTAL_SPART)
1779 : ix = 0;
1780 : }
1781 : }
1782 : if (re->re_dst_nsegs > 1) {
1783 : ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1784 : sizeof(struct safe_pdesc);
1785 : for (nsegs = re->re_dst_nsegs; nsegs; nsegs--) {
1786 : printf(" dpd[%u] %p: %p\n", ix,
1787 : &sc->sc_dpring[ix],
1788 : (caddr_t) sc->sc_dpring[ix].pd_addr);
1789 : if (++ix == SAFE_TOTAL_DPART)
1790 : ix = 0;
1791 : }
1792 : }
1793 : printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
1794 : re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1795 : printf("sa: key %x %x %x %x %x %x %x %x\n", re->re_sa.sa_key[0],
1796 : re->re_sa.sa_key[1], re->re_sa.sa_key[2], re->re_sa.sa_key[3],
1797 : re->re_sa.sa_key[4], re->re_sa.sa_key[5], re->re_sa.sa_key[6],
1798 : re->re_sa.sa_key[7]);
1799 : printf("sa: indigest %x %x %x %x %x\n", re->re_sa.sa_indigest[0],
1800 : re->re_sa.sa_indigest[1], re->re_sa.sa_indigest[2],
1801 : re->re_sa.sa_indigest[3], re->re_sa.sa_indigest[4]);
1802 : printf("sa: outdigest %x %x %x %x %x\n", re->re_sa.sa_outdigest[0],
1803 : re->re_sa.sa_outdigest[1], re->re_sa.sa_outdigest[2],
1804 : re->re_sa.sa_outdigest[3], re->re_sa.sa_outdigest[4]);
1805 : printf("sr: iv %x %x %x %x\n",
1806 : re->re_sastate.sa_saved_iv[0], re->re_sastate.sa_saved_iv[1],
1807 : re->re_sastate.sa_saved_iv[2], re->re_sastate.sa_saved_iv[3]);
1808 : printf("sr: hashbc %u indigest %x %x %x %x %x\n",
1809 : re->re_sastate.sa_saved_hashbc,
1810 : re->re_sastate.sa_saved_indigest[0],
1811 : re->re_sastate.sa_saved_indigest[1],
1812 : re->re_sastate.sa_saved_indigest[2],
1813 : re->re_sastate.sa_saved_indigest[3],
1814 : re->re_sastate.sa_saved_indigest[4]);
1815 : }
1816 :
1817 : void
1818 : safe_dump_ring(struct safe_softc *sc, const char *tag)
1819 : {
1820 : printf("\nSafeNet Ring State:\n");
1821 : safe_dump_intrstate(sc, tag);
1822 : safe_dump_dmastatus(sc, tag);
1823 : safe_dump_ringstate(sc, tag);
1824 : if (sc->sc_nqchip) {
1825 : struct safe_ringentry *re = sc->sc_back;
1826 : do {
1827 : safe_dump_request(sc, tag, re);
1828 : if (++re == sc->sc_ringtop)
1829 : re = sc->sc_ring;
1830 : } while (re != sc->sc_front);
1831 : }
1832 : }
1833 :
1834 : #endif /* SAFE_DEBUG */
|
