Line data Source code
1 : /* $OpenBSD: uipc_mbuf.c,v 1.259 2018/09/13 19:53:58 bluhm Exp $ */
2 : /* $NetBSD: uipc_mbuf.c,v 1.15.4.1 1996/06/13 17:11:44 cgd Exp $ */
3 :
4 : /*
5 : * Copyright (c) 1982, 1986, 1988, 1991, 1993
6 : * The Regents of the University of California. All rights reserved.
7 : *
8 : * Redistribution and use in source and binary forms, with or without
9 : * modification, are permitted provided that the following conditions
10 : * are met:
11 : * 1. Redistributions of source code must retain the above copyright
12 : * notice, this list of conditions and the following disclaimer.
13 : * 2. Redistributions in binary form must reproduce the above copyright
14 : * notice, this list of conditions and the following disclaimer in the
15 : * documentation and/or other materials provided with the distribution.
16 : * 3. Neither the name of the University nor the names of its contributors
17 : * may be used to endorse or promote products derived from this software
18 : * without specific prior written permission.
19 : *
20 : * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 : * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 : * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 : * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 : * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 : * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 : * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 : * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 : * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 : * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 : * SUCH DAMAGE.
31 : *
32 : * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
33 : */
34 :
35 : /*
36 : * @(#)COPYRIGHT 1.1 (NRL) 17 January 1995
37 : *
38 : * NRL grants permission for redistribution and use in source and binary
39 : * forms, with or without modification, of the software and documentation
40 : * created at NRL provided that the following conditions are met:
41 : *
42 : * 1. Redistributions of source code must retain the above copyright
43 : * notice, this list of conditions and the following disclaimer.
44 : * 2. Redistributions in binary form must reproduce the above copyright
45 : * notice, this list of conditions and the following disclaimer in the
46 : * documentation and/or other materials provided with the distribution.
47 : * 3. All advertising materials mentioning features or use of this software
48 : * must display the following acknowledgements:
49 : * This product includes software developed by the University of
50 : * California, Berkeley and its contributors.
51 : * This product includes software developed at the Information
52 : * Technology Division, US Naval Research Laboratory.
53 : * 4. Neither the name of the NRL nor the names of its contributors
54 : * may be used to endorse or promote products derived from this software
55 : * without specific prior written permission.
56 : *
57 : * THE SOFTWARE PROVIDED BY NRL IS PROVIDED BY NRL AND CONTRIBUTORS ``AS
58 : * IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
59 : * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
60 : * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NRL OR
61 : * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
62 : * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
63 : * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
64 : * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
65 : * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
66 : * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
67 : * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
68 : *
69 : * The views and conclusions contained in the software and documentation
70 : * are those of the authors and should not be interpreted as representing
71 : * official policies, either expressed or implied, of the US Naval
72 : * Research Laboratory (NRL).
73 : */
74 :
75 : #include "pf.h"
76 :
77 : #include <sys/param.h>
78 : #include <sys/systm.h>
79 : #include <sys/malloc.h>
80 : #include <sys/mbuf.h>
81 : #include <sys/kernel.h>
82 : #include <sys/syslog.h>
83 : #include <sys/domain.h>
84 : #include <sys/protosw.h>
85 : #include <sys/pool.h>
86 : #include <sys/percpu.h>
87 : #include <sys/sysctl.h>
88 :
89 : #include <sys/socket.h>
90 : #include <sys/socketvar.h>
91 : #include <net/if.h>
92 :
93 :
94 : #include <uvm/uvm_extern.h>
95 :
96 : #ifdef DDB
97 : #include <machine/db_machdep.h>
98 : #endif
99 :
100 : #if NPF > 0
101 : #include <net/pfvar.h>
102 : #endif /* NPF > 0 */
103 :
104 : /* mbuf stats */
105 : COUNTERS_BOOT_MEMORY(mbstat_boot, MBSTAT_COUNT);
106 : struct cpumem *mbstat = COUNTERS_BOOT_INITIALIZER(mbstat_boot);
107 : /* mbuf pools */
108 : struct pool mbpool;
109 : struct pool mtagpool;
110 :
111 : /* mbuf cluster pools */
112 : u_int mclsizes[MCLPOOLS] = {
113 : MCLBYTES, /* must be at slot 0 */
114 : MCLBYTES + 2, /* ETHER_ALIGNED 2k mbufs */
115 : 4 * 1024,
116 : 8 * 1024,
117 : 9 * 1024,
118 : 12 * 1024,
119 : 16 * 1024,
120 : 64 * 1024
121 : };
122 : static char mclnames[MCLPOOLS][8];
123 : struct pool mclpools[MCLPOOLS];
124 :
125 : struct pool *m_clpool(u_int);
126 :
127 : int max_linkhdr; /* largest link-level header */
128 : int max_protohdr; /* largest protocol header */
129 : int max_hdr; /* largest link+protocol header */
130 :
131 : struct mutex m_extref_mtx = MUTEX_INITIALIZER(IPL_NET);
132 :
133 : void m_extfree(struct mbuf *);
134 : void nmbclust_update(void);
135 : void m_zero(struct mbuf *);
136 :
137 : struct mutex m_pool_mtx = MUTEX_INITIALIZER(IPL_NET);
138 : unsigned int mbuf_mem_limit; /* how much memory can be allocated */
139 : unsigned int mbuf_mem_alloc; /* how much memory has been allocated */
140 :
141 : void *m_pool_alloc(struct pool *, int, int *);
142 : void m_pool_free(struct pool *, void *);
143 :
144 : struct pool_allocator m_pool_allocator = {
145 : m_pool_alloc,
146 : m_pool_free,
147 : 0 /* will be copied from pool_allocator_multi */
148 : };
149 :
150 : static void (*mextfree_fns[4])(caddr_t, u_int, void *);
151 : static u_int num_extfree_fns;
152 :
153 : /*
154 : * Initialize the mbuf allocator.
155 : */
156 : void
157 0 : mbinit(void)
158 : {
159 : int i;
160 : unsigned int lowbits;
161 :
162 : CTASSERT(MSIZE == sizeof(struct mbuf));
163 :
164 0 : m_pool_allocator.pa_pagesz = pool_allocator_multi.pa_pagesz;
165 :
166 0 : nmbclust_update();
167 0 : mbuf_mem_alloc = 0;
168 :
169 : #if DIAGNOSTIC
170 0 : if (mclsizes[0] != MCLBYTES)
171 0 : panic("mbinit: the smallest cluster size != MCLBYTES");
172 0 : if (mclsizes[nitems(mclsizes) - 1] != MAXMCLBYTES)
173 0 : panic("mbinit: the largest cluster size != MAXMCLBYTES");
174 : #endif
175 :
176 0 : m_pool_init(&mbpool, MSIZE, 64, "mbufpl");
177 :
178 0 : pool_init(&mtagpool, PACKET_TAG_MAXSIZE + sizeof(struct m_tag), 0,
179 : IPL_NET, 0, "mtagpl", NULL);
180 :
181 0 : for (i = 0; i < nitems(mclsizes); i++) {
182 0 : lowbits = mclsizes[i] & ((1 << 10) - 1);
183 0 : if (lowbits) {
184 0 : snprintf(mclnames[i], sizeof(mclnames[0]),
185 : "mcl%dk%u", mclsizes[i] >> 10, lowbits);
186 0 : } else {
187 0 : snprintf(mclnames[i], sizeof(mclnames[0]), "mcl%dk",
188 : mclsizes[i] >> 10);
189 : }
190 :
191 0 : m_pool_init(&mclpools[i], mclsizes[i], 64, mclnames[i]);
192 : }
193 :
194 0 : (void)mextfree_register(m_extfree_pool);
195 0 : KASSERT(num_extfree_fns == 1);
196 0 : }
197 :
198 : void
199 0 : mbcpuinit()
200 : {
201 : int i;
202 :
203 0 : mbstat = counters_alloc_ncpus(mbstat, MBSTAT_COUNT);
204 :
205 0 : pool_cache_init(&mbpool);
206 0 : pool_cache_init(&mtagpool);
207 :
208 0 : for (i = 0; i < nitems(mclsizes); i++)
209 0 : pool_cache_init(&mclpools[i]);
210 0 : }
211 :
212 : void
213 0 : nmbclust_update(void)
214 : {
215 : /* update the global mbuf memory limit */
216 0 : mbuf_mem_limit = nmbclust * MCLBYTES;
217 0 : }
218 :
219 : /*
220 : * Space allocation routines.
221 : */
222 : struct mbuf *
223 0 : m_get(int nowait, int type)
224 : {
225 : struct mbuf *m;
226 0 : struct counters_ref cr;
227 : uint64_t *counters;
228 : int s;
229 :
230 : KDASSERT(type < MT_NTYPES);
231 :
232 0 : m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT);
233 0 : if (m == NULL)
234 0 : return (NULL);
235 :
236 0 : s = splnet();
237 0 : counters = counters_enter(&cr, mbstat);
238 0 : counters[type]++;
239 0 : counters_leave(&cr, mbstat);
240 0 : splx(s);
241 :
242 0 : m->m_type = type;
243 0 : m->m_next = NULL;
244 0 : m->m_nextpkt = NULL;
245 0 : m->m_data = m->m_dat;
246 0 : m->m_flags = 0;
247 :
248 0 : return (m);
249 0 : }
250 :
251 : /*
252 : * ATTN: When changing anything here check m_inithdr() and m_defrag() those
253 : * may need to change as well.
254 : */
255 : struct mbuf *
256 0 : m_gethdr(int nowait, int type)
257 : {
258 : struct mbuf *m;
259 0 : struct counters_ref cr;
260 : uint64_t *counters;
261 : int s;
262 :
263 : KDASSERT(type < MT_NTYPES);
264 :
265 0 : m = pool_get(&mbpool, nowait == M_WAIT ? PR_WAITOK : PR_NOWAIT);
266 0 : if (m == NULL)
267 0 : return (NULL);
268 :
269 0 : s = splnet();
270 0 : counters = counters_enter(&cr, mbstat);
271 0 : counters[type]++;
272 0 : counters_leave(&cr, mbstat);
273 0 : splx(s);
274 :
275 0 : m->m_type = type;
276 :
277 0 : return (m_inithdr(m));
278 0 : }
279 :
280 : struct mbuf *
281 0 : m_inithdr(struct mbuf *m)
282 : {
283 : /* keep in sync with m_gethdr */
284 0 : m->m_next = NULL;
285 0 : m->m_nextpkt = NULL;
286 0 : m->m_data = m->m_pktdat;
287 0 : m->m_flags = M_PKTHDR;
288 0 : memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr));
289 0 : m->m_pkthdr.pf.prio = IFQ_DEFPRIO;
290 :
291 0 : return (m);
292 : }
293 :
294 : static inline void
295 0 : m_clearhdr(struct mbuf *m)
296 : {
297 : /* delete all mbuf tags to reset the state */
298 0 : m_tag_delete_chain(m);
299 : #if NPF > 0
300 0 : pf_mbuf_unlink_state_key(m);
301 0 : pf_mbuf_unlink_inpcb(m);
302 : #endif /* NPF > 0 */
303 :
304 0 : memset(&m->m_pkthdr, 0, sizeof(m->m_pkthdr));
305 0 : }
306 :
307 : void
308 0 : m_removehdr(struct mbuf *m)
309 : {
310 0 : KASSERT(m->m_flags & M_PKTHDR);
311 0 : m_clearhdr(m);
312 0 : m->m_flags &= ~M_PKTHDR;
313 0 : }
314 :
315 : void
316 0 : m_resethdr(struct mbuf *m)
317 : {
318 0 : int len = m->m_pkthdr.len;
319 0 : u_int8_t loopcnt = m->m_pkthdr.ph_loopcnt;
320 :
321 0 : KASSERT(m->m_flags & M_PKTHDR);
322 0 : m->m_flags &= (M_EXT|M_PKTHDR|M_EOR|M_EXTWR|M_ZEROIZE);
323 0 : m_clearhdr(m);
324 : /* like m_inithdr(), but keep any associated data and mbufs */
325 0 : m->m_pkthdr.pf.prio = IFQ_DEFPRIO;
326 0 : m->m_pkthdr.len = len;
327 0 : m->m_pkthdr.ph_loopcnt = loopcnt;
328 0 : }
329 :
330 : void
331 0 : m_calchdrlen(struct mbuf *m)
332 : {
333 : struct mbuf *n;
334 : int plen = 0;
335 :
336 0 : KASSERT(m->m_flags & M_PKTHDR);
337 0 : for (n = m; n; n = n->m_next)
338 0 : plen += n->m_len;
339 0 : m->m_pkthdr.len = plen;
340 0 : }
341 :
342 : struct mbuf *
343 0 : m_getclr(int nowait, int type)
344 : {
345 : struct mbuf *m;
346 :
347 0 : MGET(m, nowait, type);
348 0 : if (m == NULL)
349 0 : return (NULL);
350 0 : memset(mtod(m, caddr_t), 0, MLEN);
351 0 : return (m);
352 0 : }
353 :
354 : struct pool *
355 0 : m_clpool(u_int pktlen)
356 : {
357 : struct pool *pp;
358 : int pi;
359 :
360 0 : for (pi = 0; pi < nitems(mclpools); pi++) {
361 0 : pp = &mclpools[pi];
362 0 : if (pktlen <= pp->pr_size)
363 0 : return (pp);
364 : }
365 :
366 0 : return (NULL);
367 0 : }
368 :
369 : struct mbuf *
370 0 : m_clget(struct mbuf *m, int how, u_int pktlen)
371 : {
372 : struct mbuf *m0 = NULL;
373 : struct pool *pp;
374 : caddr_t buf;
375 :
376 0 : pp = m_clpool(pktlen);
377 : #ifdef DIAGNOSTIC
378 0 : if (pp == NULL)
379 0 : panic("m_clget: request for %u byte cluster", pktlen);
380 : #endif
381 :
382 0 : if (m == NULL) {
383 0 : m0 = m_gethdr(how, MT_DATA);
384 0 : if (m0 == NULL)
385 0 : return (NULL);
386 :
387 : m = m0;
388 0 : }
389 0 : buf = pool_get(pp, how == M_WAIT ? PR_WAITOK : PR_NOWAIT);
390 0 : if (buf == NULL) {
391 0 : m_freem(m0);
392 0 : return (NULL);
393 : }
394 :
395 0 : MEXTADD(m, buf, pp->pr_size, M_EXTWR, MEXTFREE_POOL, pp);
396 0 : return (m);
397 0 : }
398 :
399 : void
400 0 : m_extfree_pool(caddr_t buf, u_int size, void *pp)
401 : {
402 0 : pool_put(pp, buf);
403 0 : }
404 :
405 : struct mbuf *
406 0 : m_free(struct mbuf *m)
407 : {
408 : struct mbuf *n;
409 0 : struct counters_ref cr;
410 : uint64_t *counters;
411 : int s;
412 :
413 0 : if (m == NULL)
414 0 : return (NULL);
415 :
416 0 : s = splnet();
417 0 : counters = counters_enter(&cr, mbstat);
418 0 : counters[m->m_type]--;
419 0 : counters_leave(&cr, mbstat);
420 0 : splx(s);
421 :
422 0 : n = m->m_next;
423 0 : if (m->m_flags & M_ZEROIZE) {
424 0 : m_zero(m);
425 : /* propagate M_ZEROIZE to the next mbuf in the chain */
426 0 : if (n)
427 0 : n->m_flags |= M_ZEROIZE;
428 : }
429 0 : if (m->m_flags & M_PKTHDR) {
430 0 : m_tag_delete_chain(m);
431 : #if NPF > 0
432 0 : pf_mbuf_unlink_state_key(m);
433 0 : pf_mbuf_unlink_inpcb(m);
434 : #endif /* NPF > 0 */
435 0 : }
436 0 : if (m->m_flags & M_EXT)
437 0 : m_extfree(m);
438 :
439 0 : pool_put(&mbpool, m);
440 :
441 0 : return (n);
442 0 : }
443 :
444 : void
445 0 : m_extref(struct mbuf *o, struct mbuf *n)
446 : {
447 0 : int refs = MCLISREFERENCED(o);
448 :
449 0 : n->m_flags |= o->m_flags & (M_EXT|M_EXTWR);
450 :
451 0 : if (refs)
452 0 : mtx_enter(&m_extref_mtx);
453 0 : n->m_ext.ext_nextref = o->m_ext.ext_nextref;
454 0 : n->m_ext.ext_prevref = o;
455 0 : o->m_ext.ext_nextref = n;
456 0 : n->m_ext.ext_nextref->m_ext.ext_prevref = n;
457 0 : if (refs)
458 0 : mtx_leave(&m_extref_mtx);
459 :
460 : MCLREFDEBUGN((n), __FILE__, __LINE__);
461 0 : }
462 :
463 : static inline u_int
464 0 : m_extunref(struct mbuf *m)
465 : {
466 : int refs = 1;
467 :
468 0 : if (!MCLISREFERENCED(m))
469 0 : return (0);
470 :
471 0 : mtx_enter(&m_extref_mtx);
472 0 : if (MCLISREFERENCED(m)) {
473 0 : m->m_ext.ext_nextref->m_ext.ext_prevref =
474 0 : m->m_ext.ext_prevref;
475 0 : m->m_ext.ext_prevref->m_ext.ext_nextref =
476 0 : m->m_ext.ext_nextref;
477 0 : } else
478 : refs = 0;
479 0 : mtx_leave(&m_extref_mtx);
480 :
481 0 : return (refs);
482 0 : }
483 :
484 : /*
485 : * Returns a number for use with MEXTADD.
486 : * Should only be called once per function.
487 : * Drivers can be assured that the index will be non zero.
488 : */
489 : u_int
490 0 : mextfree_register(void (*fn)(caddr_t, u_int, void *))
491 : {
492 0 : KASSERT(num_extfree_fns < nitems(mextfree_fns));
493 0 : mextfree_fns[num_extfree_fns] = fn;
494 0 : return num_extfree_fns++;
495 : }
496 :
497 : void
498 0 : m_extfree(struct mbuf *m)
499 : {
500 0 : if (m_extunref(m) == 0) {
501 0 : KASSERT(m->m_ext.ext_free_fn < num_extfree_fns);
502 0 : mextfree_fns[m->m_ext.ext_free_fn](m->m_ext.ext_buf,
503 0 : m->m_ext.ext_size, m->m_ext.ext_arg);
504 0 : }
505 :
506 0 : m->m_flags &= ~(M_EXT|M_EXTWR);
507 0 : }
508 :
509 : struct mbuf *
510 0 : m_freem(struct mbuf *m)
511 : {
512 : struct mbuf *n;
513 :
514 0 : if (m == NULL)
515 0 : return (NULL);
516 :
517 0 : n = m->m_nextpkt;
518 :
519 0 : do
520 0 : m = m_free(m);
521 0 : while (m != NULL);
522 :
523 0 : return (n);
524 0 : }
525 :
526 : void
527 0 : m_purge(struct mbuf *m)
528 : {
529 0 : while (m != NULL)
530 0 : m = m_freem(m);
531 0 : }
532 :
533 : /*
534 : * mbuf chain defragmenter. This function uses some evil tricks to defragment
535 : * an mbuf chain into a single buffer without changing the mbuf pointer.
536 : * This needs to know a lot of the mbuf internals to make this work.
537 : */
538 : int
539 0 : m_defrag(struct mbuf *m, int how)
540 : {
541 : struct mbuf *m0;
542 :
543 0 : if (m->m_next == NULL)
544 0 : return (0);
545 :
546 0 : KASSERT(m->m_flags & M_PKTHDR);
547 :
548 0 : if ((m0 = m_gethdr(how, m->m_type)) == NULL)
549 0 : return (ENOBUFS);
550 0 : if (m->m_pkthdr.len > MHLEN) {
551 0 : MCLGETI(m0, how, NULL, m->m_pkthdr.len);
552 0 : if (!(m0->m_flags & M_EXT)) {
553 0 : m_free(m0);
554 0 : return (ENOBUFS);
555 : }
556 : }
557 0 : m_copydata(m, 0, m->m_pkthdr.len, mtod(m0, caddr_t));
558 0 : m0->m_pkthdr.len = m0->m_len = m->m_pkthdr.len;
559 :
560 : /* free chain behind and possible ext buf on the first mbuf */
561 0 : m_freem(m->m_next);
562 0 : m->m_next = NULL;
563 0 : if (m->m_flags & M_EXT)
564 0 : m_extfree(m);
565 :
566 : /*
567 : * Bounce copy mbuf over to the original mbuf and set everything up.
568 : * This needs to reset or clear all pointers that may go into the
569 : * original mbuf chain.
570 : */
571 0 : if (m0->m_flags & M_EXT) {
572 0 : memcpy(&m->m_ext, &m0->m_ext, sizeof(struct mbuf_ext));
573 0 : MCLINITREFERENCE(m);
574 0 : m->m_flags |= m0->m_flags & (M_EXT|M_EXTWR);
575 0 : m->m_data = m->m_ext.ext_buf;
576 0 : } else {
577 0 : m->m_data = m->m_pktdat;
578 0 : memcpy(m->m_data, m0->m_data, m0->m_len);
579 : }
580 0 : m->m_pkthdr.len = m->m_len = m0->m_len;
581 :
582 0 : m0->m_flags &= ~(M_EXT|M_EXTWR); /* cluster is gone */
583 0 : m_free(m0);
584 :
585 0 : return (0);
586 0 : }
587 :
588 : /*
589 : * Mbuffer utility routines.
590 : */
591 :
592 : /*
593 : * Ensure len bytes of contiguous space at the beginning of the mbuf chain
594 : */
595 : struct mbuf *
596 0 : m_prepend(struct mbuf *m, int len, int how)
597 : {
598 : struct mbuf *mn;
599 :
600 0 : if (len > MHLEN)
601 0 : panic("mbuf prepend length too big");
602 :
603 0 : if (M_LEADINGSPACE(m) >= len) {
604 0 : m->m_data -= len;
605 0 : m->m_len += len;
606 0 : } else {
607 0 : MGET(mn, how, m->m_type);
608 0 : if (mn == NULL) {
609 0 : m_freem(m);
610 0 : return (NULL);
611 : }
612 0 : if (m->m_flags & M_PKTHDR)
613 0 : M_MOVE_PKTHDR(mn, m);
614 0 : mn->m_next = m;
615 : m = mn;
616 0 : MH_ALIGN(m, len);
617 0 : m->m_len = len;
618 : }
619 0 : if (m->m_flags & M_PKTHDR)
620 0 : m->m_pkthdr.len += len;
621 0 : return (m);
622 0 : }
623 :
624 : /*
625 : * Make a copy of an mbuf chain starting "off" bytes from the beginning,
626 : * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
627 : * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
628 : */
629 : struct mbuf *
630 0 : m_copym(struct mbuf *m0, int off, int len, int wait)
631 : {
632 : struct mbuf *m, *n, **np;
633 0 : struct mbuf *top;
634 : int copyhdr = 0;
635 :
636 0 : if (off < 0 || len < 0)
637 0 : panic("m_copym0: off %d, len %d", off, len);
638 0 : if (off == 0 && m0->m_flags & M_PKTHDR)
639 0 : copyhdr = 1;
640 0 : if ((m = m_getptr(m0, off, &off)) == NULL)
641 0 : panic("m_copym0: short mbuf chain");
642 : np = ⊤
643 0 : top = NULL;
644 0 : while (len > 0) {
645 0 : if (m == NULL) {
646 0 : if (len != M_COPYALL)
647 0 : panic("m_copym0: m == NULL and not COPYALL");
648 : break;
649 : }
650 0 : MGET(n, wait, m->m_type);
651 0 : *np = n;
652 0 : if (n == NULL)
653 : goto nospace;
654 0 : if (copyhdr) {
655 0 : if (m_dup_pkthdr(n, m0, wait))
656 : goto nospace;
657 0 : if (len != M_COPYALL)
658 0 : n->m_pkthdr.len = len;
659 : copyhdr = 0;
660 0 : }
661 0 : n->m_len = min(len, m->m_len - off);
662 0 : if (m->m_flags & M_EXT) {
663 0 : n->m_data = m->m_data + off;
664 0 : n->m_ext = m->m_ext;
665 0 : MCLADDREFERENCE(m, n);
666 0 : } else {
667 0 : n->m_data += m->m_data -
668 0 : (m->m_flags & M_PKTHDR ? m->m_pktdat : m->m_dat);
669 0 : n->m_data += off;
670 0 : memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off,
671 : n->m_len);
672 : }
673 0 : if (len != M_COPYALL)
674 0 : len -= n->m_len;
675 0 : off += n->m_len;
676 : #ifdef DIAGNOSTIC
677 0 : if (off > m->m_len)
678 0 : panic("m_copym0 overrun");
679 : #endif
680 0 : if (off == m->m_len) {
681 0 : m = m->m_next;
682 0 : off = 0;
683 0 : }
684 0 : np = &n->m_next;
685 : }
686 0 : return (top);
687 : nospace:
688 0 : m_freem(top);
689 0 : return (NULL);
690 0 : }
691 :
692 : /*
693 : * Copy data from an mbuf chain starting "off" bytes from the beginning,
694 : * continuing for "len" bytes, into the indicated buffer.
695 : */
696 : void
697 0 : m_copydata(struct mbuf *m, int off, int len, caddr_t cp)
698 : {
699 : unsigned count;
700 :
701 0 : if (off < 0)
702 0 : panic("m_copydata: off %d < 0", off);
703 0 : if (len < 0)
704 0 : panic("m_copydata: len %d < 0", len);
705 0 : if ((m = m_getptr(m, off, &off)) == NULL)
706 0 : panic("m_copydata: short mbuf chain");
707 0 : while (len > 0) {
708 0 : if (m == NULL)
709 0 : panic("m_copydata: null mbuf");
710 0 : count = min(m->m_len - off, len);
711 0 : memmove(cp, mtod(m, caddr_t) + off, count);
712 0 : len -= count;
713 0 : cp += count;
714 0 : off = 0;
715 0 : m = m->m_next;
716 : }
717 0 : }
718 :
719 : /*
720 : * Copy data from a buffer back into the indicated mbuf chain,
721 : * starting "off" bytes from the beginning, extending the mbuf
722 : * chain if necessary. The mbuf needs to be properly initialized
723 : * including the setting of m_len.
724 : */
725 : int
726 0 : m_copyback(struct mbuf *m0, int off, int len, const void *_cp, int wait)
727 : {
728 : int mlen, totlen = 0;
729 : struct mbuf *m = m0, *n;
730 : caddr_t cp = (caddr_t)_cp;
731 : int error = 0;
732 :
733 0 : if (m0 == NULL)
734 0 : return (0);
735 0 : while (off > (mlen = m->m_len)) {
736 0 : off -= mlen;
737 0 : totlen += mlen;
738 0 : if (m->m_next == NULL) {
739 0 : if ((n = m_get(wait, m->m_type)) == NULL) {
740 : error = ENOBUFS;
741 0 : goto out;
742 : }
743 :
744 0 : if (off + len > MLEN) {
745 0 : MCLGETI(n, wait, NULL, off + len);
746 0 : if (!(n->m_flags & M_EXT)) {
747 0 : m_free(n);
748 : error = ENOBUFS;
749 0 : goto out;
750 : }
751 : }
752 0 : memset(mtod(n, caddr_t), 0, off);
753 0 : n->m_len = len + off;
754 0 : m->m_next = n;
755 0 : }
756 0 : m = m->m_next;
757 : }
758 0 : while (len > 0) {
759 : /* extend last packet to be filled fully */
760 0 : if (m->m_next == NULL && (len > m->m_len - off))
761 0 : m->m_len += min(len - (m->m_len - off),
762 0 : M_TRAILINGSPACE(m));
763 0 : mlen = min(m->m_len - off, len);
764 0 : memmove(mtod(m, caddr_t) + off, cp, mlen);
765 0 : cp += mlen;
766 0 : len -= mlen;
767 0 : totlen += mlen + off;
768 0 : if (len == 0)
769 : break;
770 : off = 0;
771 :
772 0 : if (m->m_next == NULL) {
773 0 : if ((n = m_get(wait, m->m_type)) == NULL) {
774 : error = ENOBUFS;
775 0 : goto out;
776 : }
777 :
778 0 : if (len > MLEN) {
779 0 : MCLGETI(n, wait, NULL, len);
780 0 : if (!(n->m_flags & M_EXT)) {
781 0 : m_free(n);
782 : error = ENOBUFS;
783 0 : goto out;
784 : }
785 : }
786 0 : n->m_len = len;
787 0 : m->m_next = n;
788 0 : }
789 0 : m = m->m_next;
790 : }
791 : out:
792 0 : if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
793 0 : m->m_pkthdr.len = totlen;
794 :
795 0 : return (error);
796 0 : }
797 :
798 : /*
799 : * Concatenate mbuf chain n to m.
800 : * n might be copied into m (when n->m_len is small), therefore data portion of
801 : * n could be copied into an mbuf of different mbuf type.
802 : * Therefore both chains should be of the same type (e.g. MT_DATA).
803 : * Any m_pkthdr is not updated.
804 : */
805 : void
806 0 : m_cat(struct mbuf *m, struct mbuf *n)
807 : {
808 0 : while (m->m_next)
809 : m = m->m_next;
810 0 : while (n) {
811 0 : if (M_READONLY(m) || n->m_len > M_TRAILINGSPACE(m)) {
812 : /* just join the two chains */
813 0 : m->m_next = n;
814 0 : return;
815 : }
816 : /* splat the data from one into the other */
817 0 : memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
818 : n->m_len);
819 0 : m->m_len += n->m_len;
820 0 : n = m_free(n);
821 : }
822 0 : }
823 :
824 : void
825 0 : m_adj(struct mbuf *mp, int req_len)
826 : {
827 : int len = req_len;
828 : struct mbuf *m;
829 : int count;
830 :
831 0 : if (mp == NULL)
832 0 : return;
833 0 : if (len >= 0) {
834 : /*
835 : * Trim from head.
836 : */
837 : m = mp;
838 0 : while (m != NULL && len > 0) {
839 0 : if (m->m_len <= len) {
840 0 : len -= m->m_len;
841 0 : m->m_data += m->m_len;
842 0 : m->m_len = 0;
843 0 : m = m->m_next;
844 0 : } else {
845 0 : m->m_data += len;
846 0 : m->m_len -= len;
847 : len = 0;
848 : }
849 : }
850 0 : if (mp->m_flags & M_PKTHDR)
851 0 : mp->m_pkthdr.len -= (req_len - len);
852 : } else {
853 : /*
854 : * Trim from tail. Scan the mbuf chain,
855 : * calculating its length and finding the last mbuf.
856 : * If the adjustment only affects this mbuf, then just
857 : * adjust and return. Otherwise, rescan and truncate
858 : * after the remaining size.
859 : */
860 0 : len = -len;
861 : count = 0;
862 : m = mp;
863 0 : for (;;) {
864 0 : count += m->m_len;
865 0 : if (m->m_next == NULL)
866 : break;
867 : m = m->m_next;
868 : }
869 0 : if (m->m_len >= len) {
870 0 : m->m_len -= len;
871 0 : if (mp->m_flags & M_PKTHDR)
872 0 : mp->m_pkthdr.len -= len;
873 0 : return;
874 : }
875 0 : count -= len;
876 0 : if (count < 0)
877 : count = 0;
878 : /*
879 : * Correct length for chain is "count".
880 : * Find the mbuf with last data, adjust its length,
881 : * and toss data from remaining mbufs on chain.
882 : */
883 0 : if (mp->m_flags & M_PKTHDR)
884 0 : mp->m_pkthdr.len = count;
885 : m = mp;
886 0 : for (;;) {
887 0 : if (m->m_len >= count) {
888 0 : m->m_len = count;
889 : break;
890 : }
891 0 : count -= m->m_len;
892 0 : m = m->m_next;
893 : }
894 0 : while ((m = m->m_next) != NULL)
895 0 : m->m_len = 0;
896 : }
897 0 : }
898 :
899 : /*
900 : * Rearrange an mbuf chain so that len bytes are contiguous
901 : * and in the data area of an mbuf (so that mtod will work
902 : * for a structure of size len). Returns the resulting
903 : * mbuf chain on success, frees it and returns null on failure.
904 : */
905 : struct mbuf *
906 0 : m_pullup(struct mbuf *n, int len)
907 : {
908 : struct mbuf *m;
909 : unsigned int adj;
910 : caddr_t head, tail;
911 : unsigned int space;
912 :
913 : /* if n is already contig then don't do any work */
914 0 : if (len <= n->m_len)
915 0 : return (n);
916 :
917 0 : adj = (unsigned long)n->m_data & ALIGNBYTES;
918 0 : head = (caddr_t)ALIGN(mtod(n, caddr_t) - M_LEADINGSPACE(n)) + adj;
919 0 : tail = mtod(n, caddr_t) + n->m_len + M_TRAILINGSPACE(n);
920 :
921 0 : if (head < tail && len <= tail - head) {
922 : /* there's enough space in the first mbuf */
923 :
924 0 : if (len > tail - mtod(n, caddr_t)) {
925 : /* need to memmove to make space at the end */
926 0 : memmove(head, mtod(n, caddr_t), n->m_len);
927 0 : n->m_data = head;
928 0 : }
929 :
930 0 : len -= n->m_len;
931 : m = n;
932 0 : n = m->m_next;
933 0 : } else {
934 : /* the first mbuf is too small so prepend one with space */
935 0 : space = adj + len;
936 :
937 0 : if (space > MAXMCLBYTES)
938 : goto bad;
939 :
940 0 : MGET(m, M_DONTWAIT, n->m_type);
941 0 : if (m == NULL)
942 : goto bad;
943 0 : if (space > MHLEN) {
944 0 : MCLGETI(m, M_DONTWAIT, NULL, space);
945 0 : if ((m->m_flags & M_EXT) == 0) {
946 0 : m_free(m);
947 0 : goto bad;
948 : }
949 : }
950 :
951 0 : if (n->m_flags & M_PKTHDR)
952 0 : M_MOVE_PKTHDR(m, n);
953 :
954 0 : m->m_len = 0;
955 0 : m->m_data += adj;
956 : }
957 :
958 0 : KASSERT(M_TRAILINGSPACE(m) >= len);
959 :
960 0 : do {
961 0 : if (n == NULL) {
962 0 : (void)m_free(m);
963 0 : goto bad;
964 : }
965 :
966 0 : space = min(len, n->m_len);
967 0 : memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), space);
968 0 : len -= space;
969 0 : m->m_len += space;
970 0 : n->m_len -= space;
971 :
972 0 : if (n->m_len > 0)
973 0 : n->m_data += space;
974 : else
975 0 : n = m_free(n);
976 0 : } while (len > 0);
977 :
978 0 : m->m_next = n;
979 :
980 0 : return (m);
981 :
982 : bad:
983 0 : m_freem(n);
984 0 : return (NULL);
985 0 : }
986 :
987 : /*
988 : * Return a pointer to mbuf/offset of location in mbuf chain.
989 : */
990 : struct mbuf *
991 0 : m_getptr(struct mbuf *m, int loc, int *off)
992 : {
993 0 : while (loc >= 0) {
994 : /* Normal end of search */
995 0 : if (m->m_len > loc) {
996 0 : *off = loc;
997 0 : return (m);
998 : } else {
999 0 : loc -= m->m_len;
1000 :
1001 0 : if (m->m_next == NULL) {
1002 0 : if (loc == 0) {
1003 : /* Point at the end of valid data */
1004 0 : *off = m->m_len;
1005 0 : return (m);
1006 : } else {
1007 0 : return (NULL);
1008 : }
1009 : } else {
1010 : m = m->m_next;
1011 : }
1012 : }
1013 : }
1014 :
1015 0 : return (NULL);
1016 0 : }
1017 :
1018 : /*
1019 : * Partition an mbuf chain in two pieces, returning the tail --
1020 : * all but the first len0 bytes. In case of failure, it returns NULL and
1021 : * attempts to restore the chain to its original state.
1022 : */
1023 : struct mbuf *
1024 0 : m_split(struct mbuf *m0, int len0, int wait)
1025 : {
1026 : struct mbuf *m, *n;
1027 : unsigned len = len0, remain, olen;
1028 :
1029 0 : for (m = m0; m && len > m->m_len; m = m->m_next)
1030 0 : len -= m->m_len;
1031 0 : if (m == NULL)
1032 0 : return (NULL);
1033 0 : remain = m->m_len - len;
1034 0 : if (m0->m_flags & M_PKTHDR) {
1035 0 : MGETHDR(n, wait, m0->m_type);
1036 0 : if (n == NULL)
1037 0 : return (NULL);
1038 0 : if (m_dup_pkthdr(n, m0, wait)) {
1039 0 : m_freem(n);
1040 0 : return (NULL);
1041 : }
1042 0 : n->m_pkthdr.len -= len0;
1043 0 : olen = m0->m_pkthdr.len;
1044 0 : m0->m_pkthdr.len = len0;
1045 0 : if (remain == 0) {
1046 0 : n->m_next = m->m_next;
1047 0 : m->m_next = NULL;
1048 0 : n->m_len = 0;
1049 0 : return (n);
1050 : }
1051 0 : if (m->m_flags & M_EXT)
1052 : goto extpacket;
1053 0 : if (remain > MHLEN) {
1054 : /* m can't be the lead packet */
1055 0 : MH_ALIGN(n, 0);
1056 0 : n->m_next = m_split(m, len, wait);
1057 0 : if (n->m_next == NULL) {
1058 0 : (void) m_free(n);
1059 0 : m0->m_pkthdr.len = olen;
1060 0 : return (NULL);
1061 : } else {
1062 0 : n->m_len = 0;
1063 0 : return (n);
1064 : }
1065 : } else
1066 0 : MH_ALIGN(n, remain);
1067 0 : } else if (remain == 0) {
1068 0 : n = m->m_next;
1069 0 : m->m_next = NULL;
1070 0 : return (n);
1071 : } else {
1072 0 : MGET(n, wait, m->m_type);
1073 0 : if (n == NULL)
1074 0 : return (NULL);
1075 0 : M_ALIGN(n, remain);
1076 : }
1077 : extpacket:
1078 0 : if (m->m_flags & M_EXT) {
1079 0 : n->m_ext = m->m_ext;
1080 0 : MCLADDREFERENCE(m, n);
1081 0 : n->m_data = m->m_data + len;
1082 0 : } else {
1083 0 : memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain);
1084 : }
1085 0 : n->m_len = remain;
1086 0 : m->m_len = len;
1087 0 : n->m_next = m->m_next;
1088 0 : m->m_next = NULL;
1089 0 : return (n);
1090 0 : }
1091 :
1092 : /*
1093 : * Make space for a new header of length hlen at skip bytes
1094 : * into the packet. When doing this we allocate new mbufs only
1095 : * when absolutely necessary. The mbuf where the new header
1096 : * is to go is returned together with an offset into the mbuf.
1097 : * If NULL is returned then the mbuf chain may have been modified;
1098 : * the caller is assumed to always free the chain.
1099 : */
1100 : struct mbuf *
1101 0 : m_makespace(struct mbuf *m0, int skip, int hlen, int *off)
1102 : {
1103 : struct mbuf *m;
1104 : unsigned remain;
1105 :
1106 0 : KASSERT(m0->m_flags & M_PKTHDR);
1107 : /*
1108 : * Limit the size of the new header to MHLEN. In case
1109 : * skip = 0 and the first buffer is not a cluster this
1110 : * is the maximum space available in that mbuf.
1111 : * In other words this code never prepends a mbuf.
1112 : */
1113 0 : KASSERT(hlen < MHLEN);
1114 :
1115 0 : for (m = m0; m && skip > m->m_len; m = m->m_next)
1116 0 : skip -= m->m_len;
1117 0 : if (m == NULL)
1118 0 : return (NULL);
1119 : /*
1120 : * At this point skip is the offset into the mbuf m
1121 : * where the new header should be placed. Figure out
1122 : * if there's space to insert the new header. If so,
1123 : * and copying the remainder makes sense then do so.
1124 : * Otherwise insert a new mbuf in the chain, splitting
1125 : * the contents of m as needed.
1126 : */
1127 0 : remain = m->m_len - skip; /* data to move */
1128 0 : if (skip < remain && hlen <= M_LEADINGSPACE(m)) {
1129 0 : if (skip)
1130 0 : memmove(m->m_data-hlen, m->m_data, skip);
1131 0 : m->m_data -= hlen;
1132 0 : m->m_len += hlen;
1133 0 : *off = skip;
1134 0 : } else if (hlen > M_TRAILINGSPACE(m)) {
1135 : struct mbuf *n;
1136 :
1137 0 : if (remain > 0) {
1138 0 : MGET(n, M_DONTWAIT, m->m_type);
1139 0 : if (n && remain > MLEN) {
1140 0 : MCLGETI(n, M_DONTWAIT, NULL, remain);
1141 0 : if ((n->m_flags & M_EXT) == 0) {
1142 0 : m_free(n);
1143 : n = NULL;
1144 0 : }
1145 : }
1146 0 : if (n == NULL)
1147 0 : return (NULL);
1148 :
1149 0 : memcpy(n->m_data, mtod(m, char *) + skip, remain);
1150 0 : n->m_len = remain;
1151 0 : m->m_len -= remain;
1152 :
1153 0 : n->m_next = m->m_next;
1154 0 : m->m_next = n;
1155 0 : }
1156 :
1157 0 : if (hlen <= M_TRAILINGSPACE(m)) {
1158 0 : m->m_len += hlen;
1159 0 : *off = skip;
1160 0 : } else {
1161 0 : n = m_get(M_DONTWAIT, m->m_type);
1162 0 : if (n == NULL)
1163 0 : return NULL;
1164 :
1165 0 : n->m_len = hlen;
1166 :
1167 0 : n->m_next = m->m_next;
1168 0 : m->m_next = n;
1169 :
1170 0 : *off = 0; /* header is at front ... */
1171 : m = n; /* ... of new mbuf */
1172 : }
1173 0 : } else {
1174 : /*
1175 : * Copy the remainder to the back of the mbuf
1176 : * so there's space to write the new header.
1177 : */
1178 0 : if (remain > 0)
1179 0 : memmove(mtod(m, caddr_t) + skip + hlen,
1180 : mtod(m, caddr_t) + skip, remain);
1181 0 : m->m_len += hlen;
1182 0 : *off = skip;
1183 : }
1184 0 : m0->m_pkthdr.len += hlen; /* adjust packet length */
1185 0 : return m;
1186 0 : }
1187 :
1188 :
1189 : /*
1190 : * Routine to copy from device local memory into mbufs.
1191 : */
1192 : struct mbuf *
1193 0 : m_devget(char *buf, int totlen, int off)
1194 : {
1195 : struct mbuf *m;
1196 0 : struct mbuf *top, **mp;
1197 : int len;
1198 :
1199 0 : top = NULL;
1200 : mp = ⊤
1201 :
1202 0 : if (off < 0 || off > MHLEN)
1203 0 : return (NULL);
1204 :
1205 0 : MGETHDR(m, M_DONTWAIT, MT_DATA);
1206 0 : if (m == NULL)
1207 0 : return (NULL);
1208 :
1209 0 : m->m_pkthdr.len = totlen;
1210 :
1211 : len = MHLEN;
1212 :
1213 0 : while (totlen > 0) {
1214 0 : if (top != NULL) {
1215 0 : MGET(m, M_DONTWAIT, MT_DATA);
1216 0 : if (m == NULL) {
1217 : /*
1218 : * As we might get called by pfkey, make sure
1219 : * we do not leak sensitive data.
1220 : */
1221 0 : top->m_flags |= M_ZEROIZE;
1222 0 : m_freem(top);
1223 0 : return (NULL);
1224 : }
1225 : len = MLEN;
1226 0 : }
1227 :
1228 0 : if (totlen + off >= MINCLSIZE) {
1229 0 : MCLGET(m, M_DONTWAIT);
1230 0 : if (m->m_flags & M_EXT)
1231 0 : len = MCLBYTES;
1232 : } else {
1233 : /* Place initial small packet/header at end of mbuf. */
1234 0 : if (top == NULL && totlen + off + max_linkhdr <= len) {
1235 0 : m->m_data += max_linkhdr;
1236 0 : len -= max_linkhdr;
1237 0 : }
1238 : }
1239 :
1240 0 : if (off) {
1241 0 : m->m_data += off;
1242 0 : len -= off;
1243 : off = 0;
1244 0 : }
1245 :
1246 0 : m->m_len = len = min(totlen, len);
1247 0 : memcpy(mtod(m, void *), buf, (size_t)len);
1248 :
1249 0 : buf += len;
1250 0 : *mp = m;
1251 0 : mp = &m->m_next;
1252 0 : totlen -= len;
1253 : }
1254 0 : return (top);
1255 0 : }
1256 :
1257 : void
1258 0 : m_zero(struct mbuf *m)
1259 : {
1260 : #ifdef DIAGNOSTIC
1261 0 : if (M_READONLY(m))
1262 0 : panic("m_zero: M_READONLY");
1263 : #endif /* DIAGNOSTIC */
1264 :
1265 0 : if (m->m_flags & M_EXT)
1266 0 : explicit_bzero(m->m_ext.ext_buf, m->m_ext.ext_size);
1267 : else {
1268 0 : if (m->m_flags & M_PKTHDR)
1269 0 : explicit_bzero(m->m_pktdat, MHLEN);
1270 : else
1271 0 : explicit_bzero(m->m_dat, MLEN);
1272 : }
1273 0 : }
1274 :
1275 : /*
1276 : * Apply function f to the data in an mbuf chain starting "off" bytes from the
1277 : * beginning, continuing for "len" bytes.
1278 : */
1279 : int
1280 0 : m_apply(struct mbuf *m, int off, int len,
1281 : int (*f)(caddr_t, caddr_t, unsigned int), caddr_t fstate)
1282 : {
1283 : int rval;
1284 : unsigned int count;
1285 :
1286 0 : if (len < 0)
1287 0 : panic("m_apply: len %d < 0", len);
1288 0 : if (off < 0)
1289 0 : panic("m_apply: off %d < 0", off);
1290 0 : while (off > 0) {
1291 0 : if (m == NULL)
1292 0 : panic("m_apply: null mbuf in skip");
1293 0 : if (off < m->m_len)
1294 : break;
1295 0 : off -= m->m_len;
1296 0 : m = m->m_next;
1297 : }
1298 0 : while (len > 0) {
1299 0 : if (m == NULL)
1300 0 : panic("m_apply: null mbuf");
1301 0 : count = min(m->m_len - off, len);
1302 :
1303 0 : rval = f(fstate, mtod(m, caddr_t) + off, count);
1304 0 : if (rval)
1305 0 : return (rval);
1306 :
1307 0 : len -= count;
1308 : off = 0;
1309 0 : m = m->m_next;
1310 : }
1311 :
1312 0 : return (0);
1313 0 : }
1314 :
1315 : int
1316 0 : m_leadingspace(struct mbuf *m)
1317 : {
1318 0 : if (M_READONLY(m))
1319 0 : return 0;
1320 0 : return (m->m_flags & M_EXT ? m->m_data - m->m_ext.ext_buf :
1321 0 : m->m_flags & M_PKTHDR ? m->m_data - m->m_pktdat :
1322 0 : m->m_data - m->m_dat);
1323 0 : }
1324 :
1325 : int
1326 0 : m_trailingspace(struct mbuf *m)
1327 : {
1328 0 : if (M_READONLY(m))
1329 0 : return 0;
1330 0 : return (m->m_flags & M_EXT ? m->m_ext.ext_buf +
1331 0 : m->m_ext.ext_size - (m->m_data + m->m_len) :
1332 0 : &m->m_dat[MLEN] - (m->m_data + m->m_len));
1333 0 : }
1334 :
1335 :
1336 : /*
1337 : * Duplicate mbuf pkthdr from from to to.
1338 : * from must have M_PKTHDR set, and to must be empty.
1339 : */
1340 : int
1341 0 : m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int wait)
1342 : {
1343 : int error;
1344 :
1345 0 : KASSERT(from->m_flags & M_PKTHDR);
1346 :
1347 0 : to->m_flags = (to->m_flags & (M_EXT | M_EXTWR));
1348 0 : to->m_flags |= (from->m_flags & M_COPYFLAGS);
1349 0 : to->m_pkthdr = from->m_pkthdr;
1350 :
1351 : #if NPF > 0
1352 0 : to->m_pkthdr.pf.statekey = NULL;
1353 0 : pf_mbuf_link_state_key(to, from->m_pkthdr.pf.statekey);
1354 0 : to->m_pkthdr.pf.inp = NULL;
1355 0 : pf_mbuf_link_inpcb(to, from->m_pkthdr.pf.inp);
1356 : #endif /* NPF > 0 */
1357 :
1358 0 : SLIST_INIT(&to->m_pkthdr.ph_tags);
1359 :
1360 0 : if ((error = m_tag_copy_chain(to, from, wait)) != 0)
1361 0 : return (error);
1362 :
1363 0 : if ((to->m_flags & M_EXT) == 0)
1364 0 : to->m_data = to->m_pktdat;
1365 :
1366 0 : return (0);
1367 0 : }
1368 :
1369 : struct mbuf *
1370 0 : m_dup_pkt(struct mbuf *m0, unsigned int adj, int wait)
1371 : {
1372 : struct mbuf *m;
1373 : int len;
1374 :
1375 0 : KASSERT(m0->m_flags & M_PKTHDR);
1376 :
1377 0 : len = m0->m_pkthdr.len + adj;
1378 0 : if (len > MAXMCLBYTES) /* XXX */
1379 0 : return (NULL);
1380 :
1381 0 : m = m_get(wait, m0->m_type);
1382 0 : if (m == NULL)
1383 0 : return (NULL);
1384 :
1385 0 : if (m_dup_pkthdr(m, m0, wait) != 0)
1386 : goto fail;
1387 :
1388 0 : if (len > MHLEN) {
1389 0 : MCLGETI(m, wait, NULL, len);
1390 0 : if (!ISSET(m->m_flags, M_EXT))
1391 : goto fail;
1392 : }
1393 :
1394 0 : m->m_len = m->m_pkthdr.len = len;
1395 0 : m_adj(m, adj);
1396 0 : m_copydata(m0, 0, m0->m_pkthdr.len, mtod(m, caddr_t));
1397 :
1398 0 : return (m);
1399 :
1400 : fail:
1401 0 : m_freem(m);
1402 0 : return (NULL);
1403 0 : }
1404 :
1405 : void *
1406 0 : m_pool_alloc(struct pool *pp, int flags, int *slowdown)
1407 : {
1408 : void *v = NULL;
1409 : int avail = 1;
1410 :
1411 0 : if (mbuf_mem_alloc + pp->pr_pgsize > mbuf_mem_limit)
1412 0 : return (NULL);
1413 :
1414 0 : mtx_enter(&m_pool_mtx);
1415 0 : if (mbuf_mem_alloc + pp->pr_pgsize > mbuf_mem_limit)
1416 0 : avail = 0;
1417 : else
1418 0 : mbuf_mem_alloc += pp->pr_pgsize;
1419 0 : mtx_leave(&m_pool_mtx);
1420 :
1421 0 : if (avail) {
1422 0 : v = (*pool_allocator_multi.pa_alloc)(pp, flags, slowdown);
1423 :
1424 0 : if (v == NULL) {
1425 0 : mtx_enter(&m_pool_mtx);
1426 0 : mbuf_mem_alloc -= pp->pr_pgsize;
1427 0 : mtx_leave(&m_pool_mtx);
1428 0 : }
1429 : }
1430 :
1431 0 : return (v);
1432 0 : }
1433 :
1434 : void
1435 0 : m_pool_free(struct pool *pp, void *v)
1436 : {
1437 0 : (*pool_allocator_multi.pa_free)(pp, v);
1438 :
1439 0 : mtx_enter(&m_pool_mtx);
1440 0 : mbuf_mem_alloc -= pp->pr_pgsize;
1441 0 : mtx_leave(&m_pool_mtx);
1442 0 : }
1443 :
1444 : void
1445 0 : m_pool_init(struct pool *pp, u_int size, u_int align, const char *wmesg)
1446 : {
1447 0 : pool_init(pp, size, align, IPL_NET, 0, wmesg, &m_pool_allocator);
1448 0 : pool_set_constraints(pp, &kp_dma_contig);
1449 0 : }
1450 :
1451 : #ifdef DDB
1452 : void
1453 0 : m_print(void *v,
1454 : int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2))))
1455 : {
1456 0 : struct mbuf *m = v;
1457 :
1458 0 : (*pr)("mbuf %p\n", m);
1459 0 : (*pr)("m_type: %i\tm_flags: %b\n", m->m_type, m->m_flags, M_BITS);
1460 0 : (*pr)("m_next: %p\tm_nextpkt: %p\n", m->m_next, m->m_nextpkt);
1461 0 : (*pr)("m_data: %p\tm_len: %u\n", m->m_data, m->m_len);
1462 0 : (*pr)("m_dat: %p\tm_pktdat: %p\n", m->m_dat, m->m_pktdat);
1463 0 : if (m->m_flags & M_PKTHDR) {
1464 0 : (*pr)("m_ptkhdr.ph_ifidx: %u\tm_pkthdr.len: %i\n",
1465 0 : m->m_pkthdr.ph_ifidx, m->m_pkthdr.len);
1466 0 : (*pr)("m_ptkhdr.ph_tags: %p\tm_pkthdr.ph_tagsset: %b\n",
1467 0 : SLIST_FIRST(&m->m_pkthdr.ph_tags),
1468 0 : m->m_pkthdr.ph_tagsset, MTAG_BITS);
1469 0 : (*pr)("m_pkthdr.ph_flowid: %u\tm_pkthdr.ph_loopcnt: %u\n",
1470 0 : m->m_pkthdr.ph_flowid, m->m_pkthdr.ph_loopcnt);
1471 0 : (*pr)("m_pkthdr.csum_flags: %b\n",
1472 0 : m->m_pkthdr.csum_flags, MCS_BITS);
1473 0 : (*pr)("m_pkthdr.ether_vtag: %u\tm_ptkhdr.ph_rtableid: %u\n",
1474 0 : m->m_pkthdr.ether_vtag, m->m_pkthdr.ph_rtableid);
1475 0 : (*pr)("m_pkthdr.pf.statekey: %p\tm_pkthdr.pf.inp %p\n",
1476 0 : m->m_pkthdr.pf.statekey, m->m_pkthdr.pf.inp);
1477 0 : (*pr)("m_pkthdr.pf.qid: %u\tm_pkthdr.pf.tag: %u\n",
1478 0 : m->m_pkthdr.pf.qid, m->m_pkthdr.pf.tag);
1479 0 : (*pr)("m_pkthdr.pf.flags: %b\n",
1480 0 : m->m_pkthdr.pf.flags, MPF_BITS);
1481 0 : (*pr)("m_pkthdr.pf.routed: %u\tm_pkthdr.pf.prio: %u\n",
1482 0 : m->m_pkthdr.pf.routed, m->m_pkthdr.pf.prio);
1483 0 : }
1484 0 : if (m->m_flags & M_EXT) {
1485 0 : (*pr)("m_ext.ext_buf: %p\tm_ext.ext_size: %u\n",
1486 0 : m->m_ext.ext_buf, m->m_ext.ext_size);
1487 0 : (*pr)("m_ext.ext_free_fn: %u\tm_ext.ext_arg: %p\n",
1488 0 : m->m_ext.ext_free_fn, m->m_ext.ext_arg);
1489 0 : (*pr)("m_ext.ext_nextref: %p\tm_ext.ext_prevref: %p\n",
1490 0 : m->m_ext.ext_nextref, m->m_ext.ext_prevref);
1491 :
1492 0 : }
1493 0 : }
1494 : #endif
1495 :
1496 : /*
1497 : * mbuf lists
1498 : */
1499 :
1500 : void
1501 0 : ml_init(struct mbuf_list *ml)
1502 : {
1503 0 : ml->ml_head = ml->ml_tail = NULL;
1504 0 : ml->ml_len = 0;
1505 0 : }
1506 :
1507 : void
1508 0 : ml_enqueue(struct mbuf_list *ml, struct mbuf *m)
1509 : {
1510 0 : if (ml->ml_tail == NULL)
1511 0 : ml->ml_head = ml->ml_tail = m;
1512 : else {
1513 0 : ml->ml_tail->m_nextpkt = m;
1514 0 : ml->ml_tail = m;
1515 : }
1516 :
1517 0 : m->m_nextpkt = NULL;
1518 0 : ml->ml_len++;
1519 0 : }
1520 :
1521 : void
1522 0 : ml_enlist(struct mbuf_list *mla, struct mbuf_list *mlb)
1523 : {
1524 0 : if (!ml_empty(mlb)) {
1525 0 : if (ml_empty(mla))
1526 0 : mla->ml_head = mlb->ml_head;
1527 : else
1528 0 : mla->ml_tail->m_nextpkt = mlb->ml_head;
1529 0 : mla->ml_tail = mlb->ml_tail;
1530 0 : mla->ml_len += mlb->ml_len;
1531 :
1532 0 : ml_init(mlb);
1533 0 : }
1534 0 : }
1535 :
1536 : struct mbuf *
1537 0 : ml_dequeue(struct mbuf_list *ml)
1538 : {
1539 : struct mbuf *m;
1540 :
1541 0 : m = ml->ml_head;
1542 0 : if (m != NULL) {
1543 0 : ml->ml_head = m->m_nextpkt;
1544 0 : if (ml->ml_head == NULL)
1545 0 : ml->ml_tail = NULL;
1546 :
1547 0 : m->m_nextpkt = NULL;
1548 0 : ml->ml_len--;
1549 0 : }
1550 :
1551 0 : return (m);
1552 : }
1553 :
1554 : struct mbuf *
1555 0 : ml_dechain(struct mbuf_list *ml)
1556 : {
1557 : struct mbuf *m0;
1558 :
1559 0 : m0 = ml->ml_head;
1560 :
1561 0 : ml_init(ml);
1562 :
1563 0 : return (m0);
1564 : }
1565 :
1566 : unsigned int
1567 0 : ml_purge(struct mbuf_list *ml)
1568 : {
1569 : struct mbuf *m, *n;
1570 : unsigned int len;
1571 :
1572 0 : for (m = ml->ml_head; m != NULL; m = n) {
1573 0 : n = m->m_nextpkt;
1574 0 : m_freem(m);
1575 : }
1576 :
1577 0 : len = ml->ml_len;
1578 0 : ml_init(ml);
1579 :
1580 0 : return (len);
1581 : }
1582 :
1583 : /*
1584 : * mbuf queues
1585 : */
1586 :
1587 : void
1588 0 : mq_init(struct mbuf_queue *mq, u_int maxlen, int ipl)
1589 : {
1590 0 : mtx_init(&mq->mq_mtx, ipl);
1591 0 : ml_init(&mq->mq_list);
1592 0 : mq->mq_maxlen = maxlen;
1593 0 : }
1594 :
1595 : int
1596 0 : mq_enqueue(struct mbuf_queue *mq, struct mbuf *m)
1597 : {
1598 : int dropped = 0;
1599 :
1600 0 : mtx_enter(&mq->mq_mtx);
1601 0 : if (mq_len(mq) < mq->mq_maxlen)
1602 0 : ml_enqueue(&mq->mq_list, m);
1603 : else {
1604 0 : mq->mq_drops++;
1605 : dropped = 1;
1606 : }
1607 0 : mtx_leave(&mq->mq_mtx);
1608 :
1609 0 : if (dropped)
1610 0 : m_freem(m);
1611 :
1612 0 : return (dropped);
1613 : }
1614 :
1615 : struct mbuf *
1616 0 : mq_dequeue(struct mbuf_queue *mq)
1617 : {
1618 : struct mbuf *m;
1619 :
1620 0 : mtx_enter(&mq->mq_mtx);
1621 0 : m = ml_dequeue(&mq->mq_list);
1622 0 : mtx_leave(&mq->mq_mtx);
1623 :
1624 0 : return (m);
1625 : }
1626 :
1627 : int
1628 0 : mq_enlist(struct mbuf_queue *mq, struct mbuf_list *ml)
1629 : {
1630 : struct mbuf *m;
1631 : int dropped = 0;
1632 :
1633 0 : mtx_enter(&mq->mq_mtx);
1634 0 : if (mq_len(mq) < mq->mq_maxlen)
1635 0 : ml_enlist(&mq->mq_list, ml);
1636 : else {
1637 0 : dropped = ml_len(ml);
1638 0 : mq->mq_drops += dropped;
1639 : }
1640 0 : mtx_leave(&mq->mq_mtx);
1641 :
1642 0 : if (dropped) {
1643 0 : while ((m = ml_dequeue(ml)) != NULL)
1644 0 : m_freem(m);
1645 : }
1646 :
1647 0 : return (dropped);
1648 : }
1649 :
1650 : void
1651 0 : mq_delist(struct mbuf_queue *mq, struct mbuf_list *ml)
1652 : {
1653 0 : mtx_enter(&mq->mq_mtx);
1654 0 : *ml = mq->mq_list;
1655 0 : ml_init(&mq->mq_list);
1656 0 : mtx_leave(&mq->mq_mtx);
1657 0 : }
1658 :
1659 : struct mbuf *
1660 0 : mq_dechain(struct mbuf_queue *mq)
1661 : {
1662 : struct mbuf *m0;
1663 :
1664 0 : mtx_enter(&mq->mq_mtx);
1665 0 : m0 = ml_dechain(&mq->mq_list);
1666 0 : mtx_leave(&mq->mq_mtx);
1667 :
1668 0 : return (m0);
1669 : }
1670 :
1671 : unsigned int
1672 0 : mq_purge(struct mbuf_queue *mq)
1673 : {
1674 0 : struct mbuf_list ml;
1675 :
1676 0 : mq_delist(mq, &ml);
1677 :
1678 0 : return (ml_purge(&ml));
1679 0 : }
1680 :
1681 : int
1682 0 : sysctl_mq(int *name, u_int namelen, void *oldp, size_t *oldlenp,
1683 : void *newp, size_t newlen, struct mbuf_queue *mq)
1684 : {
1685 0 : unsigned int maxlen;
1686 : int error;
1687 :
1688 : /* All sysctl names at this level are terminal. */
1689 0 : if (namelen != 1)
1690 0 : return (ENOTDIR);
1691 :
1692 0 : switch (name[0]) {
1693 : case IFQCTL_LEN:
1694 0 : return (sysctl_rdint(oldp, oldlenp, newp, mq_len(mq)));
1695 : case IFQCTL_MAXLEN:
1696 0 : maxlen = mq->mq_maxlen;
1697 0 : error = sysctl_int(oldp, oldlenp, newp, newlen, &maxlen);
1698 0 : if (!error && maxlen != mq->mq_maxlen) {
1699 0 : mtx_enter(&mq->mq_mtx);
1700 0 : mq->mq_maxlen = maxlen;
1701 0 : mtx_leave(&mq->mq_mtx);
1702 0 : }
1703 0 : return (error);
1704 : case IFQCTL_DROPS:
1705 0 : return (sysctl_rdint(oldp, oldlenp, newp, mq_drops(mq)));
1706 : default:
1707 0 : return (EOPNOTSUPP);
1708 : }
1709 : /* NOTREACHED */
1710 0 : }
|
