Line data Source code
1 : /* $OpenBSD: percpu.h,v 1.8 2018/08/28 15:15:02 mpi Exp $ */
2 :
3 : /*
4 : * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
5 : *
6 : * Permission to use, copy, modify, and distribute this software for any
7 : * purpose with or without fee is hereby granted, provided that the above
8 : * copyright notice and this permission notice appear in all copies.
9 : *
10 : * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 : * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 : * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 : * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 : * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 : * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 : * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 : */
18 :
19 : #ifndef _SYS_PERCPU_H_
20 : #define _SYS_PERCPU_H_
21 :
22 : #ifndef CACHELINESIZE
23 : #define CACHELINESIZE 64
24 : #endif
25 :
26 : #ifndef __upunused /* this should go in param.h */
27 : #ifdef MULTIPROCESSOR
28 : #define __upunused
29 : #else
30 : #define __upunused __attribute__((__unused__))
31 : #endif
32 : #endif
33 :
34 : struct cpumem {
35 : void *mem;
36 : };
37 :
38 : struct cpumem_iter {
39 : unsigned int cpu;
40 : } __upunused;
41 :
42 : struct counters_ref {
43 : uint64_t g;
44 : uint64_t *c;
45 : };
46 :
47 : #ifdef _KERNEL
48 :
49 : #include <sys/atomic.h>
50 :
51 : struct pool;
52 :
53 : struct cpumem *cpumem_get(struct pool *);
54 : void cpumem_put(struct pool *, struct cpumem *);
55 :
56 : struct cpumem *cpumem_malloc(size_t, int);
57 : struct cpumem *cpumem_malloc_ncpus(struct cpumem *, size_t, int);
58 : void cpumem_free(struct cpumem *, int, size_t);
59 :
60 : void *cpumem_first(struct cpumem_iter *, struct cpumem *);
61 : void *cpumem_next(struct cpumem_iter *, struct cpumem *);
62 :
63 : static inline void *
64 0 : cpumem_enter(struct cpumem *cm)
65 : {
66 : #ifdef MULTIPROCESSOR
67 0 : return (cm[cpu_number()].mem);
68 : #else
69 : return (cm);
70 : #endif
71 : }
72 :
73 : static inline void
74 0 : cpumem_leave(struct cpumem *cm, void *mem)
75 : {
76 : /* KDASSERT? */
77 0 : }
78 :
79 : #ifdef MULTIPROCESSOR
80 :
81 : #define CPUMEM_BOOT_MEMORY(_name, _sz) \
82 : static struct { \
83 : unsigned char mem[_sz]; \
84 : struct cpumem cpumem; \
85 : } __aligned(CACHELINESIZE) _name##_boot_cpumem = { \
86 : .cpumem = { _name##_boot_cpumem.mem } \
87 : }
88 :
89 : #define CPUMEM_BOOT_INITIALIZER(_name) \
90 : { &_name##_boot_cpumem.cpumem }
91 :
92 : #else /* MULTIPROCESSOR */
93 :
94 : #define CPUMEM_BOOT_MEMORY(_name, _sz) \
95 : static struct { \
96 : unsigned char mem[_sz]; \
97 : } __aligned(sizeof(uint64_t)) _name##_boot_cpumem
98 :
99 : #define CPUMEM_BOOT_INITIALIZER(_name) \
100 : { (struct cpumem *)&_name##_boot_cpumem.mem }
101 :
102 : #endif /* MULTIPROCESSOR */
103 :
104 : #define CPUMEM_FOREACH(_var, _iter, _cpumem) \
105 : for ((_var) = cpumem_first((_iter), (_cpumem)); \
106 : (_var) != NULL; \
107 : (_var) = cpumem_next((_iter), (_cpumem)))
108 :
109 : /*
110 : * per cpu counters
111 : */
112 :
113 : struct cpumem *counters_alloc(unsigned int);
114 : struct cpumem *counters_alloc_ncpus(struct cpumem *, unsigned int);
115 : void counters_free(struct cpumem *, unsigned int);
116 : void counters_read(struct cpumem *, uint64_t *, unsigned int);
117 : void counters_zero(struct cpumem *, unsigned int);
118 :
119 : static inline uint64_t *
120 0 : counters_enter(struct counters_ref *ref, struct cpumem *cm)
121 : {
122 0 : ref->c = cpumem_enter(cm);
123 : #ifdef MULTIPROCESSOR
124 0 : ref->g = ++(*ref->c); /* make the generation number odd */
125 0 : membar_producer();
126 0 : return (ref->c + 1);
127 : #else
128 : return (ref->c);
129 : #endif
130 : }
131 :
132 : static inline void
133 0 : counters_leave(struct counters_ref *ref, struct cpumem *cm)
134 : {
135 : #ifdef MULTIPROCESSOR
136 0 : membar_producer();
137 0 : (*ref->c) = ++ref->g; /* make the generation number even again */
138 : #endif
139 0 : cpumem_leave(cm, ref->c);
140 0 : }
141 :
142 : static inline void
143 0 : counters_inc(struct cpumem *cm, unsigned int c)
144 : {
145 0 : struct counters_ref ref;
146 : uint64_t *counters;
147 :
148 0 : counters = counters_enter(&ref, cm);
149 0 : counters[c]++;
150 0 : counters_leave(&ref, cm);
151 0 : }
152 :
153 : static inline void
154 0 : counters_dec(struct cpumem *cm, unsigned int c)
155 : {
156 0 : struct counters_ref ref;
157 : uint64_t *counters;
158 :
159 0 : counters = counters_enter(&ref, cm);
160 0 : counters[c]--;
161 0 : counters_leave(&ref, cm);
162 0 : }
163 :
164 : static inline void
165 0 : counters_add(struct cpumem *cm, unsigned int c, uint64_t v)
166 : {
167 0 : struct counters_ref ref;
168 : uint64_t *counters;
169 :
170 0 : counters = counters_enter(&ref, cm);
171 0 : counters[c] += v;
172 0 : counters_leave(&ref, cm);
173 0 : }
174 :
175 : static inline void
176 0 : counters_pkt(struct cpumem *cm, unsigned int c, unsigned int b, uint64_t v)
177 : {
178 0 : struct counters_ref ref;
179 : uint64_t *counters;
180 :
181 0 : counters = counters_enter(&ref, cm);
182 0 : counters[c]++;
183 0 : counters[b] += v;
184 0 : counters_leave(&ref, cm);
185 0 : }
186 :
187 : #ifdef MULTIPROCESSOR
188 : #define COUNTERS_BOOT_MEMORY(_name, _n) \
189 : CPUMEM_BOOT_MEMORY(_name, ((_n) + 1) * sizeof(uint64_t))
190 : #else
191 : #define COUNTERS_BOOT_MEMORY(_name, _n) \
192 : CPUMEM_BOOT_MEMORY(_name, (_n) * sizeof(uint64_t))
193 : #endif
194 :
195 : #define COUNTERS_BOOT_INITIALIZER(_name) CPUMEM_BOOT_INITIALIZER(_name)
196 :
197 : #endif /* _KERNEL */
198 : #endif /* _SYS_PERCPU_H_ */
|
