Line data Source code
1 : /* $OpenBSD: subr_percpu.c,v 1.8 2017/09/08 05:36:53 deraadt Exp $ */
2 :
3 : /*
4 : * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
5 : *
6 : * Permission to use, copy, modify, and distribute this software for any
7 : * purpose with or without fee is hereby granted, provided that the above
8 : * copyright notice and this permission notice appear in all copies.
9 : *
10 : * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 : * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 : * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 : * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 : * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 : * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 : * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 : */
18 :
19 : #include <sys/param.h>
20 : #include <sys/systm.h>
21 : #include <sys/pool.h>
22 : #include <sys/malloc.h>
23 :
24 : #include <sys/percpu.h>
25 :
26 : #ifdef MULTIPROCESSOR
27 : struct pool cpumem_pl;
28 :
29 : void
30 0 : percpu_init(void)
31 : {
32 0 : pool_init(&cpumem_pl, sizeof(struct cpumem) * ncpusfound, 0,
33 : IPL_NONE, PR_WAITOK, "percpumem", &pool_allocator_single);
34 0 : }
35 :
36 : struct cpumem *
37 0 : cpumem_get(struct pool *pp)
38 : {
39 : struct cpumem *cm;
40 : unsigned int cpu;
41 :
42 0 : cm = pool_get(&cpumem_pl, PR_WAITOK);
43 :
44 0 : for (cpu = 0; cpu < ncpusfound; cpu++)
45 0 : cm[cpu].mem = pool_get(pp, PR_WAITOK | PR_ZERO);
46 :
47 0 : return (cm);
48 : }
49 :
50 : void
51 0 : cpumem_put(struct pool *pp, struct cpumem *cm)
52 : {
53 : unsigned int cpu;
54 :
55 0 : for (cpu = 0; cpu < ncpusfound; cpu++)
56 0 : pool_put(pp, cm[cpu].mem);
57 :
58 0 : pool_put(&cpumem_pl, cm);
59 0 : }
60 :
61 : struct cpumem *
62 0 : cpumem_malloc(size_t sz, int type)
63 : {
64 : struct cpumem *cm;
65 : unsigned int cpu;
66 :
67 0 : sz = roundup(sz, CACHELINESIZE);
68 :
69 0 : cm = pool_get(&cpumem_pl, PR_WAITOK);
70 :
71 0 : for (cpu = 0; cpu < ncpusfound; cpu++)
72 0 : cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO);
73 :
74 0 : return (cm);
75 : }
76 :
77 : struct cpumem *
78 0 : cpumem_malloc_ncpus(struct cpumem *bootcm, size_t sz, int type)
79 : {
80 : struct cpumem *cm;
81 : unsigned int cpu;
82 :
83 0 : sz = roundup(sz, CACHELINESIZE);
84 :
85 0 : cm = pool_get(&cpumem_pl, PR_WAITOK);
86 :
87 0 : cm[0].mem = bootcm[0].mem;
88 0 : for (cpu = 1; cpu < ncpusfound; cpu++)
89 0 : cm[cpu].mem = malloc(sz, type, M_WAITOK | M_ZERO);
90 :
91 0 : return (cm);
92 : }
93 :
94 : void
95 0 : cpumem_free(struct cpumem *cm, int type, size_t sz)
96 : {
97 : unsigned int cpu;
98 :
99 0 : sz = roundup(sz, CACHELINESIZE);
100 :
101 0 : for (cpu = 0; cpu < ncpusfound; cpu++)
102 0 : free(cm[cpu].mem, type, sz);
103 :
104 0 : pool_put(&cpumem_pl, cm);
105 0 : }
106 :
107 : void *
108 0 : cpumem_first(struct cpumem_iter *i, struct cpumem *cm)
109 : {
110 0 : i->cpu = 0;
111 :
112 0 : return (cm[0].mem);
113 : }
114 :
115 : void *
116 0 : cpumem_next(struct cpumem_iter *i, struct cpumem *cm)
117 : {
118 0 : unsigned int cpu = ++i->cpu;
119 :
120 0 : if (cpu >= ncpusfound)
121 0 : return (NULL);
122 :
123 0 : return (cm[cpu].mem);
124 0 : }
125 :
126 : struct cpumem *
127 0 : counters_alloc(unsigned int n)
128 : {
129 : struct cpumem *cm;
130 0 : struct cpumem_iter cmi;
131 : uint64_t *counters;
132 : unsigned int i;
133 :
134 0 : KASSERT(n > 0);
135 :
136 0 : n++; /* add space for a generation number */
137 0 : cm = cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS);
138 :
139 0 : CPUMEM_FOREACH(counters, &cmi, cm) {
140 0 : for (i = 0; i < n; i++)
141 0 : counters[i] = 0;
142 : }
143 :
144 0 : return (cm);
145 0 : }
146 :
147 : struct cpumem *
148 0 : counters_alloc_ncpus(struct cpumem *cm, unsigned int n)
149 : {
150 0 : n++; /* the generation number */
151 0 : return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS));
152 : }
153 :
154 : void
155 0 : counters_free(struct cpumem *cm, unsigned int n)
156 : {
157 0 : n++; /* generation number */
158 0 : cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t));
159 0 : }
160 :
161 : void
162 0 : counters_read(struct cpumem *cm, uint64_t *output, unsigned int n)
163 : {
164 0 : struct cpumem_iter cmi;
165 : uint64_t *gen, *counters, *temp;
166 : uint64_t enter, leave;
167 : unsigned int i;
168 :
169 0 : for (i = 0; i < n; i++)
170 0 : output[i] = 0;
171 :
172 0 : temp = mallocarray(n, sizeof(uint64_t), M_TEMP, M_WAITOK);
173 :
174 0 : gen = cpumem_first(&cmi, cm);
175 0 : do {
176 0 : counters = gen + 1;
177 :
178 0 : enter = *gen;
179 0 : for (;;) {
180 : /* the generation number is odd during an update */
181 0 : while (enter & 1) {
182 0 : yield();
183 0 : enter = *gen;
184 : }
185 :
186 0 : membar_consumer();
187 0 : for (i = 0; i < n; i++)
188 0 : temp[i] = counters[i];
189 :
190 0 : membar_consumer();
191 0 : leave = *gen;
192 :
193 0 : if (enter == leave)
194 : break;
195 :
196 : enter = leave;
197 : }
198 :
199 0 : for (i = 0; i < n; i++)
200 0 : output[i] += temp[i];
201 :
202 0 : gen = cpumem_next(&cmi, cm);
203 0 : } while (gen != NULL);
204 :
205 0 : free(temp, M_TEMP, n * sizeof(uint64_t));
206 0 : }
207 :
208 : void
209 0 : counters_zero(struct cpumem *cm, unsigned int n)
210 : {
211 0 : struct cpumem_iter cmi;
212 : uint64_t *counters;
213 : unsigned int i;
214 :
215 0 : counters = cpumem_first(&cmi, cm);
216 0 : do {
217 0 : for (i = 0; i < n; i++)
218 0 : counters[i] = 0;
219 : /* zero the generation numbers too */
220 0 : membar_producer();
221 0 : counters[i] = 0;
222 :
223 0 : counters = cpumem_next(&cmi, cm);
224 0 : } while (counters != NULL);
225 0 : }
226 :
227 : #else /* MULTIPROCESSOR */
228 :
229 : /*
230 : * Uniprocessor implementation of per-CPU data structures.
231 : *
232 : * UP percpu memory is a single memory allocation cast to/from the
233 : * cpumem struct. It is not scaled up to the size of cacheline because
234 : * there's no other cache to contend with.
235 : */
236 :
237 : void
238 : percpu_init(void)
239 : {
240 : /* nop */
241 : }
242 :
243 : struct cpumem *
244 : cpumem_get(struct pool *pp)
245 : {
246 : return (pool_get(pp, PR_WAITOK | PR_ZERO));
247 : }
248 :
249 : void
250 : cpumem_put(struct pool *pp, struct cpumem *cm)
251 : {
252 : pool_put(pp, cm);
253 : }
254 :
255 : struct cpumem *
256 : cpumem_malloc(size_t sz, int type)
257 : {
258 : return (malloc(sz, type, M_WAITOK | M_ZERO));
259 : }
260 :
261 : struct cpumem *
262 : cpumem_malloc_ncpus(struct cpumem *cm, size_t sz, int type)
263 : {
264 : return (cm);
265 : }
266 :
267 : void
268 : cpumem_free(struct cpumem *cm, int type, size_t sz)
269 : {
270 : free(cm, type, sz);
271 : }
272 :
273 : void *
274 : cpumem_first(struct cpumem_iter *i, struct cpumem *cm)
275 : {
276 : return (cm);
277 : }
278 :
279 : void *
280 : cpumem_next(struct cpumem_iter *i, struct cpumem *cm)
281 : {
282 : return (NULL);
283 : }
284 :
285 : struct cpumem *
286 : counters_alloc(unsigned int n)
287 : {
288 : KASSERT(n > 0);
289 :
290 : return (cpumem_malloc(n * sizeof(uint64_t), M_COUNTERS));
291 : }
292 :
293 : struct cpumem *
294 : counters_alloc_ncpus(struct cpumem *cm, unsigned int n)
295 : {
296 : /* this is unecessary, but symmetrical */
297 : return (cpumem_malloc_ncpus(cm, n * sizeof(uint64_t), M_COUNTERS));
298 : }
299 :
300 : void
301 : counters_free(struct cpumem *cm, unsigned int n)
302 : {
303 : cpumem_free(cm, M_COUNTERS, n * sizeof(uint64_t));
304 : }
305 :
306 : void
307 : counters_read(struct cpumem *cm, uint64_t *output, unsigned int n)
308 : {
309 : uint64_t *counters;
310 : unsigned int i;
311 : int s;
312 :
313 : counters = (uint64_t *)cm;
314 :
315 : s = splhigh();
316 : for (i = 0; i < n; i++)
317 : output[i] = counters[i];
318 : splx(s);
319 : }
320 :
321 : void
322 : counters_zero(struct cpumem *cm, unsigned int n)
323 : {
324 : uint64_t *counters;
325 : unsigned int i;
326 : int s;
327 :
328 : counters = (uint64_t *)cm;
329 :
330 : s = splhigh();
331 : for (i = 0; i < n; i++)
332 : counters[i] = 0;
333 : splx(s);
334 : }
335 :
336 : #endif /* MULTIPROCESSOR */
|
