Line data Source code
1 : /* $OpenBSD: subr_hibernate.c,v 1.125 2018/06/21 07:49:13 mlarkin Exp $ */
2 :
3 : /*
4 : * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl>
5 : * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org>
6 : *
7 : * Permission to use, copy, modify, and distribute this software for any
8 : * purpose with or without fee is hereby granted, provided that the above
9 : * copyright notice and this permission notice appear in all copies.
10 : *
11 : * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 : * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 : * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 : * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 : * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 : * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 : * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 : */
19 :
20 : #include <sys/hibernate.h>
21 : #include <sys/malloc.h>
22 : #include <sys/param.h>
23 : #include <sys/tree.h>
24 : #include <sys/systm.h>
25 : #include <sys/disklabel.h>
26 : #include <sys/disk.h>
27 : #include <sys/conf.h>
28 : #include <sys/buf.h>
29 : #include <sys/fcntl.h>
30 : #include <sys/stat.h>
31 : #include <sys/atomic.h>
32 :
33 : #include <uvm/uvm.h>
34 : #include <uvm/uvm_swap.h>
35 :
36 : #include <machine/hibernate.h>
37 :
38 : /*
39 : * Hibernate piglet layout information
40 : *
41 : * The piglet is a scratch area of memory allocated by the suspending kernel.
42 : * Its phys and virt addrs are recorded in the signature block. The piglet is
43 : * used to guarantee an unused area of memory that can be used by the resuming
44 : * kernel for various things. The piglet is excluded during unpack operations.
45 : * The piglet size is presently 4*HIBERNATE_CHUNK_SIZE (typically 4*4MB).
46 : *
47 : * Offset from piglet_base Purpose
48 : * ----------------------------------------------------------------------------
49 : * 0 Private page for suspend I/O write functions
50 : * 1*PAGE_SIZE I/O page used during hibernate suspend
51 : * 2*PAGE_SIZE I/O page used during hibernate suspend
52 : * 3*PAGE_SIZE copy page used during hibernate suspend
53 : * 4*PAGE_SIZE final chunk ordering list (24 pages)
54 : * 28*PAGE_SIZE RLE utility page
55 : * 29*PAGE_SIZE start of hiballoc area
56 : * 30*PAGE_SIZE preserved entropy
57 : * 110*PAGE_SIZE end of hiballoc area (80 pages)
58 : * 366*PAGE_SIZE end of retguard preservation region (256 pages)
59 : * ... unused
60 : * HIBERNATE_CHUNK_SIZE start of hibernate chunk table
61 : * 2*HIBERNATE_CHUNK_SIZE bounce area for chunks being unpacked
62 : * 4*HIBERNATE_CHUNK_SIZE end of piglet
63 : */
64 :
65 : /* Temporary vaddr ranges used during hibernate */
66 : vaddr_t hibernate_temp_page;
67 : vaddr_t hibernate_copy_page;
68 : vaddr_t hibernate_rle_page;
69 :
70 : /* Hibernate info as read from disk during resume */
71 : union hibernate_info disk_hib;
72 :
73 : /*
74 : * Global copy of the pig start address. This needs to be a global as we
75 : * switch stacks after computing it - it can't be stored on the stack.
76 : */
77 : paddr_t global_pig_start;
78 :
79 : /*
80 : * Global copies of the piglet start addresses (PA/VA). We store these
81 : * as globals to avoid having to carry them around as parameters, as the
82 : * piglet is allocated early and freed late - its lifecycle extends beyond
83 : * that of the hibernate info union which is calculated on suspend/resume.
84 : */
85 : vaddr_t global_piglet_va;
86 : paddr_t global_piglet_pa;
87 :
88 : /* #define HIB_DEBUG */
89 : #ifdef HIB_DEBUG
90 : int hib_debug = 99;
91 : #define DPRINTF(x...) do { if (hib_debug) printf(x); } while (0)
92 : #define DNPRINTF(n,x...) do { if (hib_debug > (n)) printf(x); } while (0)
93 : #else
94 : #define DPRINTF(x...)
95 : #define DNPRINTF(n,x...)
96 : #endif
97 :
98 : #ifndef NO_PROPOLICE
99 : extern long __guard_local;
100 : #endif /* ! NO_PROPOLICE */
101 :
102 : void hibernate_copy_chunk_to_piglet(paddr_t, vaddr_t, size_t);
103 : int hibernate_calc_rle(paddr_t, paddr_t);
104 : int hibernate_write_rle(union hibernate_info *, paddr_t, paddr_t, daddr_t *,
105 : size_t *);
106 :
107 : #define MAX_RLE (HIBERNATE_CHUNK_SIZE / PAGE_SIZE)
108 :
109 : /*
110 : * Hib alloc enforced alignment.
111 : */
112 : #define HIB_ALIGN 8 /* bytes alignment */
113 :
114 : /*
115 : * sizeof builtin operation, but with alignment constraint.
116 : */
117 : #define HIB_SIZEOF(_type) roundup(sizeof(_type), HIB_ALIGN)
118 :
119 : struct hiballoc_entry {
120 : size_t hibe_use;
121 : size_t hibe_space;
122 : RBT_ENTRY(hiballoc_entry) hibe_entry;
123 : };
124 :
125 : /*
126 : * Sort hibernate memory ranges by ascending PA
127 : */
128 : void
129 0 : hibernate_sort_ranges(union hibernate_info *hib_info)
130 : {
131 : int i, j;
132 : struct hibernate_memory_range *ranges;
133 : paddr_t base, end;
134 :
135 0 : ranges = hib_info->ranges;
136 :
137 0 : for (i = 1; i < hib_info->nranges; i++) {
138 : j = i;
139 0 : while (j > 0 && ranges[j - 1].base > ranges[j].base) {
140 : base = ranges[j].base;
141 0 : end = ranges[j].end;
142 0 : ranges[j].base = ranges[j - 1].base;
143 0 : ranges[j].end = ranges[j - 1].end;
144 0 : ranges[j - 1].base = base;
145 0 : ranges[j - 1].end = end;
146 0 : j--;
147 : }
148 : }
149 0 : }
150 :
151 : /*
152 : * Compare hiballoc entries based on the address they manage.
153 : *
154 : * Since the address is fixed, relative to struct hiballoc_entry,
155 : * we just compare the hiballoc_entry pointers.
156 : */
157 : static __inline int
158 0 : hibe_cmp(const struct hiballoc_entry *l, const struct hiballoc_entry *r)
159 : {
160 0 : vaddr_t vl = (vaddr_t)l;
161 0 : vaddr_t vr = (vaddr_t)r;
162 :
163 0 : return vl < vr ? -1 : (vl > vr);
164 : }
165 :
166 0 : RBT_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp)
167 :
168 : /*
169 : * Given a hiballoc entry, return the address it manages.
170 : */
171 : static __inline void *
172 0 : hib_entry_to_addr(struct hiballoc_entry *entry)
173 : {
174 : caddr_t addr;
175 :
176 0 : addr = (caddr_t)entry;
177 0 : addr += HIB_SIZEOF(struct hiballoc_entry);
178 0 : return addr;
179 : }
180 :
181 : /*
182 : * Given an address, find the hiballoc that corresponds.
183 : */
184 : static __inline struct hiballoc_entry*
185 0 : hib_addr_to_entry(void *addr_param)
186 : {
187 : caddr_t addr;
188 :
189 : addr = (caddr_t)addr_param;
190 0 : addr -= HIB_SIZEOF(struct hiballoc_entry);
191 0 : return (struct hiballoc_entry*)addr;
192 : }
193 :
194 0 : RBT_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp);
195 :
196 : /*
197 : * Allocate memory from the arena.
198 : *
199 : * Returns NULL if no memory is available.
200 : */
201 : void *
202 0 : hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz)
203 : {
204 : struct hiballoc_entry *entry, *new_entry;
205 : size_t find_sz;
206 :
207 : /*
208 : * Enforce alignment of HIB_ALIGN bytes.
209 : *
210 : * Note that, because the entry is put in front of the allocation,
211 : * 0-byte allocations are guaranteed a unique address.
212 : */
213 0 : alloc_sz = roundup(alloc_sz, HIB_ALIGN);
214 :
215 : /*
216 : * Find an entry with hibe_space >= find_sz.
217 : *
218 : * If the root node is not large enough, we switch to tree traversal.
219 : * Because all entries are made at the bottom of the free space,
220 : * traversal from the end has a slightly better chance of yielding
221 : * a sufficiently large space.
222 : */
223 0 : find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry);
224 0 : entry = RBT_ROOT(hiballoc_addr, &arena->hib_addrs);
225 0 : if (entry != NULL && entry->hibe_space < find_sz) {
226 0 : RBT_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) {
227 0 : if (entry->hibe_space >= find_sz)
228 : break;
229 : }
230 : }
231 :
232 : /*
233 : * Insufficient or too fragmented memory.
234 : */
235 0 : if (entry == NULL)
236 0 : return NULL;
237 :
238 : /*
239 : * Create new entry in allocated space.
240 : */
241 0 : new_entry = (struct hiballoc_entry*)(
242 0 : (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use);
243 0 : new_entry->hibe_space = entry->hibe_space - find_sz;
244 0 : new_entry->hibe_use = alloc_sz;
245 :
246 : /*
247 : * Insert entry.
248 : */
249 0 : if (RBT_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL)
250 0 : panic("hib_alloc: insert failure");
251 0 : entry->hibe_space = 0;
252 :
253 : /* Return address managed by entry. */
254 0 : return hib_entry_to_addr(new_entry);
255 0 : }
256 :
257 : void
258 0 : hib_getentropy(char **bufp, size_t *bufplen)
259 : {
260 0 : if (!bufp || !bufplen)
261 : return;
262 :
263 0 : *bufp = (char *)(global_piglet_va + (29 * PAGE_SIZE));
264 0 : *bufplen = PAGE_SIZE;
265 0 : }
266 :
267 : /*
268 : * Free a pointer previously allocated from this arena.
269 : *
270 : * If addr is NULL, this will be silently accepted.
271 : */
272 : void
273 0 : hib_free(struct hiballoc_arena *arena, void *addr)
274 : {
275 : struct hiballoc_entry *entry, *prev;
276 :
277 0 : if (addr == NULL)
278 0 : return;
279 :
280 : /*
281 : * Derive entry from addr and check it is really in this arena.
282 : */
283 0 : entry = hib_addr_to_entry(addr);
284 0 : if (RBT_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry)
285 0 : panic("hib_free: freed item %p not in hib arena", addr);
286 :
287 : /*
288 : * Give the space in entry to its predecessor.
289 : *
290 : * If entry has no predecessor, change its used space into free space
291 : * instead.
292 : */
293 0 : prev = RBT_PREV(hiballoc_addr, entry);
294 0 : if (prev != NULL &&
295 0 : (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) +
296 0 : prev->hibe_use + prev->hibe_space) == entry) {
297 : /* Merge entry. */
298 0 : RBT_REMOVE(hiballoc_addr, &arena->hib_addrs, entry);
299 0 : prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) +
300 0 : entry->hibe_use + entry->hibe_space;
301 0 : } else {
302 : /* Flip used memory to free space. */
303 0 : entry->hibe_space += entry->hibe_use;
304 0 : entry->hibe_use = 0;
305 : }
306 0 : }
307 :
308 : /*
309 : * Initialize hiballoc.
310 : *
311 : * The allocator will manage memmory at ptr, which is len bytes.
312 : */
313 : int
314 0 : hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len)
315 : {
316 : struct hiballoc_entry *entry;
317 : caddr_t ptr;
318 : size_t len;
319 :
320 0 : RBT_INIT(hiballoc_addr, &arena->hib_addrs);
321 :
322 : /*
323 : * Hib allocator enforces HIB_ALIGN alignment.
324 : * Fixup ptr and len.
325 : */
326 0 : ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN);
327 0 : len = p_len - ((size_t)ptr - (size_t)p_ptr);
328 0 : len &= ~((size_t)HIB_ALIGN - 1);
329 :
330 : /*
331 : * Insufficient memory to be able to allocate and also do bookkeeping.
332 : */
333 0 : if (len <= HIB_SIZEOF(struct hiballoc_entry))
334 0 : return ENOMEM;
335 :
336 : /*
337 : * Create entry describing space.
338 : */
339 0 : entry = (struct hiballoc_entry*)ptr;
340 0 : entry->hibe_use = 0;
341 0 : entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry);
342 0 : RBT_INSERT(hiballoc_addr, &arena->hib_addrs, entry);
343 :
344 0 : return 0;
345 0 : }
346 :
347 : /*
348 : * Zero all free memory.
349 : */
350 : void
351 0 : uvm_pmr_zero_everything(void)
352 : {
353 : struct uvm_pmemrange *pmr;
354 : struct vm_page *pg;
355 : int i;
356 :
357 0 : uvm_lock_fpageq();
358 0 : TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
359 : /* Zero single pages. */
360 0 : while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY]))
361 0 : != NULL) {
362 0 : uvm_pmr_remove(pmr, pg);
363 0 : uvm_pagezero(pg);
364 0 : atomic_setbits_int(&pg->pg_flags, PG_ZERO);
365 0 : uvmexp.zeropages++;
366 0 : uvm_pmr_insert(pmr, pg, 0);
367 : }
368 :
369 : /* Zero multi page ranges. */
370 0 : while ((pg = RBT_ROOT(uvm_pmr_size,
371 0 : &pmr->size[UVM_PMR_MEMTYPE_DIRTY])) != NULL) {
372 0 : pg--; /* Size tree always has second page. */
373 0 : uvm_pmr_remove(pmr, pg);
374 0 : for (i = 0; i < pg->fpgsz; i++) {
375 0 : uvm_pagezero(&pg[i]);
376 0 : atomic_setbits_int(&pg[i].pg_flags, PG_ZERO);
377 0 : uvmexp.zeropages++;
378 : }
379 0 : uvm_pmr_insert(pmr, pg, 0);
380 : }
381 : }
382 0 : uvm_unlock_fpageq();
383 0 : }
384 :
385 : /*
386 : * Mark all memory as dirty.
387 : *
388 : * Used to inform the system that the clean memory isn't clean for some
389 : * reason, for example because we just came back from hibernate.
390 : */
391 : void
392 0 : uvm_pmr_dirty_everything(void)
393 : {
394 : struct uvm_pmemrange *pmr;
395 : struct vm_page *pg;
396 : int i;
397 :
398 0 : uvm_lock_fpageq();
399 0 : TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
400 : /* Dirty single pages. */
401 0 : while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO]))
402 0 : != NULL) {
403 0 : uvm_pmr_remove(pmr, pg);
404 0 : atomic_clearbits_int(&pg->pg_flags, PG_ZERO);
405 0 : uvm_pmr_insert(pmr, pg, 0);
406 : }
407 :
408 : /* Dirty multi page ranges. */
409 0 : while ((pg = RBT_ROOT(uvm_pmr_size,
410 0 : &pmr->size[UVM_PMR_MEMTYPE_ZERO])) != NULL) {
411 0 : pg--; /* Size tree always has second page. */
412 0 : uvm_pmr_remove(pmr, pg);
413 0 : for (i = 0; i < pg->fpgsz; i++)
414 0 : atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO);
415 0 : uvm_pmr_insert(pmr, pg, 0);
416 : }
417 : }
418 :
419 0 : uvmexp.zeropages = 0;
420 0 : uvm_unlock_fpageq();
421 0 : }
422 :
423 : /*
424 : * Allocate an area that can hold sz bytes and doesn't overlap with
425 : * the piglet at piglet_pa.
426 : */
427 : int
428 0 : uvm_pmr_alloc_pig(paddr_t *pa, psize_t sz, paddr_t piglet_pa)
429 : {
430 0 : struct uvm_constraint_range pig_constraint;
431 0 : struct kmem_pa_mode kp_pig = {
432 : .kp_constraint = &pig_constraint,
433 : .kp_maxseg = 1
434 : };
435 : vaddr_t va;
436 :
437 0 : sz = round_page(sz);
438 :
439 0 : pig_constraint.ucr_low = piglet_pa + 4 * HIBERNATE_CHUNK_SIZE;
440 0 : pig_constraint.ucr_high = -1;
441 :
442 0 : va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait);
443 0 : if (va == 0) {
444 0 : pig_constraint.ucr_low = 0;
445 0 : pig_constraint.ucr_high = piglet_pa - 1;
446 :
447 0 : va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait);
448 0 : if (va == 0)
449 0 : return ENOMEM;
450 : }
451 :
452 0 : pmap_extract(pmap_kernel(), va, pa);
453 0 : return 0;
454 0 : }
455 :
456 : /*
457 : * Allocate a piglet area.
458 : *
459 : * This needs to be in DMA-safe memory.
460 : * Piglets are aligned.
461 : *
462 : * sz and align in bytes.
463 : *
464 : * The call will sleep for the pagedaemon to attempt to free memory.
465 : * The pagedaemon may decide its not possible to free enough memory, causing
466 : * the allocation to fail.
467 : */
468 : int
469 0 : uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align)
470 : {
471 0 : struct kmem_pa_mode kp_piglet = {
472 : .kp_constraint = &dma_constraint,
473 : .kp_align = align,
474 : .kp_maxseg = 1
475 : };
476 :
477 : /* Ensure align is a power of 2 */
478 0 : KASSERT((align & (align - 1)) == 0);
479 :
480 : /*
481 : * Fixup arguments: align must be at least PAGE_SIZE,
482 : * sz will be converted to pagecount, since that is what
483 : * pmemrange uses internally.
484 : */
485 0 : if (align < PAGE_SIZE)
486 0 : kp_piglet.kp_align = PAGE_SIZE;
487 :
488 0 : sz = round_page(sz);
489 :
490 0 : *va = (vaddr_t)km_alloc(sz, &kv_any, &kp_piglet, &kd_nowait);
491 0 : if (*va == 0)
492 0 : return ENOMEM;
493 :
494 0 : pmap_extract(pmap_kernel(), *va, pa);
495 0 : return 0;
496 0 : }
497 :
498 : /*
499 : * Free a piglet area.
500 : */
501 : void
502 0 : uvm_pmr_free_piglet(vaddr_t va, vsize_t sz)
503 : {
504 : /*
505 : * Fix parameters.
506 : */
507 0 : sz = round_page(sz);
508 :
509 : /*
510 : * Free the physical and virtual memory.
511 : */
512 0 : km_free((void *)va, sz, &kv_any, &kp_dma_contig);
513 0 : }
514 :
515 : /*
516 : * Physmem RLE compression support.
517 : *
518 : * Given a physical page address, return the number of pages starting at the
519 : * address that are free. Clamps to the number of pages in
520 : * HIBERNATE_CHUNK_SIZE. Returns 0 if the page at addr is not free.
521 : */
522 : int
523 0 : uvm_page_rle(paddr_t addr)
524 : {
525 : struct vm_page *pg, *pg_end;
526 : struct vm_physseg *vmp;
527 0 : int pseg_idx, off_idx;
528 :
529 0 : pseg_idx = vm_physseg_find(atop(addr), &off_idx);
530 0 : if (pseg_idx == -1)
531 0 : return 0;
532 :
533 0 : vmp = &vm_physmem[pseg_idx];
534 0 : pg = &vmp->pgs[off_idx];
535 0 : if (!(pg->pg_flags & PQ_FREE))
536 0 : return 0;
537 :
538 : /*
539 : * Search for the first non-free page after pg.
540 : * Note that the page may not be the first page in a free pmemrange,
541 : * therefore pg->fpgsz cannot be used.
542 : */
543 0 : for (pg_end = pg; pg_end <= vmp->lastpg &&
544 0 : (pg_end->pg_flags & PQ_FREE) == PQ_FREE; pg_end++)
545 : ;
546 0 : return min((pg_end - pg), HIBERNATE_CHUNK_SIZE/PAGE_SIZE);
547 0 : }
548 :
549 : /*
550 : * Calculate a hopefully unique version # for this kernel, based upon
551 : * how it was linked.
552 : */
553 : u_int32_t
554 0 : hibsum(void)
555 : {
556 0 : return ((long)malloc ^ (long)km_alloc ^ (long)printf ^ (long)strlen);
557 : }
558 :
559 :
560 : /*
561 : * Fills out the hibernate_info union pointed to by hib
562 : * with information about this machine (swap signature block
563 : * offsets, number of memory ranges, kernel in use, etc)
564 : */
565 : int
566 0 : get_hibernate_info(union hibernate_info *hib, int suspend)
567 : {
568 0 : struct disklabel dl;
569 0 : char err_string[128], *dl_ret;
570 :
571 : #ifndef NO_PROPOLICE
572 : /* Save propolice guard */
573 0 : hib->guard = __guard_local;
574 : #endif /* ! NO_PROPOLICE */
575 :
576 : /* Determine I/O function to use */
577 0 : hib->io_func = get_hibernate_io_function(swdevt[0].sw_dev);
578 0 : if (hib->io_func == NULL)
579 0 : return (1);
580 :
581 : /* Calculate hibernate device */
582 0 : hib->dev = swdevt[0].sw_dev;
583 :
584 : /* Read disklabel (used to calculate signature and image offsets) */
585 0 : dl_ret = disk_readlabel(&dl, hib->dev, err_string, sizeof(err_string));
586 :
587 0 : if (dl_ret) {
588 0 : printf("Hibernate error reading disklabel: %s\n", dl_ret);
589 0 : return (1);
590 : }
591 :
592 : /* Make sure we have a swap partition. */
593 0 : if (dl.d_partitions[1].p_fstype != FS_SWAP ||
594 0 : DL_GETPSIZE(&dl.d_partitions[1]) == 0)
595 0 : return (1);
596 :
597 : /* Make sure the signature can fit in one block */
598 : if (sizeof(union hibernate_info) > DEV_BSIZE)
599 : return (1);
600 :
601 : /* Magic number */
602 0 : hib->magic = HIBERNATE_MAGIC;
603 :
604 : /* Calculate signature block location */
605 0 : hib->sig_offset = DL_GETPSIZE(&dl.d_partitions[1]) -
606 : sizeof(union hibernate_info)/DEV_BSIZE;
607 :
608 : /* Stash kernel version information */
609 0 : memset(&hib->kernel_version, 0, 128);
610 0 : bcopy(version, &hib->kernel_version,
611 0 : min(strlen(version), sizeof(hib->kernel_version)-1));
612 0 : hib->kernel_sum = hibsum();
613 :
614 0 : if (suspend) {
615 : /* Grab the previously-allocated piglet addresses */
616 0 : hib->piglet_va = global_piglet_va;
617 0 : hib->piglet_pa = global_piglet_pa;
618 0 : hib->io_page = (void *)hib->piglet_va;
619 :
620 : /*
621 : * Initialization of the hibernate IO function for drivers
622 : * that need to do prep work (such as allocating memory or
623 : * setting up data structures that cannot safely be done
624 : * during suspend without causing side effects). There is
625 : * a matching HIB_DONE call performed after the write is
626 : * completed.
627 : */
628 0 : if (hib->io_func(hib->dev, DL_GETPOFFSET(&dl.d_partitions[1]),
629 0 : (vaddr_t)NULL, DL_GETPSIZE(&dl.d_partitions[1]),
630 : HIB_INIT, hib->io_page))
631 : goto fail;
632 :
633 : } else {
634 : /*
635 : * Resuming kernels use a regular private page for the driver
636 : * No need to free this I/O page as it will vanish as part of
637 : * the resume.
638 : */
639 0 : hib->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
640 0 : if (!hib->io_page)
641 : goto fail;
642 : }
643 :
644 0 : if (get_hibernate_info_md(hib))
645 : goto fail;
646 :
647 0 : return (0);
648 :
649 : fail:
650 0 : return (1);
651 0 : }
652 :
653 : /*
654 : * Allocate nitems*size bytes from the hiballoc area presently in use
655 : */
656 : void *
657 0 : hibernate_zlib_alloc(void *unused, int nitems, int size)
658 : {
659 : struct hibernate_zlib_state *hibernate_state;
660 :
661 : hibernate_state =
662 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
663 :
664 0 : return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size);
665 : }
666 :
667 : /*
668 : * Free the memory pointed to by addr in the hiballoc area presently in
669 : * use
670 : */
671 : void
672 0 : hibernate_zlib_free(void *unused, void *addr)
673 : {
674 : struct hibernate_zlib_state *hibernate_state;
675 :
676 : hibernate_state =
677 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
678 :
679 0 : hib_free(&hibernate_state->hiballoc_arena, addr);
680 0 : }
681 :
682 : /*
683 : * Inflate next page of data from the image stream.
684 : * The rle parameter is modified on exit to contain the number of pages to
685 : * skip in the output stream (or 0 if this page was inflated into).
686 : *
687 : * Returns 0 if the stream contains additional data, or 1 if the stream is
688 : * finished.
689 : */
690 : int
691 0 : hibernate_inflate_page(int *rle)
692 : {
693 : struct hibernate_zlib_state *hibernate_state;
694 : int i;
695 :
696 : hibernate_state =
697 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
698 :
699 : /* Set up the stream for RLE code inflate */
700 0 : hibernate_state->hib_stream.next_out = (unsigned char *)rle;
701 0 : hibernate_state->hib_stream.avail_out = sizeof(*rle);
702 :
703 : /* Inflate RLE code */
704 0 : i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH);
705 0 : if (i != Z_OK && i != Z_STREAM_END) {
706 : /*
707 : * XXX - this will likely reboot/hang most machines
708 : * since the console output buffer will be unmapped,
709 : * but there's not much else we can do here.
710 : */
711 0 : panic("rle inflate stream error");
712 : }
713 :
714 0 : if (hibernate_state->hib_stream.avail_out != 0) {
715 : /*
716 : * XXX - this will likely reboot/hang most machines
717 : * since the console output buffer will be unmapped,
718 : * but there's not much else we can do here.
719 : */
720 0 : panic("rle short inflate error");
721 : }
722 :
723 0 : if (*rle < 0 || *rle > 1024) {
724 : /*
725 : * XXX - this will likely reboot/hang most machines
726 : * since the console output buffer will be unmapped,
727 : * but there's not much else we can do here.
728 : */
729 0 : panic("invalid rle count");
730 : }
731 :
732 0 : if (i == Z_STREAM_END)
733 0 : return (1);
734 :
735 0 : if (*rle != 0)
736 0 : return (0);
737 :
738 : /* Set up the stream for page inflate */
739 0 : hibernate_state->hib_stream.next_out =
740 : (unsigned char *)HIBERNATE_INFLATE_PAGE;
741 0 : hibernate_state->hib_stream.avail_out = PAGE_SIZE;
742 :
743 : /* Process next block of data */
744 0 : i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH);
745 0 : if (i != Z_OK && i != Z_STREAM_END) {
746 : /*
747 : * XXX - this will likely reboot/hang most machines
748 : * since the console output buffer will be unmapped,
749 : * but there's not much else we can do here.
750 : */
751 0 : panic("inflate error");
752 : }
753 :
754 : /* We should always have extracted a full page ... */
755 0 : if (hibernate_state->hib_stream.avail_out != 0) {
756 : /*
757 : * XXX - this will likely reboot/hang most machines
758 : * since the console output buffer will be unmapped,
759 : * but there's not much else we can do here.
760 : */
761 0 : panic("incomplete page");
762 : }
763 :
764 0 : return (i == Z_STREAM_END);
765 0 : }
766 :
767 : /*
768 : * Inflate size bytes from src into dest, skipping any pages in
769 : * [src..dest] that are special (see hibernate_inflate_skip)
770 : *
771 : * This function executes while using the resume-time stack
772 : * and pmap, and therefore cannot use ddb/printf/etc. Doing so
773 : * will likely hang or reset the machine since the console output buffer
774 : * will be unmapped.
775 : */
776 : void
777 0 : hibernate_inflate_region(union hibernate_info *hib, paddr_t dest,
778 : paddr_t src, size_t size)
779 : {
780 0 : int end_stream = 0, rle, skip;
781 : struct hibernate_zlib_state *hibernate_state;
782 :
783 : hibernate_state =
784 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
785 :
786 0 : hibernate_state->hib_stream.next_in = (unsigned char *)src;
787 0 : hibernate_state->hib_stream.avail_in = size;
788 :
789 0 : do {
790 : /*
791 : * Is this a special page? If yes, redirect the
792 : * inflate output to a scratch page (eg, discard it)
793 : */
794 0 : skip = hibernate_inflate_skip(hib, dest);
795 0 : if (skip == HIB_SKIP) {
796 0 : hibernate_enter_resume_mapping(
797 : HIBERNATE_INFLATE_PAGE,
798 : HIBERNATE_INFLATE_PAGE, 0);
799 0 : } else if (skip == HIB_MOVE) {
800 : /*
801 : * Special case : retguard region. This gets moved
802 : * temporarily into the piglet region and copied into
803 : * place immediately before resume
804 : */
805 0 : hibernate_enter_resume_mapping(
806 : HIBERNATE_INFLATE_PAGE,
807 0 : hib->piglet_pa + (110 * PAGE_SIZE) +
808 0 : hib->retguard_ofs, 0);
809 0 : hib->retguard_ofs += PAGE_SIZE;
810 0 : if (hib->retguard_ofs > 255 * PAGE_SIZE) {
811 : /*
812 : * XXX - this will likely reboot/hang most
813 : * machines since the console output
814 : * buffer will be unmapped, but there's
815 : * not much else we can do here.
816 : */
817 0 : panic("retguard move error, out of space");
818 : }
819 : } else {
820 0 : hibernate_enter_resume_mapping(
821 : HIBERNATE_INFLATE_PAGE, dest, 0);
822 : }
823 :
824 0 : hibernate_flush();
825 0 : end_stream = hibernate_inflate_page(&rle);
826 :
827 0 : if (rle == 0)
828 0 : dest += PAGE_SIZE;
829 : else
830 0 : dest += (rle * PAGE_SIZE);
831 0 : } while (!end_stream);
832 0 : }
833 :
834 : /*
835 : * deflate from src into the I/O page, up to 'remaining' bytes
836 : *
837 : * Returns number of input bytes consumed, and may reset
838 : * the 'remaining' parameter if not all the output space was consumed
839 : * (this information is needed to know how much to write to disk
840 : */
841 : size_t
842 0 : hibernate_deflate(union hibernate_info *hib, paddr_t src,
843 : size_t *remaining)
844 : {
845 0 : vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
846 : struct hibernate_zlib_state *hibernate_state;
847 :
848 : hibernate_state =
849 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
850 :
851 : /* Set up the stream for deflate */
852 0 : hibernate_state->hib_stream.next_in = (unsigned char *)src;
853 0 : hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK);
854 0 : hibernate_state->hib_stream.next_out =
855 0 : (unsigned char *)hibernate_io_page + (PAGE_SIZE - *remaining);
856 0 : hibernate_state->hib_stream.avail_out = *remaining;
857 :
858 : /* Process next block of data */
859 0 : if (deflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH) != Z_OK)
860 0 : panic("hibernate zlib deflate error");
861 :
862 : /* Update pointers and return number of bytes consumed */
863 0 : *remaining = hibernate_state->hib_stream.avail_out;
864 0 : return (PAGE_SIZE - (src & PAGE_MASK)) -
865 0 : hibernate_state->hib_stream.avail_in;
866 : }
867 :
868 : /*
869 : * Write the hibernation information specified in hiber_info
870 : * to the location in swap previously calculated (last block of
871 : * swap), called the "signature block".
872 : */
873 : int
874 0 : hibernate_write_signature(union hibernate_info *hib)
875 : {
876 : /* Write hibernate info to disk */
877 0 : return (hib->io_func(hib->dev, hib->sig_offset,
878 0 : (vaddr_t)hib, DEV_BSIZE, HIB_W,
879 0 : hib->io_page));
880 : }
881 :
882 : /*
883 : * Write the memory chunk table to the area in swap immediately
884 : * preceding the signature block. The chunk table is stored
885 : * in the piglet when this function is called. Returns errno.
886 : */
887 : int
888 0 : hibernate_write_chunktable(union hibernate_info *hib)
889 : {
890 : vaddr_t hibernate_chunk_table_start;
891 : size_t hibernate_chunk_table_size;
892 : int i, err;
893 :
894 : hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE;
895 :
896 0 : hibernate_chunk_table_start = hib->piglet_va +
897 : HIBERNATE_CHUNK_SIZE;
898 :
899 : /* Write chunk table */
900 0 : for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) {
901 0 : if ((err = hib->io_func(hib->dev,
902 0 : hib->chunktable_offset + (i/DEV_BSIZE),
903 0 : (vaddr_t)(hibernate_chunk_table_start + i),
904 0 : MAXPHYS, HIB_W, hib->io_page))) {
905 : DPRINTF("chunktable write error: %d\n", err);
906 0 : return (err);
907 : }
908 : }
909 :
910 0 : return (0);
911 0 : }
912 :
913 : /*
914 : * Write an empty hiber_info to the swap signature block, which is
915 : * guaranteed to not match any valid hib.
916 : */
917 : int
918 0 : hibernate_clear_signature(void)
919 : {
920 0 : union hibernate_info blank_hiber_info;
921 0 : union hibernate_info hib;
922 :
923 : /* Zero out a blank hiber_info */
924 0 : memset(&blank_hiber_info, 0, sizeof(union hibernate_info));
925 :
926 : /* Get the signature block location */
927 0 : if (get_hibernate_info(&hib, 0))
928 0 : return (1);
929 :
930 : /* Write (zeroed) hibernate info to disk */
931 : DPRINTF("clearing hibernate signature block location: %lld\n",
932 : hib.sig_offset);
933 0 : if (hibernate_block_io(&hib,
934 0 : hib.sig_offset,
935 0 : DEV_BSIZE, (vaddr_t)&blank_hiber_info, 1))
936 0 : printf("Warning: could not clear hibernate signature\n");
937 :
938 0 : return (0);
939 0 : }
940 :
941 : /*
942 : * Compare two hibernate_infos to determine if they are the same (eg,
943 : * we should be performing a hibernate resume on this machine.
944 : * Not all fields are checked - just enough to verify that the machine
945 : * has the same memory configuration and kernel as the one that
946 : * wrote the signature previously.
947 : */
948 : int
949 0 : hibernate_compare_signature(union hibernate_info *mine,
950 : union hibernate_info *disk)
951 : {
952 : u_int i;
953 :
954 0 : if (mine->nranges != disk->nranges) {
955 0 : printf("unhibernate failed: memory layout changed\n");
956 0 : return (1);
957 : }
958 :
959 0 : if (strcmp(mine->kernel_version, disk->kernel_version) != 0) {
960 0 : printf("unhibernate failed: original kernel changed\n");
961 0 : return (1);
962 : }
963 :
964 0 : if (hibsum() != disk->kernel_sum) {
965 0 : printf("unhibernate failed: original kernel changed\n");
966 0 : return (1);
967 : }
968 :
969 0 : for (i = 0; i < mine->nranges; i++) {
970 0 : if ((mine->ranges[i].base != disk->ranges[i].base) ||
971 0 : (mine->ranges[i].end != disk->ranges[i].end) ) {
972 : DPRINTF("hib range %d mismatch [%p-%p != %p-%p]\n",
973 : i,
974 : (void *)mine->ranges[i].base,
975 : (void *)mine->ranges[i].end,
976 : (void *)disk->ranges[i].base,
977 : (void *)disk->ranges[i].end);
978 0 : printf("unhibernate failed: memory size changed\n");
979 0 : return (1);
980 : }
981 : }
982 :
983 0 : return (0);
984 0 : }
985 :
986 : /*
987 : * Transfers xfer_size bytes between the hibernate device specified in
988 : * hib_info at offset blkctr and the vaddr specified at dest.
989 : *
990 : * Separate offsets and pages are used to handle misaligned reads (reads
991 : * that span a page boundary).
992 : *
993 : * blkctr specifies a relative offset (relative to the start of swap),
994 : * not an absolute disk offset
995 : *
996 : */
997 : int
998 0 : hibernate_block_io(union hibernate_info *hib, daddr_t blkctr,
999 : size_t xfer_size, vaddr_t dest, int iswrite)
1000 : {
1001 : struct buf *bp;
1002 : struct bdevsw *bdsw;
1003 : int error;
1004 :
1005 0 : bp = geteblk(xfer_size);
1006 0 : bdsw = &bdevsw[major(hib->dev)];
1007 :
1008 0 : error = (*bdsw->d_open)(hib->dev, FREAD, S_IFCHR, curproc);
1009 0 : if (error) {
1010 0 : printf("hibernate_block_io open failed\n");
1011 0 : return (1);
1012 : }
1013 :
1014 0 : if (iswrite)
1015 0 : bcopy((caddr_t)dest, bp->b_data, xfer_size);
1016 :
1017 0 : bp->b_bcount = xfer_size;
1018 0 : bp->b_blkno = blkctr;
1019 0 : CLR(bp->b_flags, B_READ | B_WRITE | B_DONE);
1020 0 : SET(bp->b_flags, B_BUSY | (iswrite ? B_WRITE : B_READ) | B_RAW);
1021 0 : bp->b_dev = hib->dev;
1022 0 : (*bdsw->d_strategy)(bp);
1023 :
1024 0 : error = biowait(bp);
1025 0 : if (error) {
1026 0 : printf("hib block_io biowait error %d blk %lld size %zu\n",
1027 : error, (long long)blkctr, xfer_size);
1028 0 : error = (*bdsw->d_close)(hib->dev, 0, S_IFCHR,
1029 0 : curproc);
1030 0 : if (error)
1031 0 : printf("hibernate_block_io error close failed\n");
1032 0 : return (1);
1033 : }
1034 :
1035 0 : error = (*bdsw->d_close)(hib->dev, FREAD, S_IFCHR, curproc);
1036 0 : if (error) {
1037 0 : printf("hibernate_block_io close failed\n");
1038 0 : return (1);
1039 : }
1040 :
1041 0 : if (!iswrite)
1042 0 : bcopy(bp->b_data, (caddr_t)dest, xfer_size);
1043 :
1044 0 : bp->b_flags |= B_INVAL;
1045 0 : brelse(bp);
1046 :
1047 0 : return (0);
1048 0 : }
1049 :
1050 : /*
1051 : * Preserve one page worth of random data, generated from the resuming
1052 : * kernel's arc4random. After resume, this preserved entropy can be used
1053 : * to further improve the un-hibernated machine's entropy pool. This
1054 : * random data is stored in the piglet, which is preserved across the
1055 : * unpack operation, and is restored later in the resume process (see
1056 : * hib_getentropy)
1057 : */
1058 : void
1059 0 : hibernate_preserve_entropy(union hibernate_info *hib)
1060 : {
1061 : void *entropy;
1062 :
1063 0 : entropy = km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_nowait);
1064 :
1065 0 : if (!entropy)
1066 0 : return;
1067 :
1068 0 : pmap_activate(curproc);
1069 0 : pmap_kenter_pa((vaddr_t)entropy,
1070 0 : (paddr_t)(hib->piglet_pa + (29 * PAGE_SIZE)),
1071 : PROT_READ | PROT_WRITE);
1072 :
1073 0 : arc4random_buf((void *)entropy, PAGE_SIZE);
1074 0 : pmap_kremove((vaddr_t)entropy, PAGE_SIZE);
1075 0 : km_free(entropy, PAGE_SIZE, &kv_any, &kp_none);
1076 0 : }
1077 :
1078 : #ifndef NO_PROPOLICE
1079 : vaddr_t
1080 0 : hibernate_unprotect_ssp(void)
1081 : {
1082 0 : struct kmem_dyn_mode kd_avoidalias;
1083 : vaddr_t va = trunc_page((vaddr_t)&__guard_local);
1084 0 : paddr_t pa;
1085 :
1086 0 : pmap_extract(pmap_kernel(), va, &pa);
1087 :
1088 0 : memset(&kd_avoidalias, 0, sizeof kd_avoidalias);
1089 0 : kd_avoidalias.kd_prefer = pa;
1090 0 : kd_avoidalias.kd_waitok = 1;
1091 0 : va = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_avoidalias);
1092 0 : if (!va)
1093 0 : panic("hibernate_unprotect_ssp");
1094 :
1095 0 : pmap_kenter_pa(va, pa, PROT_READ | PROT_WRITE);
1096 : pmap_update(pmap_kernel());
1097 :
1098 0 : return va;
1099 0 : }
1100 :
1101 : void
1102 0 : hibernate_reprotect_ssp(vaddr_t va)
1103 : {
1104 0 : pmap_kremove(va, PAGE_SIZE);
1105 0 : km_free((void *)va, PAGE_SIZE, &kv_any, &kp_none);
1106 0 : }
1107 : #endif /* NO_PROPOLICE */
1108 :
1109 : /*
1110 : * Reads the signature block from swap, checks against the current machine's
1111 : * information. If the information matches, perform a resume by reading the
1112 : * saved image into the pig area, and unpacking.
1113 : *
1114 : * Must be called with interrupts enabled.
1115 : */
1116 : void
1117 0 : hibernate_resume(void)
1118 : {
1119 0 : union hibernate_info hib;
1120 : int s;
1121 : #ifndef NO_PROPOLICE
1122 : vsize_t off = (vaddr_t)&__guard_local -
1123 : trunc_page((vaddr_t)&__guard_local);
1124 : vaddr_t guard_va;
1125 : #endif
1126 :
1127 : /* Get current running machine's hibernate info */
1128 0 : memset(&hib, 0, sizeof(hib));
1129 0 : if (get_hibernate_info(&hib, 0)) {
1130 : DPRINTF("couldn't retrieve machine's hibernate info\n");
1131 0 : return;
1132 : }
1133 :
1134 : /* Read hibernate info from disk */
1135 0 : s = splbio();
1136 :
1137 : DPRINTF("reading hibernate signature block location: %lld\n",
1138 : hib.sig_offset);
1139 :
1140 0 : if (hibernate_block_io(&hib,
1141 0 : hib.sig_offset,
1142 : DEV_BSIZE, (vaddr_t)&disk_hib, 0)) {
1143 : DPRINTF("error in hibernate read");
1144 0 : splx(s);
1145 0 : return;
1146 : }
1147 :
1148 : /* Check magic number */
1149 0 : if (disk_hib.magic != HIBERNATE_MAGIC) {
1150 : DPRINTF("wrong magic number in hibernate signature: %x\n",
1151 : disk_hib.magic);
1152 0 : splx(s);
1153 0 : return;
1154 : }
1155 :
1156 : /*
1157 : * We (possibly) found a hibernate signature. Clear signature first,
1158 : * to prevent accidental resume or endless resume cycles later.
1159 : */
1160 0 : if (hibernate_clear_signature()) {
1161 : DPRINTF("error clearing hibernate signature block\n");
1162 0 : splx(s);
1163 0 : return;
1164 : }
1165 :
1166 : /*
1167 : * If on-disk and in-memory hibernate signatures match,
1168 : * this means we should do a resume from hibernate.
1169 : */
1170 0 : if (hibernate_compare_signature(&hib, &disk_hib)) {
1171 : DPRINTF("mismatched hibernate signature block\n");
1172 0 : splx(s);
1173 0 : return;
1174 : }
1175 :
1176 : #ifdef MULTIPROCESSOR
1177 : /* XXX - if we fail later, we may need to rehatch APs on some archs */
1178 : DPRINTF("hibernate: quiescing APs\n");
1179 0 : hibernate_quiesce_cpus();
1180 : #endif /* MULTIPROCESSOR */
1181 :
1182 : /* Read the image from disk into the image (pig) area */
1183 0 : if (hibernate_read_image(&disk_hib))
1184 : goto fail;
1185 :
1186 : DPRINTF("hibernate: quiescing devices\n");
1187 0 : if (config_suspend_all(DVACT_QUIESCE) != 0)
1188 : goto fail;
1189 :
1190 : #ifndef NO_PROPOLICE
1191 0 : guard_va = hibernate_unprotect_ssp();
1192 : #endif /* NO_PROPOLICE */
1193 :
1194 0 : (void) splhigh();
1195 0 : hibernate_disable_intr_machdep();
1196 0 : cold = 1;
1197 :
1198 : DPRINTF("hibernate: suspending devices\n");
1199 0 : if (config_suspend_all(DVACT_SUSPEND) != 0) {
1200 0 : cold = 0;
1201 0 : hibernate_enable_intr_machdep();
1202 : #ifndef NO_PROPOLICE
1203 0 : hibernate_reprotect_ssp(guard_va);
1204 : #endif /* ! NO_PROPOLICE */
1205 0 : goto fail;
1206 : }
1207 :
1208 0 : hibernate_preserve_entropy(&disk_hib);
1209 :
1210 0 : printf("Unpacking image...\n");
1211 :
1212 : /* Switch stacks */
1213 : DPRINTF("hibernate: switching stacks\n");
1214 0 : hibernate_switch_stack_machdep();
1215 :
1216 : #ifndef NO_PROPOLICE
1217 : /* Start using suspended kernel's propolice guard */
1218 0 : *(long *)(guard_va + off) = disk_hib.guard;
1219 0 : hibernate_reprotect_ssp(guard_va);
1220 : #endif /* ! NO_PROPOLICE */
1221 :
1222 : /* Unpack and resume */
1223 0 : hibernate_unpack_image(&disk_hib);
1224 :
1225 : fail:
1226 0 : splx(s);
1227 0 : printf("\nUnable to resume hibernated image\n");
1228 0 : }
1229 :
1230 : /*
1231 : * Unpack image from pig area to original location by looping through the
1232 : * list of output chunks in the order they should be restored (fchunks).
1233 : *
1234 : * Note that due to the stack smash protector and the fact that we have
1235 : * switched stacks, it is not permitted to return from this function.
1236 : */
1237 : void
1238 0 : hibernate_unpack_image(union hibernate_info *hib)
1239 : {
1240 : struct hibernate_disk_chunk *chunks;
1241 0 : union hibernate_info local_hib;
1242 0 : paddr_t image_cur = global_pig_start;
1243 : short i, *fchunks;
1244 : char *pva;
1245 :
1246 : /* Piglet will be identity mapped (VA == PA) */
1247 0 : pva = (char *)hib->piglet_pa;
1248 :
1249 0 : fchunks = (short *)(pva + (4 * PAGE_SIZE));
1250 :
1251 0 : chunks = (struct hibernate_disk_chunk *)(pva + HIBERNATE_CHUNK_SIZE);
1252 :
1253 : /* Can't use hiber_info that's passed in after this point */
1254 0 : bcopy(hib, &local_hib, sizeof(union hibernate_info));
1255 0 : local_hib.retguard_ofs = 0;
1256 :
1257 : /* VA == PA */
1258 0 : local_hib.piglet_va = local_hib.piglet_pa;
1259 :
1260 : /*
1261 : * Point of no return. Once we pass this point, only kernel code can
1262 : * be accessed. No global variables or other kernel data structures
1263 : * are guaranteed to be coherent after unpack starts.
1264 : *
1265 : * The image is now in high memory (pig area), we unpack from the pig
1266 : * to the correct location in memory. We'll eventually end up copying
1267 : * on top of ourself, but we are assured the kernel code here is the
1268 : * same between the hibernated and resuming kernel, and we are running
1269 : * on our own stack, so the overwrite is ok.
1270 : */
1271 : DPRINTF("hibernate: activating alt. pagetable and starting unpack\n");
1272 0 : hibernate_activate_resume_pt_machdep();
1273 :
1274 0 : for (i = 0; i < local_hib.chunk_ctr; i++) {
1275 : /* Reset zlib for inflate */
1276 0 : if (hibernate_zlib_reset(&local_hib, 0) != Z_OK)
1277 0 : panic("hibernate failed to reset zlib for inflate");
1278 :
1279 0 : hibernate_process_chunk(&local_hib, &chunks[fchunks[i]],
1280 : image_cur);
1281 :
1282 0 : image_cur += chunks[fchunks[i]].compressed_size;
1283 :
1284 : }
1285 :
1286 : /*
1287 : * Resume the loaded kernel by jumping to the MD resume vector.
1288 : * We won't be returning from this call. We pass the location of
1289 : * the retguard save area so the MD code can replace it before
1290 : * resuming. See the piglet layout at the top of this file for
1291 : * more information on the layout of the piglet area.
1292 : *
1293 : * We use 'global_piglet_va' here since by the time we are at
1294 : * this point, we have already unpacked the image, and we want
1295 : * the suspended kernel's view of what the piglet was, before
1296 : * suspend occurred (since we will need to use that in the retguard
1297 : * copy code in hibernate_resume_machdep.)
1298 : */
1299 0 : hibernate_resume_machdep(global_piglet_va + (110 * PAGE_SIZE));
1300 0 : }
1301 :
1302 : /*
1303 : * Bounce a compressed image chunk to the piglet, entering mappings for the
1304 : * copied pages as needed
1305 : */
1306 : void
1307 0 : hibernate_copy_chunk_to_piglet(paddr_t img_cur, vaddr_t piglet, size_t size)
1308 : {
1309 : size_t ct, ofs;
1310 : paddr_t src = img_cur;
1311 : vaddr_t dest = piglet;
1312 :
1313 : /* Copy first partial page */
1314 0 : ct = (PAGE_SIZE) - (src & PAGE_MASK);
1315 : ofs = (src & PAGE_MASK);
1316 :
1317 0 : if (ct < PAGE_SIZE) {
1318 0 : hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE,
1319 0 : (src - ofs), 0);
1320 0 : hibernate_flush();
1321 0 : bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE + ofs), (caddr_t)dest, ct);
1322 0 : src += ct;
1323 0 : dest += ct;
1324 0 : }
1325 :
1326 : /* Copy remaining pages */
1327 0 : while (src < size + img_cur) {
1328 0 : hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, src, 0);
1329 0 : hibernate_flush();
1330 : ct = PAGE_SIZE;
1331 0 : bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE), (caddr_t)dest, ct);
1332 0 : hibernate_flush();
1333 0 : src += ct;
1334 0 : dest += ct;
1335 : }
1336 0 : }
1337 :
1338 : /*
1339 : * Process a chunk by bouncing it to the piglet, followed by unpacking
1340 : */
1341 : void
1342 0 : hibernate_process_chunk(union hibernate_info *hib,
1343 : struct hibernate_disk_chunk *chunk, paddr_t img_cur)
1344 : {
1345 0 : char *pva = (char *)hib->piglet_va;
1346 :
1347 0 : hibernate_copy_chunk_to_piglet(img_cur,
1348 0 : (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), chunk->compressed_size);
1349 0 : hibernate_inflate_region(hib, chunk->base,
1350 : (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)),
1351 0 : chunk->compressed_size);
1352 0 : }
1353 :
1354 : /*
1355 : * Calculate RLE component for 'inaddr'. Clamps to max RLE pages between
1356 : * inaddr and range_end.
1357 : */
1358 : int
1359 0 : hibernate_calc_rle(paddr_t inaddr, paddr_t range_end)
1360 : {
1361 : int rle;
1362 :
1363 0 : rle = uvm_page_rle(inaddr);
1364 0 : KASSERT(rle >= 0 && rle <= MAX_RLE);
1365 :
1366 : /* Clamp RLE to range end */
1367 0 : if (rle > 0 && inaddr + (rle * PAGE_SIZE) > range_end)
1368 0 : rle = (range_end - inaddr) / PAGE_SIZE;
1369 :
1370 0 : return (rle);
1371 : }
1372 :
1373 : /*
1374 : * Write the RLE byte for page at 'inaddr' to the output stream.
1375 : * Returns the number of pages to be skipped at 'inaddr'.
1376 : */
1377 : int
1378 0 : hibernate_write_rle(union hibernate_info *hib, paddr_t inaddr,
1379 : paddr_t range_end, daddr_t *blkctr,
1380 : size_t *out_remaining)
1381 : {
1382 : int rle, err, *rleloc;
1383 : struct hibernate_zlib_state *hibernate_state;
1384 0 : vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
1385 :
1386 : hibernate_state =
1387 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1388 :
1389 0 : rle = hibernate_calc_rle(inaddr, range_end);
1390 :
1391 0 : rleloc = (int *)hibernate_rle_page + MAX_RLE - 1;
1392 0 : *rleloc = rle;
1393 :
1394 : /* Deflate the RLE byte into the stream */
1395 0 : hibernate_deflate(hib, (paddr_t)rleloc, out_remaining);
1396 :
1397 : /* Did we fill the output page? If so, flush to disk */
1398 0 : if (*out_remaining == 0) {
1399 0 : if ((err = hib->io_func(hib->dev, *blkctr + hib->image_offset,
1400 : (vaddr_t)hibernate_io_page, PAGE_SIZE, HIB_W,
1401 0 : hib->io_page))) {
1402 : DPRINTF("hib write error %d\n", err);
1403 0 : return (err);
1404 : }
1405 :
1406 0 : *blkctr += PAGE_SIZE / DEV_BSIZE;
1407 0 : *out_remaining = PAGE_SIZE;
1408 :
1409 : /* If we didn't deflate the entire RLE byte, finish it now */
1410 0 : if (hibernate_state->hib_stream.avail_in != 0)
1411 0 : hibernate_deflate(hib,
1412 0 : (vaddr_t)hibernate_state->hib_stream.next_in,
1413 : out_remaining);
1414 : }
1415 :
1416 0 : return (rle);
1417 0 : }
1418 :
1419 : /*
1420 : * Write a compressed version of this machine's memory to disk, at the
1421 : * precalculated swap offset:
1422 : *
1423 : * end of swap - signature block size - chunk table size - memory size
1424 : *
1425 : * The function begins by looping through each phys mem range, cutting each
1426 : * one into MD sized chunks. These chunks are then compressed individually
1427 : * and written out to disk, in phys mem order. Some chunks might compress
1428 : * more than others, and for this reason, each chunk's size is recorded
1429 : * in the chunk table, which is written to disk after the image has
1430 : * properly been compressed and written (in hibernate_write_chunktable).
1431 : *
1432 : * When this function is called, the machine is nearly suspended - most
1433 : * devices are quiesced/suspended, interrupts are off, and cold has
1434 : * been set. This means that there can be no side effects once the
1435 : * write has started, and the write function itself can also have no
1436 : * side effects. This also means no printfs are permitted (since printf
1437 : * has side effects.)
1438 : *
1439 : * Return values :
1440 : *
1441 : * 0 - success
1442 : * EIO - I/O error occurred writing the chunks
1443 : * EINVAL - Failed to write a complete range
1444 : * ENOMEM - Memory allocation failure during preparation of the zlib arena
1445 : */
1446 : int
1447 0 : hibernate_write_chunks(union hibernate_info *hib)
1448 : {
1449 : paddr_t range_base, range_end, inaddr, temp_inaddr;
1450 0 : size_t nblocks, out_remaining, used;
1451 : struct hibernate_disk_chunk *chunks;
1452 0 : vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
1453 0 : daddr_t blkctr = 0;
1454 : int i, rle, err;
1455 : struct hibernate_zlib_state *hibernate_state;
1456 :
1457 : hibernate_state =
1458 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1459 :
1460 0 : hib->chunk_ctr = 0;
1461 :
1462 : /*
1463 : * Map the utility VAs to the piglet. See the piglet map at the
1464 : * top of this file for piglet layout information.
1465 : */
1466 0 : hibernate_copy_page = hib->piglet_va + 3 * PAGE_SIZE;
1467 0 : hibernate_rle_page = hib->piglet_va + 28 * PAGE_SIZE;
1468 :
1469 0 : chunks = (struct hibernate_disk_chunk *)(hib->piglet_va +
1470 : HIBERNATE_CHUNK_SIZE);
1471 :
1472 : /* Calculate the chunk regions */
1473 0 : for (i = 0; i < hib->nranges; i++) {
1474 0 : range_base = hib->ranges[i].base;
1475 0 : range_end = hib->ranges[i].end;
1476 :
1477 : inaddr = range_base;
1478 :
1479 0 : while (inaddr < range_end) {
1480 0 : chunks[hib->chunk_ctr].base = inaddr;
1481 0 : if (inaddr + HIBERNATE_CHUNK_SIZE < range_end)
1482 0 : chunks[hib->chunk_ctr].end = inaddr +
1483 : HIBERNATE_CHUNK_SIZE;
1484 : else
1485 0 : chunks[hib->chunk_ctr].end = range_end;
1486 :
1487 : inaddr += HIBERNATE_CHUNK_SIZE;
1488 0 : hib->chunk_ctr ++;
1489 : }
1490 : }
1491 :
1492 0 : uvm_pmr_dirty_everything();
1493 0 : uvm_pmr_zero_everything();
1494 :
1495 : /* Compress and write the chunks in the chunktable */
1496 0 : for (i = 0; i < hib->chunk_ctr; i++) {
1497 0 : range_base = chunks[i].base;
1498 0 : range_end = chunks[i].end;
1499 :
1500 0 : chunks[i].offset = blkctr + hib->image_offset;
1501 :
1502 : /* Reset zlib for deflate */
1503 0 : if (hibernate_zlib_reset(hib, 1) != Z_OK) {
1504 : DPRINTF("hibernate_zlib_reset failed for deflate\n");
1505 0 : return (ENOMEM);
1506 : }
1507 :
1508 : inaddr = range_base;
1509 :
1510 : /*
1511 : * For each range, loop through its phys mem region
1512 : * and write out the chunks (the last chunk might be
1513 : * smaller than the chunk size).
1514 : */
1515 0 : while (inaddr < range_end) {
1516 0 : out_remaining = PAGE_SIZE;
1517 0 : while (out_remaining > 0 && inaddr < range_end) {
1518 : /*
1519 : * Adjust for regions that are not evenly
1520 : * divisible by PAGE_SIZE or overflowed
1521 : * pages from the previous iteration.
1522 : */
1523 0 : temp_inaddr = (inaddr & PAGE_MASK) +
1524 0 : hibernate_copy_page;
1525 :
1526 : /* Deflate from temp_inaddr to IO page */
1527 0 : if (inaddr != range_end) {
1528 0 : if (inaddr % PAGE_SIZE == 0) {
1529 0 : rle = hibernate_write_rle(hib,
1530 : inaddr,
1531 : range_end,
1532 : &blkctr,
1533 : &out_remaining);
1534 0 : }
1535 :
1536 0 : if (rle == 0) {
1537 0 : pmap_kenter_pa(hibernate_temp_page,
1538 0 : inaddr & PMAP_PA_MASK,
1539 : PROT_READ);
1540 :
1541 0 : bcopy((caddr_t)hibernate_temp_page,
1542 0 : (caddr_t)hibernate_copy_page,
1543 : PAGE_SIZE);
1544 0 : inaddr += hibernate_deflate(hib,
1545 : temp_inaddr,
1546 : &out_remaining);
1547 0 : } else {
1548 0 : inaddr += rle * PAGE_SIZE;
1549 0 : if (inaddr > range_end)
1550 0 : inaddr = range_end;
1551 : }
1552 :
1553 : }
1554 :
1555 0 : if (out_remaining == 0) {
1556 : /* Filled up the page */
1557 : nblocks = PAGE_SIZE / DEV_BSIZE;
1558 :
1559 0 : if ((err = hib->io_func(hib->dev,
1560 0 : blkctr + hib->image_offset,
1561 : (vaddr_t)hibernate_io_page,
1562 0 : PAGE_SIZE, HIB_W, hib->io_page))) {
1563 : DPRINTF("hib write error %d\n",
1564 : err);
1565 0 : return (err);
1566 : }
1567 :
1568 0 : blkctr += nblocks;
1569 0 : }
1570 : }
1571 : }
1572 :
1573 0 : if (inaddr != range_end) {
1574 : DPRINTF("deflate range ended prematurely\n");
1575 0 : return (EINVAL);
1576 : }
1577 :
1578 : /*
1579 : * End of range. Round up to next secsize bytes
1580 : * after finishing compress
1581 : */
1582 0 : if (out_remaining == 0)
1583 0 : out_remaining = PAGE_SIZE;
1584 :
1585 : /* Finish compress */
1586 0 : hibernate_state->hib_stream.next_in = (unsigned char *)inaddr;
1587 0 : hibernate_state->hib_stream.avail_in = 0;
1588 0 : hibernate_state->hib_stream.next_out =
1589 0 : (unsigned char *)hibernate_io_page +
1590 0 : (PAGE_SIZE - out_remaining);
1591 :
1592 : /* We have an extra output page available for finalize */
1593 0 : hibernate_state->hib_stream.avail_out =
1594 0 : out_remaining + PAGE_SIZE;
1595 :
1596 0 : if ((err = deflate(&hibernate_state->hib_stream, Z_FINISH)) !=
1597 : Z_STREAM_END) {
1598 : DPRINTF("deflate error in output stream: %d\n", err);
1599 0 : return (err);
1600 : }
1601 :
1602 0 : out_remaining = hibernate_state->hib_stream.avail_out;
1603 :
1604 0 : used = 2 * PAGE_SIZE - out_remaining;
1605 0 : nblocks = used / DEV_BSIZE;
1606 :
1607 : /* Round up to next block if needed */
1608 0 : if (used % DEV_BSIZE != 0)
1609 0 : nblocks ++;
1610 :
1611 : /* Write final block(s) for this chunk */
1612 0 : if ((err = hib->io_func(hib->dev, blkctr + hib->image_offset,
1613 0 : (vaddr_t)hibernate_io_page, nblocks*DEV_BSIZE,
1614 0 : HIB_W, hib->io_page))) {
1615 : DPRINTF("hib final write error %d\n", err);
1616 0 : return (err);
1617 : }
1618 :
1619 0 : blkctr += nblocks;
1620 :
1621 0 : chunks[i].compressed_size = (blkctr + hib->image_offset -
1622 0 : chunks[i].offset) * DEV_BSIZE;
1623 : }
1624 :
1625 0 : hib->chunktable_offset = hib->image_offset + blkctr;
1626 0 : return (0);
1627 0 : }
1628 :
1629 : /*
1630 : * Reset the zlib stream state and allocate a new hiballoc area for either
1631 : * inflate or deflate. This function is called once for each hibernate chunk.
1632 : * Calling hiballoc_init multiple times is acceptable since the memory it is
1633 : * provided is unmanaged memory (stolen). We use the memory provided to us
1634 : * by the piglet allocated via the supplied hib.
1635 : */
1636 : int
1637 0 : hibernate_zlib_reset(union hibernate_info *hib, int deflate)
1638 : {
1639 : vaddr_t hibernate_zlib_start;
1640 : size_t hibernate_zlib_size;
1641 0 : char *pva = (char *)hib->piglet_va;
1642 : struct hibernate_zlib_state *hibernate_state;
1643 :
1644 : hibernate_state =
1645 : (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1646 :
1647 0 : if (!deflate)
1648 0 : pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK));
1649 :
1650 : /*
1651 : * See piglet layout information at the start of this file for
1652 : * information on the zlib page assignments.
1653 : */
1654 0 : hibernate_zlib_start = (vaddr_t)(pva + (30 * PAGE_SIZE));
1655 : hibernate_zlib_size = 80 * PAGE_SIZE;
1656 :
1657 0 : memset((void *)hibernate_zlib_start, 0, hibernate_zlib_size);
1658 0 : memset(hibernate_state, 0, PAGE_SIZE);
1659 :
1660 : /* Set up stream structure */
1661 0 : hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc;
1662 0 : hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free;
1663 :
1664 : /* Initialize the hiballoc arena for zlib allocs/frees */
1665 0 : hiballoc_init(&hibernate_state->hiballoc_arena,
1666 : (caddr_t)hibernate_zlib_start, hibernate_zlib_size);
1667 :
1668 0 : if (deflate) {
1669 0 : return deflateInit(&hibernate_state->hib_stream,
1670 : Z_BEST_SPEED);
1671 : } else
1672 0 : return inflateInit(&hibernate_state->hib_stream);
1673 0 : }
1674 :
1675 : /*
1676 : * Reads the hibernated memory image from disk, whose location and
1677 : * size are recorded in hib. Begin by reading the persisted
1678 : * chunk table, which records the original chunk placement location
1679 : * and compressed size for each. Next, allocate a pig region of
1680 : * sufficient size to hold the compressed image. Next, read the
1681 : * chunks into the pig area (calling hibernate_read_chunks to do this),
1682 : * and finally, if all of the above succeeds, clear the hibernate signature.
1683 : * The function will then return to hibernate_resume, which will proceed
1684 : * to unpack the pig image to the correct place in memory.
1685 : */
1686 : int
1687 0 : hibernate_read_image(union hibernate_info *hib)
1688 : {
1689 : size_t compressed_size, disk_size, chunktable_size, pig_sz;
1690 0 : paddr_t image_start, image_end, pig_start, pig_end;
1691 : struct hibernate_disk_chunk *chunks;
1692 : daddr_t blkctr;
1693 : vaddr_t chunktable = (vaddr_t)NULL;
1694 0 : paddr_t piglet_chunktable = hib->piglet_pa +
1695 : HIBERNATE_CHUNK_SIZE;
1696 : int i, status;
1697 :
1698 : status = 0;
1699 0 : pmap_activate(curproc);
1700 :
1701 : /* Calculate total chunk table size in disk blocks */
1702 : chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / DEV_BSIZE;
1703 :
1704 0 : blkctr = hib->chunktable_offset;
1705 :
1706 0 : chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any,
1707 : &kp_none, &kd_nowait);
1708 :
1709 0 : if (!chunktable)
1710 0 : return (1);
1711 :
1712 : /* Map chunktable pages */
1713 0 : for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; i += PAGE_SIZE)
1714 0 : pmap_kenter_pa(chunktable + i, piglet_chunktable + i,
1715 : PROT_READ | PROT_WRITE);
1716 : pmap_update(pmap_kernel());
1717 :
1718 : /* Read the chunktable from disk into the piglet chunktable */
1719 0 : for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE;
1720 0 : i += MAXPHYS, blkctr += MAXPHYS/DEV_BSIZE)
1721 0 : hibernate_block_io(hib, blkctr, MAXPHYS,
1722 0 : chunktable + i, 0);
1723 :
1724 0 : blkctr = hib->image_offset;
1725 : compressed_size = 0;
1726 :
1727 0 : chunks = (struct hibernate_disk_chunk *)chunktable;
1728 :
1729 0 : for (i = 0; i < hib->chunk_ctr; i++)
1730 0 : compressed_size += chunks[i].compressed_size;
1731 :
1732 : disk_size = compressed_size;
1733 :
1734 0 : printf("unhibernating @ block %lld length %lu bytes\n",
1735 0 : hib->sig_offset - chunktable_size,
1736 : compressed_size);
1737 :
1738 : /* Allocate the pig area */
1739 0 : pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE;
1740 0 : if (uvm_pmr_alloc_pig(&pig_start, pig_sz, hib->piglet_pa) == ENOMEM) {
1741 : status = 1;
1742 0 : goto unmap;
1743 : }
1744 :
1745 0 : pig_end = pig_start + pig_sz;
1746 :
1747 : /* Calculate image extents. Pig image must end on a chunk boundary. */
1748 0 : image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1);
1749 0 : image_start = image_end - disk_size;
1750 :
1751 0 : hibernate_read_chunks(hib, image_start, image_end, disk_size,
1752 : chunks);
1753 :
1754 : /* Prepare the resume time pmap/page table */
1755 0 : hibernate_populate_resume_pt(hib, image_start, image_end);
1756 :
1757 : unmap:
1758 : /* Unmap chunktable pages */
1759 0 : pmap_kremove(chunktable, HIBERNATE_CHUNK_TABLE_SIZE);
1760 : pmap_update(pmap_kernel());
1761 :
1762 0 : return (status);
1763 0 : }
1764 :
1765 : /*
1766 : * Read the hibernated memory chunks from disk (chunk information at this
1767 : * point is stored in the piglet) into the pig area specified by
1768 : * [pig_start .. pig_end]. Order the chunks so that the final chunk is the
1769 : * only chunk with overlap possibilities.
1770 : */
1771 : int
1772 0 : hibernate_read_chunks(union hibernate_info *hib, paddr_t pig_start,
1773 : paddr_t pig_end, size_t image_compr_size,
1774 : struct hibernate_disk_chunk *chunks)
1775 : {
1776 : paddr_t img_cur, piglet_base;
1777 : daddr_t blkctr;
1778 : size_t processed, compressed_size, read_size;
1779 : int nchunks, nfchunks, num_io_pages;
1780 : vaddr_t tempva, hibernate_fchunk_area;
1781 : short *fchunks, i, j;
1782 :
1783 : tempva = (vaddr_t)NULL;
1784 : hibernate_fchunk_area = (vaddr_t)NULL;
1785 : nfchunks = 0;
1786 0 : piglet_base = hib->piglet_pa;
1787 0 : global_pig_start = pig_start;
1788 :
1789 : /*
1790 : * These mappings go into the resuming kernel's page table, and are
1791 : * used only during image read. They dissappear from existence
1792 : * when the suspended kernel is unpacked on top of us.
1793 : */
1794 0 : tempva = (vaddr_t)km_alloc(MAXPHYS + PAGE_SIZE, &kv_any, &kp_none,
1795 : &kd_nowait);
1796 0 : if (!tempva)
1797 0 : return (1);
1798 0 : hibernate_fchunk_area = (vaddr_t)km_alloc(24 * PAGE_SIZE, &kv_any,
1799 : &kp_none, &kd_nowait);
1800 0 : if (!hibernate_fchunk_area)
1801 0 : return (1);
1802 :
1803 : /* Final output chunk ordering VA */
1804 0 : fchunks = (short *)hibernate_fchunk_area;
1805 :
1806 : /* Map the chunk ordering region */
1807 0 : for(i = 0; i < 24 ; i++)
1808 0 : pmap_kenter_pa(hibernate_fchunk_area + (i * PAGE_SIZE),
1809 0 : piglet_base + ((4 + i) * PAGE_SIZE),
1810 : PROT_READ | PROT_WRITE);
1811 : pmap_update(pmap_kernel());
1812 :
1813 0 : nchunks = hib->chunk_ctr;
1814 :
1815 : /* Initially start all chunks as unplaced */
1816 0 : for (i = 0; i < nchunks; i++)
1817 0 : chunks[i].flags = 0;
1818 :
1819 : /*
1820 : * Search the list for chunks that are outside the pig area. These
1821 : * can be placed first in the final output list.
1822 : */
1823 0 : for (i = 0; i < nchunks; i++) {
1824 0 : if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) {
1825 0 : fchunks[nfchunks] = i;
1826 0 : nfchunks++;
1827 0 : chunks[i].flags |= HIBERNATE_CHUNK_PLACED;
1828 0 : }
1829 : }
1830 :
1831 : /*
1832 : * Walk the ordering, place the chunks in ascending memory order.
1833 : */
1834 0 : for (i = 0; i < nchunks; i++) {
1835 0 : if (chunks[i].flags != HIBERNATE_CHUNK_PLACED) {
1836 0 : fchunks[nfchunks] = i;
1837 0 : nfchunks++;
1838 0 : chunks[i].flags = HIBERNATE_CHUNK_PLACED;
1839 0 : }
1840 : }
1841 :
1842 : img_cur = pig_start;
1843 :
1844 0 : for (i = 0; i < nfchunks; i++) {
1845 0 : blkctr = chunks[fchunks[i]].offset;
1846 : processed = 0;
1847 0 : compressed_size = chunks[fchunks[i]].compressed_size;
1848 :
1849 0 : while (processed < compressed_size) {
1850 0 : if (compressed_size - processed >= MAXPHYS)
1851 0 : read_size = MAXPHYS;
1852 : else
1853 : read_size = compressed_size - processed;
1854 :
1855 : /*
1856 : * We're reading read_size bytes, offset from the
1857 : * start of a page by img_cur % PAGE_SIZE, so the
1858 : * end will be read_size + (img_cur % PAGE_SIZE)
1859 : * from the start of the first page. Round that
1860 : * up to the next page size.
1861 : */
1862 0 : num_io_pages = (read_size + (img_cur % PAGE_SIZE)
1863 0 : + PAGE_SIZE - 1) / PAGE_SIZE;
1864 :
1865 0 : KASSERT(num_io_pages <= MAXPHYS/PAGE_SIZE + 1);
1866 :
1867 : /* Map pages for this read */
1868 0 : for (j = 0; j < num_io_pages; j ++)
1869 0 : pmap_kenter_pa(tempva + j * PAGE_SIZE,
1870 0 : img_cur + j * PAGE_SIZE,
1871 : PROT_READ | PROT_WRITE);
1872 :
1873 : pmap_update(pmap_kernel());
1874 :
1875 0 : hibernate_block_io(hib, blkctr, read_size,
1876 0 : tempva + (img_cur & PAGE_MASK), 0);
1877 :
1878 0 : blkctr += (read_size / DEV_BSIZE);
1879 :
1880 0 : pmap_kremove(tempva, num_io_pages * PAGE_SIZE);
1881 : pmap_update(pmap_kernel());
1882 :
1883 0 : processed += read_size;
1884 0 : img_cur += read_size;
1885 : }
1886 : }
1887 :
1888 0 : pmap_kremove(hibernate_fchunk_area, 24 * PAGE_SIZE);
1889 : pmap_update(pmap_kernel());
1890 :
1891 0 : return (0);
1892 0 : }
1893 :
1894 : /*
1895 : * Hibernating a machine comprises the following operations:
1896 : * 1. Calculating this machine's hibernate_info information
1897 : * 2. Allocating a piglet and saving the piglet's physaddr
1898 : * 3. Calculating the memory chunks
1899 : * 4. Writing the compressed chunks to disk
1900 : * 5. Writing the chunk table
1901 : * 6. Writing the signature block (hibernate_info)
1902 : *
1903 : * On most architectures, the function calling hibernate_suspend would
1904 : * then power off the machine using some MD-specific implementation.
1905 : */
1906 : int
1907 0 : hibernate_suspend(void)
1908 : {
1909 0 : union hibernate_info hib;
1910 0 : u_long start, end;
1911 :
1912 : /*
1913 : * Calculate memory ranges, swap offsets, etc.
1914 : * This also allocates a piglet whose physaddr is stored in
1915 : * hib->piglet_pa and vaddr stored in hib->piglet_va
1916 : */
1917 0 : if (get_hibernate_info(&hib, 1)) {
1918 : DPRINTF("failed to obtain hibernate info\n");
1919 0 : return (1);
1920 : }
1921 :
1922 : /* Find a page-addressed region in swap [start,end] */
1923 0 : if (uvm_hibswap(hib.dev, &start, &end)) {
1924 0 : printf("hibernate: cannot find any swap\n");
1925 0 : return (1);
1926 : }
1927 :
1928 0 : if (end - start < 1000) {
1929 0 : printf("hibernate: insufficient swap (%lu is too small)\n",
1930 : end - start);
1931 0 : return (1);
1932 : }
1933 :
1934 : /* Calculate block offsets in swap */
1935 0 : hib.image_offset = ctod(start);
1936 :
1937 : DPRINTF("hibernate @ block %lld max-length %lu blocks\n",
1938 : hib.image_offset, ctod(end) - ctod(start));
1939 :
1940 0 : pmap_activate(curproc);
1941 : DPRINTF("hibernate: writing chunks\n");
1942 0 : if (hibernate_write_chunks(&hib)) {
1943 : DPRINTF("hibernate_write_chunks failed\n");
1944 0 : return (1);
1945 : }
1946 :
1947 : DPRINTF("hibernate: writing chunktable\n");
1948 0 : if (hibernate_write_chunktable(&hib)) {
1949 : DPRINTF("hibernate_write_chunktable failed\n");
1950 0 : return (1);
1951 : }
1952 :
1953 : DPRINTF("hibernate: writing signature\n");
1954 0 : if (hibernate_write_signature(&hib)) {
1955 : DPRINTF("hibernate_write_signature failed\n");
1956 0 : return (1);
1957 : }
1958 :
1959 : /* Allow the disk to settle */
1960 0 : delay(500000);
1961 :
1962 : /*
1963 : * Give the device-specific I/O function a notification that we're
1964 : * done, and that it can clean up or shutdown as needed.
1965 : */
1966 0 : hib.io_func(hib.dev, 0, (vaddr_t)NULL, 0, HIB_DONE, hib.io_page);
1967 0 : return (0);
1968 0 : }
1969 :
1970 : int
1971 0 : hibernate_alloc(void)
1972 : {
1973 0 : KASSERT(global_piglet_va == 0);
1974 0 : KASSERT(hibernate_temp_page == 0);
1975 :
1976 0 : pmap_activate(curproc);
1977 0 : pmap_kenter_pa(HIBERNATE_HIBALLOC_PAGE, HIBERNATE_HIBALLOC_PAGE,
1978 : PROT_READ | PROT_WRITE);
1979 :
1980 : /* Allocate a piglet, store its addresses in the supplied globals */
1981 0 : if (uvm_pmr_alloc_piglet(&global_piglet_va, &global_piglet_pa,
1982 : HIBERNATE_CHUNK_SIZE * 4, HIBERNATE_CHUNK_SIZE))
1983 : goto unmap;
1984 :
1985 : /*
1986 : * Allocate VA for the temp page.
1987 : *
1988 : * This will become part of the suspended kernel and will
1989 : * be freed in hibernate_free, upon resume (or hibernate
1990 : * failure)
1991 : */
1992 0 : hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any,
1993 : &kp_none, &kd_nowait);
1994 0 : if (!hibernate_temp_page) {
1995 0 : uvm_pmr_free_piglet(global_piglet_va,
1996 : 4 * HIBERNATE_CHUNK_SIZE);
1997 0 : global_piglet_va = 0;
1998 0 : goto unmap;
1999 : }
2000 0 : return (0);
2001 : unmap:
2002 0 : pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE);
2003 : pmap_update(pmap_kernel());
2004 0 : return (ENOMEM);
2005 0 : }
2006 :
2007 : /*
2008 : * Free items allocated by hibernate_alloc()
2009 : */
2010 : void
2011 0 : hibernate_free(void)
2012 : {
2013 0 : pmap_activate(curproc);
2014 :
2015 0 : if (global_piglet_va)
2016 0 : uvm_pmr_free_piglet(global_piglet_va,
2017 : 4 * HIBERNATE_CHUNK_SIZE);
2018 :
2019 0 : if (hibernate_temp_page) {
2020 0 : pmap_kremove(hibernate_temp_page, PAGE_SIZE);
2021 0 : km_free((void *)hibernate_temp_page, PAGE_SIZE,
2022 : &kv_any, &kp_none);
2023 0 : }
2024 :
2025 0 : global_piglet_va = 0;
2026 0 : hibernate_temp_page = 0;
2027 0 : pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE);
2028 : pmap_update(pmap_kernel());
2029 0 : }
|
