?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
include: reduce default stack size
[minix.git] / servers / vm / region.c
blob21b6e70441824e1f04aa315ab2e4ab881b7fce50
1
2 #include <minix/com.h>
3 #include <minix/callnr.h>
4 #include <minix/type.h>
5 #include <minix/config.h>
6 #include <minix/const.h>
7 #include <minix/sysutil.h>
8 #include <minix/syslib.h>
9 #include <minix/debug.h>
10 #include <minix/bitmap.h>
11 #include <minix/hash.h>
12 #include <machine/multiboot.h>
13
14 #include <sys/mman.h>
15
16 #include <limits.h>
17 #include <stdlib.h>
18 #include <string.h>
19 #include <assert.h>
20 #include <stdint.h>
21 #include <sys/param.h>
22
23 #include "vm.h"
24 #include "proto.h"
25 #include "util.h"
26 #include "glo.h"
27 #include "region.h"
28 #include "sanitycheck.h"
29 #include "yieldedavl.h"
30 #include "memlist.h"
31 #include "memtype.h"
32
33 /* LRU list. */
34 static yielded_t *lru_youngest = NULL, *lru_oldest = NULL;
35
36 static int map_ph_writept(struct vmproc *vmp, struct vir_region *vr,
37 struct phys_region *pr);
38
39 static phys_bytes freeyieldednode(yielded_t *node, int freemem);
40
41 static struct vir_region *map_copy_region(struct vmproc *vmp, struct
42 vir_region *vr);
43
44 #if SANITYCHECKS
45 static void lrucheck(void);
46 #endif
47
48 /* hash table of yielded blocks */
49 #define YIELD_HASHSIZE 65536
50 static yielded_avl vm_yielded_blocks[YIELD_HASHSIZE];
51
52 static int avl_inited = 0;
53
54 void map_region_init(void)
55 {
56 int h;
57 assert(!avl_inited);
58 for(h = 0; h < YIELD_HASHSIZE; h++)
59 yielded_init(&vm_yielded_blocks[h]);
60 avl_inited = 1;
61 }
62
63 static yielded_avl *get_yielded_avl(block_id_t id)
64 {
65 u32_t h;
66
67 assert(avl_inited);
68
69 hash_i_64(id.owner, id.id, h);
70 h = h % YIELD_HASHSIZE;
71
72 assert(h >= 0);
73 assert(h < YIELD_HASHSIZE);
74
75 return &vm_yielded_blocks[h];
76 }
77
78 void map_printregion(struct vir_region *vr)
79 {
80 int i;
81 struct phys_region *ph;
82 printf("map_printmap: map_name: %s\n", vr->memtype->name);
83 printf("\t%lx (len 0x%lx, %lukB), %p\n",
84 vr->vaddr, vr->length, vr->length/1024, vr->memtype->name);
85 printf("\t\tphysblocks:\n");
86 for(i = 0; i < vr->length/VM_PAGE_SIZE; i++) {
87 if(!(ph=vr->physblocks[i])) continue;
88 printf("\t\t@ %lx (refs %d): phys 0x%lx\n",
89 (vr->vaddr + ph->offset),
90 ph->ph->refcount, ph->ph->phys);
91 }
92 }
93
94 struct phys_region *physblock_get(struct vir_region *region, vir_bytes offset)
95 {
96 int i;
97 struct phys_region *foundregion;
98 assert(!(offset % VM_PAGE_SIZE));
99 assert(offset >= 0 && offset < region->length);
100 i = offset/VM_PAGE_SIZE;
101 if((foundregion = region->physblocks[i]))
102 assert(foundregion->offset == offset);
103 return foundregion;
104 }
105
106 void physblock_set(struct vir_region *region, vir_bytes offset,
107 struct phys_region *newphysr)
108 {
109 int i;
110 assert(!(offset % VM_PAGE_SIZE));
111 assert(offset >= 0 && offset < region->length);
112 i = offset/VM_PAGE_SIZE;
113 if(newphysr) {
114 assert(!region->physblocks[i]);
115 assert(newphysr->offset == offset);
116 } else {
117 assert(region->physblocks[i]);
118 }
119 region->physblocks[i] = newphysr;
120 }
121
122 /*===========================================================================*
123 * map_printmap *
124 *===========================================================================*/
125 void map_printmap(vmp)
126 struct vmproc *vmp;
127 {
128 struct vir_region *vr;
129 region_iter iter;
130
131 printf("memory regions in process %d:\n", vmp->vm_endpoint);
132
133 region_start_iter_least(&vmp->vm_regions_avl, &iter);
134 while((vr = region_get_iter(&iter))) {
135 map_printregion(vr);
136 region_incr_iter(&iter);
137 }
138 }
139
140 static struct vir_region *getnextvr(struct vir_region *vr)
141 {
142 struct vir_region *nextvr;
143 region_iter v_iter;
144 SLABSANE(vr);
145 region_start_iter(&vr->parent->vm_regions_avl, &v_iter, vr->vaddr, AVL_EQUAL);
146 assert(region_get_iter(&v_iter));
147 assert(region_get_iter(&v_iter) == vr);
148 region_incr_iter(&v_iter);
149 nextvr = region_get_iter(&v_iter);
150 if(!nextvr) return NULL;
151 SLABSANE(nextvr);
152 assert(vr->parent == nextvr->parent);
153 assert(vr->vaddr < nextvr->vaddr);
154 assert(vr->vaddr + vr->length <= nextvr->vaddr);
155 return nextvr;
156 }
157
158 int pr_writable(struct vir_region *vr, struct phys_region *pr)
159 {
160 assert(vr->memtype->writable);
161 return ((vr->flags & VR_WRITABLE) && vr->memtype->writable(pr));
162 }
163
164 #if SANITYCHECKS
165
166 /*===========================================================================*
167 * map_sanitycheck_pt *
168 *===========================================================================*/
169 static int map_sanitycheck_pt(struct vmproc *vmp,
170 struct vir_region *vr, struct phys_region *pr)
171 {
172 struct phys_block *pb = pr->ph;
173 int rw;
174 int r;
175
176 if(pr_writable(vr, pr))
177 rw = PTF_WRITE;
178 else
179 rw = PTF_READ;
180
181 r = pt_writemap(vmp, &vmp->vm_pt, vr->vaddr + pr->offset,
182 pb->phys, VM_PAGE_SIZE, PTF_PRESENT | PTF_USER | rw, WMF_VERIFY);
183
184 if(r != OK) {
185 printf("proc %d phys_region 0x%lx sanity check failed\n",
186 vmp->vm_endpoint, pr->offset);
187 map_printregion(vr);
188 }
189
190 return r;
191 }
192
193 /*===========================================================================*
194 * map_sanitycheck *
195 *===========================================================================*/
196 void map_sanitycheck(char *file, int line)
197 {
198 struct vmproc *vmp;
199
200 lrucheck();
201
202 /* Macro for looping over all physical blocks of all regions of
203 * all processes.
204 */
205 #define ALLREGIONS(regioncode, physcode) \
206 for(vmp = vmproc; vmp < &vmproc[VMP_NR]; vmp++) { \
207 vir_bytes voffset; \
208 region_iter v_iter; \
209 struct vir_region *vr; \
210 if(!(vmp->vm_flags & VMF_INUSE)) \
211 continue; \
212 region_start_iter_least(&vmp->vm_regions_avl, &v_iter); \
213 while((vr = region_get_iter(&v_iter))) { \
214 struct phys_region *pr; \
215 regioncode; \
216 for(voffset = 0; voffset < vr->length; \
217 voffset += VM_PAGE_SIZE) { \
218 if(!(pr = physblock_get(vr, voffset))) \
219 continue; \
220 physcode; \
221 } \
222 region_incr_iter(&v_iter); \
223 } \
224 }
225
226 #define MYSLABSANE(s) MYASSERT(slabsane_f(__FILE__, __LINE__, s, sizeof(*(s))))
227 /* Basic pointers check. */
228 ALLREGIONS(MYSLABSANE(vr),MYSLABSANE(pr); MYSLABSANE(pr->ph);MYSLABSANE(pr->parent));
229 ALLREGIONS(/* MYASSERT(vr->parent == vmp) */,MYASSERT(pr->parent == vr););
230
231 /* Do counting for consistency check. */
232 ALLREGIONS(;,USE(pr->ph, pr->ph->seencount = 0;););
233 ALLREGIONS(;,MYASSERT(pr->offset == voffset););
234 ALLREGIONS(;,USE(pr->ph, pr->ph->seencount++;);
235 if(pr->ph->seencount == 1) {
236 if(pr->parent->memtype->ev_sanitycheck)
237 pr->parent->memtype->ev_sanitycheck(pr, file, line);
238 }
239 );
240
241 /* Do consistency check. */
242 ALLREGIONS({ struct vir_region *nextvr = getnextvr(vr);
243 if(nextvr) {
244 MYASSERT(vr->vaddr < nextvr->vaddr);
245 MYASSERT(vr->vaddr + vr->length <= nextvr->vaddr);
246 }
247 }
248 MYASSERT(!(vr->vaddr % VM_PAGE_SIZE));,
249 if(pr->ph->refcount != pr->ph->seencount) {
250 map_printmap(vmp);
251 printf("ph in vr %p: 0x%lx refcount %u "
252 "but seencount %u\n",
253 vr, pr->offset,
254 pr->ph->refcount, pr->ph->seencount);
255 }
256 {
257 int n_others = 0;
258 struct phys_region *others;
259 if(pr->ph->refcount > 0) {
260 MYASSERT(pr->ph->firstregion);
261 if(pr->ph->refcount == 1) {
262 MYASSERT(pr->ph->firstregion == pr);
263 }
264 } else {
265 MYASSERT(!pr->ph->firstregion);
266 }
267 for(others = pr->ph->firstregion; others;
268 others = others->next_ph_list) {
269 MYSLABSANE(others);
270 MYASSERT(others->ph == pr->ph);
271 n_others++;
272 }
273 MYASSERT(pr->ph->refcount == n_others);
274 }
275 MYASSERT(pr->ph->refcount == pr->ph->seencount);
276 MYASSERT(!(pr->offset % VM_PAGE_SIZE)););
277 ALLREGIONS(,MYASSERT(map_sanitycheck_pt(vmp, vr, pr) == OK));
278 }
279
280 #define LRUCHECK lrucheck()
281
282 static void lrucheck(void)
283 {
284 yielded_t *list;
285
286 /* list is empty and ok if both ends point to null. */
287 if(!lru_youngest && !lru_oldest)
288 return;
289
290 /* if not, both should point to something. */
291 SLABSANE(lru_youngest);
292 SLABSANE(lru_oldest);
293
294 assert(!lru_youngest->younger);
295 assert(!lru_oldest->older);
296
297 for(list = lru_youngest; list; list = list->older) {
298 SLABSANE(list);
299 if(list->younger) {
300 SLABSANE(list->younger);
301 assert(list->younger->older == list);
302 } else assert(list == lru_youngest);
303 if(list->older) {
304 SLABSANE(list->older);
305 assert(list->older->younger == list);
306 } else assert(list == lru_oldest);
307 }
308 }
309
310 void blockstats(void)
311 {
312 yielded_t *list;
313 int blocks = 0;
314 phys_bytes mem = 0;
315 clock_t ticks;
316 int s;
317
318 s = getuptime(&ticks);
319
320 assert(s == OK);
321
322 LRUCHECK;
323
324 for(list = lru_youngest; list; list = list->older) {
325 mem += VM_PAGE_SIZE;
326 blocks++;
327 }
328
329 if(blocks > 0)
330 printf("%d blocks, %lukB; ", blocks, mem/1024);
331
332 printmemstats();
333 }
334 #else
335 #define LRUCHECK
336 #endif
337
338
339 /*=========================================================================*
340 * map_ph_writept *
341 *=========================================================================*/
342 static int map_ph_writept(struct vmproc *vmp, struct vir_region *vr,
343 struct phys_region *pr)
344 {
345 int rw;
346 struct phys_block *pb = pr->ph;
347
348 assert(vr);
349 assert(pr);
350 assert(pb);
351
352 assert(!(vr->vaddr % VM_PAGE_SIZE));
353 assert(!(pr->offset % VM_PAGE_SIZE));
354 assert(pb->refcount > 0);
355
356 if(pr_writable(vr, pr))
357 rw = PTF_WRITE;
358 else
359 rw = PTF_READ;
360
361 if(pt_writemap(vmp, &vmp->vm_pt, vr->vaddr + pr->offset,
362 pb->phys, VM_PAGE_SIZE, PTF_PRESENT | PTF_USER | rw,
363 #if SANITYCHECKS
364 !pr->written ? 0 :
365 #endif
366 WMF_OVERWRITE) != OK) {
367 printf("VM: map_writept: pt_writemap failed\n");
368 return ENOMEM;
369 }
370
371 #if SANITYCHECKS
372 USE(pr, pr->written = 1;);
373 #endif
374
375 return OK;
376 }
377
378 #define SLOT_FAIL ((vir_bytes) -1)
379
380 /*===========================================================================*
381 * region_find_slot_range *
382 *===========================================================================*/
383 static vir_bytes region_find_slot_range(struct vmproc *vmp,
384 vir_bytes minv, vir_bytes maxv, vir_bytes length)
385 {
386 struct vir_region *lastregion;
387 vir_bytes startv = 0;
388 int foundflag = 0;
389 region_iter iter;
390
391 SANITYCHECK(SCL_FUNCTIONS);
392
393 /* Length must be reasonable. */
394 assert(length > 0);
395
396 /* Special case: allow caller to set maxv to 0 meaning 'I want
397 * it to be mapped in right here.'
398 */
399 if(maxv == 0) {
400 maxv = minv + length;
401
402 /* Sanity check. */
403 if(maxv <= minv) {
404 printf("region_find_slot: minv 0x%lx and bytes 0x%lx\n",
405 minv, length);
406 return SLOT_FAIL;
407 }
408 }
409
410 /* Basic input sanity checks. */
411 assert(!(length % VM_PAGE_SIZE));
412 if(minv >= maxv) {
413 printf("VM: 1 minv: 0x%lx maxv: 0x%lx length: 0x%lx\n",
414 minv, maxv, length);
415 }
416
417 assert(minv < maxv);
418
419 if(minv + length > maxv)
420 return SLOT_FAIL;
421
422 #define FREEVRANGE_TRY(rangestart, rangeend) { \
423 vir_bytes frstart = (rangestart), frend = (rangeend); \
424 frstart = MAX(frstart, minv); \
425 frend = MIN(frend, maxv); \
426 if(frend > frstart && (frend - frstart) >= length) { \
427 startv = frend-length; \
428 foundflag = 1; \
429 } }
430
431 #define FREEVRANGE(start, end) { \
432 assert(!foundflag); \
433 FREEVRANGE_TRY(((start)+VM_PAGE_SIZE), ((end)-VM_PAGE_SIZE)); \
434 if(!foundflag) { \
435 FREEVRANGE_TRY((start), (end)); \
436 } \
437 }
438
439 /* find region after maxv. */
440 region_start_iter(&vmp->vm_regions_avl, &iter, maxv, AVL_GREATER_EQUAL);
441 lastregion = region_get_iter(&iter);
442
443 if(!lastregion) {
444 /* This is the free virtual address space after the last region. */
445 region_start_iter(&vmp->vm_regions_avl, &iter, maxv, AVL_LESS);
446 lastregion = region_get_iter(&iter);
447 FREEVRANGE(lastregion ?
448 lastregion->vaddr+lastregion->length : 0, VM_DATATOP);
449 }
450
451 if(!foundflag) {
452 struct vir_region *vr;
453 while((vr = region_get_iter(&iter)) && !foundflag) {
454 struct vir_region *nextvr;
455 region_decr_iter(&iter);
456 nextvr = region_get_iter(&iter);
457 FREEVRANGE(nextvr ? nextvr->vaddr+nextvr->length : 0,
458 vr->vaddr);
459 }
460 }
461
462 if(!foundflag) {
463 return SLOT_FAIL;
464 }
465
466 /* However we got it, startv must be in the requested range. */
467 assert(startv >= minv);
468 assert(startv < maxv);
469 assert(startv + length <= maxv);
470
471 /* remember this position as a hint for next time. */
472 vmp->vm_region_top = startv + length;
473
474 return startv;
475 }
476
477 /*===========================================================================*
478 * region_find_slot *
479 *===========================================================================*/
480 static vir_bytes region_find_slot(struct vmproc *vmp,
481 vir_bytes minv, vir_bytes maxv, vir_bytes length)
482 {
483 vir_bytes v, hint = vmp->vm_region_top;
484
485 /* use the top of the last inserted region as a minv hint if
486 * possible. remember that a zero maxv is a special case.
487 */
488
489 if(maxv && hint < maxv && hint >= minv) {
490 v = region_find_slot_range(vmp, minv, hint, length);
491
492 if(v != SLOT_FAIL)
493 return v;
494 }
495
496 return region_find_slot_range(vmp, minv, maxv, length);
497 }
498
499 static int phys_slot(vir_bytes len)
500 {
501 assert(!(len % VM_PAGE_SIZE));
502 return len / VM_PAGE_SIZE;
503 }
504
505 struct vir_region *region_new(struct vmproc *vmp, vir_bytes startv, vir_bytes length,
506 int flags, mem_type_t *memtype)
507 {
508 struct vir_region *newregion;
509 struct phys_region **physregions;
510 static u32_t id;
511 int slots = phys_slot(length);
512
513 if(!(SLABALLOC(newregion))) {
514 printf("vm: region_new: could not allocate\n");
515 return NULL;
516 }
517
518 /* Fill in node details. */
519 USE(newregion,
520 memset(newregion, 0, sizeof(*newregion));
521 newregion->vaddr = startv;
522 newregion->length = length;
523 newregion->flags = flags;
524 newregion->memtype = memtype;
525 newregion->remaps = 0;
526 newregion->id = id++;
527 newregion->lower = newregion->higher = NULL;
528 newregion->parent = vmp;);
529
530 if(!(physregions = calloc(slots, sizeof(struct phys_region *)))) {
531 printf("VM: region_new: allocating phys blocks failed\n");
532 SLABFREE(newregion);
533 return NULL;
534 }
535
536 USE(newregion, newregion->physblocks = physregions;);
537
538 return newregion;
539 }
540
541 /*===========================================================================*
542 * map_page_region *
543 *===========================================================================*/
544 struct vir_region *map_page_region(vmp, minv, maxv, length,
545 flags, mapflags, memtype)
546 struct vmproc *vmp;
547 vir_bytes minv;
548 vir_bytes maxv;
549 vir_bytes length;
550 u32_t flags;
551 int mapflags;
552 mem_type_t *memtype;
553 {
554 struct vir_region *newregion;
555 vir_bytes startv;
556
557 assert(!(length % VM_PAGE_SIZE));
558
559 SANITYCHECK(SCL_FUNCTIONS);
560
561 startv = region_find_slot(vmp, minv, maxv, length);
562 if (startv == SLOT_FAIL)
563 return NULL;
564
565 /* Now we want a new region. */
566 if(!(newregion = region_new(vmp, startv, length, flags, memtype))) {
567 printf("VM: map_page_region: allocating region failed\n");
568 return NULL;
569 }
570
571 /* If a new event is specified, invoke it. */
572 if(newregion->memtype->ev_new) {
573 if(newregion->memtype->ev_new(newregion) != OK) {
574 /* ev_new will have freed and removed the region */
575 return NULL;
576 }
577 }
578
579 if(mapflags & MF_PREALLOC) {
580 if(map_handle_memory(vmp, newregion, 0, length, 1) != OK) {
581 printf("VM: map_page_region: prealloc failed\n");
582 free(newregion->physblocks);
583 USE(newregion,
584 newregion->physblocks = NULL;);
585 SLABFREE(newregion);
586 return NULL;
587 }
588 }
589
590 /* Pre-allocations should be uninitialized, but after that it's a
591 * different story.
592 */
593 USE(newregion, newregion->flags &= ~VR_UNINITIALIZED;);
594
595 /* Link it. */
596 region_insert(&vmp->vm_regions_avl, newregion);
597
598 #if SANITYCHECKS
599 assert(startv == newregion->vaddr);
600 {
601 struct vir_region *nextvr;
602 if((nextvr = getnextvr(newregion))) {
603 assert(newregion->vaddr < nextvr->vaddr);
604 }
605 }
606 #endif
607
608 SANITYCHECK(SCL_FUNCTIONS);
609
610 return newregion;
611 }
612
613 /*===========================================================================*
614 * map_subfree *
615 *===========================================================================*/
616 static int map_subfree(struct vir_region *region,
617 vir_bytes start, vir_bytes len)
618 {
619 struct phys_region *pr;
620 vir_bytes end = start+len;
621 vir_bytes voffset;
622
623 #if SANITYCHECKS
624 SLABSANE(region);
625 for(voffset = 0; voffset < phys_slot(region->length);
626 voffset += VM_PAGE_SIZE) {
627 struct phys_region *others;
628 struct phys_block *pb;
629
630 if(!(pr = physblock_get(region, voffset)))
631 continue;
632
633 pb = pr->ph;
634
635 for(others = pb->firstregion; others;
636 others = others->next_ph_list) {
637 assert(others->ph == pb);
638 }
639 }
640 #endif
641
642 for(voffset = start; voffset < end; voffset+=VM_PAGE_SIZE) {
643 if(!(pr = physblock_get(region, voffset)))
644 continue;
645 assert(pr->offset >= start);
646 assert(pr->offset < end);
647 pb_unreferenced(region, pr, 1);
648 SLABFREE(pr);
649 }
650
651 return OK;
652 }
653
654 /*===========================================================================*
655 * map_free *
656 *===========================================================================*/
657 int map_free(struct vir_region *region)
658 {
659 int r;
660
661 if((r=map_subfree(region, 0, region->length)) != OK) {
662 printf("%d\n", __LINE__);
663 return r;
664 }
665
666 if(region->memtype->ev_delete)
667 region->memtype->ev_delete(region);
668 free(region->physblocks);
669 region->physblocks = NULL;
670 SLABFREE(region);
671
672 return OK;
673 }
674
675 /*===========================================================================*
676 * yielded_block_cmp *
677 *===========================================================================*/
678 int yielded_block_cmp(struct block_id *id1, struct block_id *id2)
679 {
680 if(id1->owner < id2->owner)
681 return -1;
682 if(id1->owner > id2->owner)
683 return 1;
684 return cmp64(id1->id, id2->id);
685 }
686
687
688 /*===========================================================================*
689 * free_yielded_proc *
690 *===========================================================================*/
691 static vir_bytes free_yielded_proc(struct vmproc *vmp)
692 {
693 vir_bytes total = 0;
694 int h;
695
696 SANITYCHECK(SCL_FUNCTIONS);
697
698 /* Free associated regions. */
699 for(h = 0; h < YIELD_HASHSIZE && vmp->vm_yielded > 0; h++) {
700 yielded_t *yb;
701 yielded_iter iter;
702 yielded_avl *avl = &vm_yielded_blocks[h];
703 yielded_start_iter_least(avl, &iter);
704 while((yb = yielded_get_iter(&iter))) {
705 yielded_t *next_yb;
706 SLABSANE(yb);
707 yielded_incr_iter(&iter);
708 if(yb->id.owner != vmp->vm_endpoint)
709 continue;
710 next_yb = yielded_get_iter(&iter);
711 total += freeyieldednode(yb, 1);
712 /* the above removal invalidated our iter; restart it
713 * for the node we want to start at.
714 */
715 if(!next_yb) break;
716 yielded_start_iter(avl, &iter, next_yb->id, AVL_EQUAL);
717 assert(yielded_get_iter(&iter) == next_yb);
718 }
719 }
720
721 return total;
722 }
723
724
725 static phys_bytes freeyieldednode(yielded_t *node, int freemem)
726 {
727 yielded_t *older, *younger, *removed;
728 yielded_avl *avl;
729 int p;
730
731 SLABSANE(node);
732
733 LRUCHECK;
734
735 /* Update LRU. */
736
737 younger = node->younger;
738 older = node->older;
739
740 if(younger) {
741 SLABSANE(younger);
742 assert(younger->older == node);
743 USE(younger, younger->older = node->older;);
744 } else {
745 assert(node == lru_youngest);
746 lru_youngest = node->older;
747 }
748
749 if(older) {
750 SLABSANE(older);
751 assert(older->younger == node);
752 USE(older, older->younger = node->younger;);
753 } else {
754 assert(node == lru_oldest);
755 lru_oldest = node->younger;
756 }
757
758 LRUCHECK;
759
760 /* Update AVL. */
761
762 if(vm_isokendpt(node->id.owner, &p) != OK)
763 panic("out of date owner of yielded block %d", node->id.owner);
764 avl = get_yielded_avl(node->id);
765 removed = yielded_remove(avl, node->id);
766 assert(removed == node);
767 assert(vmproc[p].vm_yielded > 0);
768 vmproc[p].vm_yielded--;
769
770 /* Free associated memory if requested. */
771
772 if(freemem) {
773 free_mem(ABS2CLICK(node->physaddr), node->pages);
774 }
775
776 /* Free node. */
777 SLABFREE(node);
778
779 return VM_PAGE_SIZE;
780 }
781
782 /*========================================================================*
783 * free_yielded *
784 *========================================================================*/
785 vir_bytes free_yielded(vir_bytes max_bytes)
786 {
787
788 /* PRIVATE yielded_t *lru_youngest = NULL, *lru_oldest = NULL; */
789 vir_bytes freed = 0;
790 int blocks = 0;
791
792 while(freed < max_bytes && lru_oldest) {
793 SLABSANE(lru_oldest);
794 freed += freeyieldednode(lru_oldest, 1);
795 blocks++;
796 }
797
798 return freed;
799 }
800
801 /*========================================================================*
802 * map_free_proc *
803 *========================================================================*/
804 int map_free_proc(vmp)
805 struct vmproc *vmp;
806 {
807 struct vir_region *r;
808
809 while((r = region_search_root(&vmp->vm_regions_avl))) {
810 SANITYCHECK(SCL_DETAIL);
811 #if SANITYCHECKS
812 nocheck++;
813 #endif
814 region_remove(&vmp->vm_regions_avl, r->vaddr); /* For sanity checks. */
815 map_free(r);
816 #if SANITYCHECKS
817 nocheck--;
818 #endif
819 SANITYCHECK(SCL_DETAIL);
820 }
821
822 region_init(&vmp->vm_regions_avl);
823
824 /* Free associated yielded blocks. */
825 free_yielded_proc(vmp);
826
827 SANITYCHECK(SCL_FUNCTIONS);
828
829 return OK;
830 }
831
832 /*===========================================================================*
833 * map_lookup *
834 *===========================================================================*/
835 struct vir_region *map_lookup(vmp, offset, physr)
836 struct vmproc *vmp;
837 vir_bytes offset;
838 struct phys_region **physr;
839 {
840 struct vir_region *r;
841
842 SANITYCHECK(SCL_FUNCTIONS);
843
844 #if SANITYCHECKS
845 if(!region_search_root(&vmp->vm_regions_avl))
846 panic("process has no regions: %d", vmp->vm_endpoint);
847 #endif
848
849 if((r = region_search(&vmp->vm_regions_avl, offset, AVL_LESS_EQUAL))) {
850 vir_bytes ph;
851 if(offset >= r->vaddr && offset < r->vaddr + r->length) {
852 ph = offset - r->vaddr;
853 if(physr) {
854 *physr = physblock_get(r, ph);
855 if(*physr) assert((*physr)->offset == ph);
856 }
857 return r;
858 }
859 }
860
861 SANITYCHECK(SCL_FUNCTIONS);
862
863 return NULL;
864 }
865
866 u32_t vrallocflags(u32_t flags)
867 {
868 u32_t allocflags = 0;
869
870 if(flags & VR_PHYS64K)
871 allocflags |= PAF_ALIGN64K;
872 if(flags & VR_LOWER16MB)
873 allocflags |= PAF_LOWER16MB;
874 if(flags & VR_LOWER1MB)
875 allocflags |= PAF_LOWER1MB;
876 if(flags & VR_CONTIG)
877 allocflags |= PAF_CONTIG;
878 if(!(flags & VR_UNINITIALIZED))
879 allocflags |= PAF_CLEAR;
880
881 return allocflags;
882 }
883
884 /*===========================================================================*
885 * map_clone_ph_block *
886 *===========================================================================*/
887 struct phys_region *map_clone_ph_block(vmp, region, ph)
888 struct vmproc *vmp;
889 struct vir_region *region;
890 struct phys_region *ph;
891 {
892 vir_bytes offset;
893 u32_t allocflags;
894 phys_bytes physaddr;
895 struct phys_region *newpr;
896 int region_has_single_block;
897 SANITYCHECK(SCL_FUNCTIONS);
898
899 /* Warning: this function will free the passed
900 * phys_region *ph and replace it (in the same offset)
901 * with another! So both the pointer to it
902 * and any iterators over the phys_regions in the vir_region
903 * will be invalid on successful return. (Iterators over
904 * the vir_region could be invalid on unsuccessful return too.)
905 */
906
907 /* This is only to be done if there is more than one copy. */
908 assert(ph->ph->refcount > 1);
909
910 /* This function takes a physical block, copies its contents
911 * into newly allocated memory, and replaces the single physical
912 * block by one or more physical blocks with refcount 1 with the
913 * same contents as the original. In other words, a fragmentable
914 * version of map_copy_ph_block().
915 */
916
917 /* Remember where and how much. */
918 offset = ph->offset;
919 physaddr = ph->ph->phys;
920
921 /* Now unlink the original physical block so we can replace
922 * it with new ones.
923 */
924
925 SLABSANE(ph);
926 SLABSANE(ph->ph);
927 assert(ph->ph->refcount > 1);
928 pb_unreferenced(region, ph, 1);
929 SLABFREE(ph);
930
931 SANITYCHECK(SCL_DETAIL);
932
933 /* Put new free memory in. */
934 allocflags = vrallocflags(region->flags | VR_UNINITIALIZED);
935 region_has_single_block = (offset == 0 && region->length == VM_PAGE_SIZE);
936 assert(region_has_single_block || !(allocflags & PAF_CONTIG));
937 assert(!(allocflags & PAF_CLEAR));
938
939 if(map_pf(vmp, region, offset, 1) != OK) {
940 /* XXX original range now gone. */
941 printf("VM: map_clone_ph_block: map_pf failed.\n");
942 return NULL;
943 }
944
945 /* Copy the block to the new memory.
946 * Can only fail if map_new_physblock didn't do what we asked.
947 */
948 if(copy_abs2region(physaddr, region, offset, VM_PAGE_SIZE) != OK)
949 panic("copy_abs2region failed, no good reason for that");
950
951 newpr = physblock_get(region, offset);
952 assert(newpr);
953 assert(newpr->offset == offset);
954
955 SANITYCHECK(SCL_FUNCTIONS);
956
957 return newpr;
958 }
959
960
961 /*===========================================================================*
962 * map_pf *
963 *===========================================================================*/
964 int map_pf(vmp, region, offset, write)
965 struct vmproc *vmp;
966 struct vir_region *region;
967 vir_bytes offset;
968 int write;
969 {
970 struct phys_region *ph;
971 int r = OK;
972
973 offset -= offset % VM_PAGE_SIZE;
974
975 assert(offset >= 0);
976 assert(offset < region->length);
977
978 assert(!(region->vaddr % VM_PAGE_SIZE));
979 assert(!(write && !(region->flags & VR_WRITABLE)));
980
981 SANITYCHECK(SCL_FUNCTIONS);
982
983 if(!(ph = physblock_get(region, offset))) {
984 struct phys_block *pb;
985
986 /* New block. */
987
988 if(!(pb = pb_new(MAP_NONE))) {
989 printf("map_pf: pb_new failed\n");
990 return ENOMEM;
991 }
992
993 if(!(ph = pb_reference(pb, offset, region))) {
994 printf("map_pf: pb_reference failed\n");
995 pb_free(pb);
996 return ENOMEM;
997 }
998 }
999
1000 assert(ph);
1001 assert(ph->ph);
1002
1003 /* If we're writing and the block is already
1004 * writable, nothing to do.
1005 */
1006
1007 assert(region->memtype->writable);
1008
1009 if(!write || !region->memtype->writable(ph)) {
1010 assert(region->memtype->ev_pagefault);
1011 assert(ph->ph);
1012
1013 if((r = region->memtype->ev_pagefault(vmp,
1014 region, ph, write)) == SUSPEND) {
1015 panic("map_pf: memtype->ev_pagefault returned SUSPEND\n");
1016 return SUSPEND;
1017 }
1018
1019 if(r != OK) {
1020 printf("map_pf: memtype->ev_pagefault failed\n");
1021 if(ph)
1022 pb_unreferenced(region, ph, 1);
1023 return r;
1024 }
1025
1026 assert(ph);
1027 assert(ph->ph);
1028 }
1029
1030 assert(ph->ph);
1031
1032 if((r = map_ph_writept(vmp, region, ph)) != OK) {
1033 printf("map_pf: writept failed\n");
1034 return r;
1035 }
1036
1037 SANITYCHECK(SCL_FUNCTIONS);
1038
1039 #if SANITYCHECKS
1040 if(OK != pt_checkrange(&vmp->vm_pt, region->vaddr+offset,
1041 VM_PAGE_SIZE, write)) {
1042 panic("map_pf: pt_checkrange failed: %d", r);
1043 }
1044 #endif
1045
1046 return r;
1047 }
1048
1049 int map_handle_memory(vmp, region, start_offset, length, write)
1050 struct vmproc *vmp;
1051 struct vir_region *region;
1052 vir_bytes start_offset;
1053 vir_bytes length;
1054 int write;
1055 {
1056 vir_bytes offset, lim;
1057 int r;
1058
1059 assert(length > 0);
1060 lim = start_offset + length;
1061 assert(lim > start_offset);
1062
1063 for(offset = start_offset; offset < lim; offset += VM_PAGE_SIZE)
1064 if((r = map_pf(vmp, region, offset, write)) != OK)
1065 return r;
1066
1067 return OK;
1068 }
1069
1070 /*===========================================================================*
1071 * map_pin_memory *
1072 *===========================================================================*/
1073 int map_pin_memory(struct vmproc *vmp)
1074 {
1075 struct vir_region *vr;
1076 int r;
1077 region_iter iter;
1078 region_start_iter_least(&vmp->vm_regions_avl, &iter);
1079 /* Scan all memory regions. */
1080 while((vr = region_get_iter(&iter))) {
1081 /* Make sure region is mapped to physical memory and writable.*/
1082 r = map_handle_memory(vmp, vr, 0, vr->length, 1);
1083 if(r != OK) {
1084 panic("map_pin_memory: map_handle_memory failed: %d", r);
1085 }
1086 region_incr_iter(&iter);
1087 }
1088 return OK;
1089 }
1090
1091 /*===========================================================================*
1092 * map_copy_region *
1093 *===========================================================================*/
1094 static struct vir_region *map_copy_region(struct vmproc *vmp, struct vir_region *vr)
1095 {
1096 /* map_copy_region creates a complete copy of the vir_region
1097 * data structure, linking in the same phys_blocks directly,
1098 * but all in limbo, i.e., the caller has to link the vir_region
1099 * to a process. Therefore it doesn't increase the refcount in
1100 * the phys_block; the caller has to do this once it's linked.
1101 * The reason for this is to keep the sanity checks working
1102 * within this function.
1103 */
1104 struct vir_region *newvr;
1105 struct phys_region *ph;
1106 int r;
1107 #if SANITYCHECKS
1108 int cr;
1109 cr = physregions(vr);
1110 #endif
1111 vir_bytes p;
1112
1113 if(!(newvr = region_new(vr->parent, vr->vaddr, vr->length, vr->flags, vr->memtype)))
1114 return NULL;
1115
1116 if(vr->memtype->ev_copy && (r=vr->memtype->ev_copy(vr, newvr)) != OK) {
1117 map_free(newvr);
1118 printf("VM: memtype-specific copy failed (%d)\n", r);
1119 return NULL;
1120 }
1121
1122 for(p = 0; p < phys_slot(vr->length); p++) {
1123 if(!(ph = physblock_get(vr, p*VM_PAGE_SIZE))) continue;
1124 struct phys_region *newph = pb_reference(ph->ph, ph->offset, newvr);
1125
1126 if(!newph) { map_free(newvr); return NULL; }
1127
1128 #if SANITYCHECKS
1129 USE(newph, newph->written = 0;);
1130 assert(physregions(vr) == cr);
1131 #endif
1132 }
1133
1134 #if SANITYCHECKS
1135 assert(physregions(vr) == physregions(newvr));
1136 #endif
1137
1138 return newvr;
1139 }
1140
1141 /*===========================================================================*
1142 * copy_abs2region *
1143 *===========================================================================*/
1144 int copy_abs2region(phys_bytes abs, struct vir_region *destregion,
1145 phys_bytes offset, phys_bytes len)
1146
1147 {
1148 assert(destregion);
1149 assert(destregion->physblocks);
1150 while(len > 0) {
1151 phys_bytes sublen, suboffset;
1152 struct phys_region *ph;
1153 assert(destregion);
1154 assert(destregion->physblocks);
1155 if(!(ph = physblock_get(destregion, offset))) {
1156 printf("VM: copy_abs2region: no phys region found (1).\n");
1157 return EFAULT;
1158 }
1159 assert(ph->offset <= offset);
1160 if(ph->offset+VM_PAGE_SIZE <= offset) {
1161 printf("VM: copy_abs2region: no phys region found (2).\n");
1162 return EFAULT;
1163 }
1164 suboffset = offset - ph->offset;
1165 assert(suboffset < VM_PAGE_SIZE);
1166 sublen = len;
1167 if(sublen > VM_PAGE_SIZE - suboffset)
1168 sublen = VM_PAGE_SIZE - suboffset;
1169 assert(suboffset + sublen <= VM_PAGE_SIZE);
1170 if(ph->ph->refcount != 1) {
1171 printf("VM: copy_abs2region: refcount not 1.\n");
1172 return EFAULT;
1173 }
1174
1175 if(sys_abscopy(abs, ph->ph->phys + suboffset, sublen) != OK) {
1176 printf("VM: copy_abs2region: abscopy failed.\n");
1177 return EFAULT;
1178 }
1179 abs += sublen;
1180 offset += sublen;
1181 len -= sublen;
1182 }
1183
1184 return OK;
1185 }
1186
1187 /*=========================================================================*
1188 * map_writept *
1189 *=========================================================================*/
1190 int map_writept(struct vmproc *vmp)
1191 {
1192 struct vir_region *vr;
1193 struct phys_region *ph;
1194 int r;
1195 region_iter v_iter;
1196 region_start_iter_least(&vmp->vm_regions_avl, &v_iter);
1197
1198 while((vr = region_get_iter(&v_iter))) {
1199 vir_bytes p;
1200 for(p = 0; p < vr->length; p += VM_PAGE_SIZE) {
1201 if(!(ph = physblock_get(vr, p))) continue;
1202
1203 if((r=map_ph_writept(vmp, vr, ph)) != OK) {
1204 printf("VM: map_writept: failed\n");
1205 return r;
1206 }
1207 }
1208 region_incr_iter(&v_iter);
1209 }
1210
1211 return OK;
1212 }
1213
1214 /*========================================================================*
1215 * map_proc_copy *
1216 *========================================================================*/
1217 int map_proc_copy(dst, src)
1218 struct vmproc *dst;
1219 struct vmproc *src;
1220 {
1221 /* Copy all the memory regions from the src process to the dst process. */
1222 region_init(&dst->vm_regions_avl);
1223
1224 return map_proc_copy_from(dst, src, NULL);
1225 }
1226
1227 /*========================================================================*
1228 * map_proc_copy_from *
1229 *========================================================================*/
1230 int map_proc_copy_from(dst, src, start_src_vr)
1231 struct vmproc *dst;
1232 struct vmproc *src;
1233 struct vir_region *start_src_vr;
1234 {
1235 struct vir_region *vr;
1236 region_iter v_iter;
1237
1238 if(!start_src_vr)
1239 start_src_vr = region_search_least(&src->vm_regions_avl);
1240
1241 assert(start_src_vr);
1242 assert(start_src_vr->parent == src);
1243 region_start_iter(&src->vm_regions_avl, &v_iter,
1244 start_src_vr->vaddr, AVL_EQUAL);
1245 assert(region_get_iter(&v_iter) == start_src_vr);
1246
1247 /* Copy source regions after the destination's last region (if any). */
1248
1249 SANITYCHECK(SCL_FUNCTIONS);
1250
1251 while((vr = region_get_iter(&v_iter))) {
1252 struct vir_region *newvr;
1253 if(!(newvr = map_copy_region(dst, vr))) {
1254 map_free_proc(dst);
1255 return ENOMEM;
1256 }
1257 USE(newvr, newvr->parent = dst;);
1258 region_insert(&dst->vm_regions_avl, newvr);
1259 assert(vr->length == newvr->length);
1260
1261 #if SANITYCHECKS
1262 {
1263 vir_bytes vaddr;
1264 struct phys_region *orig_ph, *new_ph;
1265 assert(vr->physblocks != newvr->physblocks);
1266 for(vaddr = 0; vaddr < vr->length; vaddr += VM_PAGE_SIZE) {
1267 orig_ph = physblock_get(vr, vaddr);
1268 new_ph = physblock_get(newvr, vaddr);
1269 if(!orig_ph) { assert(!new_ph); continue;}
1270 assert(new_ph);
1271 assert(orig_ph != new_ph);
1272 assert(orig_ph->ph == new_ph->ph);
1273 }
1274 }
1275 #endif
1276 region_incr_iter(&v_iter);
1277 }
1278
1279 map_writept(src);
1280 map_writept(dst);
1281
1282 SANITYCHECK(SCL_FUNCTIONS);
1283 return OK;
1284 }
1285
1286 int map_region_extend_upto_v(struct vmproc *vmp, vir_bytes v)
1287 {
1288 vir_bytes offset = v;
1289 struct vir_region *vr, *nextvr;
1290 struct phys_region **newpr;
1291 int newslots, prevslots, addedslots;
1292
1293 offset = roundup(offset, VM_PAGE_SIZE);
1294
1295 if(!(vr = region_search(&vmp->vm_regions_avl, offset, AVL_LESS))) {
1296 printf("VM: nothing to extend\n");
1297 return ENOMEM;
1298 }
1299
1300 if(vr->vaddr + vr->length >= v) return OK;
1301
1302 assert(vr->vaddr <= offset);
1303 newslots = phys_slot(offset - vr->vaddr);
1304 prevslots = phys_slot(vr->length);
1305 assert(newslots >= prevslots);
1306 addedslots = newslots - prevslots;
1307
1308 if(!(newpr = realloc(vr->physblocks,
1309 newslots * sizeof(struct phys_region *)))) {
1310 printf("VM: map_region_extend_upto_v: realloc failed\n");
1311 return ENOMEM;
1312 }
1313
1314 vr->physblocks = newpr;
1315 memset(vr->physblocks + prevslots, 0,
1316 addedslots * sizeof(struct phys_region *));
1317
1318 if((nextvr = getnextvr(vr))) {
1319 assert(offset <= nextvr->vaddr);
1320 }
1321
1322 if(nextvr && nextvr->vaddr < offset) {
1323 printf("VM: can't grow into next region\n");
1324 return ENOMEM;
1325 }
1326
1327 if(!vr->memtype->ev_resize) {
1328 printf("VM: can't resize this type of memory\n");
1329 return ENOMEM;
1330 }
1331
1332 return vr->memtype->ev_resize(vmp, vr, offset - vr->vaddr);
1333 }
1334
1335 /*========================================================================*
1336 * map_unmap_region *
1337 *========================================================================*/
1338 int map_unmap_region(struct vmproc *vmp, struct vir_region *r,
1339 vir_bytes offset, vir_bytes len)
1340 {
1341 /* Shrink the region by 'len' bytes, from the start. Unreference
1342 * memory it used to reference if any.
1343 */
1344 vir_bytes regionstart;
1345 int freeslots = phys_slot(len);
1346
1347 SANITYCHECK(SCL_FUNCTIONS);
1348
1349 if(offset+len > r->length || (len % VM_PAGE_SIZE)) {
1350 printf("VM: bogus length 0x%lx\n", len);
1351 return EINVAL;
1352 }
1353
1354 regionstart = r->vaddr + offset;
1355
1356 /* unreference its memory */
1357 map_subfree(r, offset, len);
1358
1359 /* if unmap was at start/end of this region, it actually shrinks */
1360 if(offset == 0) {
1361 struct phys_region *pr;
1362 vir_bytes voffset;
1363 int remslots;
1364
1365 region_remove(&vmp->vm_regions_avl, r->vaddr);
1366
1367 USE(r,
1368 r->vaddr += len;
1369 r->length -= len;);
1370
1371 remslots = phys_slot(r->length);
1372
1373 region_insert(&vmp->vm_regions_avl, r);
1374
1375 /* vaddr has increased; to make all the phys_regions
1376 * point to the same addresses, make them shrink by the
1377 * same amount.
1378 */
1379 for(voffset = offset; voffset < r->length;
1380 voffset += VM_PAGE_SIZE) {
1381 if(!(pr = physblock_get(r, voffset))) continue;
1382 assert(pr->offset >= offset);
1383 USE(pr, pr->offset -= len;);
1384 }
1385 if(remslots)
1386 memmove(r->physblocks, r->physblocks + freeslots,
1387 remslots * sizeof(struct phys_region *));
1388 } else if(offset + len == r->length) {
1389 assert(len <= r->length);
1390 r->length -= len;
1391 }
1392
1393 if(r->length == 0) {
1394 /* Whole region disappears. Unlink and free it. */
1395 region_remove(&vmp->vm_regions_avl, r->vaddr);
1396 map_free(r);
1397 }
1398
1399 SANITYCHECK(SCL_DETAIL);
1400
1401 if(pt_writemap(vmp, &vmp->vm_pt, regionstart,
1402 MAP_NONE, len, 0, WMF_OVERWRITE) != OK) {
1403 printf("VM: map_unmap_region: pt_writemap failed\n");
1404 return ENOMEM;
1405 }
1406
1407 SANITYCHECK(SCL_FUNCTIONS);
1408
1409 return OK;
1410 }
1411
1412 /*========================================================================*
1413 * map_get_phys *
1414 *========================================================================*/
1415 int map_get_phys(struct vmproc *vmp, vir_bytes addr, phys_bytes *r)
1416 {
1417 struct vir_region *vr;
1418
1419 if (!(vr = map_lookup(vmp, addr, NULL)) ||
1420 (vr->vaddr != addr))
1421 return EINVAL;
1422
1423 if (!vr->memtype->regionid)
1424 return EINVAL;
1425
1426 if(r)
1427 *r = vr->memtype->regionid(vr);
1428
1429 return OK;
1430 }
1431
1432 /*========================================================================*
1433 * map_get_ref *
1434 *========================================================================*/
1435 int map_get_ref(struct vmproc *vmp, vir_bytes addr, u8_t *cnt)
1436 {
1437 struct vir_region *vr;
1438
1439 if (!(vr = map_lookup(vmp, addr, NULL)) ||
1440 (vr->vaddr != addr) || !vr->memtype->refcount)
1441 return EINVAL;
1442
1443 if (cnt)
1444 *cnt = vr->memtype->refcount(vr);
1445
1446 return OK;
1447 }
1448
1449 /*========================================================================*
1450 * get_stats_info *
1451 *========================================================================*/
1452 void get_stats_info(struct vm_stats_info *vsi)
1453 {
1454 yielded_t *yb;
1455
1456 vsi->vsi_cached = 0L;
1457
1458 for(yb = lru_youngest; yb; yb = yb->older)
1459 vsi->vsi_cached++;
1460 }
1461
1462 void get_usage_info_kernel(struct vm_usage_info *vui)
1463 {
1464 memset(vui, 0, sizeof(*vui));
1465 vui->vui_total = kernel_boot_info.kernel_allocated_bytes;
1466 }
1467
1468 static void get_usage_info_vm(struct vm_usage_info *vui)
1469 {
1470 memset(vui, 0, sizeof(*vui));
1471 vui->vui_total = kernel_boot_info.vm_allocated_bytes +
1472 get_vm_self_pages() * VM_PAGE_SIZE;
1473 }
1474
1475 /*========================================================================*
1476 * get_usage_info *
1477 *========================================================================*/
1478 void get_usage_info(struct vmproc *vmp, struct vm_usage_info *vui)
1479 {
1480 struct vir_region *vr;
1481 struct phys_region *ph;
1482 region_iter v_iter;
1483 region_start_iter_least(&vmp->vm_regions_avl, &v_iter);
1484 vir_bytes voffset;
1485
1486 memset(vui, 0, sizeof(*vui));
1487
1488 if(vmp->vm_endpoint == VM_PROC_NR) {
1489 get_usage_info_vm(vui);
1490 return;
1491 }
1492
1493 if(vmp->vm_endpoint < 0) {
1494 get_usage_info_kernel(vui);
1495 return;
1496 }
1497
1498 while((vr = region_get_iter(&v_iter))) {
1499 for(voffset = 0; voffset < vr->length; voffset += VM_PAGE_SIZE) {
1500 if(!(ph = physblock_get(vr, voffset))) continue;
1501 /* All present pages are counted towards the total. */
1502 vui->vui_total += VM_PAGE_SIZE;
1503
1504 if (ph->ph->refcount > 1) {
1505 /* Any page with a refcount > 1 is common. */
1506 vui->vui_common += VM_PAGE_SIZE;
1507
1508 /* Any common, non-COW page is shared. */
1509 if (vr->flags & VR_SHARED)
1510 vui->vui_shared += VM_PAGE_SIZE;
1511 }
1512 }
1513 region_incr_iter(&v_iter);
1514 }
1515 }
1516
1517 /*===========================================================================*
1518 * get_region_info *
1519 *===========================================================================*/
1520 int get_region_info(struct vmproc *vmp, struct vm_region_info *vri,
1521 int max, vir_bytes *nextp)
1522 {
1523 struct vir_region *vr;
1524 vir_bytes next;
1525 int count;
1526 region_iter v_iter;
1527
1528 next = *nextp;
1529
1530 if (!max) return 0;
1531
1532 region_start_iter(&vmp->vm_regions_avl, &v_iter, next, AVL_GREATER_EQUAL);
1533 if(!(vr = region_get_iter(&v_iter))) return 0;
1534
1535 for(count = 0; (vr = region_get_iter(&v_iter)) && count < max; count++, vri++) {
1536 struct phys_region *ph1 = NULL, *ph2 = NULL;
1537 vir_bytes voffset;
1538
1539 /* Report part of the region that's actually in use. */
1540
1541 /* Get first and last phys_regions, if any */
1542 for(voffset = 0; voffset > vr->length; voffset += VM_PAGE_SIZE) {
1543 struct phys_region *ph;
1544 if(!(ph = physblock_get(vr, voffset))) continue;
1545 if(!ph1) ph1 = ph;
1546 ph2 = ph;
1547 }
1548 if(!ph1 || !ph2) { assert(!ph1 && !ph2); continue; }
1549
1550 /* Report start+length of region starting from lowest use. */
1551 vri->vri_addr = vr->vaddr + ph1->offset;
1552 vri->vri_prot = 0;
1553 vri->vri_length = ph2->offset + VM_PAGE_SIZE - ph1->offset;
1554
1555 /* "AND" the provided protection with per-page protection. */
1556 if (!(vr->flags & VR_WRITABLE))
1557 vri->vri_prot &= ~PROT_WRITE;
1558
1559 next = vr->vaddr + vr->length;
1560 region_incr_iter(&v_iter);
1561 }
1562
1563 *nextp = next;
1564 return count;
1565 }
1566
1567 /*========================================================================*
1568 * regionprintstats *
1569 *========================================================================*/
1570 void printregionstats(struct vmproc *vmp)
1571 {
1572 struct vir_region *vr;
1573 struct phys_region *pr;
1574 vir_bytes used = 0, weighted = 0;
1575 region_iter v_iter;
1576 region_start_iter_least(&vmp->vm_regions_avl, &v_iter);
1577
1578 while((vr = region_get_iter(&v_iter))) {
1579 vir_bytes voffset;
1580 region_incr_iter(&v_iter);
1581 if(vr->flags & VR_DIRECT)
1582 continue;
1583 for(voffset = 0; voffset < vr->length; voffset+=VM_PAGE_SIZE) {
1584 if(!(pr = physblock_get(vr, voffset))) continue;
1585 used += VM_PAGE_SIZE;
1586 weighted += VM_PAGE_SIZE / pr->ph->refcount;
1587 }
1588 }
1589
1590 printf("%6lukB %6lukB\n", used/1024, weighted/1024);
1591
1592 return;
1593 }
1594
1595 /*===========================================================================*
1596 * get_clean_phys_region *
1597 *===========================================================================*/
1598 static struct phys_region *
1599 get_clean_phys_region(struct vmproc *vmp, vir_bytes vaddr, struct vir_region **ret_region)
1600 {
1601 struct vir_region *region;
1602 vir_bytes mapaddr;
1603 struct phys_region *ph;
1604
1605 mapaddr = vaddr;
1606
1607 if(!(region = map_lookup(vmp, mapaddr, &ph)) || !ph) {
1608 printf("VM: get_clean_phys_region: 0x%lx not found\n", vaddr);
1609 return NULL;
1610 }
1611
1612 assert(mapaddr >= region->vaddr);
1613 assert(mapaddr < region->vaddr + region->length);
1614
1615 /* If it's mapped more than once, make a copy. */
1616 assert(ph->ph->refcount > 0);
1617 if(ph->ph->refcount > 1) {
1618 if(!(ph = map_clone_ph_block(vmp, region,
1619 ph))) {
1620 printf("VM: get_clean_phys_region: ph copy failed\n");
1621 return NULL;
1622 }
1623 }
1624
1625 assert(ph->ph->refcount == 1);
1626
1627 *ret_region = region;
1628
1629 return ph;
1630 }
1631
1632 static int getblock(struct vmproc *vmp, u64_t id, vir_bytes vaddr, int pages)
1633 {
1634 yielded_t *yb;
1635 struct phys_region *ph;
1636 struct vir_region *region;
1637 yielded_avl *avl;
1638 block_id_t blockid;
1639 phys_bytes phaddr;
1640 int p;
1641
1642 /* Try to get the yielded block */
1643 blockid.owner = vmp->vm_endpoint;
1644 blockid.id = id;
1645 avl = get_yielded_avl(blockid);
1646 if(!(yb = yielded_search(avl, blockid, AVL_EQUAL))) {
1647 return ESRCH;
1648 }
1649
1650 if(yb->pages != pages) {
1651 printf("VM: getblock: length mismatch (%d != %d)\n",
1652 pages, yb->pages);
1653 return EFAULT;
1654 }
1655
1656 phaddr = yb->physaddr;
1657
1658 for(p = 0; p < pages; p++) {
1659 /* Get the intended phys region, make sure refcount is 1. */
1660 if(!(ph = get_clean_phys_region(vmp, vaddr, &region))) {
1661 printf("VM: getblock: not found for %d\n", vmp->vm_endpoint);
1662 return EINVAL;
1663 }
1664
1665 assert(ph->ph->refcount == 1);
1666
1667 /* Free the block that is currently there. */
1668 free_mem(ABS2CLICK(ph->ph->phys), 1);
1669
1670 /* Set the phys block to new addr and update pagetable. */
1671 USE(ph->ph, ph->ph->phys = phaddr;);
1672 if(map_ph_writept(vmp, region, ph) != OK) {
1673 /* Presumably it was mapped, so there is no reason
1674 * updating should fail.
1675 */
1676 panic("do_get_block: couldn't write pt");
1677 }
1678
1679 vaddr += VM_PAGE_SIZE;
1680 phaddr += VM_PAGE_SIZE;
1681 }
1682
1683 /* Forget about the yielded block and free the struct. */
1684 freeyieldednode(yb, 0);
1685
1686 return OK;
1687 }
1688
1689 static int yieldblock(struct vmproc *vmp, u64_t id,
1690 vir_bytes vaddr, yielded_t **retyb, int pages)
1691 {
1692 yielded_t *newyb;
1693 vir_bytes mem_clicks, v, p, new_phaddr;
1694 struct vir_region *region;
1695 struct phys_region *ph = NULL, *prev_ph = NULL, *first_ph = NULL;
1696 yielded_avl *avl;
1697 block_id_t blockid;
1698
1699 /* Makes no sense if yielded block ID already exists, and
1700 * is likely a serious bug in the caller.
1701 */
1702 blockid.id = id;
1703 blockid.owner = vmp->vm_endpoint;
1704 avl = get_yielded_avl(blockid);
1705 if(yielded_search(avl, blockid, AVL_EQUAL)) {
1706 printf("!");
1707 return EINVAL;
1708 }
1709
1710 if((vaddr % VM_PAGE_SIZE) || pages < 1) return EFAULT;
1711
1712 v = vaddr;
1713 for(p = 0; p < pages; p++) {
1714 if(!(region = map_lookup(vmp, v, &ph)) || !ph) {
1715 printf("VM: do_yield_block: not found for %d\n",
1716 vmp->vm_endpoint);
1717 return EINVAL;
1718 }
1719 if(!(region->flags & VR_ANON)) {
1720 printf("VM: yieldblock: non-anon 0x%lx\n", v);
1721 return EFAULT;
1722 }
1723 if(ph->ph->refcount != 1) {
1724 printf("VM: do_yield_block: mapped not once for %d\n",
1725 vmp->vm_endpoint);
1726 return EFAULT;
1727 }
1728 if(prev_ph) {
1729 if(ph->ph->phys != prev_ph->ph->phys + VM_PAGE_SIZE) {
1730 printf("VM: physically discontiguous yield\n");
1731 return EINVAL;
1732 }
1733 }
1734 prev_ph = ph;
1735 if(!first_ph) first_ph = ph;
1736 v += VM_PAGE_SIZE;
1737 }
1738
1739 /* Make a new block to record the yielding in. */
1740 if(!SLABALLOC(newyb)) {
1741 return ENOMEM;
1742 }
1743
1744 assert(!(ph->ph->phys % VM_PAGE_SIZE));
1745
1746 if((mem_clicks = alloc_mem(pages, PAF_CLEAR)) == NO_MEM) {
1747 SLABFREE(newyb);
1748 return ENOMEM;
1749 }
1750
1751 /* Update yielded block info. */
1752 USE(newyb,
1753 newyb->id = blockid;
1754 newyb->physaddr = first_ph->ph->phys;
1755 newyb->pages = pages;
1756 newyb->younger = NULL;);
1757
1758 new_phaddr = CLICK2ABS(mem_clicks);
1759
1760 /* Set new phys block to new addr and update pagetable. */
1761 v = vaddr;
1762 for(p = 0; p < pages; p++) {
1763 region = map_lookup(vmp, v, &ph);
1764 assert(region && ph);
1765 assert(ph->ph->refcount == 1);
1766 USE(ph->ph,
1767 ph->ph->phys = new_phaddr;);
1768 if(map_ph_writept(vmp, region, ph) != OK) {
1769 /* Presumably it was mapped, so there is no reason
1770 * updating should fail.
1771 */
1772 panic("yield_block: couldn't write pt");
1773 }
1774 v += VM_PAGE_SIZE;
1775 new_phaddr += VM_PAGE_SIZE;
1776 }
1777
1778 /* Remember yielded block. */
1779
1780 yielded_insert(avl, newyb);
1781 vmp->vm_yielded++;
1782
1783 /* Add to LRU list too. It's the youngest block. */
1784 LRUCHECK;
1785
1786 if(lru_youngest) {
1787 USE(lru_youngest,
1788 lru_youngest->younger = newyb;);
1789 } else {
1790 lru_oldest = newyb;
1791 }
1792
1793 USE(newyb,
1794 newyb->older = lru_youngest;);
1795
1796 lru_youngest = newyb;
1797
1798 LRUCHECK;
1799
1800 if(retyb)
1801 *retyb = newyb;
1802
1803 return OK;
1804 }
1805
1806 /*===========================================================================*
1807 * do_forgetblocks *
1808 *===========================================================================*/
1809 int do_forgetblocks(message *m)
1810 {
1811 int n;
1812 struct vmproc *vmp;
1813 endpoint_t caller = m->m_source;
1814
1815 if(vm_isokendpt(caller, &n) != OK)
1816 panic("do_yield_block: message from strange source: %d",
1817 m->m_source);
1818
1819 vmp = &vmproc[n];
1820
1821 free_yielded_proc(vmp);
1822
1823 return OK;
1824 }
1825
1826 /*===========================================================================*
1827 * do_forgetblock *
1828 *===========================================================================*/
1829 int do_forgetblock(message *m)
1830 {
1831 int n;
1832 struct vmproc *vmp;
1833 endpoint_t caller = m->m_source;
1834 yielded_t *yb;
1835 u64_t id;
1836 block_id_t blockid;
1837 yielded_avl *avl;
1838
1839 if(vm_isokendpt(caller, &n) != OK)
1840 panic("do_yield_block: message from strange source: %d",
1841 m->m_source);
1842
1843 vmp = &vmproc[n];
1844
1845 id = make64(m->VMFB_IDLO, m->VMFB_IDHI);
1846
1847 blockid.id = id;
1848 blockid.owner = vmp->vm_endpoint;
1849 avl = get_yielded_avl(blockid);
1850 if((yb = yielded_search(avl, blockid, AVL_EQUAL))) {
1851 freeyieldednode(yb, 1);
1852 }
1853
1854 return OK;
1855 }
1856
1857 /*===========================================================================*
1858 * do_yieldblockgetblock *
1859 *===========================================================================*/
1860 int do_yieldblockgetblock(message *m)
1861 {
1862 u64_t yieldid, getid;
1863 int n;
1864 endpoint_t caller = m->m_source;
1865 struct vmproc *vmp;
1866 yielded_t *yb = NULL;
1867 int r = ESRCH;
1868 int pages;
1869
1870 if(vm_isokendpt(caller, &n) != OK)
1871 panic("do_yieldblockgetblock: message from strange source: %d",
1872 m->m_source);
1873
1874 vmp = &vmproc[n];
1875
1876 pages = m->VMYBGB_LEN / VM_PAGE_SIZE;
1877
1878 if((m->VMYBGB_LEN % VM_PAGE_SIZE) || pages < 1) {
1879 static int printed;
1880 if(!printed) {
1881 printed = 1;
1882 printf("vm: non-page-aligned or short block length\n");
1883 }
1884 return EFAULT;
1885 }
1886
1887 yieldid = make64(m->VMYBGB_YIELDIDLO, m->VMYBGB_YIELDIDHI);
1888 getid = make64(m->VMYBGB_GETIDLO, m->VMYBGB_GETIDHI);
1889
1890 if(cmp64(yieldid, VM_BLOCKID_NONE) != 0) {
1891 /* A block was given to yield. */
1892 yieldblock(vmp, yieldid, (vir_bytes) m->VMYBGB_VADDR, &yb,
1893 pages);
1894 }
1895
1896 if(cmp64(getid, VM_BLOCKID_NONE) != 0) {
1897 /* A block was given to get. */
1898 r = getblock(vmp, getid, (vir_bytes) m->VMYBGB_VADDR, pages);
1899 }
1900
1901 return r;
1902 }
1903
1904 void map_setparent(struct vmproc *vmp)
1905 {
1906 region_iter iter;
1907 struct vir_region *vr;
1908 region_start_iter_least(&vmp->vm_regions_avl, &iter);
1909 while((vr = region_get_iter(&iter))) {
1910 USE(vr, vr->parent = vmp;);
1911 region_incr_iter(&iter);
1912 }
1913 }
1914
1915 int physregions(struct vir_region *vr)
1916 {
1917 int n = 0;
1918 vir_bytes voffset;
1919 for(voffset = 0; voffset < vr->length; voffset += VM_PAGE_SIZE) {
1920 if(physblock_get(vr, voffset))
1921 n++;
1922 }
1923 return n;
1924 }
MINIX 3 (repo.or.cz mirror)