?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
vm: use assert() instead of vm_assert(); remove vm_assert().
[minix.git] / servers / vm / region.c
blobb3f2b35650bbd1a8b64ca0847b07375561b92002
1
2 #define _SYSTEM 1
3
4 #include <minix/com.h>
5 #include <minix/callnr.h>
6 #include <minix/type.h>
7 #include <minix/config.h>
8 #include <minix/const.h>
9 #include <minix/sysutil.h>
10 #include <minix/syslib.h>
11 #include <minix/debug.h>
12 #include <minix/bitmap.h>
13
14 #include <sys/mman.h>
15
16 #include <limits.h>
17 #include <string.h>
18 #include <errno.h>
19 #include <assert.h>
20 #include <stdint.h>
21 #include <memory.h>
22
23 #include "vm.h"
24 #include "proto.h"
25 #include "util.h"
26 #include "glo.h"
27 #include "region.h"
28 #include "sanitycheck.h"
29 #include "physravl.h"
30 #include "memlist.h"
31
32 /* Should a physblock be mapped writable? */
33 #define WRITABLE(r, pb) \
34 (((r)->flags & (VR_DIRECT | VR_SHARED)) || \
35 (((r)->flags & VR_WRITABLE) && (pb)->refcount == 1))
36
37 FORWARD _PROTOTYPE(int map_new_physblock, (struct vmproc *vmp,
38 struct vir_region *region, vir_bytes offset, vir_bytes length,
39 phys_bytes what, u32_t allocflags, int written));
40
41 FORWARD _PROTOTYPE(int map_ph_writept, (struct vmproc *vmp, struct vir_region *vr,
42 struct phys_region *pr));
43
44 FORWARD _PROTOTYPE(struct vir_region *map_copy_region, (struct vmproc *vmp, struct vir_region *vr));
45
46 FORWARD _PROTOTYPE(struct phys_region *map_clone_ph_block, (struct vmproc *vmp,
47 struct vir_region *region, struct phys_region *ph, physr_iter *iter));
48
49 PRIVATE char *map_name(struct vir_region *vr)
50 {
51 static char name[100];
52 char *typename, *tag;
53 int type = vr->flags & (VR_ANON|VR_DIRECT);
54 switch(type) {
55 case VR_ANON:
56 typename = "anonymous";
57 break;
58 case VR_DIRECT:
59 typename = "direct";
60 break;
61 default:
62 panic("unknown mapping type: %d", type);
63 }
64
65 switch(vr->tag) {
66 case VRT_TEXT:
67 tag = "text";
68 break;
69 case VRT_STACK:
70 tag = "stack";
71 break;
72 case VRT_HEAP:
73 tag = "heap";
74 break;
75 case VRT_NONE:
76 tag = "untagged";
77 break;
78 default:
79 tag = "unknown tag value";
80 break;
81 }
82
83 sprintf(name, "%s, %s", typename, tag);
84
85 return name;
86 }
87
88 PUBLIC void map_printregion(struct vmproc *vmp, struct vir_region *vr)
89 {
90 physr_iter iter;
91 struct phys_region *ph;
92 printf("map_printmap: map_name: %s\n", map_name(vr));
93 printf("\t%s (len 0x%lx, %dkB), %s\n",
94 arch_map2str(vmp, vr->vaddr), vr->length,
95 vr->length/1024, map_name(vr));
96 printf("\t\tphysblocks:\n");
97 physr_start_iter_least(vr->phys, &iter);
98 while((ph = physr_get_iter(&iter))) {
99 printf("\t\t@ %s (refs %d): phys 0x%lx len 0x%lx\n",
100 arch_map2str(vmp, vr->vaddr + ph->offset),
101 ph->ph->refcount, ph->ph->phys, ph->ph->length);
102 physr_incr_iter(&iter);
103 }
104 }
105
106 /*===========================================================================*
107 * map_printmap *
108 *===========================================================================*/
109 PUBLIC void map_printmap(vmp)
110 struct vmproc *vmp;
111 {
112 struct vir_region *vr;
113
114 printf("memory regions in process %d:\n", vmp->vm_endpoint);
115 for(vr = vmp->vm_regions; vr; vr = vr->next) {
116 map_printregion(vmp, vr);
117 }
118 }
119
120
121 #if SANITYCHECKS
122
123
124 /*===========================================================================*
125 * map_sanitycheck_pt *
126 *===========================================================================*/
127 PRIVATE int map_sanitycheck_pt(struct vmproc *vmp,
128 struct vir_region *vr, struct phys_region *pr)
129 {
130 struct phys_block *pb = pr->ph;
131 int rw;
132 int r;
133
134 if(!(vmp->vm_flags & VMF_HASPT))
135 return OK;
136
137 if(WRITABLE(vr, pb))
138 rw = PTF_WRITE;
139 else
140 rw = 0;
141
142 r = pt_writemap(&vmp->vm_pt, vr->vaddr + pr->offset,
143 pb->phys, pb->length, PTF_PRESENT | PTF_USER | rw, WMF_VERIFY);
144
145 if(r != OK) {
146 printf("proc %d phys_region 0x%lx sanity check failed\n",
147 vmp->vm_endpoint, pr->offset);
148 map_printregion(vmp, vr);
149 }
150
151 return r;
152 }
153
154 /*===========================================================================*
155 * map_sanitycheck *
156 *===========================================================================*/
157 PUBLIC void map_sanitycheck(char *file, int line)
158 {
159 struct vmproc *vmp;
160
161 /* Macro for looping over all physical blocks of all regions of
162 * all processes.
163 */
164 #define ALLREGIONS(regioncode, physcode) \
165 for(vmp = vmproc; vmp < &vmproc[VMP_NR]; vmp++) { \
166 struct vir_region *vr; \
167 if(!(vmp->vm_flags & VMF_INUSE)) \
168 continue; \
169 for(vr = vmp->vm_regions; vr; vr = vr->next) { \
170 physr_iter iter; \
171 struct phys_region *pr; \
172 regioncode; \
173 physr_start_iter_least(vr->phys, &iter); \
174 while((pr = physr_get_iter(&iter))) { \
175 physcode; \
176 physr_incr_iter(&iter); \
177 } \
178 } \
179 }
180
181 #define MYSLABSANE(s) MYASSERT(slabsane_f(__FILE__, __LINE__, s, sizeof(*(s))))
182 /* Basic pointers check. */
183 ALLREGIONS(MYSLABSANE(vr),MYSLABSANE(pr); MYSLABSANE(pr->ph);MYSLABSANE(pr->parent));
184 ALLREGIONS(/* MYASSERT(vr->parent == vmp) */,MYASSERT(pr->parent == vr););
185
186 /* Do counting for consistency check. */
187 ALLREGIONS(;,USE(pr->ph, pr->ph->seencount = 0;););
188 ALLREGIONS(;,USE(pr->ph, pr->ph->seencount++;);
189 if(pr->ph->seencount == 1) {
190 MYASSERT(usedpages_add(pr->ph->phys,
191 pr->ph->length) == OK);
192 }
193 );
194
195 /* Do consistency check. */
196 ALLREGIONS(if(vr->next) {
197 MYASSERT(vr->vaddr < vr->next->vaddr);
198 MYASSERT(vr->vaddr + vr->length <= vr->next->vaddr);
199 }
200 MYASSERT(!(vr->vaddr % VM_PAGE_SIZE));,
201 if(pr->ph->refcount != pr->ph->seencount) {
202 map_printmap(vmp);
203 printf("ph in vr 0x%lx: 0x%lx-0x%lx refcount %d "
204 "but seencount %lu\n",
205 vr, pr->offset,
206 pr->offset + pr->ph->length,
207 pr->ph->refcount, pr->ph->seencount);
208 }
209 {
210 int n_others = 0;
211 struct phys_region *others;
212 if(pr->ph->refcount > 0) {
213 MYASSERT(pr->ph->firstregion);
214 if(pr->ph->refcount == 1) {
215 MYASSERT(pr->ph->firstregion == pr);
216 }
217 } else {
218 MYASSERT(!pr->ph->firstregion);
219 }
220 for(others = pr->ph->firstregion; others;
221 others = others->next_ph_list) {
222 MYSLABSANE(others);
223 MYASSERT(others->ph == pr->ph);
224 n_others++;
225 }
226 MYASSERT(pr->ph->refcount == n_others);
227 }
228 MYASSERT(pr->ph->refcount == pr->ph->seencount);
229 MYASSERT(!(pr->offset % VM_PAGE_SIZE));
230 MYASSERT(!(pr->ph->length % VM_PAGE_SIZE)););
231 ALLREGIONS(,MYASSERT(map_sanitycheck_pt(vmp, vr, pr) == OK));
232 }
233 #endif
234
235
236 /*=========================================================================*
237 * map_ph_writept *
238 *=========================================================================*/
239 PRIVATE int map_ph_writept(struct vmproc *vmp, struct vir_region *vr,
240 struct phys_region *pr)
241 {
242 int rw;
243 struct phys_block *pb = pr->ph;
244
245 assert(!(vr->vaddr % VM_PAGE_SIZE));
246 assert(!(pb->length % VM_PAGE_SIZE));
247 assert(!(pr->offset % VM_PAGE_SIZE));
248 assert(pb->refcount > 0);
249
250 if(WRITABLE(vr, pb))
251 rw = PTF_WRITE;
252 else
253 rw = 0;
254
255 if(pt_writemap(&vmp->vm_pt, vr->vaddr + pr->offset,
256 pb->phys, pb->length, PTF_PRESENT | PTF_USER | rw,
257 #if SANITYCHECKS
258 !pr->written ? 0 :
259 #endif
260 WMF_OVERWRITE) != OK) {
261 printf("VM: map_writept: pt_writemap failed\n");
262 return ENOMEM;
263 }
264
265 #if SANITYCHECKS
266 USE(pr, pr->written = 1;);
267 #endif
268
269 return OK;
270 }
271
272 /*===========================================================================*
273 * region_find_slot *
274 *===========================================================================*/
275 PRIVATE vir_bytes region_find_slot(struct vmproc *vmp,
276 vir_bytes minv, vir_bytes maxv, vir_bytes length,
277 struct vir_region **prev)
278 {
279 struct vir_region *firstregion = vmp->vm_regions, *prevregion = NULL;
280 vir_bytes startv;
281 int foundflag = 0;
282
283 SANITYCHECK(SCL_FUNCTIONS);
284
285 /* We must be in paged mode to be able to do this. */
286 assert(vm_paged);
287
288 /* Length must be reasonable. */
289 assert(length > 0);
290
291 /* Special case: allow caller to set maxv to 0 meaning 'I want
292 * it to be mapped in right here.'
293 */
294 if(maxv == 0) {
295 maxv = minv + length;
296
297 /* Sanity check. */
298 if(maxv <= minv) {
299 printf("region_find_slot: minv 0x%lx and bytes 0x%lx\n",
300 minv, length);
301 map_printmap(vmp);
302 return (vir_bytes) -1;
303 }
304 }
305
306 /* Basic input sanity checks. */
307 assert(!(length % VM_PAGE_SIZE));
308 if(minv >= maxv) {
309 printf("VM: 1 minv: 0x%lx maxv: 0x%lx length: 0x%lx\n",
310 minv, maxv, length);
311 }
312 assert(minv < maxv);
313 assert(minv + length <= maxv);
314
315 #define FREEVRANGE(rangestart, rangeend, foundcode) { \
316 vir_bytes frstart = (rangestart), frend = (rangeend); \
317 frstart = MAX(frstart, minv); \
318 frend = MIN(frend, maxv); \
319 if(frend > frstart && (frend - frstart) >= length) { \
320 startv = frstart; \
321 foundflag = 1; \
322 foundcode; \
323 } }
324
325 /* This is the free virtual address space before the first region. */
326 FREEVRANGE(0, firstregion ? firstregion->vaddr : VM_DATATOP, ;);
327
328 if(!foundflag) {
329 struct vir_region *vr;
330 for(vr = vmp->vm_regions; vr && !foundflag; vr = vr->next) {
331 FREEVRANGE(vr->vaddr + vr->length,
332 vr->next ? vr->next->vaddr : VM_DATATOP,
333 prevregion = vr;);
334 }
335 }
336
337 if(!foundflag) {
338 printf("VM: region_find_slot: no 0x%lx bytes found for %d between 0x%lx and 0x%lx\n",
339 length, vmp->vm_endpoint, minv, maxv);
340 map_printmap(vmp);
341 return (vir_bytes) -1;
342 }
343
344 #if SANITYCHECKS
345 if(prevregion) assert(prevregion->vaddr < startv);
346 #endif
347
348 /* However we got it, startv must be in the requested range. */
349 assert(startv >= minv);
350 assert(startv < maxv);
351 assert(startv + length <= maxv);
352
353 if (prev)
354 *prev = prevregion;
355 return startv;
356 }
357
358 /*===========================================================================*
359 * map_page_region *
360 *===========================================================================*/
361 PUBLIC struct vir_region *map_page_region(vmp, minv, maxv, length,
362 what, flags, mapflags)
363 struct vmproc *vmp;
364 vir_bytes minv;
365 vir_bytes maxv;
366 vir_bytes length;
367 vir_bytes what;
368 u32_t flags;
369 int mapflags;
370 {
371 struct vir_region *prevregion = NULL, *newregion;
372 vir_bytes startv;
373 struct phys_region *ph;
374 physr_avl *phavl;
375
376 assert(!(length % VM_PAGE_SIZE));
377
378 SANITYCHECK(SCL_FUNCTIONS);
379
380 startv = region_find_slot(vmp, minv, maxv, length, &prevregion);
381 if (startv == (vir_bytes) -1)
382 return NULL;
383
384 /* Now we want a new region. */
385 if(!SLABALLOC(newregion)) {
386 printf("VM: map_page_region: allocating region failed\n");
387 return NULL;
388 }
389
390 /* Fill in node details. */
391 USE(newregion,
392 newregion->vaddr = startv;
393 newregion->length = length;
394 newregion->flags = flags;
395 newregion->tag = VRT_NONE;
396 newregion->parent = vmp;);
397
398 SLABALLOC(phavl);
399 if(!phavl) {
400 printf("VM: map_page_region: allocating phys avl failed\n");
401 SLABFREE(newregion);
402 return NULL;
403 }
404 USE(newregion, newregion->phys = phavl;);
405
406 physr_init(newregion->phys);
407
408 /* If we know what we're going to map to, map it right away. */
409 if(what != MAP_NONE) {
410 assert(what); /* mapping in 0 is unlikely to be right */
411 assert(!(what % VM_PAGE_SIZE));
412 assert(!(startv % VM_PAGE_SIZE));
413 assert(!(mapflags & MF_PREALLOC));
414 if(map_new_physblock(vmp, newregion, 0, length,
415 what, PAF_CLEAR, 0) != OK) {
416 printf("VM: map_new_physblock failed\n");
417 USE(newregion,
418 SLABFREE(newregion->phys););
419 SLABFREE(newregion);
420 return NULL;
421 }
422 }
423
424 if((flags & VR_ANON) && (mapflags & MF_PREALLOC)) {
425 if(map_handle_memory(vmp, newregion, 0, length, 1) != OK) {
426 printf("VM: map_page_region: prealloc failed\n");
427 USE(newregion,
428 SLABFREE(newregion->phys););
429 SLABFREE(newregion);
430 return NULL;
431 }
432 }
433
434 /* Link it. */
435 if(prevregion) {
436 assert(prevregion->vaddr < newregion->vaddr);
437 USE(newregion, newregion->next = prevregion->next;);
438 USE(prevregion, prevregion->next = newregion;);
439 } else {
440 USE(newregion, newregion->next = vmp->vm_regions;);
441 vmp->vm_regions = newregion;
442 }
443
444 #if SANITYCHECKS
445 assert(startv == newregion->vaddr);
446 if(newregion->next) {
447 assert(newregion->vaddr < newregion->next->vaddr);
448 }
449 #endif
450
451 SANITYCHECK(SCL_FUNCTIONS);
452
453 return newregion;
454 }
455
456 /*===========================================================================*
457 * pb_unreferenced *
458 *===========================================================================*/
459 PUBLIC void pb_unreferenced(struct vir_region *region, struct phys_region *pr)
460 {
461 struct phys_block *pb;
462 int remap = 0;
463
464 pb = pr->ph;
465 assert(pb->refcount > 0);
466 USE(pb, pb->refcount--;);
467 assert(pb->refcount >= 0);
468
469 if(pb->firstregion == pr) {
470 USE(pb, pb->firstregion = pr->next_ph_list;);
471 } else {
472 struct phys_region *others;
473
474 for(others = pb->firstregion; others;
475 others = others->next_ph_list) {
476 assert(others->ph == pb);
477 if(others->next_ph_list == pr) {
478 USE(others, others->next_ph_list = pr->next_ph_list;);
479 break;
480 }
481 }
482
483 assert(others); /* Otherwise, wasn't on the list. */
484 }
485
486 if(pb->refcount == 0) {
487 assert(!pb->firstregion);
488 if(region->flags & VR_ANON) {
489 free_mem(ABS2CLICK(pb->phys),
490 ABS2CLICK(pb->length));
491 } else if(region->flags & VR_DIRECT) {
492 ; /* No action required. */
493 } else {
494 panic("strange phys flags");
495 }
496 SLABFREE(pb);
497 } else {
498 struct phys_region *others;
499 int n = 0;
500
501 for(others = pb->firstregion; others;
502 others = others->next_ph_list) {
503 if(WRITABLE(region, others->ph)) {
504 if(map_ph_writept(others->parent->parent,
505 others->parent, others) != OK) {
506 printf("VM: map_ph_writept failed unexpectedly\n");
507 }
508 }
509 n++;
510 }
511 assert(n == pb->refcount);
512 }
513 }
514
515 PRIVATE struct phys_region *reset_physr_iter(struct vir_region *region,
516 physr_iter *iter, vir_bytes offset)
517 {
518 struct phys_region *ph;
519
520 physr_start_iter(region->phys, iter, offset, AVL_EQUAL);
521 ph = physr_get_iter(iter);
522 assert(ph);
523 assert(ph->offset == offset);
524
525 return ph;
526 }
527
528 /*===========================================================================*
529 * map_subfree *
530 *===========================================================================*/
531 PRIVATE int map_subfree(struct vmproc *vmp,
532 struct vir_region *region, vir_bytes len)
533 {
534 struct phys_region *pr, *nextpr;
535 physr_iter iter;
536
537
538 #if SANITYCHECKS
539 {
540 physr_start_iter_least(region->phys, &iter);
541 while((pr = physr_get_iter(&iter))) {
542 struct phys_region *others;
543 struct phys_block *pb;
544
545 pb = pr->ph;
546
547 for(others = pb->firstregion; others;
548 others = others->next_ph_list) {
549 assert(others->ph == pb);
550 }
551 physr_incr_iter(&iter);
552 }
553 }
554 #endif
555
556 physr_start_iter_least(region->phys, &iter);
557 while((pr = physr_get_iter(&iter))) {
558 physr_incr_iter(&iter);
559 if(pr->offset >= len)
560 break;
561 if(pr->offset + pr->ph->length <= len) {
562 pb_unreferenced(region, pr);
563 physr_remove(region->phys, pr->offset);
564 physr_start_iter_least(region->phys, &iter);
565 SLABFREE(pr);
566 } else {
567 vir_bytes sublen;
568 assert(len > pr->offset);
569 assert(len < pr->offset + pr->ph->length);
570 assert(pr->ph->refcount > 0);
571 sublen = len - pr->offset;
572 assert(!(sublen % VM_PAGE_SIZE));
573 assert(sublen < pr->ph->length);
574 if(pr->ph->refcount > 1) {
575 int r;
576 if(!(pr = map_clone_ph_block(vmp, region,
577 pr, &iter)))
578 return ENOMEM;
579 }
580 assert(pr->ph->refcount == 1);
581 if(!(region->flags & VR_DIRECT)) {
582 free_mem(ABS2CLICK(pr->ph->phys), ABS2CLICK(sublen));
583 }
584 USE(pr, pr->offset += sublen;);
585 USE(pr->ph,
586 pr->ph->phys += sublen;
587 pr->ph->length -= sublen;);
588 assert(!(pr->offset % VM_PAGE_SIZE));
589 assert(!(pr->ph->phys % VM_PAGE_SIZE));
590 assert(!(pr->ph->length % VM_PAGE_SIZE));
591 }
592 }
593
594 return OK;
595 }
596
597 /*===========================================================================*
598 * map_free *
599 *===========================================================================*/
600 PRIVATE int map_free(struct vmproc *vmp, struct vir_region *region)
601 {
602 int r;
603
604 if((r=map_subfree(vmp, region, region->length)) != OK) {
605 printf("%d\n", __LINE__);
606 return r;
607 }
608
609 USE(region,
610 SLABFREE(region->phys););
611 SLABFREE(region);
612
613 return OK;
614 }
615
616 /*========================================================================*
617 * map_free_proc *
618 *========================================================================*/
619 PUBLIC int map_free_proc(vmp)
620 struct vmproc *vmp;
621 {
622 struct vir_region *r, *nextr;
623
624 SANITYCHECK(SCL_FUNCTIONS);
625
626 for(r = vmp->vm_regions; r; r = nextr) {
627 nextr = r->next;
628 SANITYCHECK(SCL_DETAIL);
629 #if SANITYCHECKS
630 nocheck++;
631 #endif
632 map_free(vmp, r);
633 vmp->vm_regions = nextr; /* For sanity checks. */
634 #if SANITYCHECKS
635 nocheck--;
636 #endif
637 SANITYCHECK(SCL_DETAIL);
638 }
639
640 vmp->vm_regions = NULL;
641
642 SANITYCHECK(SCL_FUNCTIONS);
643
644 return OK;
645 }
646
647 /*===========================================================================*
648 * map_lookup *
649 *===========================================================================*/
650 PUBLIC struct vir_region *map_lookup(vmp, offset)
651 struct vmproc *vmp;
652 vir_bytes offset;
653 {
654 struct vir_region *r;
655
656 SANITYCHECK(SCL_FUNCTIONS);
657
658 if(!vmp->vm_regions)
659 panic("process has no regions: %d", vmp->vm_endpoint);
660
661 for(r = vmp->vm_regions; r; r = r->next) {
662 if(offset >= r->vaddr && offset < r->vaddr + r->length)
663 return r;
664 }
665
666 SANITYCHECK(SCL_FUNCTIONS);
667
668 return NULL;
669 }
670
671 PRIVATE u32_t vrallocflags(u32_t flags)
672 {
673 u32_t allocflags = 0;
674
675 if(flags & VR_PHYS64K)
676 allocflags |= PAF_ALIGN64K;
677 if(flags & VR_LOWER16MB)
678 allocflags |= PAF_LOWER16MB;
679 if(flags & VR_LOWER1MB)
680 allocflags |= PAF_LOWER1MB;
681 if(flags & VR_CONTIG)
682 allocflags |= PAF_CONTIG;
683
684 return allocflags;
685 }
686
687 /*===========================================================================*
688 * map_new_physblock *
689 *===========================================================================*/
690 PRIVATE int map_new_physblock(vmp, region, start_offset, length,
691 what_mem, allocflags, written)
692 struct vmproc *vmp;
693 struct vir_region *region;
694 vir_bytes start_offset;
695 vir_bytes length;
696 phys_bytes what_mem;
697 u32_t allocflags;
698 int written;
699 {
700 struct memlist *memlist, given, *ml;
701 int used_memlist, r;
702 vir_bytes mapped = 0;
703 vir_bytes offset = start_offset;
704
705 SANITYCHECK(SCL_FUNCTIONS);
706
707 assert(!(length % VM_PAGE_SIZE));
708
709 if((region->flags & VR_CONTIG) &&
710 (start_offset > 0 || length < region->length)) {
711 printf("VM: map_new_physblock: non-full allocation requested\n");
712 return EFAULT;
713 }
714
715 /* Memory for new physical block. */
716 if(what_mem == MAP_NONE) {
717 allocflags |= vrallocflags(region->flags);
718
719 if(!(memlist = alloc_mem_in_list(length, allocflags))) {
720 printf("map_new_physblock: couldn't allocate\n");
721 return ENOMEM;
722 }
723 used_memlist = 1;
724 } else {
725 given.phys = what_mem;
726 given.length = length;
727 given.next = NULL;
728 memlist = &given;
729 used_memlist = 0;
730 assert(given.phys);
731 assert(given.length);
732 }
733
734 r = OK;
735
736 for(ml = memlist; ml; ml = ml->next) {
737 assert(ml->phys);
738 assert(ml->length);
739 }
740
741 for(ml = memlist; ml; ml = ml->next) {
742 struct phys_region *newphysr = NULL;
743 struct phys_block *newpb = NULL;
744
745 /* Allocate things necessary for this chunk of memory. */
746 if(!SLABALLOC(newphysr) || !SLABALLOC(newpb)) {
747 printf("map_new_physblock: no memory for the ph slabs\n");
748 if(newphysr) SLABFREE(newphysr);
749 if(newpb) SLABFREE(newpb);
750 r = ENOMEM;
751 break;
752 }
753
754 assert(ml->phys);
755 assert(ml->length);
756
757 /* New physical block. */
758 assert(!(ml->phys % VM_PAGE_SIZE));
759
760 USE(newpb,
761 newpb->phys = ml->phys;
762 newpb->refcount = 1;
763 newpb->length = ml->length;
764 newpb->firstregion = newphysr;);
765
766 /* New physical region. */
767 USE(newphysr,
768 newphysr->offset = offset;
769 newphysr->ph = newpb;
770 newphysr->parent = region;
771 /* No other references to this block. */
772 newphysr->next_ph_list = NULL;);
773 #if SANITYCHECKS
774 USE(newphysr, newphysr->written = written;);
775 #endif
776
777 /* Update pagetable. */
778 if(map_ph_writept(vmp, region, newphysr) != OK) {
779 printf("map_new_physblock: map_ph_writept failed\n");
780 r = ENOMEM;
781 break;
782 }
783
784 physr_insert(region->phys, newphysr);
785
786 offset += ml->length;
787 mapped += ml->length;
788 }
789
790 if(used_memlist) {
791 if(r != OK) {
792 offset = start_offset;
793 /* Things did not go well. Undo everything. */
794 for(ml = memlist; ml; ml = ml->next) {
795 struct phys_region *physr;
796 offset += ml->length;
797 if((physr = physr_search(region->phys, offset,
798 AVL_EQUAL))) {
799 assert(physr->ph->refcount == 1);
800 pb_unreferenced(region, physr);
801 physr_remove(region->phys, physr->offset);
802 SLABFREE(physr);
803 }
804 }
805 } else assert(mapped == length);
806
807 /* Always clean up the memlist itself, even if everything
808 * worked we're not using the memlist nodes any more. And
809 * the memory they reference is either freed above or in use.
810 */
811 free_mem_list(memlist, 0);
812 }
813
814 SANITYCHECK(SCL_FUNCTIONS);
815
816 return r;
817 }
818
819 /*===========================================================================*
820 * map_clone_ph_block *
821 *===========================================================================*/
822 PRIVATE struct phys_region *map_clone_ph_block(vmp, region, ph, iter)
823 struct vmproc *vmp;
824 struct vir_region *region;
825 struct phys_region *ph;
826 physr_iter *iter;
827 {
828 vir_bytes offset, length;
829 struct memlist *ml;
830 u32_t allocflags;
831 phys_bytes physaddr;
832 struct phys_region *newpr;
833 int written = 0;
834 #if SANITYCHECKS
835 written = ph->written;
836 #endif
837 SANITYCHECK(SCL_FUNCTIONS);
838
839 /* Warning: this function will free the passed
840 * phys_region *ph and replace it (in the same offset)
841 * with one or more others! So both the pointer to it
842 * and any iterators over the phys_regions in the vir_region
843 * will be invalid on successful return. (Iterators over
844 * the vir_region could be invalid on unsuccessful return too.)
845 */
846
847 /* This function takes a physical block, copies its contents
848 * into newly allocated memory, and replaces the single physical
849 * block by one or more physical blocks with refcount 1 with the
850 * same contents as the original. In other words, a fragmentable
851 * version of map_copy_ph_block().
852 */
853
854 /* Remember where and how much. */
855 offset = ph->offset;
856 length = ph->ph->length;
857 physaddr = ph->ph->phys;
858
859 /* Now unlink the original physical block so we can replace
860 * it with new ones.
861 */
862
863 SANITYCHECK(SCL_DETAIL);
864 SLABSANE(ph);
865 SLABSANE(ph->ph);
866 assert(ph->ph->refcount > 1);
867 pb_unreferenced(region, ph);
868 assert(ph->ph->refcount >= 1);
869 physr_remove(region->phys, offset);
870 SLABFREE(ph);
871
872 SANITYCHECK(SCL_DETAIL);
873
874 /* Put new free memory in. */
875 allocflags = vrallocflags(region->flags);
876 assert(!(allocflags & PAF_CONTIG));
877 assert(!(allocflags & PAF_CLEAR));
878
879 if(map_new_physblock(vmp, region, offset, length,
880 MAP_NONE, allocflags, written) != OK) {
881 /* XXX original range now gone. */
882 free_mem_list(ml, 0);
883 printf("VM: map_clone_ph_block: map_new_physblock failed.\n");
884 return NULL;
885 }
886
887 /* Copy the block to the new memory.
888 * Can only fail if map_new_physblock didn't do what we asked.
889 */
890 if(copy_abs2region(physaddr, region, offset, length) != OK)
891 panic("copy_abs2region failed, no good reason for that");
892
893 newpr = physr_search(region->phys, offset, AVL_EQUAL);
894 assert(newpr);
895 assert(newpr->offset == offset);
896
897 if(iter) {
898 physr_start_iter(region->phys, iter, offset, AVL_EQUAL);
899 assert(physr_get_iter(iter) == newpr);
900 }
901
902 SANITYCHECK(SCL_FUNCTIONS);
903
904 return newpr;
905 }
906
907
908 /*===========================================================================*
909 * map_pf *
910 *===========================================================================*/
911 PUBLIC int map_pf(vmp, region, offset, write)
912 struct vmproc *vmp;
913 struct vir_region *region;
914 vir_bytes offset;
915 int write;
916 {
917 vir_bytes virpage;
918 struct phys_region *ph;
919 int r = OK;
920
921 assert(offset >= 0);
922 assert(offset < region->length);
923
924 assert(region->flags & VR_ANON);
925 assert(!(region->vaddr % VM_PAGE_SIZE));
926
927 virpage = offset - offset % VM_PAGE_SIZE;
928
929 SANITYCHECK(SCL_FUNCTIONS);
930
931 if((ph = physr_search(region->phys, offset, AVL_LESS_EQUAL)) &&
932 (ph->offset <= offset && offset < ph->offset + ph->ph->length)) {
933 phys_bytes blockoffset = ph->offset;
934 /* Pagefault in existing block. Do copy-on-write. */
935 assert(write);
936 assert(region->flags & VR_WRITABLE);
937 assert(ph->ph->refcount > 0);
938
939 if(WRITABLE(region, ph->ph)) {
940 r = map_ph_writept(vmp, region, ph);
941 if(r != OK)
942 printf("map_ph_writept failed\n");
943 } else {
944 if(ph->ph->refcount > 0
945 && ph->ph->share_flag != PBSH_COW) {
946 printf("VM: write RO mapped pages.\n");
947 return EFAULT;
948 } else {
949 if(!map_clone_ph_block(vmp, region, ph, NULL))
950 r = ENOMEM;
951 }
952 }
953 } else {
954 /* Pagefault in non-existing block. Map in new block. */
955 if(map_new_physblock(vmp, region, virpage,
956 VM_PAGE_SIZE, MAP_NONE, PAF_CLEAR, 0) != OK) {
957 printf("map_new_physblock failed\n");
958 r = ENOMEM;
959 }
960 }
961
962 SANITYCHECK(SCL_FUNCTIONS);
963
964 if(r != OK) {
965 printf("VM: map_pf: failed (%d)\n", r);
966 return r;
967 }
968
969 #if SANITYCHECKS
970 if(OK != pt_checkrange(&vmp->vm_pt, region->vaddr+virpage,
971 VM_PAGE_SIZE, write)) {
972 panic("map_pf: pt_checkrange failed: %d", r);
973 }
974 #endif
975
976 return r;
977 }
978
979 /*===========================================================================*
980 * map_handle_memory *
981 *===========================================================================*/
982 PUBLIC int map_handle_memory(vmp, region, offset, length, write)
983 struct vmproc *vmp;
984 struct vir_region *region;
985 vir_bytes offset, length;
986 int write;
987 {
988 struct phys_region *physr, *nextphysr;
989 int changes = 0;
990 physr_iter iter;
991
992 #define FREE_RANGE_HERE(er1, er2) { \
993 struct phys_region *r1 = (er1), *r2 = (er2); \
994 vir_bytes start = offset, end = offset + length; \
995 if(r1) { \
996 start = MAX(start, r1->offset + r1->ph->length); } \
997 if(r2) { \
998 end = MIN(end, r2->offset); } \
999 if(start < end) { \
1000 int r; \
1001 SANITYCHECK(SCL_DETAIL); \
1002 if(map_new_physblock(vmp, region, start, \
1003 end-start, MAP_NONE, PAF_CLEAR, 0) != OK) { \
1004 SANITYCHECK(SCL_DETAIL); \
1005 return ENOMEM; \
1006 } \
1007 changes++; \
1008 } }
1009
1010
1011 SANITYCHECK(SCL_FUNCTIONS);
1012
1013 assert(region->flags & VR_ANON);
1014 assert(!(region->vaddr % VM_PAGE_SIZE));
1015 assert(!(offset % VM_PAGE_SIZE));
1016 assert(!(length % VM_PAGE_SIZE));
1017 assert(!write || (region->flags & VR_WRITABLE));
1018
1019 physr_start_iter(region->phys, &iter, offset, AVL_LESS_EQUAL);
1020 physr = physr_get_iter(&iter);
1021
1022 if(!physr) {
1023 physr_start_iter(region->phys, &iter, offset, AVL_GREATER_EQUAL);
1024 physr = physr_get_iter(&iter);
1025 }
1026
1027 FREE_RANGE_HERE(NULL, physr);
1028
1029 if(physr) {
1030 physr = reset_physr_iter(region, &iter, physr->offset);
1031 if(physr->offset + physr->ph->length <= offset) {
1032 physr_incr_iter(&iter);
1033 physr = physr_get_iter(&iter);
1034
1035 FREE_RANGE_HERE(NULL, physr);
1036 if(physr) {
1037 physr = reset_physr_iter(region, &iter,
1038 physr->offset);
1039 }
1040 }
1041 }
1042
1043 while(physr) {
1044 int r;
1045
1046 SANITYCHECK(SCL_DETAIL);
1047
1048 if(write) {
1049 assert(physr->ph->refcount > 0);
1050 if(!WRITABLE(region, physr->ph)) {
1051 if(!(physr = map_clone_ph_block(vmp, region,
1052 physr, &iter))) {
1053 printf("VM: map_handle_memory: no copy\n");
1054 return ENOMEM;
1055 }
1056 changes++;
1057 } else {
1058 SANITYCHECK(SCL_DETAIL);
1059 if((r=map_ph_writept(vmp, region, physr)) != OK) {
1060 printf("VM: map_ph_writept failed\n");
1061 return r;
1062 }
1063 changes++;
1064 SANITYCHECK(SCL_DETAIL);
1065 }
1066 }
1067
1068 SANITYCHECK(SCL_DETAIL);
1069 physr_incr_iter(&iter);
1070 nextphysr = physr_get_iter(&iter);
1071 FREE_RANGE_HERE(physr, nextphysr);
1072 SANITYCHECK(SCL_DETAIL);
1073 if(nextphysr) {
1074 if(nextphysr->offset >= offset + length)
1075 break;
1076 nextphysr = reset_physr_iter(region, &iter,
1077 nextphysr->offset);
1078 }
1079 physr = nextphysr;
1080 }
1081
1082 SANITYCHECK(SCL_FUNCTIONS);
1083
1084 if(changes < 1) {
1085 #if VERBOSE
1086 printf("region start at 0x%lx offset 0x%lx len 0x%lx write %d\n",
1087 region->vaddr, offset, length, write);
1088 printf("no changes in map_handle_memory\n");
1089 #endif
1090 return EFAULT;
1091 }
1092
1093 #if SANITYCHECKS
1094 if(OK != pt_checkrange(&vmp->vm_pt, region->vaddr+offset, length, write)) {
1095 printf("handle mem %s-", arch_map2str(vmp, region->vaddr+offset));
1096 printf("%s failed\n", arch_map2str(vmp, region->vaddr+offset+length));
1097 map_printregion(vmp, region);
1098 panic("checkrange failed");
1099 }
1100 #endif
1101
1102 return OK;
1103 }
1104
1105 #if SANITYCHECKS
1106 static int countregions(struct vir_region *vr)
1107 {
1108 int n = 0;
1109 struct phys_region *ph;
1110 physr_iter iter;
1111 physr_start_iter_least(vr->phys, &iter);
1112 while((ph = physr_get_iter(&iter))) {
1113 n++;
1114 physr_incr_iter(&iter);
1115 }
1116 return n;
1117 }
1118 #endif
1119
1120 /*===========================================================================*
1121 * map_copy_region *
1122 *===========================================================================*/
1123 PRIVATE struct vir_region *map_copy_region(struct vmproc *vmp, struct vir_region *vr)
1124 {
1125 /* map_copy_region creates a complete copy of the vir_region
1126 * data structure, linking in the same phys_blocks directly,
1127 * but all in limbo, i.e., the caller has to link the vir_region
1128 * to a process. Therefore it doesn't increase the refcount in
1129 * the phys_block; the caller has to do this once it's linked.
1130 * The reason for this is to keep the sanity checks working
1131 * within this function.
1132 */
1133 struct vir_region *newvr;
1134 struct phys_region *ph;
1135 physr_iter iter;
1136 physr_avl *phavl;
1137 #if SANITYCHECKS
1138 int cr;
1139 cr = countregions(vr);
1140 #endif
1141
1142 if(!SLABALLOC(newvr))
1143 return NULL;
1144 SLABALLOC(phavl);
1145 if(!phavl) {
1146 SLABFREE(newvr);
1147 return NULL;
1148 }
1149 USE(newvr,
1150 *newvr = *vr;
1151 newvr->next = NULL;
1152 newvr->phys = phavl;
1153 );
1154 physr_init(newvr->phys);
1155
1156 physr_start_iter_least(vr->phys, &iter);
1157 while((ph = physr_get_iter(&iter))) {
1158 struct phys_region *newph;
1159 if(!SLABALLOC(newph)) {
1160 map_free(vmp, newvr);
1161 return NULL;
1162 }
1163 USE(newph,
1164 newph->ph = ph->ph;
1165 newph->next_ph_list = NULL;
1166 newph->parent = newvr;
1167 newph->offset = ph->offset;);
1168 #if SANITYCHECKS
1169 USE(newph, newph->written = 0;);
1170 #endif
1171 physr_insert(newvr->phys, newph);
1172 #if SANITYCHECKS
1173 assert(countregions(vr) == cr);
1174 #endif
1175 physr_incr_iter(&iter);
1176 }
1177
1178 #if SANITYCHECKS
1179 assert(countregions(vr) == countregions(newvr));
1180 #endif
1181
1182 return newvr;
1183 }
1184
1185 /*===========================================================================*
1186 * copy_abs2region *
1187 *===========================================================================*/
1188 PUBLIC int copy_abs2region(phys_bytes abs, struct vir_region *destregion,
1189 phys_bytes offset, phys_bytes len)
1190
1191 {
1192 assert(destregion);
1193 assert(destregion->phys);
1194 while(len > 0) {
1195 phys_bytes sublen, suboffset;
1196 struct phys_region *ph;
1197 assert(destregion);
1198 assert(destregion->phys);
1199 if(!(ph = physr_search(destregion->phys, offset, AVL_LESS_EQUAL))) {
1200 printf("VM: copy_abs2region: no phys region found (1).\n");
1201 return EFAULT;
1202 }
1203 assert(ph->offset <= offset);
1204 if(ph->offset+ph->ph->length <= offset) {
1205 printf("VM: copy_abs2region: no phys region found (2).\n");
1206 return EFAULT;
1207 }
1208 suboffset = offset - ph->offset;
1209 assert(suboffset < ph->ph->length);
1210 sublen = len;
1211 if(sublen > ph->ph->length - suboffset)
1212 sublen = ph->ph->length - suboffset;
1213 assert(suboffset + sublen <= ph->ph->length);
1214 if(ph->ph->refcount != 1) {
1215 printf("VM: copy_abs2region: no phys region found (3).\n");
1216 return EFAULT;
1217 }
1218
1219 if(sys_abscopy(abs, ph->ph->phys + suboffset, sublen) != OK) {
1220 printf("VM: copy_abs2region: abscopy failed.\n");
1221 return EFAULT;
1222 }
1223 abs += sublen;
1224 offset += sublen;
1225 len -= sublen;
1226 }
1227
1228 return OK;
1229 }
1230
1231 /*=========================================================================*
1232 * map_writept *
1233 *=========================================================================*/
1234 PUBLIC int map_writept(struct vmproc *vmp)
1235 {
1236 struct vir_region *vr;
1237 struct phys_region *ph;
1238 int r;
1239
1240 for(vr = vmp->vm_regions; vr; vr = vr->next) {
1241 physr_iter iter;
1242 physr_start_iter_least(vr->phys, &iter);
1243 while((ph = physr_get_iter(&iter))) {
1244 physr_incr_iter(&iter);
1245
1246 /* If this phys block is shared as SMAP, then do
1247 * not update the page table. */
1248 if(ph->ph->refcount > 1
1249 && ph->ph->share_flag == PBSH_SMAP) {
1250 continue;
1251 }
1252
1253 if((r=map_ph_writept(vmp, vr, ph)) != OK) {
1254 printf("VM: map_writept: failed\n");
1255 return r;
1256 }
1257 }
1258 }
1259
1260 return OK;
1261 }
1262
1263 /*========================================================================*
1264 * map_proc_copy *
1265 *========================================================================*/
1266 PUBLIC int map_proc_copy(dst, src)
1267 struct vmproc *dst;
1268 struct vmproc *src;
1269 {
1270 struct vir_region *vr, *prevvr = NULL;
1271 dst->vm_regions = NULL;
1272
1273 SANITYCHECK(SCL_FUNCTIONS);
1274
1275
1276 for(vr = src->vm_regions; vr; vr = vr->next) {
1277 physr_iter iter_orig, iter_new;
1278 struct vir_region *newvr;
1279 struct phys_region *orig_ph, *new_ph;
1280 if(!(newvr = map_copy_region(dst, vr))) {
1281 map_free_proc(dst);
1282 return ENOMEM;
1283 }
1284 USE(newvr, newvr->parent = dst;);
1285 if(prevvr) { USE(prevvr, prevvr->next = newvr;); }
1286 else { dst->vm_regions = newvr; }
1287 physr_start_iter_least(vr->phys, &iter_orig);
1288 physr_start_iter_least(newvr->phys, &iter_new);
1289 while((orig_ph = physr_get_iter(&iter_orig))) {
1290 struct phys_block *pb;
1291 new_ph = physr_get_iter(&iter_new);
1292 /* Check two physregions both are nonnull,
1293 * are different, and match physblocks.
1294 */
1295 assert(new_ph);
1296 assert(orig_ph);
1297 assert(orig_ph != new_ph);
1298 pb = orig_ph->ph;
1299 assert(pb == new_ph->ph);
1300
1301 /* Link in new physregion. */
1302 assert(!new_ph->next_ph_list);
1303 USE(new_ph, new_ph->next_ph_list = pb->firstregion;);
1304 USE(pb, pb->firstregion = new_ph;);
1305
1306 /* Increase phys block refcount */
1307 assert(pb->refcount > 0);
1308 USE(pb, pb->refcount++;);
1309 assert(pb->refcount > 1);
1310
1311 /* If the phys block has been shared as SMAP,
1312 * do the regular copy. */
1313 if(pb->refcount > 2 && pb->share_flag == PBSH_SMAP) {
1314 map_clone_ph_block(dst, newvr,new_ph,
1315 &iter_new);
1316 } else {
1317 USE(pb, pb->share_flag = PBSH_COW;);
1318 }
1319
1320 /* Get next new physregion */
1321 physr_incr_iter(&iter_orig);
1322 physr_incr_iter(&iter_new);
1323 }
1324 assert(!physr_get_iter(&iter_new));
1325 prevvr = newvr;
1326 }
1327
1328 map_writept(src);
1329 map_writept(dst);
1330
1331 SANITYCHECK(SCL_FUNCTIONS);
1332 return OK;
1333 }
1334
1335 /*========================================================================*
1336 * map_proc_kernel *
1337 *========================================================================*/
1338 PUBLIC struct vir_region *map_proc_kernel(struct vmproc *vmp)
1339 {
1340 struct vir_region *vr;
1341
1342 /* We assume these are the first regions to be mapped to
1343 * make the function a bit simpler (free all regions on error).
1344 */
1345 assert(!vmp->vm_regions);
1346 assert(vmproc[VMP_SYSTEM].vm_flags & VMF_INUSE);
1347 assert(!(KERNEL_TEXT % VM_PAGE_SIZE));
1348 assert(!(KERNEL_TEXT_LEN % VM_PAGE_SIZE));
1349 assert(!(KERNEL_DATA % VM_PAGE_SIZE));
1350 assert(!(KERNEL_DATA_LEN % VM_PAGE_SIZE));
1351
1352 if(!(vr = map_page_region(vmp, KERNEL_TEXT, 0, KERNEL_TEXT_LEN,
1353 KERNEL_TEXT, VR_DIRECT | VR_WRITABLE | VR_NOPF, 0)) ||
1354 !(vr = map_page_region(vmp, KERNEL_DATA, 0, KERNEL_DATA_LEN,
1355 KERNEL_DATA, VR_DIRECT | VR_WRITABLE | VR_NOPF, 0))) {
1356 map_free_proc(vmp);
1357 return NULL;
1358 }
1359
1360 return vr; /* Return pointer not useful, just non-NULL. */
1361 }
1362
1363 /*========================================================================*
1364 * map_region_extend *
1365 *========================================================================*/
1366 PUBLIC int map_region_extend(struct vmproc *vmp, struct vir_region *vr,
1367 vir_bytes delta)
1368 {
1369 vir_bytes end;
1370
1371 assert(vr);
1372 assert(vr->flags & VR_ANON);
1373 assert(!(delta % VM_PAGE_SIZE));
1374
1375 if(!delta) return OK;
1376 end = vr->vaddr + vr->length;
1377 assert(end >= vr->vaddr);
1378
1379 if(end + delta <= end) {
1380 printf("VM: strange delta 0x%lx\n", delta);
1381 return ENOMEM;
1382 }
1383
1384 if(!vr->next || end + delta <= vr->next->vaddr) {
1385 USE(vr, vr->length += delta;);
1386 return OK;
1387 }
1388
1389 map_printmap(vmp);
1390
1391 return ENOMEM;
1392 }
1393
1394 /*========================================================================*
1395 * map_region_shrink *
1396 *========================================================================*/
1397 PUBLIC int map_region_shrink(struct vir_region *vr, vir_bytes delta)
1398 {
1399 assert(vr);
1400 assert(vr->flags & VR_ANON);
1401 assert(!(delta % VM_PAGE_SIZE));
1402
1403 #if 0
1404 printf("VM: ignoring region shrink\n");
1405 #endif
1406
1407 return OK;
1408 }
1409
1410 PUBLIC struct vir_region *map_region_lookup_tag(vmp, tag)
1411 struct vmproc *vmp;
1412 u32_t tag;
1413 {
1414 struct vir_region *vr;
1415
1416 for(vr = vmp->vm_regions; vr; vr = vr->next)
1417 if(vr->tag == tag)
1418 return vr;
1419
1420 return NULL;
1421 }
1422
1423 PUBLIC void map_region_set_tag(struct vir_region *vr, u32_t tag)
1424 {
1425 USE(vr, vr->tag = tag;);
1426 }
1427
1428 PUBLIC u32_t map_region_get_tag(struct vir_region *vr)
1429 {
1430 return vr->tag;
1431 }
1432
1433 /*========================================================================*
1434 * map_unmap_region *
1435 *========================================================================*/
1436 PUBLIC int map_unmap_region(struct vmproc *vmp, struct vir_region *region,
1437 vir_bytes len)
1438 {
1439 /* Shrink the region by 'len' bytes, from the start. Unreference
1440 * memory it used to reference if any.
1441 */
1442 struct vir_region *r, *nextr, *prev = NULL;
1443 vir_bytes regionstart;
1444
1445 SANITYCHECK(SCL_FUNCTIONS);
1446
1447 for(r = vmp->vm_regions; r; r = r->next) {
1448 if(r == region)
1449 break;
1450
1451 prev = r;
1452 }
1453
1454 SANITYCHECK(SCL_DETAIL);
1455
1456 if(r == NULL)
1457 panic("map_unmap_region: region not found");
1458
1459 if(len > r->length || (len % VM_PAGE_SIZE)) {
1460 printf("VM: bogus length 0x%lx\n", len);
1461 return EINVAL;
1462 }
1463
1464 if(!(r->flags & (VR_ANON|VR_DIRECT))) {
1465 printf("VM: only unmap anonymous or direct memory\n");
1466 return EINVAL;
1467 }
1468
1469 regionstart = r->vaddr;
1470
1471 if(len == r->length) {
1472 /* Whole region disappears. Unlink and free it. */
1473 if(!prev) {
1474 vmp->vm_regions = r->next;
1475 } else {
1476 USE(prev, prev->next = r->next;);
1477 }
1478 map_free(vmp, r);
1479 } else {
1480 struct phys_region *pr;
1481 physr_iter iter;
1482 /* Region shrinks. First unreference its memory
1483 * and then shrink the region.
1484 */
1485 map_subfree(vmp, r, len);
1486 USE(r,
1487 r->vaddr += len;
1488 r->length -= len;);
1489 physr_start_iter_least(r->phys, &iter);
1490
1491 /* vaddr has increased; to make all the phys_regions
1492 * point to the same addresses, make them shrink by the
1493 * same amount.
1494 */
1495 while((pr = physr_get_iter(&iter))) {
1496 assert(pr->offset >= len);
1497 USE(pr, pr->offset -= len;);
1498 physr_incr_iter(&iter);
1499 }
1500 }
1501
1502 SANITYCHECK(SCL_DETAIL);
1503
1504 if(pt_writemap(&vmp->vm_pt, regionstart,
1505 MAP_NONE, len, 0, WMF_OVERWRITE) != OK) {
1506 printf("VM: map_unmap_region: pt_writemap failed\n");
1507 return ENOMEM;
1508 }
1509
1510 SANITYCHECK(SCL_FUNCTIONS);
1511
1512 return OK;
1513 }
1514
1515 /*========================================================================*
1516 * map_remap *
1517 *========================================================================*/
1518 PUBLIC int map_remap(struct vmproc *dvmp, vir_bytes da, size_t size,
1519 struct vir_region *region, vir_bytes *r)
1520 {
1521 struct vir_region *vr, *prev;
1522 struct phys_region *ph;
1523 vir_bytes startv, dst_addr;
1524 physr_iter iter;
1525
1526 SANITYCHECK(SCL_FUNCTIONS);
1527
1528 assert(region->flags & VR_SHARED);
1529
1530 /* da is handled differently */
1531 if (!da)
1532 dst_addr = dvmp->vm_stacktop;
1533 else
1534 dst_addr = da;
1535 dst_addr = arch_vir2map(dvmp, dst_addr);
1536
1537 prev = NULL;
1538 /* round up to page size */
1539 assert(!(size % VM_PAGE_SIZE));
1540 startv = region_find_slot(dvmp, dst_addr, VM_DATATOP, size, &prev);
1541 if (startv == (vir_bytes) -1) {
1542 printf("map_remap: search 0x%x...\n", dst_addr);
1543 map_printmap(dvmp);
1544 return ENOMEM;
1545 }
1546 /* when the user specifies the address, we cannot change it */
1547 if (da && (startv != dst_addr))
1548 return EINVAL;
1549
1550 vr = map_copy_region(dvmp, region);
1551 if(!vr)
1552 return ENOMEM;
1553
1554 USE(vr,
1555 vr->vaddr = startv;
1556 vr->length = size;
1557 vr->flags = region->flags;
1558 vr->tag = VRT_NONE;
1559 vr->parent = dvmp;);
1560 assert(vr->flags & VR_SHARED);
1561
1562 if (prev) {
1563 USE(vr,
1564 vr->next = prev->next;);
1565 USE(prev, prev->next = vr;);
1566 } else {
1567 USE(vr,
1568 vr->next = dvmp->vm_regions;);
1569 dvmp->vm_regions = vr;
1570 }
1571
1572 physr_start_iter_least(vr->phys, &iter);
1573 while((ph = physr_get_iter(&iter))) {
1574 struct phys_block *pb = ph->ph;
1575 assert(!ph->next_ph_list);
1576 USE(ph, ph->next_ph_list = pb->firstregion;);
1577 USE(pb, pb->firstregion = ph;);
1578 USE(pb, pb->refcount++;);
1579 if(map_ph_writept(dvmp, vr, ph) != OK) {
1580 panic("map_remap: map_ph_writept failed");
1581 }
1582 physr_incr_iter(&iter);
1583 }
1584
1585 *r = startv;
1586
1587 SANITYCHECK(SCL_FUNCTIONS);
1588
1589 return OK;
1590 }
1591
1592 /*========================================================================*
1593 * map_get_phys *
1594 *========================================================================*/
1595 PUBLIC int map_get_phys(struct vmproc *vmp, vir_bytes addr, phys_bytes *r)
1596 {
1597 struct vir_region *vr;
1598 struct phys_region *ph;
1599 physr_iter iter;
1600
1601 if (!(vr = map_lookup(vmp, addr)) ||
1602 (vr->vaddr != addr))
1603 return EINVAL;
1604
1605 if (!(vr->flags & VR_SHARED))
1606 return EINVAL;
1607
1608 physr_start_iter_least(vr->phys, &iter);
1609 ph = physr_get_iter(&iter);
1610
1611 assert(ph);
1612 assert(ph->ph);
1613 if (r)
1614 *r = ph->ph->phys;
1615
1616 return OK;
1617 }
1618
1619 /*========================================================================*
1620 * map_get_ref *
1621 *========================================================================*/
1622 PUBLIC int map_get_ref(struct vmproc *vmp, vir_bytes addr, u8_t *cnt)
1623 {
1624 struct vir_region *vr;
1625 struct phys_region *ph;
1626 physr_iter iter;
1627
1628 if (!(vr = map_lookup(vmp, addr)) ||
1629 (vr->vaddr != addr))
1630 return EINVAL;
1631
1632 if (!(vr->flags & VR_SHARED))
1633 return EINVAL;
1634
1635 physr_start_iter_least(vr->phys, &iter);
1636 ph = physr_get_iter(&iter);
1637
1638 assert(ph);
1639 assert(ph->ph);
1640 if (cnt)
1641 *cnt = ph->ph->refcount;
1642
1643 return OK;
1644 }
1645
1646 /*========================================================================*
1647 * get_usage_info *
1648 *========================================================================*/
1649 PUBLIC void get_usage_info(struct vmproc *vmp, struct vm_usage_info *vui)
1650 {
1651 struct vir_region *vr;
1652 physr_iter iter;
1653 struct phys_region *ph;
1654 vir_bytes len;
1655
1656 memset(vui, 0, sizeof(*vui));
1657
1658 for(vr = vmp->vm_regions; vr; vr = vr->next) {
1659 physr_start_iter_least(vr->phys, &iter);
1660 while((ph = physr_get_iter(&iter))) {
1661 len = ph->ph->length;
1662
1663 /* All present pages are counted towards the total. */
1664 vui->vui_total += len;
1665
1666 if (ph->ph->refcount > 1) {
1667 /* Any page with a refcount > 1 is common. */
1668 vui->vui_common += len;
1669
1670 /* Any common, non-COW page is shared. */
1671 if (vr->flags & VR_SHARED ||
1672 ph->ph->share_flag == PBSH_SMAP)
1673 vui->vui_shared += len;
1674 }
1675 physr_incr_iter(&iter);
1676 }
1677 }
1678 }
1679
1680 /*===========================================================================*
1681 * get_region_info *
1682 *===========================================================================*/
1683 PUBLIC int get_region_info(struct vmproc *vmp, struct vm_region_info *vri,
1684 int max, vir_bytes *nextp)
1685 {
1686 struct vir_region *vr;
1687 vir_bytes next;
1688 int count;
1689
1690 next = *nextp;
1691
1692 if (!max) return 0;
1693
1694 for(vr = vmp->vm_regions; vr; vr = vr->next)
1695 if (vr->vaddr >= next) break;
1696
1697 if (!vr) return 0;
1698
1699 for(count = 0; vr && count < max; vr = vr->next, count++, vri++) {
1700 vri->vri_addr = arch_map2info(vmp, vr->vaddr, &vri->vri_seg,
1701 &vri->vri_prot);
1702 vri->vri_length = vr->length;
1703
1704 /* "AND" the provided protection with per-page protection. */
1705 if (!(vr->flags & VR_WRITABLE))
1706 vri->vri_prot &= ~PROT_WRITE;
1707
1708 vri->vri_flags = (vr->flags & VR_SHARED) ? MAP_SHARED : 0;
1709
1710 next = vr->vaddr + vr->length;
1711 }
1712
1713 *nextp = next;
1714 return count;
1715 }
1716
1717 /*========================================================================*
1718 * regionprintstats *
1719 *========================================================================*/
1720 PUBLIC void printregionstats(struct vmproc *vmp)
1721 {
1722 struct vir_region *vr;
1723 struct phys_region *pr;
1724 physr_iter iter;
1725 vir_bytes used = 0, weighted = 0;
1726
1727 for(vr = vmp->vm_regions; vr; vr = vr->next) {
1728 if(vr->flags & VR_DIRECT)
1729 continue;
1730 physr_start_iter_least(vr->phys, &iter);
1731 while((pr = physr_get_iter(&iter))) {
1732 physr_incr_iter(&iter);
1733 used += pr->ph->length;
1734 weighted += pr->ph->length / pr->ph->refcount;
1735 }
1736 }
1737
1738 printf("%6dkB %6dkB\n", used/1024, weighted/1024);
1739
1740 return;
1741 }
1742
1743 /*===========================================================================*
1744 * do_map_memory *
1745 *===========================================================================*/
1746 PRIVATE int do_map_memory(struct vmproc *vms, struct vmproc *vmd,
1747 struct vir_region *vrs, struct vir_region *vrd,
1748 vir_bytes offset_s, vir_bytes offset_d,
1749 vir_bytes length, int flag)
1750 {
1751 struct phys_region *prs;
1752 struct phys_region *newphysr;
1753 struct phys_block *pb;
1754 physr_iter iter;
1755 u32_t pt_flag = PTF_PRESENT | PTF_USER;
1756 vir_bytes end;
1757
1758 SANITYCHECK(SCL_FUNCTIONS);
1759
1760 /* Search for the first phys region in the source process. */
1761 physr_start_iter(vrs->phys, &iter, offset_s, AVL_EQUAL);
1762 prs = physr_get_iter(&iter);
1763 if(!prs)
1764 panic("do_map_memory: no aligned phys region: %d", 0);
1765
1766 /* flag: 0 -> read-only
1767 * 1 -> writable
1768 * -1 -> share as COW, so read-only
1769 */
1770 if(flag > 0)
1771 pt_flag |= PTF_WRITE;
1772
1773 /* Map phys blocks in the source process to the destination process. */
1774 end = offset_d + length;
1775 while((prs = physr_get_iter(&iter)) && offset_d < end) {
1776 /* If a SMAP share was requested but the phys block has already
1777 * been shared as COW, copy the block for the source phys region
1778 * first.
1779 */
1780 pb = prs->ph;
1781 if(flag >= 0 && pb->refcount > 1
1782 && pb->share_flag == PBSH_COW) {
1783 if(!(prs = map_clone_ph_block(vms, vrs, prs, &iter)))
1784 return ENOMEM;
1785 pb = prs->ph;
1786 }
1787
1788 /* Allocate a new phys region. */
1789 if(!SLABALLOC(newphysr))
1790 return ENOMEM;
1791
1792 /* Set and link the new phys region to the block. */
1793 newphysr->ph = pb;
1794 newphysr->offset = offset_d;
1795 newphysr->parent = vrd;
1796 newphysr->next_ph_list = pb->firstregion;
1797 pb->firstregion = newphysr;
1798 physr_insert(newphysr->parent->phys, newphysr);
1799 pb->refcount++;
1800
1801 /* If a COW share was requested but the phys block has already
1802 * been shared as SMAP, give up on COW and copy the block for
1803 * the destination phys region now.
1804 */
1805 if(flag < 0 && pb->refcount > 1
1806 && pb->share_flag == PBSH_SMAP) {
1807 if(!(newphysr = map_clone_ph_block(vmd, vrd,
1808 newphysr, NULL))) {
1809 return ENOMEM;
1810 }
1811 }
1812 else {
1813 /* See if this is a COW share or SMAP share. */
1814 if(flag < 0) { /* COW share */
1815 pb->share_flag = PBSH_COW;
1816 /* Update the page table for the src process. */
1817 pt_writemap(&vms->vm_pt, offset_s + vrs->vaddr,
1818 pb->phys, pb->length,
1819 pt_flag, WMF_OVERWRITE);
1820 }
1821 else { /* SMAP share */
1822 pb->share_flag = PBSH_SMAP;
1823 }
1824 /* Update the page table for the destination process. */
1825 pt_writemap(&vmd->vm_pt, offset_d + vrd->vaddr,
1826 pb->phys, pb->length, pt_flag, WMF_OVERWRITE);
1827 }
1828
1829 physr_incr_iter(&iter);
1830 offset_d += pb->length;
1831 offset_s += pb->length;
1832 }
1833
1834 SANITYCHECK(SCL_FUNCTIONS);
1835
1836 return OK;
1837 }
1838
1839 /*===========================================================================*
1840 * unmap_memory *
1841 *===========================================================================*/
1842 PUBLIC int unmap_memory(endpoint_t sour, endpoint_t dest,
1843 vir_bytes virt_s, vir_bytes virt_d, vir_bytes length, int flag)
1844 {
1845 struct vmproc *vmd;
1846 struct vir_region *vrd;
1847 struct phys_region *pr;
1848 struct phys_block *pb;
1849 physr_iter iter;
1850 vir_bytes off, end;
1851 int p;
1852
1853 /* Use information on the destination process to unmap. */
1854 if(vm_isokendpt(dest, &p) != OK)
1855 panic("unmap_memory: bad endpoint: %d", dest);
1856 vmd = &vmproc[p];
1857
1858 vrd = map_lookup(vmd, virt_d);
1859 assert(vrd);
1860
1861 /* Search for the first phys region in the destination process. */
1862 off = virt_d - vrd->vaddr;
1863 physr_start_iter(vrd->phys, &iter, off, AVL_EQUAL);
1864 pr = physr_get_iter(&iter);
1865 if(!pr)
1866 panic("unmap_memory: no aligned phys region: %d", 0);
1867
1868 /* Copy the phys block now rather than doing COW. */
1869 end = off + length;
1870 while((pr = physr_get_iter(&iter)) && off < end) {
1871 pb = pr->ph;
1872 assert(pb->refcount > 1);
1873 assert(pb->share_flag == PBSH_SMAP);
1874
1875 if(!(pr = map_clone_ph_block(vmd, vrd, pr, &iter)))
1876 return ENOMEM;
1877
1878 physr_incr_iter(&iter);
1879 off += pb->length;
1880 }
1881
1882 return OK;
1883 }
1884
1885 /*===========================================================================*
1886 * split_phys *
1887 *===========================================================================*/
1888 PRIVATE int split_phys(struct phys_region *pr, vir_bytes point)
1889 {
1890 struct phys_region *newpr, *q, *prev;
1891 struct phys_block *newpb;
1892 struct phys_block *pb = pr->ph;
1893 /* Split the phys region into 2 parts by @point. */
1894
1895 if(pr->offset >= point || pr->offset + pb->length <= point)
1896 return OK;
1897 if(!SLABALLOC(newpb))
1898 return ENOMEM;
1899
1900 /* Split phys block. */
1901 *newpb = *pb;
1902 pb->length = point - pr->offset;
1903 newpb->length -= pb->length;
1904 newpb->phys += pb->length;
1905
1906 /* Split phys regions in a list. */
1907 for(q = pb->firstregion; q; q = q->next_ph_list) {
1908 if(!SLABALLOC(newpr))
1909 return ENOMEM;
1910
1911 *newpr = *q;
1912 newpr->ph = newpb;
1913 newpr->offset += pb->length;
1914
1915 /* Link to the vir region's phys region list. */
1916 physr_insert(newpr->parent->phys, newpr);
1917
1918 /* Link to the next_ph_list. */
1919 if(q == pb->firstregion) {
1920 newpb->firstregion = newpr;
1921 prev = newpr;
1922 } else {
1923 prev->next_ph_list = newpr;
1924 prev = newpr;
1925 }
1926 }
1927 prev->next_ph_list = NULL;
1928
1929 return OK;
1930 }
1931
1932 /*===========================================================================*
1933 * clean_phys_regions *
1934 *===========================================================================*/
1935 PRIVATE void clean_phys_regions(struct vir_region *region,
1936 vir_bytes offset, vir_bytes length)
1937 {
1938 /* Consider @offset as the start address and @offset+length as the end address.
1939 * If there are phys regions crossing the start address or the end address,
1940 * split them into 2 parts.
1941 *
1942 * We assume that the phys regions are listed in order and don't overlap.
1943 */
1944 struct phys_region *pr;
1945 physr_iter iter;
1946
1947 physr_start_iter_least(region->phys, &iter);
1948 while((pr = physr_get_iter(&iter))) {
1949 /* If this phys region crosses the start address, split it. */
1950 if(pr->offset < offset
1951 && pr->offset + pr->ph->length > offset) {
1952 split_phys(pr, offset);
1953 physr_start_iter_least(region->phys, &iter);
1954 }
1955 /* If this phys region crosses the end address, split it. */
1956 else if(pr->offset < offset + length
1957 && pr->offset + pr->ph->length > offset + length) {
1958 split_phys(pr, offset + length);
1959 physr_start_iter_least(region->phys, &iter);
1960 }
1961 else {
1962 physr_incr_iter(&iter);
1963 }
1964 }
1965 }
1966
1967 /*===========================================================================*
1968 * rm_phys_regions *
1969 *===========================================================================*/
1970 PRIVATE void rm_phys_regions(struct vir_region *region,
1971 vir_bytes begin, vir_bytes length)
1972 {
1973 /* Remove all phys regions between @begin and @begin+length.
1974 *
1975 * Don't update the page table, because we will update it at map_memory()
1976 * later.
1977 */
1978 struct phys_region *pr;
1979 physr_iter iter;
1980
1981 physr_start_iter(region->phys, &iter, begin, AVL_GREATER_EQUAL);
1982 while((pr = physr_get_iter(&iter)) && pr->offset < begin + length) {
1983 pb_unreferenced(region, pr);
1984 physr_remove(region->phys, pr->offset);
1985 physr_start_iter(region->phys, &iter, begin,
1986 AVL_GREATER_EQUAL);
1987 SLABFREE(pr);
1988 }
1989 }
1990
1991 /*===========================================================================*
1992 * map_memory *
1993 *===========================================================================*/
1994 PUBLIC int map_memory(endpoint_t sour, endpoint_t dest,
1995 vir_bytes virt_s, vir_bytes virt_d, vir_bytes length, int flag)
1996 {
1997 /* This is the entry point. This function will be called by handle_memory() when
1998 * VM recieves a map-memory request.
1999 */
2000 struct vmproc *vms, *vmd;
2001 struct vir_region *vrs, *vrd;
2002 physr_iter iterd;
2003 vir_bytes offset_s, offset_d;
2004 int p;
2005 int r;
2006
2007 if(vm_isokendpt(sour, &p) != OK)
2008 panic("map_memory: bad endpoint: %d", sour);
2009 vms = &vmproc[p];
2010 if(vm_isokendpt(dest, &p) != OK)
2011 panic("map_memory: bad endpoint: %d", dest);
2012 vmd = &vmproc[p];
2013
2014 vrs = map_lookup(vms, virt_s);
2015 assert(vrs);
2016 vrd = map_lookup(vmd, virt_d);
2017 assert(vrd);
2018
2019 /* Linear address -> offset from start of vir region. */
2020 offset_s = virt_s - vrs->vaddr;
2021 offset_d = virt_d - vrd->vaddr;
2022
2023 /* Make sure that the range in the source process has been mapped
2024 * to physical memory.
2025 */
2026 map_handle_memory(vms, vrs, offset_s, length, 0);
2027
2028 /* Prepare work. */
2029 clean_phys_regions(vrs, offset_s, length);
2030 clean_phys_regions(vrd, offset_d, length);
2031 rm_phys_regions(vrd, offset_d, length);
2032
2033 /* Map memory. */
2034 r = do_map_memory(vms, vmd, vrs, vrd, offset_s, offset_d, length, flag);
2035
2036 return r;
2037 }
2038
2039 /*========================================================================*
2040 * map_lookup_phys *
2041 *========================================================================*/
2042 phys_bytes
2043 map_lookup_phys(struct vmproc *vmp, u32_t tag)
2044 {
2045 struct vir_region *vr;
2046 struct phys_region *pr;
2047 physr_iter iter;
2048
2049 if(!(vr = map_region_lookup_tag(vmp, tag))) {
2050 printf("VM: request for phys of missing region\n");
2051 return MAP_NONE;
2052 }
2053
2054 physr_start_iter_least(vr->phys, &iter);
2055
2056 if(!(pr = physr_get_iter(&iter))) {
2057 printf("VM: request for phys of unmapped region\n");
2058 return MAP_NONE;
2059 }
2060
2061 if(pr->offset != 0 || pr->ph->length != vr->length) {
2062 printf("VM: request for phys of partially mapped region\n");
2063 return MAP_NONE;
2064 }
2065
2066 return pr->ph->phys;
2067 }
2068
2069
2070
2071
MINIX 3 (repo.or.cz mirror)