?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
fix a warning fredf pointed out on irc
[newos.git] / kernel / vm / vm.c
blobcae41061b51c7d5c44cdf308068a83374b9b6888
1 /*
2 ** Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
3 ** Distributed under the terms of the NewOS License.
4 */
5 #include <kernel/kernel.h>
6 #include <kernel/vm.h>
7 #include <kernel/vm_priv.h>
8 #include <kernel/vm_page.h>
9 #include <kernel/vm_cache.h>
10 #include <kernel/vm_store_anonymous_noswap.h>
11 #include <kernel/vm_store_device.h>
12 #include <kernel/vm_store_null.h>
13 #include <kernel/vm_store_vnode.h>
14 #include <kernel/heap.h>
15 #include <kernel/debug.h>
16 #include <kernel/console.h>
17 #include <kernel/int.h>
18 #include <kernel/smp.h>
19 #include <kernel/sem.h>
20 #include <kernel/lock.h>
21 #include <kernel/khash.h>
22 #include <kernel/time.h>
23 #include <newos/errors.h>
24
25 #include <boot/stage2.h>
26
27 #include <kernel/arch/cpu.h>
28 #include <kernel/arch/vm.h>
29
30 #include <string.h>
31 #include <ctype.h>
32 #include <stdlib.h>
33 #include <stdio.h>
34
35
36 /* global data about the vm, used for informational purposes */
37 vm_info_t vm_info;
38
39 static vm_address_space *kernel_aspace;
40
41 #define REGION_HASH_TABLE_SIZE 1024
42 static region_id next_region_id;
43 static void *region_table;
44 static sem_id region_hash_sem;
45
46 #define ASPACE_HASH_TABLE_SIZE 1024
47 static aspace_id next_aspace_id;
48 static void *aspace_table;
49 static sem_id aspace_hash_sem;
50
51 static int max_commit;
52 static spinlock_t max_commit_lock;
53
54 // function declarations
55 static vm_region *_vm_create_region_struct(vm_address_space *aspace, const char *name, int wiring, int lock);
56 static int map_backing_store(vm_address_space *aspace, vm_store *store, void **vaddr,
57 off_t offset, addr_t size, int addr_type, int wiring, int lock, int mapping, vm_region **_region, const char *region_name);
58 static int vm_soft_fault(addr_t address, bool is_write, bool is_user);
59 static vm_region *vm_virtual_map_lookup(vm_virtual_map *map, addr_t address);
60
61 static int region_compare(void *_r, const void *key)
62 {
63 vm_region *r = _r;
64 const region_id *id = key;
65
66 #if DEBUG > 1
67 VERIFY_VM_REGION(r);
68 #endif
69
70 if(r->id == *id)
71 return 0;
72 else
73 return -1;
74 }
75
76 static unsigned int region_hash(void *_r, const void *key, unsigned int range)
77 {
78 vm_region *r = _r;
79 const region_id *id = key;
80
81 #if DEBUG > 1
82 if(r != NULL)
83 VERIFY_VM_REGION(r);
84 #endif
85
86 if(r != NULL)
87 return (r->id % range);
88 else
89 return (*id % range);
90 }
91
92 static int aspace_compare(void *_a, const void *key)
93 {
94 vm_address_space *aspace = _a;
95 const aspace_id *id = key;
96
97 #if DEBUG > 1
98 VERIFY_VM_ASPACE(aspace);
99 #endif
100
101 if(aspace->id == *id)
102 return 0;
103 else
104 return -1;
105 }
106
107 static unsigned int aspace_hash(void *_a, const void *key, unsigned int range)
108 {
109 vm_address_space *aspace = _a;
110 const aspace_id *id = key;
111
112 #if DEBUG > 1
113 if(aspace != NULL)
114 VERIFY_VM_ASPACE(aspace);
115 #endif
116
117 if(aspace != NULL)
118 return (aspace->id % range);
119 else
120 return (*id % range);
121 }
122
123 vm_address_space *vm_get_aspace_by_id(aspace_id aid)
124 {
125 vm_address_space *aspace;
126
127 sem_acquire(aspace_hash_sem, READ_COUNT);
128 aspace = hash_lookup(aspace_table, &aid);
129 if(aspace) {
130 VERIFY_VM_ASPACE(aspace);
131 atomic_add(&aspace->ref_count, 1);
132 }
133 sem_release(aspace_hash_sem, READ_COUNT);
134
135 return aspace;
136 }
137
138 vm_region *vm_get_region_by_id(region_id rid)
139 {
140 vm_region *region;
141
142 sem_acquire(region_hash_sem, READ_COUNT);
143 region = hash_lookup(region_table, &rid);
144 if(region) {
145 VERIFY_VM_REGION(region);
146 atomic_add(&region->ref_count, 1);
147 }
148 sem_release(region_hash_sem, READ_COUNT);
149
150 return region;
151 }
152
153 region_id vm_find_region_by_name(aspace_id aid, const char *name)
154 {
155 vm_region *region = NULL;
156 vm_address_space *aspace;
157 region_id id = ERR_NOT_FOUND;
158
159 ASSERT(name != NULL);
160
161 aspace = vm_get_aspace_by_id(aid);
162 if(aspace == NULL)
163 return ERR_VM_INVALID_ASPACE;
164
165 sem_acquire(aspace->virtual_map.sem, READ_COUNT);
166
167 region = aspace->virtual_map.region_list;
168 while(region != NULL) {
169 VERIFY_VM_REGION(region);
170 ASSERT(region->name != NULL);
171 if(strcmp(region->name, name) == 0) {
172 id = region->id;
173 break;
174 }
175 region = region->aspace_next;
176 }
177
178 sem_release(aspace->virtual_map.sem, READ_COUNT);
179 vm_put_aspace(aspace);
180 return id;
181 }
182
183 static vm_region *_vm_create_region_struct(vm_address_space *aspace, const char *name, int wiring, int lock)
184 {
185 vm_region *region = NULL;
186
187 VERIFY_VM_ASPACE(aspace);
188 ASSERT(name != NULL);
189
190 region = (vm_region *)kmalloc(sizeof(vm_region));
191 if(region == NULL)
192 return NULL;
193 region->name = (char *)kmalloc(strlen(name) + 1);
194 if(region->name == NULL) {
195 kfree(region);
196 return NULL;
197 }
198 strcpy(region->name, name);
199 region->magic = VM_REGION_MAGIC;
200 region->id = atomic_add(&next_region_id, 1);
201 region->base = 0;
202 region->size = 0;
203 region->lock = lock;
204 region->wiring = wiring;
205 region->ref_count = 1;
206
207 region->cache_ref = NULL;
208 region->cache_offset = 0;
209
210 region->aspace = aspace;
211 region->aspace_next = NULL;
212 region->map = &aspace->virtual_map;
213 region->cache_next = region->cache_prev = NULL;
214 region->hash_next = NULL;
215
216 return region;
217 }
218
219 // must be called with this address space's virtual_map.sem held
220 static int find_and_insert_region_slot(vm_virtual_map *map, addr_t start, addr_t size, addr_t end, int addr_type, vm_region *region)
221 {
222 vm_region *last_r = NULL;
223 vm_region *next_r;
224 bool foundspot = false;
225
226 dprintf("find_and_insert_region_slot: map %p, start 0x%lx, size %ld, end 0x%lx, addr_type %d, region %p\n",
227 map, start, size, end, addr_type, region);
228 // dprintf("map->base 0x%x, map->size 0x%x\n", map->base, map->size);
229
230 // do some sanity checking
231 if(start < map->base || size == 0 || (end - 1) > (map->base + (map->size - 1)) || start + size > end)
232 return ERR_VM_BAD_ADDRESS;
233
234 // walk up to the spot where we should start searching
235 next_r = map->region_list;
236 while(next_r) {
237 if(next_r->base >= start + size) {
238 // we have a winner
239 break;
240 }
241 last_r = next_r;
242 next_r = next_r->aspace_next;
243 }
244
245 #if 0
246 dprintf("last_r 0x%x, next_r 0x%x\n", last_r, next_r);
247 if(last_r) dprintf("last_r->base 0x%x, last_r->size 0x%x\n", last_r->base, last_r->size);
248 if(next_r) dprintf("next_r->base 0x%x, next_r->size 0x%x\n", next_r->base, next_r->size);
249 #endif
250
251 switch(addr_type) {
252 case REGION_ADDR_ANY_ADDRESS:
253 // find a hole big enough for a new region
254 if(!last_r) {
255 // see if we can build it at the beginning of the virtual map
256 if(!next_r || (next_r->base >= map->base + size)) {
257 foundspot = true;
258 region->base = map->base;
259 break;
260 }
261 last_r = next_r;
262 next_r = next_r->aspace_next;
263 }
264 // keep walking
265 while(next_r) {
266 if(next_r->base >= last_r->base + last_r->size + size) {
267 // we found a spot
268 foundspot = true;
269 region->base = last_r->base + last_r->size;
270 break;
271 }
272 last_r = next_r;
273 next_r = next_r->aspace_next;
274 }
275 if((map->base + (map->size - 1)) >= (last_r->base + last_r->size + (size - 1))) {
276 // found a spot
277 foundspot = true;
278 region->base = last_r->base + last_r->size;
279 break;
280 }
281 break;
282 case REGION_ADDR_EXACT_ADDRESS:
283 // see if we can create it exactly here
284 if(!last_r) {
285 if(!next_r || (next_r->base >= start + size)) {
286 foundspot = true;
287 region->base = start;
288 break;
289 }
290 } else {
291 if(next_r) {
292 if(last_r->base + last_r->size <= start && next_r->base >= start + size) {
293 foundspot = true;
294 region->base = start;
295 break;
296 }
297 } else {
298 if((last_r->base + (last_r->size - 1)) <= start - 1) {
299 foundspot = true;
300 region->base = start;
301 }
302 }
303 }
304 break;
305 default:
306 return ERR_INVALID_ARGS;
307 }
308
309 if(foundspot) {
310 region->size = size;
311 if(last_r) {
312 region->aspace_next = last_r->aspace_next;
313 last_r->aspace_next = region;
314 } else {
315 region->aspace_next = map->region_list;
316 map->region_list = region;
317 }
318 map->change_count++;
319 return NO_ERROR;
320 } else {
321 return ERR_VM_NO_REGION_SLOT;
322 }
323 }
324
325 // a ref to the cache holding this store must be held before entering here
326 static int map_backing_store(vm_address_space *aspace, vm_store *store, void **vaddr,
327 off_t offset, addr_t size, int addr_type, int wiring, int lock, int mapping, vm_region **_region, const char *region_name)
328 {
329 vm_cache *cache;
330 vm_cache_ref *cache_ref;
331 vm_region *region;
332 vm_cache *nu_cache;
333 vm_cache_ref *nu_cache_ref = NULL;
334 vm_store *nu_store;
335
336 int err;
337
338 VERIFY_VM_ASPACE(aspace);
339 VERIFY_VM_STORE(store);
340
341 // dprintf("map_backing_store: aspace %p, store %p, *vaddr %p, offset 0x%Lx, size 0x%lx, addr_type %d, wiring %d, lock %d, _region %p, region_name '%s'\n",
342 // aspace, store, *vaddr, offset, size, addr_type, wiring, lock, _region, region_name);
343
344 region = _vm_create_region_struct(aspace, region_name, wiring, lock);
345 if(!region)
346 return ERR_NO_MEMORY;
347
348 cache = store->cache;
349 VERIFY_VM_CACHE(cache);
350 cache_ref = cache->ref;
351 VERIFY_VM_CACHE_REF(cache_ref);
352
353 // if this is a private map, we need to create a new cache & store object
354 // pair to handle the private copies of pages as they are written to
355 if(mapping == REGION_PRIVATE_MAP) {
356 // create an anonymous store object
357 nu_store = vm_store_create_anonymous_noswap();
358 if(nu_store == NULL)
359 panic("map_backing_store: vm_create_store_anonymous_noswap returned NULL");
360 nu_cache = vm_cache_create(nu_store);
361 if(nu_cache == NULL)
362 panic("map_backing_store: vm_cache_create returned NULL");
363 nu_cache_ref = vm_cache_ref_create(nu_cache);
364 if(nu_cache_ref == NULL)
365 panic("map_backing_store: vm_cache_ref_create returned NULL");
366 nu_cache->temporary = 1;
367 nu_cache->scan_skip = cache->scan_skip;
368
369 nu_cache->source = cache;
370
371 // grab a ref to the cache object we're now linked to as a source
372 vm_cache_acquire_ref(cache_ref, true);
373
374 cache = nu_cache;
375 cache_ref = cache->ref;
376 store = nu_store;
377 }
378
379 mutex_lock(&cache_ref->lock);
380 if(store->committed_size < offset + size) {
381 // try to commit more memory
382 off_t old_store_commitment = store->committed_size;
383 off_t commitment = (store->ops->commit)(store, offset + size);
384 if(commitment < offset + size) {
385 if(cache->temporary) {
386 int_disable_interrupts();
387 acquire_spinlock(&max_commit_lock);
388
389 if(max_commit - old_store_commitment + commitment < offset + size) {
390 release_spinlock(&max_commit_lock);
391 int_restore_interrupts();
392 mutex_unlock(&cache_ref->lock);
393 err = ERR_VM_WOULD_OVERCOMMIT;
394 goto err1a;
395 }
396
397 // dprintf("map_backing_store: adding %d to max_commit\n",
398 // (commitment - old_store_commitment) - (offset + size - cache->committed_size));
399
400 max_commit += (commitment - old_store_commitment) - (offset + size - cache->virtual_size);
401 cache->virtual_size = offset + size;
402 release_spinlock(&max_commit_lock);
403 int_restore_interrupts();
404 } else {
405 mutex_unlock(&cache_ref->lock);
406 err = ERR_NO_MEMORY;
407 goto err1a;
408 }
409 }
410 }
411
412 mutex_unlock(&cache_ref->lock);
413
414 vm_cache_acquire_ref(cache_ref, true);
415
416 sem_acquire(aspace->virtual_map.sem, WRITE_COUNT);
417
418 // check to see if this aspace has entered DELETE state
419 if(aspace->state == VM_ASPACE_STATE_DELETION) {
420 // okay, someone is trying to delete this aspace now, so we can't
421 // insert the region, so back out
422 err = ERR_VM_INVALID_ASPACE;
423 goto err1b;
424 }
425
426 {
427 addr_t search_addr, search_end;
428
429 if(addr_type == REGION_ADDR_EXACT_ADDRESS) {
430 search_addr = (addr_t)*vaddr;
431 search_end = (addr_t)*vaddr + size;
432 } else if(addr_type == REGION_ADDR_ANY_ADDRESS) {
433 search_addr = aspace->virtual_map.base;
434 search_end = aspace->virtual_map.base + (aspace->virtual_map.size - 1);
435 } else {
436 err = ERR_INVALID_ARGS;
437 goto err1b;
438 }
439
440 err = find_and_insert_region_slot(&aspace->virtual_map, search_addr, size, search_end, addr_type, region);
441 if(err < 0)
442 goto err1b;
443 *vaddr = (addr_t *)region->base;
444 }
445
446 // attach the cache to the region
447 region->cache_ref = cache_ref;
448 region->cache_offset = offset;
449 // point the cache back to the region
450 vm_cache_insert_region(cache_ref, region);
451
452 // insert the region in the global region hash table
453 sem_acquire(region_hash_sem, WRITE_COUNT);
454 hash_insert(region_table, region);
455 sem_release(region_hash_sem, WRITE_COUNT);
456
457 // grab a ref to the aspace (the region holds this)
458 atomic_add(&aspace->ref_count, 1);
459
460 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
461
462 *_region = region;
463
464 return NO_ERROR;
465
466 err1b:
467 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
468 vm_cache_release_ref(cache_ref);
469 goto err;
470 err1a:
471 if(nu_cache_ref) {
472 // had never acquired it's initial ref, so acquire and then release it
473 // this should clean up all the objects it references
474 vm_cache_acquire_ref(cache_ref, true);
475 vm_cache_release_ref(cache_ref);
476 }
477 err:
478 kfree(region->name);
479 kfree(region);
480 return err;
481 }
482
483 region_id user_vm_create_anonymous_region(char *uname, void **uaddress, int addr_type,
484 addr_t size, int wiring, int lock)
485 {
486 char name[SYS_MAX_OS_NAME_LEN];
487 void *address;
488 int rc, rc2;
489
490 if((addr_t)uname >= KERNEL_BASE && (addr_t)uname <= KERNEL_TOP)
491 return ERR_VM_BAD_USER_MEMORY;
492
493 rc = user_strncpy(name, uname, SYS_MAX_OS_NAME_LEN-1);
494 if(rc < 0)
495 return rc;
496 name[SYS_MAX_OS_NAME_LEN-1] = 0;
497
498 rc = user_memcpy(&address, uaddress, sizeof(address));
499 if(rc < 0)
500 return rc;
501
502 rc = vm_create_anonymous_region(vm_get_current_user_aspace_id(), name, &address, addr_type, size, wiring, lock);
503 if(rc < 0)
504 return rc;
505
506 rc2 = user_memcpy(uaddress, &address, sizeof(address));
507 if(rc2 < 0)
508 return rc2;
509
510 return rc;
511 }
512
513 region_id vm_create_anonymous_region(aspace_id aid, char *name, void **address, int addr_type,
514 addr_t size, int wiring, int lock)
515 {
516 int err;
517 vm_region *region;
518 vm_cache *cache;
519 vm_store *store;
520 vm_address_space *aspace;
521 vm_cache_ref *cache_ref;
522
523 dprintf("create_anonymous_region: name '%s', type %d, size 0x%lx, wiring %d, lock %d\n",
524 name, addr_type, size, wiring, lock);
525
526 if(addr_type != REGION_ADDR_ANY_ADDRESS && addr_type != REGION_ADDR_EXACT_ADDRESS)
527 return ERR_INVALID_ARGS;
528 switch(wiring) {
529 case REGION_WIRING_WIRED:
530 case REGION_WIRING_WIRED_ALREADY:
531 case REGION_WIRING_WIRED_CONTIG:
532 case REGION_WIRING_LAZY:
533 break;
534 default:
535 return ERR_INVALID_ARGS;
536 }
537 aspace = vm_get_aspace_by_id(aid);
538 if(aspace == NULL)
539 return ERR_VM_INVALID_ASPACE;
540
541 size = PAGE_ALIGN(size);
542
543 // create an anonymous store object
544 store = vm_store_create_anonymous_noswap();
545 if(store == NULL)
546 panic("vm_create_anonymous_region: vm_create_store_anonymous_noswap returned NULL");
547 cache = vm_cache_create(store);
548 if(cache == NULL)
549 panic("vm_create_anonymous_region: vm_cache_create returned NULL");
550 cache_ref = vm_cache_ref_create(cache);
551 if(cache_ref == NULL)
552 panic("vm_create_anonymous_region: vm_cache_ref_create returned NULL");
553 cache->temporary = 1;
554
555 switch(wiring) {
556 case REGION_WIRING_WIRED:
557 case REGION_WIRING_WIRED_ALREADY:
558 case REGION_WIRING_WIRED_CONTIG:
559 cache->scan_skip = 1;
560 break;
561 case REGION_WIRING_LAZY:
562 cache->scan_skip = 0;
563 break;
564 }
565
566 // dprintf("create_anonymous_region: calling map_backing store\n");
567
568 vm_cache_acquire_ref(cache_ref, true);
569 err = map_backing_store(aspace, store, address, 0, size, addr_type, wiring, lock, REGION_NO_PRIVATE_MAP, &region, name);
570 vm_cache_release_ref(cache_ref);
571 if(err < 0) {
572 vm_put_aspace(aspace);
573 return err;
574 }
575
576 // dprintf("create_anonymous_region: done calling map_backing store\n");
577
578 cache_ref = store->cache->ref;
579 switch(wiring) {
580 case REGION_WIRING_LAZY:
581 break; // do nothing
582 case REGION_WIRING_WIRED: {
583 // pages aren't mapped at this point, but we just simulate a fault on
584 // every page, which should allocate them
585 addr_t va;
586 // XXX remove
587 for(va = region->base; va < region->base + region->size; va += PAGE_SIZE) {
588 dprintf("mapping wired pages: region %p, cache_ref %p %p, address 0x%lx\n", region, cache_ref, region->cache_ref, va);
589 vm_soft_fault(va, false, false);
590 }
591 break;
592 }
593 case REGION_WIRING_WIRED_ALREADY: {
594 // the pages should already be mapped. This is only really useful during
595 // boot time. Find the appropriate vm_page objects and stick them in
596 // the cache object.
597 addr_t va;
598 addr_t pa;
599 unsigned int flags;
600 int err;
601 vm_page *page;
602 off_t offset = 0;
603
604 mutex_lock(&cache_ref->lock);
605 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
606 for(va = region->base; va < region->base + region->size; va += PAGE_SIZE, offset += PAGE_SIZE) {
607 err = (*aspace->translation_map.ops->query)(&aspace->translation_map,
608 va, &pa, &flags);
609 if(err < 0) {
610 // dprintf("vm_create_anonymous_region: error looking up mapping for va 0x%x\n", va);
611 continue;
612 }
613 // dprintf("vm_create_anonymous_region: looked up page at va 0x%lx. pa 0x%lx\n", va, pa);
614 page = vm_lookup_page(pa / PAGE_SIZE);
615 if(page == NULL) {
616 // dprintf("vm_create_anonymous_region: error looking up vm_page structure for pa 0x%x\n", pa);
617 continue;
618 }
619 atomic_add(&page->ref_count, 1);
620 vm_page_set_state(page, PAGE_STATE_WIRED);
621 vm_cache_insert_page(cache_ref, page, offset);
622 }
623 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
624 mutex_unlock(&cache_ref->lock);
625 break;
626 }
627 case REGION_WIRING_WIRED_CONTIG: {
628 addr_t va;
629 addr_t phys_addr;
630 int err;
631 vm_page *page;
632 off_t offset = 0;
633
634 page = vm_page_allocate_page_run(PAGE_STATE_CLEAR, ROUNDUP(region->size, PAGE_SIZE) / PAGE_SIZE);
635 if(page == NULL) {
636 // XXX back out of this
637 panic("couldn't allocate page run of size %ld\n", region->size);
638 }
639 phys_addr = page->ppn * PAGE_SIZE;
640
641 mutex_lock(&cache_ref->lock);
642 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
643 for(va = region->base; va < region->base + region->size; va += PAGE_SIZE, offset += PAGE_SIZE, phys_addr += PAGE_SIZE) {
644 page = vm_lookup_page(phys_addr / PAGE_SIZE);
645 if(page == NULL) {
646 panic("couldn't lookup physical page just allocated\n");
647 }
648 atomic_add(&page->ref_count, 1);
649 err = (*aspace->translation_map.ops->map)(&aspace->translation_map, va, phys_addr, lock);
650 if(err < 0) {
651 panic("couldn't map physical page in page run\n");
652 }
653 vm_page_set_state(page, PAGE_STATE_WIRED);
654 vm_cache_insert_page(cache_ref, page, offset);
655 }
656 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
657 mutex_unlock(&cache_ref->lock);
658
659 break;
660 }
661 default:
662 ;
663 }
664 vm_put_aspace(aspace);
665 dprintf("create_anonymous_region: done\n");
666 if(region)
667 return region->id;
668 else
669 return ERR_NO_MEMORY;
670 }
671
672 region_id vm_map_physical_memory(aspace_id aid, char *name, void **address, int addr_type,
673 addr_t size, int lock, addr_t phys_addr)
674 {
675 vm_region *region;
676 vm_cache *cache;
677 vm_cache_ref *cache_ref;
678 vm_store *store;
679 addr_t map_offset;
680 int err;
681
682 vm_address_space *aspace = vm_get_aspace_by_id(aid);
683 if(aspace == NULL)
684 return ERR_VM_INVALID_ASPACE;
685
686 // if the physical address is somewhat inside a page,
687 // move the actual region down to align on a page boundary
688 map_offset = phys_addr % PAGE_SIZE;
689 size += map_offset;
690 phys_addr -= map_offset;
691
692 size = PAGE_ALIGN(size);
693
694 // create an device store object
695 store = vm_store_create_device(phys_addr);
696 if(store == NULL)
697 panic("vm_map_physical_memory: vm_store_create_device returned NULL");
698 cache = vm_cache_create(store);
699 if(cache == NULL)
700 panic("vm_map_physical_memory: vm_cache_create returned NULL");
701 cache_ref = vm_cache_ref_create(cache);
702 if(cache_ref == NULL)
703 panic("vm_map_physical_memory: vm_cache_ref_create returned NULL");
704
705 // tell the page scanner to skip over this region, it's pages are special
706 cache->scan_skip = 1;
707
708 vm_cache_acquire_ref(cache_ref, true);
709 err = map_backing_store(aspace, store, address, 0, size, addr_type, 0, lock, REGION_NO_PRIVATE_MAP, &region, name);
710 vm_cache_release_ref(cache_ref);
711 vm_put_aspace(aspace);
712 if(err < 0) {
713 return err;
714 }
715
716 // modify the pointer returned to be offset back into the new region
717 // the same way the physical address in was offset
718 ((addr_t)(*address)) += map_offset;
719 return region->id;
720 }
721
722 region_id vm_create_null_region(aspace_id aid, char *name, void **address, int addr_type, addr_t size)
723 {
724 vm_region *region;
725 vm_cache *cache;
726 vm_cache_ref *cache_ref;
727 vm_store *store;
728 int err;
729
730 vm_address_space *aspace = vm_get_aspace_by_id(aid);
731 if(aspace == NULL)
732 return ERR_VM_INVALID_ASPACE;
733
734 size = PAGE_ALIGN(size);
735
736 // create an null store object
737 store = vm_store_create_null();
738 if(store == NULL)
739 panic("vm_create_null_region: vm_store_create_null returned NULL");
740 cache = vm_cache_create(store);
741 if(cache == NULL)
742 panic("vm_create_null_region: vm_cache_create returned NULL");
743 cache_ref = vm_cache_ref_create(cache);
744 if(cache_ref == NULL)
745 panic("vm_create_null_region: vm_cache_ref_create returned NULL");
746 // tell the page scanner to skip over this region, no pages will be mapped here
747 cache->scan_skip = 1;
748
749 vm_cache_acquire_ref(cache_ref, true);
750 err = map_backing_store(aspace, store, address, 0, size, addr_type, 0, LOCK_RO, REGION_NO_PRIVATE_MAP, &region, name);
751 vm_cache_release_ref(cache_ref);
752 vm_put_aspace(aspace);
753 if(err < 0)
754 return err;
755
756 return region->id;
757 }
758
759 static region_id _vm_map_file(aspace_id aid, char *name, void **address, int addr_type,
760 addr_t size, int lock, int mapping, const char *path, off_t offset, bool kernel)
761 {
762 vm_region *region;
763 vm_cache *cache;
764 vm_cache_ref *cache_ref;
765 vm_store *store;
766 void *v;
767 int err;
768
769 vm_address_space *aspace = vm_get_aspace_by_id(aid);
770 if(aspace == NULL)
771 return ERR_VM_INVALID_ASPACE;
772
773 offset = ROUNDOWN(offset, PAGE_SIZE);
774 size = PAGE_ALIGN(size);
775
776 restart:
777 // get the vnode for the object, this also grabs a ref to it
778 err = vfs_get_vnode_from_path(path, kernel, &v);
779 if(err < 0) {
780 vm_put_aspace(aspace);
781 return err;
782 }
783
784 cache_ref = vfs_get_cache_ptr(v);
785 if(!cache_ref) {
786 // create a vnode store object
787 store = vm_store_create_vnode(v);
788 if(store == NULL)
789 panic("vm_map_file: couldn't create vnode store");
790 cache = vm_cache_create(store);
791 if(cache == NULL)
792 panic("vm_map_physical_memory: vm_cache_create returned NULL");
793 cache_ref = vm_cache_ref_create(cache);
794 if(cache_ref == NULL)
795 panic("vm_map_physical_memory: vm_cache_ref_create returned NULL");
796
797 // acquire the cache ref once to represent the ref that the vnode will have
798 // this is one of the only places where we dont want to ref to ripple down to the store
799 vm_cache_acquire_ref(cache_ref, false);
800
801 // try to set the cache ptr in the vnode
802 if(vfs_set_cache_ptr(v, cache_ref) < 0) {
803 // the cache pointer was set between here and then
804 // this can only happen if someone else tries to map it
805 // at the same time. Rare enough to not worry about the
806 // performance impact of undoing what we just did and retrying
807
808 // this will delete the cache object and release the ref to the vnode we have
809 vm_cache_release_ref(cache_ref);
810 goto restart;
811 }
812 } else {
813 VERIFY_VM_CACHE_REF(cache_ref);
814 cache = cache_ref->cache;
815 VERIFY_VM_CACHE(cache);
816 store = cache->store;
817 VERIFY_VM_STORE(store);
818 }
819
820 // acquire a ref to the cache before we do work on it. Dont ripple the ref acquision to the vnode
821 // below because we'll have to release it later anyway, since we grabbed a ref to the vnode at
822 // vfs_get_vnode_from_path(). This puts the ref counts in sync.
823 vm_cache_acquire_ref(cache_ref, false);
824 err = map_backing_store(aspace, store, address, offset, size, addr_type, 0, lock, mapping, &region, name);
825 vm_cache_release_ref(cache_ref);
826 vm_put_aspace(aspace);
827 if(err < 0)
828 return err;
829
830 // modify the pointer returned to be offset back into the new region
831 // the same way the physical address in was offset
832 return region->id;
833 }
834
835 region_id vm_map_file(aspace_id aid, char *name, void **address, int addr_type,
836 addr_t size, int lock, int mapping, const char *path, off_t offset)
837 {
838 return _vm_map_file(aid, name, address, addr_type, size, lock, mapping, path, offset, true);
839 }
840
841 region_id user_vm_map_file(char *uname, void **uaddress, int addr_type,
842 addr_t size, int lock, int mapping, const char *upath, off_t offset)
843 {
844 char name[SYS_MAX_OS_NAME_LEN];
845 void *address;
846 char path[SYS_MAX_PATH_LEN];
847 int rc, rc2;
848
849 if((addr_t)uname >= KERNEL_BASE && (addr_t)uname <= KERNEL_TOP)
850 return ERR_VM_BAD_USER_MEMORY;
851
852 if((addr_t)uaddress >= KERNEL_BASE && (addr_t)uaddress <= KERNEL_TOP)
853 return ERR_VM_BAD_USER_MEMORY;
854
855 if((addr_t)upath >= KERNEL_BASE && (addr_t)upath <= KERNEL_TOP)
856 return ERR_VM_BAD_USER_MEMORY;
857
858 rc = user_strncpy(name, uname, SYS_MAX_OS_NAME_LEN-1);
859 if(rc < 0)
860 return rc;
861 name[SYS_MAX_OS_NAME_LEN-1] = 0;
862
863 rc = user_strncpy(path, upath, SYS_MAX_PATH_LEN-1);
864 if(rc < 0)
865 return rc;
866 path[SYS_MAX_PATH_LEN-1] = 0;
867
868 rc = user_memcpy(&address, uaddress, sizeof(address));
869 if(rc < 0)
870 return rc;
871
872 rc = _vm_map_file(vm_get_current_user_aspace_id(), name, &address, addr_type, size, lock, mapping, path, offset, false);
873 if(rc < 0)
874 return rc;
875
876 rc2 = user_memcpy(uaddress, &address, sizeof(address));
877 if(rc2 < 0)
878 return rc2;
879
880 return rc;
881 }
882
883 region_id user_vm_clone_region(char *uname, void **uaddress, int addr_type,
884 region_id source_region, int mapping, int lock)
885 {
886 char name[SYS_MAX_OS_NAME_LEN];
887 void *address;
888 int rc, rc2;
889
890 if((addr_t)uname >= KERNEL_BASE && (addr_t)uname <= KERNEL_TOP)
891 return ERR_VM_BAD_USER_MEMORY;
892
893 if((addr_t)uaddress >= KERNEL_BASE && (addr_t)uaddress <= KERNEL_TOP)
894 return ERR_VM_BAD_USER_MEMORY;
895
896 rc = user_strncpy(name, uname, SYS_MAX_OS_NAME_LEN-1);
897 if(rc < 0)
898 return rc;
899 name[SYS_MAX_OS_NAME_LEN-1] = 0;
900
901 rc = user_memcpy(&address, uaddress, sizeof(address));
902 if(rc < 0)
903 return rc;
904
905 rc = vm_clone_region(vm_get_current_user_aspace_id(), name, &address, addr_type, source_region, mapping, lock);
906 if(rc < 0)
907 return rc;
908
909 rc2 = user_memcpy(uaddress, &address, sizeof(address));
910 if(rc2 < 0)
911 return rc2;
912
913 return rc;
914 }
915
916 region_id vm_clone_region(aspace_id aid, char *name, void **address, int addr_type,
917 region_id source_region, int mapping, int lock)
918 {
919 vm_region *new_region;
920 vm_region *src_region;
921 int err;
922
923 vm_address_space *aspace = vm_get_aspace_by_id(aid);
924 if(aspace == NULL)
925 return ERR_VM_INVALID_ASPACE;
926
927 src_region = vm_get_region_by_id(source_region);
928 if(src_region == NULL) {
929 vm_put_aspace(aspace);
930 return ERR_VM_INVALID_REGION;
931 }
932
933 vm_cache_acquire_ref(src_region->cache_ref, true);
934 err = map_backing_store(aspace, src_region->cache_ref->cache->store, address, src_region->cache_offset, src_region->size,
935 addr_type, src_region->wiring, lock, mapping, &new_region, name);
936 vm_cache_release_ref(src_region->cache_ref);
937
938 // release the ref on the old region
939 vm_put_region(src_region);
940
941 vm_put_aspace(aspace);
942
943 if(err < 0)
944 return err;
945 else
946 return new_region->id;
947 }
948
949 static int __vm_delete_region(vm_address_space *aspace, vm_region *region)
950 {
951 VERIFY_VM_ASPACE(aspace);
952 VERIFY_VM_REGION(region);
953
954 if(region->aspace == aspace)
955 vm_put_region(region);
956 return NO_ERROR;
957 }
958
959 static int _vm_delete_region(vm_address_space *aspace, region_id rid)
960 {
961 vm_region *region;
962
963 dprintf("vm_delete_region: aspace id 0x%x, region id 0x%x\n", aspace->id, rid);
964
965 VERIFY_VM_ASPACE(aspace);
966
967 region = vm_get_region_by_id(rid);
968 if(region == NULL)
969 return ERR_VM_INVALID_REGION;
970
971 __vm_delete_region(aspace, region);
972 vm_put_region(region);
973
974 return 0;
975 }
976
977 int vm_delete_region(aspace_id aid, region_id rid)
978 {
979 vm_address_space *aspace;
980 int err;
981
982 aspace = vm_get_aspace_by_id(aid);
983 if(aspace == NULL)
984 return ERR_VM_INVALID_ASPACE;
985
986 err = _vm_delete_region(aspace, rid);
987 vm_put_aspace(aspace);
988 return err;
989 }
990
991 static void _vm_put_region(vm_region *region, bool aspace_locked)
992 {
993 vm_region *temp, *last = NULL;
994 vm_address_space *aspace;
995 bool removeit = false;
996
997 VERIFY_VM_REGION(region);
998
999 sem_acquire(region_hash_sem, WRITE_COUNT);
1000 if(atomic_add(&region->ref_count, -1) == 1) {
1001 hash_remove(region_table, region);
1002 removeit = true;
1003 }
1004 sem_release(region_hash_sem, WRITE_COUNT);
1005
1006 if(!removeit)
1007 return;
1008
1009 aspace = region->aspace;
1010 VERIFY_VM_ASPACE(aspace);
1011
1012 // remove the region from the aspace's virtual map
1013 if(!aspace_locked)
1014 sem_acquire(aspace->virtual_map.sem, WRITE_COUNT);
1015 temp = aspace->virtual_map.region_list;
1016 while(temp != NULL) {
1017 if(region == temp) {
1018 if(last != NULL) {
1019 last->aspace_next = temp->aspace_next;
1020 } else {
1021 aspace->virtual_map.region_list = temp->aspace_next;
1022 }
1023 aspace->virtual_map.change_count++;
1024 break;
1025 }
1026 last = temp;
1027 temp = temp->aspace_next;
1028 }
1029 if(region == aspace->virtual_map.region_hint)
1030 aspace->virtual_map.region_hint = NULL;
1031 if(!aspace_locked)
1032 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
1033
1034 if(temp == NULL)
1035 panic("vm_region_release_ref: region not found in aspace's region_list\n");
1036
1037 vm_cache_remove_region(region->cache_ref, region);
1038 vm_cache_release_ref(region->cache_ref);
1039
1040 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
1041 (*aspace->translation_map.ops->unmap)(&aspace->translation_map, region->base,
1042 region->base + (region->size - 1));
1043 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
1044
1045 // now we can give up the last ref to the aspace
1046 vm_put_aspace(aspace);
1047
1048 if(region->name)
1049 kfree(region->name);
1050 kfree(region);
1051
1052 return;
1053 }
1054
1055 void vm_put_region(vm_region *region)
1056 {
1057 return _vm_put_region(region, false);
1058 }
1059
1060 int user_vm_get_region_info(region_id id, vm_region_info *uinfo)
1061 {
1062 vm_region_info info;
1063 int rc, rc2;
1064
1065 if((addr_t)uinfo >= KERNEL_BASE && (addr_t)uinfo <= KERNEL_TOP)
1066 return ERR_VM_BAD_USER_MEMORY;
1067
1068 rc = vm_get_region_info(id, &info);
1069 if(rc < 0)
1070 return rc;
1071
1072 rc2 = user_memcpy(uinfo, &info, sizeof(info));
1073 if(rc2 < 0)
1074 return rc2;
1075
1076 return rc;
1077 }
1078
1079 int vm_get_region_info(region_id id, vm_region_info *info)
1080 {
1081 vm_region *region;
1082
1083 if(info == NULL)
1084 return ERR_INVALID_ARGS;
1085
1086 region = vm_get_region_by_id(id);
1087 if(region == NULL)
1088 return ERR_VM_INVALID_REGION;
1089
1090 info->id = region->id;
1091 info->base = region->base;
1092 info->size = region->size;
1093 info->lock = region->lock;
1094 info->wiring = region->wiring;
1095 strncpy(info->name, region->name, SYS_MAX_OS_NAME_LEN-1);
1096 info->name[SYS_MAX_OS_NAME_LEN-1] = 0;
1097
1098 vm_put_region(region);
1099
1100 return 0;
1101 }
1102
1103 int vm_get_page_mapping(aspace_id aid, addr_t vaddr, addr_t *paddr)
1104 {
1105 vm_address_space *aspace;
1106 unsigned int null_flags;
1107 int err;
1108
1109 aspace = vm_get_aspace_by_id(aid);
1110 if(aspace == NULL)
1111 return ERR_VM_INVALID_ASPACE;
1112
1113 err = aspace->translation_map.ops->query(&aspace->translation_map,
1114 vaddr, paddr, &null_flags);
1115 vm_put_aspace(aspace);
1116 return err;
1117 }
1118
1119 static void display_mem(int argc, char **argv)
1120 {
1121 int item_size;
1122 int display_width;
1123 int num = 1;
1124 addr_t address;
1125 int i;
1126 int j;
1127
1128 if(argc < 2) {
1129 dprintf("not enough arguments\n");
1130 return;
1131 }
1132
1133 address = atoul(argv[1]);
1134
1135 if(argc >= 3) {
1136 num = -1;
1137 num = atoi(argv[2]);
1138 }
1139
1140 // build the format string
1141 if(strcmp(argv[0], "db") == 0) {
1142 item_size = 1;
1143 display_width = 16;
1144 } else if(strcmp(argv[0], "ds") == 0) {
1145 item_size = 2;
1146 display_width = 8;
1147 } else if(strcmp(argv[0], "dw") == 0) {
1148 item_size = 4;
1149 display_width = 4;
1150 } else {
1151 dprintf("display_mem called in an invalid way!\n");
1152 return;
1153 }
1154
1155 dprintf("[0x%lx] '", address);
1156 for(j=0; j<min(display_width, num) * item_size; j++) {
1157 char c = *((char *)address + j);
1158 if(!isalnum(c)) {
1159 c = '.';
1160 }
1161 dprintf("%c", c);
1162 }
1163 dprintf("'");
1164 for(i=0; i<num; i++) {
1165 if((i % display_width) == 0 && i != 0) {
1166 dprintf("\n[0x%lx] '", address + i * item_size);
1167 for(j=0; j<min(display_width, (num-i)) * item_size; j++) {
1168 char c = *((char *)address + i * item_size + j);
1169 if(!isalnum(c)) {
1170 c = '.';
1171 }
1172 dprintf("%c", c);
1173 }
1174 dprintf("'");
1175 }
1176
1177 switch(item_size) {
1178 case 1:
1179 dprintf(" 0x%02x", *((uint8 *)address + i));
1180 break;
1181 case 2:
1182 dprintf(" 0x%04x", *((uint16 *)address + i));
1183 break;
1184 case 4:
1185 dprintf(" 0x%08x", *((uint32 *)address + i));
1186 break;
1187 default:
1188 dprintf("huh?\n");
1189 }
1190 }
1191 dprintf("\n");
1192 }
1193
1194 static void dump_cache_ref(int argc, char **argv)
1195 {
1196 addr_t address;
1197 vm_region *region;
1198 vm_cache_ref *cache_ref;
1199
1200 if(argc < 2) {
1201 dprintf("cache_ref: not enough arguments\n");
1202 return;
1203 }
1204 if(strlen(argv[1]) < 2 || argv[1][0] != '0' || argv[1][1] != 'x') {
1205 dprintf("cache_ref: invalid argument, pass address\n");
1206 return;
1207 }
1208
1209 address = atoul(argv[1]);
1210 cache_ref = (vm_cache_ref *)address;
1211
1212 dprintf("cache_ref at %p:\n", cache_ref);
1213 dprintf("magic: 0x%x ", cache_ref->magic);
1214 if(cache_ref->magic == VM_CACHE_REF_MAGIC)
1215 dprintf("(GOOD)\n");
1216 else
1217 dprintf("(BAD!)\n");
1218 dprintf("cache: %p\n", cache_ref->cache);
1219 dprintf("lock.holder: %d\n", cache_ref->lock.holder);
1220 dprintf("lock.sem: 0x%x\n", cache_ref->lock.sem);
1221 dprintf("region_list:\n");
1222 for(region = cache_ref->region_list; region != NULL; region = region->cache_next) {
1223 dprintf(" region 0x%x: ", region->id);
1224 dprintf("base_addr = 0x%lx ", region->base);
1225 dprintf("size = 0x%lx ", region->size);
1226 dprintf("name = '%s' ", region->name);
1227 dprintf("lock = 0x%x\n", region->lock);
1228 }
1229 dprintf("ref_count: %d\n", cache_ref->ref_count);
1230 }
1231
1232 static const char *page_state_to_text(int state)
1233 {
1234 switch(state) {
1235 case PAGE_STATE_ACTIVE:
1236 return "active";
1237 case PAGE_STATE_INACTIVE:
1238 return "inactive";
1239 case PAGE_STATE_BUSY:
1240 return "busy";
1241 case PAGE_STATE_MODIFIED:
1242 return "modified";
1243 case PAGE_STATE_FREE:
1244 return "free";
1245 case PAGE_STATE_CLEAR:
1246 return "clear";
1247 case PAGE_STATE_WIRED:
1248 return "wired";
1249 case PAGE_STATE_UNUSED:
1250 return "unused";
1251 default:
1252 return "unknown";
1253 }
1254 }
1255
1256 static void dump_cache(int argc, char **argv)
1257 {
1258 addr_t address;
1259 vm_cache *cache;
1260 vm_page *page;
1261
1262 if(argc < 2) {
1263 dprintf("cache: not enough arguments\n");
1264 return;
1265 }
1266 if(strlen(argv[1]) < 2 || argv[1][0] != '0' || argv[1][1] != 'x') {
1267 dprintf("cache: invalid argument, pass address\n");
1268 return;
1269 }
1270
1271 address = atoul(argv[1]);
1272 cache = (vm_cache *)address;
1273
1274 dprintf("cache at %p:\n", cache);
1275 dprintf("magic: 0x%x ", cache->magic);
1276 if(cache->magic == VM_CACHE_MAGIC)
1277 dprintf("(GOOD)\n");
1278 else
1279 dprintf("(BAD!)\n");
1280 dprintf("cache_ref: %p\n", cache->ref);
1281 dprintf("source: %p\n", cache->source);
1282 dprintf("store: %p\n", cache->store);
1283 dprintf("temporary: %d\n", cache->temporary);
1284 dprintf("scan_skip: %d\n", cache->scan_skip);
1285 dprintf("virtual_size: 0x%Lx\n", cache->virtual_size);
1286 dprintf("page_list:\n");
1287 for(page = cache->page_list; page != NULL; page = page->cache_next) {
1288 if(page->type == PAGE_TYPE_PHYSICAL)
1289 dprintf(" %p ppn 0x%lx offset 0x%Lx type %d state %d (%s) ref_count %d\n",
1290 page, page->ppn, page->offset, page->type, page->state, page_state_to_text(page->state), page->ref_count);
1291 else if(page->type == PAGE_TYPE_DUMMY)
1292 dprintf(" %p DUMMY PAGE state %d (%s)\n", page, page->state, page_state_to_text(page->state));
1293 else
1294 dprintf(" %p UNKNOWN PAGE type %d\n", page, page->type);
1295 }
1296 }
1297
1298 static void _dump_region(vm_region *region)
1299 {
1300 dprintf("dump of region at %p:\n", region);
1301 dprintf("magic: 0x%x ", region->magic);
1302 if(region->magic == VM_REGION_MAGIC)
1303 dprintf("(GOOD)\n");
1304 else
1305 dprintf("(BAD!)\n");
1306 dprintf("name: '%s'\n", region->name);
1307 dprintf("id: 0x%x\n", region->id);
1308 dprintf("base: 0x%lx\n", region->base);
1309 dprintf("size: 0x%lx\n", region->size);
1310 dprintf("lock: 0x%x\n", region->lock);
1311 dprintf("wiring: 0x%x\n", region->wiring);
1312 dprintf("ref_count: %d\n", region->ref_count);
1313 dprintf("cache_offset: 0x%Lx\n", region->cache_offset);
1314 dprintf("cache_ref: %p\n", region->cache_ref);
1315 dprintf("aspace: %p\n", region->aspace);
1316 dprintf("aspace_next: %p\n", region->aspace_next);
1317 dprintf("cache_next: %p\n", region->cache_next);
1318 dprintf("cache_prev: %p\n", region->cache_prev);
1319 dprintf("hash_next: %p\n", region->hash_next);
1320 }
1321
1322 static void dump_region(int argc, char **argv)
1323 {
1324 vm_region *region;
1325
1326 if(argc < 2) {
1327 dprintf("region: not enough arguments\n");
1328 return;
1329 }
1330
1331 // if the argument looks like a hex number, treat it as such
1332 if(strlen(argv[1]) > 2 && argv[1][0] == '0' && argv[1][1] == 'x') {
1333 unsigned long num = atoul(argv[1]);
1334 region_id id = num;
1335
1336 region = hash_lookup(region_table, &id);
1337 if(region == NULL) {
1338 dprintf("invalid region id\n");
1339 } else {
1340 _dump_region(region);
1341 }
1342 return;
1343 } else {
1344 // walk through the region list, looking for the arguments as a name
1345 struct hash_iterator iter;
1346
1347 hash_open(region_table, &iter);
1348 while((region = hash_next(region_table, &iter)) != NULL) {
1349 if(region->name != NULL && strcmp(argv[1], region->name) == 0) {
1350 _dump_region(region);
1351 }
1352 }
1353 }
1354 }
1355
1356 static void dump_region_list(int argc, char **argv)
1357 {
1358 vm_region *region;
1359 struct hash_iterator iter;
1360
1361 dprintf("addr\tid\t%32s\tbase\t\tsize\tlock\twiring\n", "name");
1362
1363 hash_open(region_table, &iter);
1364 while((region = hash_next(region_table, &iter)) != NULL) {
1365 dprintf("%p\t0x%x\t%32s\t0x%lx\t\t0x%lx\t%d\t%d\n",
1366 region, region->id, region->name, region->base, region->size, region->lock, region->wiring);
1367 }
1368 hash_close(region_table, &iter, false);
1369 }
1370
1371 static void _dump_aspace(vm_address_space *aspace)
1372 {
1373 vm_region *region;
1374
1375 dprintf("dump of address space at %p:\n", aspace);
1376 dprintf("magic: 0x%x ", aspace->magic);
1377 if(aspace->magic == VM_ASPACE_MAGIC)
1378 dprintf("(GOOD)\n");
1379 else
1380 dprintf("(BAD!)\n");
1381 dprintf("name: '%s'\n", aspace->name);
1382 dprintf("id: 0x%x\n", aspace->id);
1383 dprintf("ref_count: %d\n", aspace->ref_count);
1384 dprintf("fault_count: %d\n", aspace->fault_count);
1385 dprintf("state: %d\n", aspace->state);
1386 dprintf("scan_va: 0x%lx\n", aspace->scan_va);
1387 dprintf("working_set_size: 0x%lx\n", aspace->working_set_size);
1388 dprintf("max_working_set: 0x%lx\n", aspace->max_working_set);
1389 dprintf("min_working_set: 0x%lx\n", aspace->min_working_set);
1390 dprintf("last_working_set_adjust: %Ld\n", aspace->last_working_set_adjust);
1391 dprintf("hash_next: %p\n", aspace->hash_next);
1392 dprintf("translation_map: %p\n", &aspace->translation_map);
1393 dprintf("virtual_map.base: 0x%lx\n", aspace->virtual_map.base);
1394 dprintf("virtual_map.size: 0x%lx\n", aspace->virtual_map.size);
1395 dprintf("virtual_map.change_count: 0x%x\n", aspace->virtual_map.change_count);
1396 dprintf("virtual_map.sem: 0x%x\n", aspace->virtual_map.sem);
1397 dprintf("virtual_map.region_hint: %p\n", aspace->virtual_map.region_hint);
1398 dprintf("virtual_map.region_list:\n");
1399 for(region = aspace->virtual_map.region_list; region != NULL; region = region->aspace_next) {
1400 dprintf(" region 0x%x: ", region->id);
1401 dprintf("base_addr = 0x%lx ", region->base);
1402 dprintf("size = 0x%lx ", region->size);
1403 dprintf("name = '%s' ", region->name);
1404 dprintf("lock = 0x%x\n", region->lock);
1405 }
1406 }
1407
1408 static void dump_aspace(int argc, char **argv)
1409 {
1410 vm_address_space *aspace;
1411
1412 if(argc < 2) {
1413 dprintf("aspace: not enough arguments\n");
1414 return;
1415 }
1416
1417 // if the argument looks like a hex number, treat it as such
1418 if(strlen(argv[1]) > 2 && argv[1][0] == '0' && argv[1][1] == 'x') {
1419 unsigned long num = atoul(argv[1]);
1420 aspace_id id = num;
1421
1422 aspace = hash_lookup(aspace_table, &id);
1423 if(aspace == NULL) {
1424 dprintf("invalid aspace id\n");
1425 } else {
1426 _dump_aspace(aspace);
1427 }
1428 return;
1429 } else {
1430 // walk through the aspace list, looking for the arguments as a name
1431 struct hash_iterator iter;
1432
1433 hash_open(aspace_table, &iter);
1434 while((aspace = hash_next(aspace_table, &iter)) != NULL) {
1435 if(aspace->name != NULL && strcmp(argv[1], aspace->name) == 0) {
1436 _dump_aspace(aspace);
1437 }
1438 }
1439 }
1440 }
1441
1442 static void dump_aspace_list(int argc, char **argv)
1443 {
1444 vm_address_space *as;
1445 struct hash_iterator iter;
1446
1447 dprintf("addr\tid\t%32s\tbase\t\tsize\n", "name");
1448
1449 hash_open(aspace_table, &iter);
1450 while((as = hash_next(aspace_table, &iter)) != NULL) {
1451 dprintf("%p\t0x%x\t%32s\t0x%lx\t\t0x%lx\n",
1452 as, as->id, as->name, as->virtual_map.base, as->virtual_map.size);
1453 }
1454 hash_close(aspace_table, &iter, false);
1455 }
1456
1457 vm_address_space *vm_get_kernel_aspace(void)
1458 {
1459 VERIFY_VM_ASPACE(kernel_aspace);
1460
1461 /* we can treat this one a little differently since it can't be deleted */
1462 sem_acquire(aspace_hash_sem, READ_COUNT);
1463 atomic_add(&kernel_aspace->ref_count, 1);
1464 sem_release(aspace_hash_sem, READ_COUNT);
1465 return kernel_aspace;
1466 }
1467
1468 aspace_id vm_get_kernel_aspace_id(void)
1469 {
1470 VERIFY_VM_ASPACE(kernel_aspace);
1471 return kernel_aspace->id;
1472 }
1473
1474 vm_address_space *vm_get_current_user_aspace(void)
1475 {
1476 return vm_get_aspace_by_id(vm_get_current_user_aspace_id());
1477 }
1478
1479 aspace_id vm_get_current_user_aspace_id(void)
1480 {
1481 struct thread *t = thread_get_current_thread();
1482
1483 if(t)
1484 return t->proc->aspace_id;
1485 else
1486 return -1;
1487 }
1488
1489 void vm_put_aspace(vm_address_space *aspace)
1490 {
1491 bool removeit = false;
1492
1493 VERIFY_VM_ASPACE(aspace);
1494
1495 sem_acquire(aspace_hash_sem, WRITE_COUNT);
1496 if(atomic_add(&aspace->ref_count, -1) == 1) {
1497 hash_remove(aspace_table, aspace);
1498 removeit = true;
1499 }
1500 sem_release(aspace_hash_sem, WRITE_COUNT);
1501
1502 if(!removeit)
1503 return;
1504
1505 dprintf("vm_put_aspace: reached zero ref, deleting aspace\n");
1506
1507 if(aspace == kernel_aspace)
1508 panic("vm_put_aspace: tried to delete the kernel aspace!\n");
1509
1510 if(aspace->state != VM_ASPACE_STATE_DELETION)
1511 panic("vm_put_apsace: removed the last ref to aspace %p that's not in the DELETION state\n", aspace);
1512
1513 if(aspace->virtual_map.region_list)
1514 panic("vm_put_aspace: aspace at %p has zero ref count, but region list isn't empty!\n", aspace);
1515
1516 (*aspace->translation_map.ops->destroy)(&aspace->translation_map);
1517
1518 kfree(aspace->name);
1519 sem_delete(aspace->virtual_map.sem);
1520 kfree(aspace);
1521
1522 return;
1523 }
1524
1525 aspace_id vm_create_aspace(const char *name, addr_t base, addr_t size, bool kernel)
1526 {
1527 vm_address_space *aspace;
1528 int err;
1529
1530 aspace = (vm_address_space *)kmalloc(sizeof(vm_address_space));
1531 if(aspace == NULL) {
1532 err = ERR_NO_MEMORY;
1533 goto err;
1534 }
1535
1536 aspace->name = (char *)kmalloc(strlen(name) + 1);
1537 if(aspace->name == NULL ) {
1538 err = ERR_NO_MEMORY;
1539 goto err1;
1540 }
1541 strcpy(aspace->name, name);
1542
1543 aspace->magic = VM_ASPACE_MAGIC;
1544 aspace->id = next_aspace_id++;
1545 aspace->ref_count = 1;
1546 aspace->state = VM_ASPACE_STATE_NORMAL;
1547 aspace->fault_count = 0;
1548 aspace->scan_va = base;
1549 aspace->working_set_size = kernel ? DEFAULT_KERNEL_WORKING_SET : DEFAULT_WORKING_SET;
1550 aspace->max_working_set = DEFAULT_MAX_WORKING_SET;
1551 aspace->min_working_set = DEFAULT_MIN_WORKING_SET;
1552 aspace->last_working_set_adjust = system_time();
1553
1554 // initialize the corresponding translation map
1555 err = vm_translation_map_create(&aspace->translation_map, kernel);
1556 if(err < 0)
1557 goto err2;
1558
1559 // initialize the virtual map
1560 aspace->virtual_map.base = base;
1561 aspace->virtual_map.size = size;
1562 aspace->virtual_map.region_list = NULL;
1563 aspace->virtual_map.region_hint = NULL;
1564 aspace->virtual_map.change_count = 0;
1565 aspace->virtual_map.sem = sem_create(WRITE_COUNT, "aspacelock");
1566 aspace->virtual_map.aspace = aspace;
1567
1568 // add the aspace to the global hash table
1569 sem_acquire(aspace_hash_sem, WRITE_COUNT);
1570 hash_insert(aspace_table, aspace);
1571 sem_release(aspace_hash_sem, WRITE_COUNT);
1572
1573 return aspace->id;
1574
1575 err2:
1576 kfree(aspace->name);
1577 err1:
1578 kfree(aspace);
1579 err:
1580 return err;
1581 }
1582
1583 int vm_delete_aspace(aspace_id aid)
1584 {
1585 vm_region *region;
1586 vm_region *next;
1587 vm_address_space *aspace;
1588
1589 aspace = vm_get_aspace_by_id(aid);
1590 if(aspace == NULL)
1591 return ERR_VM_INVALID_ASPACE;
1592
1593 dprintf("vm_delete_aspace: called on aspace 0x%x\n", aid);
1594
1595 // put this aspace in the deletion state
1596 // this guarantees that no one else will add regions to the list
1597 sem_acquire(aspace->virtual_map.sem, WRITE_COUNT);
1598 if(aspace->state == VM_ASPACE_STATE_DELETION) {
1599 // abort, someone else is already deleting this aspace
1600 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
1601 vm_put_aspace(aspace);
1602 return NO_ERROR;
1603 }
1604 aspace->state = VM_ASPACE_STATE_DELETION;
1605
1606 // delete all the regions in this aspace
1607 region = aspace->virtual_map.region_list;
1608 while(region) {
1609 VERIFY_VM_REGION(region);
1610 next = region->aspace_next;
1611 // decrement the ref on this region, may actually push the ref < 0, but that's okay
1612 _vm_put_region(region, true);
1613 region = next;
1614 }
1615
1616 // unlock
1617 sem_release(aspace->virtual_map.sem, WRITE_COUNT);
1618
1619 // release two refs on the address space
1620 vm_put_aspace(aspace);
1621 vm_put_aspace(aspace);
1622
1623 return NO_ERROR;
1624 }
1625
1626 int vm_aspace_walk_start(struct hash_iterator *i)
1627 {
1628 hash_open(aspace_table, i);
1629 return 0;
1630 }
1631
1632 vm_address_space *vm_aspace_walk_next(struct hash_iterator *i)
1633 {
1634 vm_address_space *aspace;
1635
1636 sem_acquire(aspace_hash_sem, READ_COUNT);
1637 aspace = hash_next(aspace_table, i);
1638 if(aspace) {
1639 VERIFY_VM_ASPACE(aspace);
1640 atomic_add(&aspace->ref_count, 1);
1641 }
1642 sem_release(aspace_hash_sem, READ_COUNT);
1643 return aspace;
1644 }
1645
1646 static int vm_thread_dump_max_commit(void *unused)
1647 {
1648 int oldmax = -1;
1649
1650 (void)(unused);
1651
1652 for(;;) {
1653 thread_snooze(1000000);
1654 if(oldmax != max_commit)
1655 dprintf("max_commit 0x%x\n", max_commit);
1656 oldmax = max_commit;
1657 }
1658 }
1659
1660 int vm_init(kernel_args *ka)
1661 {
1662 int err = 0;
1663 unsigned int i;
1664 addr_t heap_base;
1665 addr_t heap_size;
1666 void *null_addr;
1667
1668 dprintf("vm_init: entry\n");
1669 kprintf("initializing vm system...\n");
1670
1671 err = vm_translation_map_module_init(ka);
1672 err = arch_vm_init(ka);
1673
1674 // initialize some globals
1675 kernel_aspace = NULL;
1676 next_region_id = 0;
1677 region_hash_sem = -1;
1678 next_aspace_id = 0;
1679 aspace_hash_sem = -1;
1680 max_commit = 0; // will be increased in vm_page_init
1681 max_commit_lock = 0;
1682 memset(&vm_info, 0, sizeof(vm_info));
1683
1684 // figure the size of memory
1685 vm_page_init(ka);
1686
1687 // map in the new heap and initialize it
1688 heap_size = ROUNDUP(vm_get_mem_size() / 32, 1*1024*1024);
1689 if(heap_size > 16*1024*1024)
1690 heap_size = 16*1024*1024;
1691 heap_base = vm_alloc_from_ka_struct(ka, heap_size, LOCK_KERNEL|LOCK_RW);
1692 dprintf("heap at 0x%lx, size 0x%lx\n", heap_base, heap_size);
1693 kprintf("creating kernel heap at 0x%lx, size 0x%lx\n", heap_base, heap_size);
1694 heap_init(heap_base, heap_size);
1695
1696 // initialize the free page list and page allocator
1697 vm_page_init_postheap(ka);
1698
1699 // initialize the hash table that stores the pages mapped to caches
1700 vm_cache_init(ka);
1701
1702 // create the region and address space hash tables
1703 {
1704 vm_address_space *aspace;
1705 aspace_table = hash_init(ASPACE_HASH_TABLE_SIZE, (addr_t)&aspace->hash_next - (addr_t)aspace,
1706 &aspace_compare, &aspace_hash);
1707 if(aspace_table == NULL)
1708 panic("vm_init: error creating aspace hash table\n");
1709 }
1710 {
1711 vm_region *region;
1712 region_table = hash_init(REGION_HASH_TABLE_SIZE, (addr_t)&region->hash_next - (addr_t)region,
1713 &region_compare, &region_hash);
1714 if(region_table == NULL)
1715 panic("vm_init: error creating aspace hash table\n");
1716 }
1717
1718
1719 // create the initial kernel address space
1720 {
1721 aspace_id aid;
1722 aid = vm_create_aspace("kernel_land", KERNEL_BASE, KERNEL_SIZE, true);
1723 if(aid < 0)
1724 panic("vm_init: error creating kernel address space!\n");
1725 kernel_aspace = vm_get_aspace_by_id(aid);
1726 vm_put_aspace(kernel_aspace);
1727 }
1728
1729 // do any further initialization that the architecture dependant layers may need now
1730 vm_translation_map_module_init2(ka);
1731 arch_vm_init2(ka);
1732 vm_page_init2(ka);
1733
1734 // allocate regions to represent stuff that already exists
1735 null_addr = (void *)ROUNDOWN(heap_base, PAGE_SIZE);
1736 vm_create_anonymous_region(vm_get_kernel_aspace_id(), "kernel_heap", &null_addr, REGION_ADDR_EXACT_ADDRESS,
1737 heap_size, REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1738
1739 null_addr = (void *)ROUNDOWN(ka->kernel_seg0_addr.start, PAGE_SIZE);
1740 vm_create_anonymous_region(vm_get_kernel_aspace_id(), "kernel_seg0", &null_addr, REGION_ADDR_EXACT_ADDRESS,
1741 PAGE_ALIGN(ka->kernel_seg0_addr.size), REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1742
1743 if(ka->kernel_seg1_addr.size > 0) {
1744 null_addr = (void *)ROUNDOWN(ka->kernel_seg1_addr.start, PAGE_SIZE);
1745 vm_create_anonymous_region(vm_get_kernel_aspace_id(), "kernel_seg1", &null_addr, REGION_ADDR_EXACT_ADDRESS,
1746 PAGE_ALIGN(ka->kernel_seg1_addr.size), REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1747 }
1748 for(i=0; i < ka->num_cpus; i++) {
1749 char temp[64];
1750
1751 sprintf(temp, "idle_thread%d_kstack", i);
1752 null_addr = (void *)ka->cpu_kstack[i].start;
1753 vm_create_anonymous_region(vm_get_kernel_aspace_id(), temp, &null_addr, REGION_ADDR_EXACT_ADDRESS,
1754 ka->cpu_kstack[i].size, REGION_WIRING_WIRED_ALREADY, LOCK_RW|LOCK_KERNEL);
1755 }
1756
1757 arch_vm_init_existing_maps(ka);
1758
1759 // map in the boot dir
1760 {
1761 void *null;
1762 vm_map_physical_memory(vm_get_kernel_aspace_id(), "bootdir", &null, REGION_ADDR_ANY_ADDRESS,
1763 ka->bootdir_addr.size, LOCK_RO|LOCK_KERNEL, ka->bootdir_addr.start);
1764 }
1765
1766 arch_vm_init_endvm(ka);
1767
1768 // add some debugger commands
1769 dbg_add_command(&dump_region_list, "regions", "Dump a list of all regions");
1770 dbg_add_command(&dump_region, "region", "Dump info about a particular region");
1771 dbg_add_command(&dump_aspace_list, "aspaces", "Dump a list of all address spaces");
1772 dbg_add_command(&dump_aspace, "aspace", "Dump info about a particular address space");
1773 dbg_add_command(&dump_cache_ref, "cache_ref", "Dump cache_ref data structure");
1774 dbg_add_command(&dump_cache, "cache", "Dump cache_ref data structure");
1775 // dbg_add_command(&display_mem, "dl", "dump memory long words (64-bit)");
1776 dbg_add_command(&display_mem, "dw", "dump memory words (32-bit)");
1777 dbg_add_command(&display_mem, "ds", "dump memory shorts (16-bit)");
1778 dbg_add_command(&display_mem, "db", "dump memory bytes (8-bit)");
1779
1780 dprintf("vm_init: exit\n");
1781
1782 return err;
1783 }
1784
1785 int vm_init_postsem(kernel_args *ka)
1786 {
1787 vm_region *region;
1788
1789 // fill in all of the semaphores that were not allocated before
1790 // since we're still single threaded and only the kernel address space exists,
1791 // it isn't that hard to find all of the ones we need to create
1792 vm_translation_map_module_init_post_sem(ka);
1793 kernel_aspace->virtual_map.sem = sem_create(WRITE_COUNT, "kernel_aspacelock");
1794 recursive_lock_create(&kernel_aspace->translation_map.lock);
1795
1796 for(region = kernel_aspace->virtual_map.region_list; region; region = region->aspace_next) {
1797 if(region->cache_ref->lock.sem < 0) {
1798 mutex_init(&region->cache_ref->lock, "cache_ref_mutex");
1799 }
1800 }
1801
1802 region_hash_sem = sem_create(WRITE_COUNT, "region_hash_sem");
1803 aspace_hash_sem = sem_create(WRITE_COUNT, "aspace_hash_sem");
1804
1805 return heap_init_postsem(ka);
1806 }
1807
1808 int vm_init_postthread(kernel_args *ka)
1809 {
1810 vm_page_init_postthread(ka);
1811
1812 {
1813 thread_id tid = thread_create_kernel_thread("max_commit_thread", &vm_thread_dump_max_commit, NULL);
1814 thread_resume_thread(tid);
1815 }
1816
1817 vm_daemon_init();
1818
1819 return 0;
1820 }
1821
1822 int vm_page_fault(addr_t address, addr_t fault_address, bool is_write, bool is_user, addr_t *newip)
1823 {
1824 int err;
1825
1826 // dprintf("vm_page_fault: page fault at 0x%x, ip 0x%x\n", address, fault_address);
1827
1828 *newip = 0;
1829
1830 err = vm_soft_fault(address, is_write, is_user);
1831 if(err < 0) {
1832 dprintf("vm_page_fault: vm_soft_fault returned error %d on fault at 0x%lx, ip 0x%lx, write %d, user %d, thread 0x%x\n",
1833 err, address, fault_address, is_write, is_user, thread_get_current_thread_id());
1834 if(!is_user) {
1835 struct thread *t = thread_get_current_thread();
1836 if(t && t->fault_handler != 0) {
1837 // this will cause the arch dependant page fault handler to
1838 // modify the IP on the interrupt frame or whatever to return
1839 // to this address
1840 *newip = t->fault_handler;
1841 } else {
1842 // unhandled page fault in the kernel
1843 panic("vm_page_fault: unhandled page fault in kernel space at 0x%lx, ip 0x%lx\n",
1844 address, fault_address);
1845 }
1846 } else {
1847 dprintf("vm_page_fault: killing process 0x%x\n", thread_get_current_thread()->proc->id);
1848 proc_kill_proc(thread_get_current_thread()->proc->id);
1849 }
1850 }
1851
1852 return INT_NO_RESCHEDULE;
1853 }
1854
1855 #define TRACE_PFAULT 0
1856
1857 #if TRACE_PFAULT
1858 #define TRACE dprintf("in pfault at line %d\n", __LINE__)
1859 #else
1860 #define TRACE
1861 #endif
1862
1863 static int vm_soft_fault(addr_t address, bool is_write, bool is_user)
1864 {
1865 vm_address_space *aspace;
1866 vm_virtual_map *map;
1867 vm_region *region;
1868 vm_cache_ref *cache_ref;
1869 vm_cache_ref *last_cache_ref;
1870 vm_cache_ref *top_cache_ref;
1871 off_t cache_offset;
1872 vm_page dummy_page;
1873 vm_page *page = NULL;
1874 int change_count;
1875 int err;
1876
1877 // dprintf("vm_soft_fault: thid 0x%x address 0x%x, is_write %d, is_user %d\n",
1878 // thread_get_current_thread_id(), address, is_write, is_user);
1879
1880 atomic_add(&vm_info.page_faults, 1);
1881
1882 address = ROUNDOWN(address, PAGE_SIZE);
1883
1884 if(address >= KERNEL_BASE && address <= KERNEL_TOP) {
1885 aspace = vm_get_kernel_aspace();
1886 } else if(address >= USER_BASE && address <= USER_TOP) {
1887 aspace = vm_get_current_user_aspace();
1888 if(aspace == NULL) {
1889 if(is_user == false) {
1890 dprintf("vm_soft_fault: kernel thread accessing invalid user memory!\n");
1891 return ERR_VM_PF_FATAL;
1892 } else {
1893 // XXX weird state.
1894 panic("vm_soft_fault: non kernel thread accessing user memory that doesn't exist!\n");
1895 }
1896 }
1897 } else {
1898 // the hit was probably in the 64k DMZ between kernel and user space
1899 // this keeps a user space thread from passing a buffer that crosses into kernel space
1900 return ERR_VM_PF_FATAL;
1901 }
1902 map = &aspace->virtual_map;
1903 atomic_add(&aspace->fault_count, 1);
1904
1905 sem_acquire(map->sem, READ_COUNT);
1906 region = vm_virtual_map_lookup(map, address);
1907 if(region == NULL) {
1908 sem_release(map->sem, READ_COUNT);
1909 vm_put_aspace(aspace);
1910 dprintf("vm_soft_fault: va 0x%lx not covered by region in address space\n", address);
1911 return ERR_VM_PF_BAD_ADDRESS; // BAD_ADDRESS
1912 }
1913
1914 // check permissions
1915 if(is_user && (region->lock & LOCK_KERNEL) == LOCK_KERNEL) {
1916 sem_release(map->sem, READ_COUNT);
1917 vm_put_aspace(aspace);
1918 dprintf("user access on kernel region\n");
1919 return ERR_VM_PF_BAD_PERM; // BAD_PERMISSION
1920 }
1921 if(is_write && (region->lock & LOCK_RW) == 0) {
1922 sem_release(map->sem, READ_COUNT);
1923 vm_put_aspace(aspace);
1924 dprintf("write access attempted on read-only region\n");
1925 return ERR_VM_PF_BAD_PERM; // BAD_PERMISSION
1926 }
1927
1928 TRACE;
1929
1930 top_cache_ref = region->cache_ref;
1931 VERIFY_VM_CACHE_REF(top_cache_ref);
1932 cache_offset = address - region->base + region->cache_offset;
1933 vm_cache_acquire_ref(top_cache_ref, true);
1934 change_count = map->change_count;
1935 sem_release(map->sem, READ_COUNT);
1936
1937 VERIFY_VM_CACHE(top_cache_ref->cache);
1938 VERIFY_VM_STORE(top_cache_ref->cache->store);
1939
1940 // see if this cache has a fault handler
1941 if(top_cache_ref->cache->store->ops->fault) {
1942 int err = (*top_cache_ref->cache->store->ops->fault)(top_cache_ref->cache->store, aspace, cache_offset);
1943 vm_cache_release_ref(top_cache_ref);
1944 vm_put_aspace(aspace);
1945 return err;
1946 }
1947
1948 TRACE;
1949
1950 dummy_page.magic = VM_PAGE_MAGIC;
1951 dummy_page.state = PAGE_STATE_INACTIVE;
1952 dummy_page.type = PAGE_TYPE_DUMMY;
1953
1954 last_cache_ref = top_cache_ref;
1955 for(cache_ref = top_cache_ref; cache_ref; cache_ref = (cache_ref->cache->source) ? cache_ref->cache->source->ref : NULL) {
1956 VERIFY_VM_CACHE_REF(cache_ref);
1957 mutex_lock(&cache_ref->lock);
1958
1959 TRACE;
1960
1961 for(;;) {
1962 page = vm_cache_lookup_page(cache_ref, cache_offset);
1963 if(page != NULL && page->state != PAGE_STATE_BUSY) {
1964 vm_page_set_state(page, PAGE_STATE_BUSY);
1965 mutex_unlock(&cache_ref->lock);
1966 break;
1967 }
1968
1969 if(page == NULL)
1970 break;
1971
1972 TRACE;
1973
1974 // page must be busy
1975 mutex_unlock(&cache_ref->lock);
1976 thread_snooze(20000);
1977 mutex_lock(&cache_ref->lock);
1978 }
1979
1980 TRACE;
1981
1982 if(page != NULL)
1983 break;
1984
1985 TRACE;
1986
1987 // insert this dummy page here to keep other threads from faulting on the
1988 // same address and chasing us up the cache chain
1989 if(cache_ref == top_cache_ref) {
1990 dummy_page.state = PAGE_STATE_BUSY;
1991 vm_cache_insert_page(cache_ref, &dummy_page, cache_offset);
1992 }
1993
1994 VERIFY_VM_CACHE(cache_ref->cache);
1995 VERIFY_VM_STORE(cache_ref->cache->store);
1996
1997 // see if the vm_store has it
1998 if(cache_ref->cache->store->ops->has_page) {
1999 if(cache_ref->cache->store->ops->has_page(cache_ref->cache->store, cache_offset)) {
2000 IOVECS(vecs, 1);
2001
2002 TRACE;
2003
2004 mutex_unlock(&cache_ref->lock);
2005
2006 vecs->num = 1;
2007 vecs->total_len = PAGE_SIZE;
2008 vecs->vec[0].len = PAGE_SIZE;
2009
2010 page = vm_page_allocate_page(PAGE_STATE_FREE);
2011 (*aspace->translation_map.ops->get_physical_page)(page->ppn * PAGE_SIZE, (addr_t *)&vecs->vec[0].start, PHYSICAL_PAGE_CAN_WAIT);
2012 // handle errors here
2013 err = cache_ref->cache->store->ops->read(cache_ref->cache->store, cache_offset, vecs);
2014 (*aspace->translation_map.ops->put_physical_page)((addr_t)vecs->vec[0].start);
2015
2016 mutex_lock(&cache_ref->lock);
2017
2018 if(cache_ref == top_cache_ref) {
2019 vm_cache_remove_page(cache_ref, &dummy_page);
2020 dummy_page.state = PAGE_STATE_INACTIVE;
2021 }
2022 vm_cache_insert_page(cache_ref, page, cache_offset);
2023 mutex_unlock(&cache_ref->lock);
2024 break;
2025 }
2026 }
2027 mutex_unlock(&cache_ref->lock);
2028 last_cache_ref = cache_ref;
2029 TRACE;
2030 }
2031
2032 TRACE;
2033
2034 // we rolled off the end of the cache chain, so we need to decide which
2035 // cache will get the new page we're about to create
2036 if(!cache_ref) {
2037 if(!is_write)
2038 cache_ref = last_cache_ref; // put it in the deepest cache
2039 else
2040 cache_ref = top_cache_ref; // put it in the topmost cache
2041 }
2042
2043 TRACE;
2044
2045 if(page == NULL) {
2046 // still haven't found a page, so zero out a new one
2047 page = vm_page_allocate_page(PAGE_STATE_CLEAR);
2048 // dprintf("vm_soft_fault: just allocated page 0x%x\n", page->ppn);
2049 mutex_lock(&cache_ref->lock);
2050 if(dummy_page.state == PAGE_STATE_BUSY && dummy_page.cache_ref == cache_ref) {
2051 vm_cache_remove_page(cache_ref, &dummy_page);
2052 dummy_page.state = PAGE_STATE_INACTIVE;
2053 }
2054 vm_cache_insert_page(cache_ref, page, cache_offset);
2055 mutex_unlock(&cache_ref->lock);
2056 if(dummy_page.state == PAGE_STATE_BUSY) {
2057 vm_cache_ref *temp_cache = dummy_page.cache_ref;
2058 mutex_lock(&temp_cache->lock);
2059 vm_cache_remove_page(temp_cache, &dummy_page);
2060 mutex_unlock(&temp_cache->lock);
2061 dummy_page.state = PAGE_STATE_INACTIVE;
2062 }
2063 }
2064
2065 TRACE;
2066
2067 if(page->cache_ref != top_cache_ref && is_write) {
2068 // now we have a page that has the data we want, but in the wrong cache object
2069 // so we need to copy it and stick it into the top cache
2070 vm_page *src_page = page;
2071 void *src, *dest;
2072
2073 page = vm_page_allocate_page(PAGE_STATE_FREE);
2074
2075 // try to get a mapping for the src and dest page so we can copy it
2076 for(;;) {
2077 (*aspace->translation_map.ops->get_physical_page)(src_page->ppn * PAGE_SIZE, (addr_t *)&src, PHYSICAL_PAGE_CAN_WAIT);
2078 err = (*aspace->translation_map.ops->get_physical_page)(page->ppn * PAGE_SIZE, (addr_t *)&dest, PHYSICAL_PAGE_NO_WAIT);
2079 if(err == NO_ERROR)
2080 break;
2081
2082 // it couldn't map the second one, so sleep and retry
2083 // keeps an extremely rare deadlock from occuring
2084 (*aspace->translation_map.ops->put_physical_page)((addr_t)src);
2085 thread_snooze(5000);
2086 }
2087
2088 memcpy(dest, src, PAGE_SIZE);
2089 (*aspace->translation_map.ops->put_physical_page)((addr_t)src);
2090 (*aspace->translation_map.ops->put_physical_page)((addr_t)dest);
2091
2092 vm_page_set_state(src_page, PAGE_STATE_ACTIVE);
2093
2094 mutex_lock(&top_cache_ref->lock);
2095 if(dummy_page.state == PAGE_STATE_BUSY && dummy_page.cache_ref == top_cache_ref) {
2096 vm_cache_remove_page(top_cache_ref, &dummy_page);
2097 dummy_page.state = PAGE_STATE_INACTIVE;
2098 }
2099 vm_cache_insert_page(top_cache_ref, page, cache_offset);
2100 mutex_unlock(&top_cache_ref->lock);
2101
2102 if(dummy_page.state == PAGE_STATE_BUSY) {
2103 vm_cache_ref *temp_cache = dummy_page.cache_ref;
2104 mutex_lock(&temp_cache->lock);
2105 vm_cache_remove_page(temp_cache, &dummy_page);
2106 mutex_unlock(&temp_cache->lock);
2107 dummy_page.state = PAGE_STATE_INACTIVE;
2108 }
2109 }
2110
2111 TRACE;
2112
2113 err = 0;
2114 sem_acquire(map->sem, READ_COUNT);
2115 if(change_count != map->change_count) {
2116 // something may have changed, see if the address is still valid
2117 region = vm_virtual_map_lookup(map, address);
2118 if(region == NULL
2119 || region->cache_ref != top_cache_ref
2120 || (address - region->base + region->cache_offset) != cache_offset) {
2121 dprintf("vm_soft_fault: address space layout changed effecting ongoing soft fault\n");
2122 err = ERR_VM_PF_BAD_ADDRESS; // BAD_ADDRESS
2123 }
2124 }
2125
2126 TRACE;
2127
2128 if(err == 0) {
2129 int new_lock = region->lock;
2130 if(page->cache_ref != top_cache_ref && !is_write)
2131 new_lock &= ~LOCK_RW;
2132
2133 atomic_add(&page->ref_count, 1);
2134 (*aspace->translation_map.ops->lock)(&aspace->translation_map);
2135 (*aspace->translation_map.ops->map)(&aspace->translation_map, address,
2136 page->ppn * PAGE_SIZE, new_lock);
2137 (*aspace->translation_map.ops->unlock)(&aspace->translation_map);
2138 }
2139
2140 TRACE;
2141
2142 sem_release(map->sem, READ_COUNT);
2143
2144 TRACE;
2145
2146 if(dummy_page.state == PAGE_STATE_BUSY) {
2147 vm_cache_ref *temp_cache = dummy_page.cache_ref;
2148 mutex_lock(&temp_cache->lock);
2149 vm_cache_remove_page(temp_cache, &dummy_page);
2150 mutex_unlock(&temp_cache->lock);
2151 dummy_page.state = PAGE_STATE_INACTIVE;
2152 }
2153
2154 TRACE;
2155
2156 vm_page_set_state(page, PAGE_STATE_ACTIVE);
2157
2158 vm_cache_release_ref(top_cache_ref);
2159 vm_put_aspace(aspace);
2160
2161 TRACE;
2162
2163 return err;
2164 }
2165
2166 static vm_region *vm_virtual_map_lookup(vm_virtual_map *map, addr_t address)
2167 {
2168 vm_region *region;
2169
2170 // check the region_hint region first
2171 region = map->region_hint;
2172 if(region)
2173 VERIFY_VM_REGION(region);
2174 if(region && region->base <= address && (region->base + region->size) > address)
2175 return region;
2176
2177 for(region = map->region_list; region != NULL; region = region->aspace_next) {
2178 VERIFY_VM_REGION(region);
2179 if(region->base <= address && (region->base + region->size) > address)
2180 break;
2181 }
2182
2183 if(region) {
2184 map->region_hint = region;
2185 VERIFY_VM_REGION(region);
2186 }
2187 return region;
2188 }
2189
2190 int vm_get_physical_page(addr_t paddr, addr_t *vaddr, int flags)
2191 {
2192 #if DEBUG > 1
2193 VERIFY_VM_ASPACE(kernel_aspace);
2194 #endif
2195 return (*kernel_aspace->translation_map.ops->get_physical_page)(paddr, vaddr, flags);
2196 }
2197
2198 int vm_put_physical_page(addr_t vaddr)
2199 {
2200 #if DEBUG > 1
2201 VERIFY_VM_ASPACE(kernel_aspace);
2202 #endif
2203 return (*kernel_aspace->translation_map.ops->put_physical_page)(vaddr);
2204 }
2205
2206 void vm_increase_max_commit(addr_t delta)
2207 {
2208 // dprintf("vm_increase_max_commit: delta 0x%x\n", delta);
2209
2210 int_disable_interrupts();
2211 acquire_spinlock(&max_commit_lock);
2212 max_commit += delta;
2213 vm_info.committed_mem += delta;
2214 release_spinlock(&max_commit_lock);
2215 int_restore_interrupts();
2216 }
2217
2218 int user_memcpy(void *to, const void *from, size_t size)
2219 {
2220 return arch_cpu_user_memcpy(to, from, size, &thread_get_current_thread()->fault_handler);
2221 }
2222
2223 int user_strcpy(char *to, const char *from)
2224 {
2225 return arch_cpu_user_strcpy(to, from, &thread_get_current_thread()->fault_handler);
2226 }
2227
2228 int user_strncpy(char *to, const char *from, size_t size)
2229 {
2230 return arch_cpu_user_strncpy(to, from, size, &thread_get_current_thread()->fault_handler);
2231 }
2232
2233 int user_memset(void *s, char c, size_t count)
2234 {
2235 return arch_cpu_user_memset(s, c, count, &thread_get_current_thread()->fault_handler);
2236 }
2237
2238 addr_t vm_get_mem_size(void)
2239 {
2240 return vm_info.physical_page_size * vm_info.physical_pages;
2241 }
2242
A new operating system