2 #include "kernel/kernel.h"
3 #include "kernel/proc.h"
6 #include <machine/vm.h>
8 #include <minix/type.h>
9 #include <minix/syslib.h>
10 #include <minix/cpufeature.h>
16 #include <machine/vm.h>
18 #include "arch_proto.h"
19 #include "kernel/proto.h"
20 #include "kernel/debug.h"
22 phys_bytes device_mem_vaddr
= 0;
24 #define HASPT(procptr) ((procptr)->p_seg.p_ttbr != 0)
25 static int nfreepdes
= 0;
27 static int freepdes
[MAXFREEPDES
];
29 static u32_t
phys_get32(phys_bytes v
);
31 void mem_clear_mapcache(void)
34 for(i
= 0; i
< nfreepdes
; i
++) {
35 struct proc
*ptproc
= get_cpulocal_var(ptproc
);
36 int pde
= freepdes
[i
];
39 ptv
= ptproc
->p_seg
.p_ttbr_v
;
45 /* This function sets up a mapping from within the kernel's address
46 * space to any other area of memory, either straight physical
47 * memory (pr == NULL) or a process view of memory, in 1MB windows.
48 * I.e., it maps in 1MB chunks of virtual (or physical) address space
49 * to 1MB chunks of kernel virtual address space.
51 * It recognizes pr already being in memory as a special case (no
54 * The target (i.e. in-kernel) mapping area is one of the freepdes[]
55 * VM has earlier already told the kernel about that is available. It is
56 * identified as the 'pde' parameter. This value can be chosen freely
57 * by the caller, as long as it is in range (i.e. 0 or higher and corresonds
58 * to a known freepde slot). It is up to the caller to keep track of which
59 * freepde's are in use, and to determine which ones are free to use.
61 * The logical number supplied by the caller is translated into an actual
62 * pde number to be used, and a pointer to it (linear address) is returned
63 * for actual use by phys_copy or memset.
65 static phys_bytes
createpde(
66 const struct proc
*pr
, /* Requested process, NULL for physical. */
67 const phys_bytes linaddr
,/* Address after segment translation. */
68 phys_bytes
*bytes
, /* Size of chunk, function may truncate it. */
69 int free_pde_idx
, /* index of the free slot to use */
70 int *changed
/* If mapping is made, this is set to 1. */
77 assert(free_pde_idx
>= 0 && free_pde_idx
< nfreepdes
);
78 pde
= freepdes
[free_pde_idx
];
79 assert(pde
>= 0 && pde
< 4096);
81 if(pr
&& ((pr
== get_cpulocal_var(ptproc
)) || iskernelp(pr
))) {
82 /* Process memory is requested, and
83 * it's a process that is already in current page table, or
84 * the kernel, which is always there.
85 * Therefore linaddr is valid directly, with the requested
92 /* Requested address is in a process that is not currently
93 * accessible directly. Grab the PDE entry of that process'
94 * page table that corresponds to the requested address.
96 assert(pr
->p_seg
.p_ttbr_v
);
97 pdeval
= pr
->p_seg
.p_ttbr_v
[ARM_VM_PDE(linaddr
)];
99 /* Requested address is physical. Make up the PDE entry. */
100 pdeval
= (linaddr
& ARM_VM_SECTION_MASK
) |
102 ARM_VM_SECTION_DOMAIN
| ARM_VM_SECTION_USER
;
105 /* Write the pde value that we need into a pde that the kernel
106 * can access, into the currently loaded page table so it becomes
109 assert(get_cpulocal_var(ptproc
)->p_seg
.p_ttbr_v
);
110 if(get_cpulocal_var(ptproc
)->p_seg
.p_ttbr_v
[pde
] != pdeval
) {
111 get_cpulocal_var(ptproc
)->p_seg
.p_ttbr_v
[pde
] = pdeval
;
115 /* Memory is now available, but only the 1MB window of virtual
116 * address space that we have mapped; calculate how much of
117 * the requested range is visible and return that in *bytes,
118 * if that is less than the requested range.
120 offset
= linaddr
& ARM_VM_OFFSET_MASK_1MB
; /* Offset in 1MB window. */
121 *bytes
= MIN(*bytes
, ARM_BIG_PAGE_SIZE
- offset
);
123 /* Return the linear address of the start of the new mapping. */
124 return ARM_BIG_PAGE_SIZE
*pde
+ offset
;
128 /*===========================================================================*
129 * check_resumed_caller *
130 *===========================================================================*/
131 static int check_resumed_caller(struct proc
*caller
)
133 /* Returns the result from VM if caller was resumed, otherwise OK. */
134 if (caller
&& (caller
->p_misc_flags
& MF_KCALL_RESUME
)) {
135 assert(caller
->p_vmrequest
.vmresult
!= VMSUSPEND
);
136 return caller
->p_vmrequest
.vmresult
;
142 /*===========================================================================*
144 *===========================================================================*/
145 static int lin_lin_copy(struct proc
*srcproc
, vir_bytes srclinaddr
,
146 struct proc
*dstproc
, vir_bytes dstlinaddr
, vir_bytes bytes
)
151 assert(get_cpulocal_var(ptproc
));
152 assert(get_cpulocal_var(proc_ptr
));
153 assert(read_ttbr0() == get_cpulocal_var(ptproc
)->p_seg
.p_ttbr
);
155 procslot
= get_cpulocal_var(ptproc
)->p_nr
;
157 assert(procslot
>= 0 && procslot
< ARM_VM_DIR_ENTRIES
);
159 if(srcproc
) assert(!RTS_ISSET(srcproc
, RTS_SLOT_FREE
));
160 if(dstproc
) assert(!RTS_ISSET(dstproc
, RTS_SLOT_FREE
));
161 assert(!RTS_ISSET(get_cpulocal_var(ptproc
), RTS_SLOT_FREE
));
162 assert(get_cpulocal_var(ptproc
)->p_seg
.p_ttbr_v
);
163 if(srcproc
) assert(!RTS_ISSET(srcproc
, RTS_VMINHIBIT
));
164 if(dstproc
) assert(!RTS_ISSET(dstproc
, RTS_VMINHIBIT
));
167 phys_bytes srcptr
, dstptr
;
168 vir_bytes chunk
= bytes
;
172 unsigned cpu
= cpuid
;
174 if (srcproc
&& GET_BIT(srcproc
->p_stale_tlb
, cpu
)) {
176 UNSET_BIT(srcproc
->p_stale_tlb
, cpu
);
178 if (dstproc
&& GET_BIT(dstproc
->p_stale_tlb
, cpu
)) {
180 UNSET_BIT(dstproc
->p_stale_tlb
, cpu
);
184 /* Set up 1MB ranges. */
185 srcptr
= createpde(srcproc
, srclinaddr
, &chunk
, 0, &changed
);
186 dstptr
= createpde(dstproc
, dstlinaddr
, &chunk
, 1, &changed
);
192 PHYS_COPY_CATCH(srcptr
, dstptr
, chunk
, addr
);
195 /* If addr is nonzero, a page fault was caught. */
197 if(addr
>= srcptr
&& addr
< (srcptr
+ chunk
)) {
200 if(addr
>= dstptr
&& addr
< (dstptr
+ chunk
)) {
204 panic("lin_lin_copy fault out of range");
210 /* Update counter and addresses for next iteration, if any. */
216 if(srcproc
) assert(!RTS_ISSET(srcproc
, RTS_SLOT_FREE
));
217 if(dstproc
) assert(!RTS_ISSET(dstproc
, RTS_SLOT_FREE
));
218 assert(!RTS_ISSET(get_cpulocal_var(ptproc
), RTS_SLOT_FREE
));
219 assert(get_cpulocal_var(ptproc
)->p_seg
.p_ttbr_v
);
224 static u32_t
phys_get32(phys_bytes addr
)
229 if((r
=lin_lin_copy(NULL
, addr
,
230 proc_addr(SYSTEM
), (phys_bytes
) &v
, sizeof(v
))) != OK
) {
231 panic("lin_lin_copy for phys_get32 failed: %d", r
);
237 /*===========================================================================*
239 *===========================================================================*/
240 phys_bytes
umap_virtual(rp
, seg
, vir_addr
, bytes
)
241 register struct proc
*rp
; /* pointer to proc table entry for process */
242 int seg
; /* T, D, or S segment */
243 vir_bytes vir_addr
; /* virtual address in bytes within the seg */
244 vir_bytes bytes
; /* # of bytes to be copied */
248 if(vm_lookup(rp
, vir_addr
, &phys
, NULL
) != OK
) {
249 printf("SYSTEM:umap_virtual: vm_lookup of %s: seg 0x%x: 0x%lx failed\n", rp
->p_name
, seg
, vir_addr
);
253 panic("vm_lookup returned phys: %d", phys
);
257 printf("SYSTEM:umap_virtual: lookup failed\n");
261 /* Now make sure addresses are contiguous in physical memory
262 * so that the umap makes sense.
264 if(bytes
> 0 && vm_lookup_range(rp
, vir_addr
, NULL
, bytes
) != bytes
) {
265 printf("umap_virtual: %s: %lu at 0x%lx (vir 0x%lx) not contiguous\n",
266 rp
->p_name
, bytes
, vir_addr
, vir_addr
);
270 /* phys must be larger than 0 (or the caller will think the call
271 * failed), and address must not cross a page boundary.
279 /*===========================================================================*
281 *===========================================================================*/
282 int vm_lookup(const struct proc
*proc
, const vir_bytes
virtual,
283 phys_bytes
*physical
, u32_t
*ptent
)
291 assert(!isemptyp(proc
));
294 /* Retrieve page directory entry. */
295 root
= (u32_t
*) proc
->p_seg
.p_ttbr
;
296 assert(!((u32_t
) root
% ARM_PAGEDIR_SIZE
));
297 pde
= ARM_VM_PDE(virtual);
298 assert(pde
>= 0 && pde
< ARM_VM_DIR_ENTRIES
);
299 pde_v
= phys_get32((u32_t
) (root
+ pde
));
301 if(!(pde_v
& ARM_VM_PDE_PRESENT
)) {
305 /* We don't expect to ever see this. */
306 if(pde_v
& ARM_VM_BIGPAGE
) {
307 *physical
= pde_v
& ARM_VM_SECTION_MASK
;
308 if(ptent
) *ptent
= pde_v
;
309 *physical
+= virtual & ARM_VM_OFFSET_MASK_1MB
;
311 /* Retrieve page table entry. */
312 pt
= (u32_t
*) (pde_v
& ARM_VM_PDE_MASK
);
313 assert(!((u32_t
) pt
% ARM_PAGETABLE_SIZE
));
314 pte
= ARM_VM_PTE(virtual);
315 assert(pte
>= 0 && pte
< ARM_VM_PT_ENTRIES
);
316 pte_v
= phys_get32((u32_t
) (pt
+ pte
));
317 if(!(pte_v
& ARM_VM_PTE_PRESENT
)) {
321 if(ptent
) *ptent
= pte_v
;
323 /* Actual address now known; retrieve it and add page offset. */
324 *physical
= pte_v
& ARM_VM_PTE_MASK
;
325 *physical
+= virtual % ARM_PAGE_SIZE
;
331 /*===========================================================================*
333 *===========================================================================*/
334 size_t vm_lookup_range(const struct proc
*proc
, vir_bytes vir_addr
,
335 phys_bytes
*phys_addr
, size_t bytes
)
337 /* Look up the physical address corresponding to linear virtual address
338 * 'vir_addr' for process 'proc'. Return the size of the range covered
339 * by contiguous physical memory starting from that address; this may
340 * be anywhere between 0 and 'bytes' inclusive. If the return value is
341 * nonzero, and 'phys_addr' is non-NULL, 'phys_addr' will be set to the
342 * base physical address of the range. 'vir_addr' and 'bytes' need not
343 * be page-aligned, but the caller must have verified that the given
344 * linear range is valid for the given process at all.
346 phys_bytes phys
, next_phys
;
353 /* Look up the first page. */
354 if (vm_lookup(proc
, vir_addr
, &phys
, NULL
) != OK
)
357 if (phys_addr
!= NULL
)
360 len
= ARM_PAGE_SIZE
- (vir_addr
% ARM_PAGE_SIZE
);
362 next_phys
= phys
+ len
;
364 /* Look up any next pages and test physical contiguity. */
365 while (len
< bytes
) {
366 if (vm_lookup(proc
, vir_addr
, &phys
, NULL
) != OK
)
369 if (next_phys
!= phys
)
372 len
+= ARM_PAGE_SIZE
;
373 vir_addr
+= ARM_PAGE_SIZE
;
374 next_phys
+= ARM_PAGE_SIZE
;
377 /* We might now have overshot the requested length somewhat. */
378 return MIN(bytes
, len
);
381 /*===========================================================================*
383 *===========================================================================*/
384 static void vm_suspend(struct proc
*caller
, const struct proc
*target
,
385 const vir_bytes linaddr
, const vir_bytes len
, const int type
)
387 /* This range is not OK for this process. Set parameters
388 * of the request and notify VM about the pending request.
390 assert(!RTS_ISSET(caller
, RTS_VMREQUEST
));
391 assert(!RTS_ISSET(target
, RTS_VMREQUEST
));
393 RTS_SET(caller
, RTS_VMREQUEST
);
395 caller
->p_vmrequest
.req_type
= VMPTYPE_CHECK
;
396 caller
->p_vmrequest
.target
= target
->p_endpoint
;
397 caller
->p_vmrequest
.params
.check
.start
= linaddr
;
398 caller
->p_vmrequest
.params
.check
.length
= len
;
399 caller
->p_vmrequest
.params
.check
.writeflag
= 1;
400 caller
->p_vmrequest
.type
= type
;
402 /* Connect caller on vmrequest wait queue. */
403 if(!(caller
->p_vmrequest
.nextrequestor
= vmrequest
))
404 if(OK
!= send_sig(VM_PROC_NR
, SIGKMEM
))
405 panic("send_sig failed");
409 /*===========================================================================*
411 *===========================================================================*/
412 int vm_check_range(struct proc
*caller
, struct proc
*target
,
413 vir_bytes vir_addr
, size_t bytes
)
415 /* Public interface to vm_suspend(), for use by kernel calls. On behalf
416 * of 'caller', call into VM to check linear virtual address range of
417 * process 'target', starting at 'vir_addr', for 'bytes' bytes. This
418 * function assumes that it will called twice if VM returned an error
419 * the first time (since nothing has changed in that case), and will
420 * then return the error code resulting from the first call. Upon the
421 * first call, a non-success error code is returned as well.
425 if ((caller
->p_misc_flags
& MF_KCALL_RESUME
) &&
426 (r
= caller
->p_vmrequest
.vmresult
) != OK
)
429 vm_suspend(caller
, target
, vir_addr
, bytes
, VMSTYPE_KERNELCALL
);
434 /*===========================================================================*
436 *===========================================================================*/
437 void delivermsg(struct proc
*rp
)
441 assert(rp
->p_misc_flags
& MF_DELIVERMSG
);
442 assert(rp
->p_delivermsg
.m_source
!= NONE
);
444 if (copy_msg_to_user(&rp
->p_delivermsg
,
445 (message
*) rp
->p_delivermsg_vir
)) {
446 printf("WARNING wrong user pointer 0x%08lx from "
448 rp
->p_delivermsg_vir
,
454 /* Indicate message has been delivered; address is 'used'. */
455 rp
->p_delivermsg
.m_source
= NONE
;
456 rp
->p_misc_flags
&= ~MF_DELIVERMSG
;
458 if(!(rp
->p_misc_flags
& MF_CONTEXT_SET
)) {
459 rp
->p_reg
.retreg
= r
;
463 /*===========================================================================*
465 *===========================================================================*/
466 int vm_memset(struct proc
* caller
, endpoint_t who
, phys_bytes ph
, int c
,
470 struct proc
*whoptr
= NULL
;
471 phys_bytes cur_ph
= ph
;
472 phys_bytes left
= count
;
473 phys_bytes ptr
, chunk
, pfa
= 0;
474 int new_ttbr
, r
= OK
;
476 if ((r
= check_resumed_caller(caller
)) != OK
)
479 /* NONE for physical, otherwise virtual */
480 if (who
!= NONE
&& !(whoptr
= endpoint_lookup(who
)))
484 pattern
= c
| (c
<< 8) | (c
<< 16) | (c
<< 24);
486 assert(get_cpulocal_var(ptproc
)->p_seg
.p_ttbr_v
);
487 assert(!catch_pagefaults
);
488 catch_pagefaults
= 1;
490 /* We can memset as many bytes as we have remaining,
491 * or as many as remain in the 1MB chunk we mapped in.
496 ptr
= createpde(whoptr
, cur_ph
, &chunk
, 0, &new_ttbr
);
502 /* If a page fault happens, pfa is non-null */
503 if ((pfa
= phys_memset(ptr
, pattern
, chunk
))) {
505 /* If a process pagefaults, VM may help out */
507 vm_suspend(caller
, whoptr
, ph
, count
,
509 assert(catch_pagefaults
);
510 catch_pagefaults
= 0;
514 /* Pagefault when phys copying ?! */
515 panic("vm_memset: pf %lx addr=%lx len=%lu\n",
523 assert(get_cpulocal_var(ptproc
)->p_seg
.p_ttbr_v
);
524 assert(catch_pagefaults
);
525 catch_pagefaults
= 0;
530 /*===========================================================================*
532 *===========================================================================*/
533 int virtual_copy_f(caller
, src_addr
, dst_addr
, bytes
, vmcheck
)
534 struct proc
* caller
;
535 struct vir_addr
*src_addr
; /* source virtual address */
536 struct vir_addr
*dst_addr
; /* destination virtual address */
537 vir_bytes bytes
; /* # of bytes to copy */
538 int vmcheck
; /* if nonzero, can return VMSUSPEND */
540 /* Copy bytes from virtual address src_addr to virtual address dst_addr. */
541 struct vir_addr
*vir_addr
[2]; /* virtual source and destination address */
543 struct proc
*procs
[2];
545 assert((vmcheck
&& caller
) || (!vmcheck
&& !caller
));
547 /* Check copy count. */
548 if (bytes
<= 0) return(EDOM
);
550 /* Do some more checks and map virtual addresses to physical addresses. */
551 vir_addr
[_SRC_
] = src_addr
;
552 vir_addr
[_DST_
] = dst_addr
;
554 for (i
=_SRC_
; i
<=_DST_
; i
++) {
555 endpoint_t proc_e
= vir_addr
[i
]->proc_nr_e
;
562 if(!isokendpt(proc_e
, &proc_nr
)) {
563 printf("virtual_copy: no reasonable endpoint\n");
566 p
= proc_addr(proc_nr
);
572 if ((r
= check_resumed_caller(caller
)) != OK
)
575 if((r
=lin_lin_copy(procs
[_SRC_
], vir_addr
[_SRC_
]->offset
,
576 procs
[_DST_
], vir_addr
[_DST_
]->offset
, bytes
)) != OK
) {
577 struct proc
*target
= NULL
;
579 if(r
!= EFAULT_SRC
&& r
!= EFAULT_DST
)
580 panic("lin_lin_copy failed: %d", r
);
581 if(!vmcheck
|| !caller
) {
585 if(r
== EFAULT_SRC
) {
586 lin
= vir_addr
[_SRC_
]->offset
;
587 target
= procs
[_SRC_
];
588 } else if(r
== EFAULT_DST
) {
589 lin
= vir_addr
[_DST_
]->offset
;
590 target
= procs
[_DST_
];
592 panic("r strange: %d", r
);
598 vm_suspend(caller
, target
, lin
, bytes
, VMSTYPE_KERNELCALL
);
605 /*===========================================================================*
607 *===========================================================================*/
608 int data_copy(const endpoint_t from_proc
, const vir_bytes from_addr
,
609 const endpoint_t to_proc
, const vir_bytes to_addr
,
612 struct vir_addr src
, dst
;
614 src
.offset
= from_addr
;
615 dst
.offset
= to_addr
;
616 src
.proc_nr_e
= from_proc
;
617 dst
.proc_nr_e
= to_proc
;
618 assert(src
.proc_nr_e
!= NONE
);
619 assert(dst
.proc_nr_e
!= NONE
);
621 return virtual_copy(&src
, &dst
, bytes
);
624 /*===========================================================================*
625 * data_copy_vmcheck *
626 *===========================================================================*/
627 int data_copy_vmcheck(struct proc
* caller
,
628 const endpoint_t from_proc
, const vir_bytes from_addr
,
629 const endpoint_t to_proc
, const vir_bytes to_addr
,
632 struct vir_addr src
, dst
;
634 src
.offset
= from_addr
;
635 dst
.offset
= to_addr
;
636 src
.proc_nr_e
= from_proc
;
637 dst
.proc_nr_e
= to_proc
;
638 assert(src
.proc_nr_e
!= NONE
);
639 assert(dst
.proc_nr_e
!= NONE
);
641 return virtual_copy_vmcheck(caller
, &src
, &dst
, bytes
);
644 void memory_init(void)
646 assert(nfreepdes
== 0);
648 freepdes
[nfreepdes
++] = kinfo
.freepde_start
++;
649 freepdes
[nfreepdes
++] = kinfo
.freepde_start
++;
651 assert(kinfo
.freepde_start
< ARM_VM_DIR_ENTRIES
);
652 assert(nfreepdes
== 2);
653 assert(nfreepdes
<= MAXFREEPDES
);
656 /*===========================================================================*
658 *===========================================================================*/
659 void arch_proc_init(struct proc
*pr
, const u32_t ip
, const u32_t sp
, char *name
)
662 strcpy(pr
->p_name
, name
);
664 /* set custom state we know */
669 static int device_mem_mapping_index
= -1,
670 usermapped_glo_index
= -1,
671 usermapped_index
= -1, first_um_idx
= -1;
675 extern char usermapped_start
, usermapped_end
, usermapped_nonglo_start
;
677 int arch_phys_map(const int index
,
682 static int first
= 1;
684 u32_t glo_len
= (u32_t
) &usermapped_nonglo_start
-
685 (u32_t
) &usermapped_start
;
688 device_mem_mapping_index
= freeidx
++;
690 usermapped_glo_index
= freeidx
++;
693 usermapped_index
= freeidx
++;
694 first_um_idx
= usermapped_index
;
695 if(usermapped_glo_index
!= -1)
696 first_um_idx
= usermapped_glo_index
;
700 if(index
== usermapped_glo_index
) {
701 *addr
= vir2phys(&usermapped_start
);
703 *flags
= VMMF_USER
| VMMF_GLO
;
706 else if(index
== usermapped_index
) {
707 *addr
= vir2phys(&usermapped_nonglo_start
);
708 *len
= (u32_t
) &usermapped_end
-
709 (u32_t
) &usermapped_nonglo_start
;
713 else if (index
== device_mem_mapping_index
) {
714 /* map device memory */
717 *flags
= VMMF_UNCACHED
| VMMF_WRITE
;
724 int arch_phys_map_reply(const int index
, const vir_bytes addr
)
726 if(index
== first_um_idx
) {
727 u32_t usermapped_offset
;
728 assert(addr
> (u32_t
) &usermapped_start
);
729 usermapped_offset
= addr
- (u32_t
) &usermapped_start
;
730 memset(&minix_kerninfo
, 0, sizeof(minix_kerninfo
));
731 #define FIXEDPTR(ptr) (void *) ((u32_t)ptr + usermapped_offset)
732 #define FIXPTR(ptr) ptr = FIXEDPTR(ptr)
733 #define ASSIGN(minixstruct) minix_kerninfo.minixstruct = FIXEDPTR(&minixstruct)
739 /* adjust the pointers of the functions and the struct
740 * itself to the user-accessible mapping
742 minix_kerninfo
.kerninfo_magic
= KERNINFO_MAGIC
;
743 minix_kerninfo
.minix_feature_flags
= minix_feature_flags
;
744 minix_kerninfo_user
= (vir_bytes
) FIXEDPTR(&minix_kerninfo
);
749 if(index
== usermapped_index
) return OK
;
751 if (index
== device_mem_mapping_index
) {
752 device_mem_vaddr
= addr
;
759 int arch_enable_paging(struct proc
* caller
)
761 assert(caller
->p_seg
.p_ttbr
);
763 /* load caller's page table */
764 switch_address_space(caller
);
766 device_mem
= (char *) device_mem_vaddr
;
771 void release_address_space(struct proc
*pr
)
773 pr
->p_seg
.p_ttbr_v
= NULL
;