src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp

   1 /*
   2  * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
   3  * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
   4  * Distributed under the terms of the MIT License.
   5  *
   6  * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
   7  * Distributed under the terms of the NewOS License.
   8  */
   9
  10
  11 #include "paging/pae/X86PagingMethodPAE.h"
  12
  13 #include <stdlib.h>
  14 #include <string.h>
  15
  16 #include <AutoDeleter.h>
  17
  18 #include <arch/smp.h>
  19 #include <boot/kernel_args.h>
  20 #include <util/AutoLock.h>
  21 #include <vm/vm.h>
  22 #include <vm/vm_page.h>
  23 #include <vm/VMAddressSpace.h>
  24
  25 #include "paging/32bit/paging.h"
  26 #include "paging/32bit/X86PagingMethod32Bit.h"
  27 #include "paging/pae/X86PagingStructuresPAE.h"
  28 #include "paging/pae/X86VMTranslationMapPAE.h"
  29 #include "paging/x86_physical_page_mapper.h"
  30 #include "paging/x86_physical_page_mapper_large_memory.h"
  31
  32
  33 //#define TRACE_X86_PAGING_METHOD_PAE
  34 #ifdef TRACE_X86_PAGING_METHOD_PAE
  35 #       define TRACE(x...) dprintf(x)
  36 #else
  37 #       define TRACE(x...) ;
  38 #endif
  39
  40
  41 #if B_HAIKU_PHYSICAL_BITS == 64
  42
  43
  44 #define MAX_INITIAL_POOLS       \
  45         (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS,      \
  46                         kPAEPageTableEntryCount)        \
  47                 / kPAEPageTableEntryCount)
  48
  49
  50 using X86LargePhysicalPageMapper::PhysicalPageSlot;
  51
  52
  53 // number of 32 bit pages that will be cached
  54 static const page_num_t kMaxFree32BitPagesCount = 32;
  55
  56
  57 // #pragma mark - ToPAESwitcher
  58
  59
  60 struct X86PagingMethodPAE::ToPAESwitcher {
  61         ToPAESwitcher(kernel_args* args)
  62                 :
  63                 fKernelArgs(args)
  64         {
  65                 // page hole set up in the boot loader
  66                 fPageHole = (page_table_entry*)
  67                         (addr_t)fKernelArgs->arch_args.page_hole;
  68
  69                 // calculate where the page dir would be
  70                 fPageHolePageDir = (page_directory_entry*)
  71                         (((addr_t)fKernelArgs->arch_args.page_hole)
  72                                 + (B_PAGE_SIZE * 1024 - B_PAGE_SIZE));
  73
  74                 fPhysicalPageDir = fKernelArgs->arch_args.phys_pgdir;
  75
  76                 TRACE("page hole: %p\n", fPageHole);
  77                 TRACE("page dir:  %p (physical: %#" B_PRIxPHYSADDR ")\n",
  78                         fPageHolePageDir, fPhysicalPageDir);
  79         }
  80
  81         void Switch(pae_page_directory_pointer_table_entry*& _virtualPDPT,
  82                 phys_addr_t& _physicalPDPT, void*& _pageStructures,
  83                 size_t& _pageStructuresSize, pae_page_directory_entry** pageDirs,
  84                 phys_addr_t* physicalPageDirs, addr_t& _freeVirtualSlot,
  85                 pae_page_table_entry*& _freeVirtualSlotPTE)
  86         {
  87                 // count the page tables we have to translate
  88                 uint32 pageTableCount = 0;
  89                 for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
  90                         i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
  91                         page_directory_entry entry = fPageHolePageDir[i];
  92                         if ((entry & X86_PDE_PRESENT) != 0)
  93                                 pageTableCount++;
  94                 }
  95
  96                 TRACE("page tables to translate: %" B_PRIu32 "\n", pageTableCount);
  97
  98                 // The pages we need to allocate to do our job:
  99                 // + 1 page dir pointer table
 100                 // + 4 page dirs
 101                 // + 2 * page tables (each has 512 instead of 1024 entries)
 102                 // + 1 page for the free virtual slot (no physical page needed)
 103                 uint32 pagesNeeded = 1 + 4 + pageTableCount * 2 + 1;
 104
 105                 // We need additional PAE page tables for the new pages we're going to
 106                 // allocate: Two tables for every 1024 pages to map, i.e. 2 additional
 107                 // pages for every 1022 pages we want to allocate. We also need 32 bit
 108                 // page tables, but we don't need additional virtual space for them,
 109                 // since we can access then via the page hole.
 110                 pagesNeeded += ((pagesNeeded + 1021) / 1022) * 2;
 111
 112                 TRACE("pages needed: %" B_PRIu32 "\n", pagesNeeded);
 113
 114                 // allocate the pages we need
 115                 _AllocateNeededPages(pagesNeeded);
 116
 117                 // prepare the page directory pointer table
 118                 phys_addr_t physicalPDPT = 0;
 119                 pae_page_directory_pointer_table_entry* pdpt
 120                         = (pae_page_directory_pointer_table_entry*)_NextPage(true,
 121                                 physicalPDPT);
 122
 123                 for (int32 i = 0; i < 4; i++) {
 124                         fPageDirs[i] = (pae_page_directory_entry*)_NextPage(true,
 125                                 fPhysicalPageDirs[i]);
 126
 127                         pdpt[i] = X86_PAE_PDPTE_PRESENT
 128                                 | (fPhysicalPageDirs[i] & X86_PAE_PDPTE_ADDRESS_MASK);
 129                 }
 130
 131                 // Since we have to enable PAE in two steps -- setting cr3 to the PDPT
 132                 // and setting the cr4 PAE bit -- we copy the kernel page dir entries to
 133                 // the PDPT page, so after setting cr3, we continue to have working
 134                 // kernel mappings. This requires that the PDPTE registers and the
 135                 // page dir entries don't interect, obviously.
 136                 ASSERT(4 * sizeof(pae_page_directory_pointer_table_entry)
 137                         <= FIRST_KERNEL_PGDIR_ENT * sizeof(page_directory_entry));
 138
 139                 // translate the page tables
 140                 for (uint32 i = FIRST_KERNEL_PGDIR_ENT;
 141                         i < FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS; i++) {
 142                         if ((fPageHolePageDir[i] & X86_PDE_PRESENT) != 0) {
 143                                 // two PAE page tables per 32 bit page table
 144                                 _TranslatePageTable((addr_t)i * 1024 * B_PAGE_SIZE);
 145                                 _TranslatePageTable(((addr_t)i * 1024 + 512) * B_PAGE_SIZE);
 146
 147                                 // copy the page directory entry to the PDPT page
 148                                 ((page_directory_entry*)pdpt)[i] = fPageHolePageDir[i];
 149                         }
 150                 }
 151
 152                 TRACE("free virtual slot: %#" B_PRIxADDR ", PTE: %p\n",
 153                         fFreeVirtualSlot, fFreeVirtualSlotPTE);
 154
 155                 // enable PAE on all CPUs
 156                 call_all_cpus_sync(&_EnablePAE, (void*)(addr_t)physicalPDPT);
 157
 158                 // if availalbe enable NX-bit (No eXecute)
 159                 if (x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD))
 160                         call_all_cpus_sync(&_EnableExecutionDisable, NULL);
 161
 162                 // set return values
 163                 _virtualPDPT = pdpt;
 164                 _physicalPDPT = physicalPDPT;
 165                 _pageStructures = fAllocatedPages;
 166                 _pageStructuresSize = (size_t)fUsedPagesCount * B_PAGE_SIZE;
 167                 memcpy(pageDirs, fPageDirs, sizeof(fPageDirs));
 168                 memcpy(physicalPageDirs, fPhysicalPageDirs, sizeof(fPhysicalPageDirs));
 169
 170                 _freeVirtualSlot = fFreeVirtualSlot;
 171                 _freeVirtualSlotPTE = fFreeVirtualSlotPTE;
 172         }
 173
 174 private:
 175         static void _EnablePAE(void* physicalPDPT, int cpu)
 176         {
 177                 x86_write_cr3((addr_t)physicalPDPT);
 178                 x86_write_cr4(x86_read_cr4() | IA32_CR4_PAE | IA32_CR4_GLOBAL_PAGES);
 179         }
 180
 181         static void _EnableExecutionDisable(void* dummy, int cpu)
 182         {
 183                 x86_write_msr(IA32_MSR_EFER, x86_read_msr(IA32_MSR_EFER)
 184                         | IA32_MSR_EFER_NX);
 185         }
 186
 187         void _TranslatePageTable(addr_t virtualBase)
 188         {
 189                 page_table_entry* entry = &fPageHole[virtualBase / B_PAGE_SIZE];
 190
 191                 // allocate a PAE page table
 192                 phys_addr_t physicalTable = 0;
 193                 pae_page_table_entry* paeTable = (pae_page_table_entry*)_NextPage(false,
 194                         physicalTable);
 195
 196                 // enter it into the page dir
 197                 pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
 198                         fPageDirs, virtualBase);
 199                 PutPageTableInPageDir(pageDirEntry, physicalTable,
 200                         B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
 201
 202                 pae_page_table_entry* paeEntry = paeTable;
 203                 for (uint32 i = 0; i < kPAEPageTableEntryCount;
 204                                 i++, entry++, paeEntry++) {
 205                         if ((*entry & X86_PTE_PRESENT) != 0
 206                                 && _IsVirtualAddressAllocated(virtualBase + i * B_PAGE_SIZE)) {
 207                                 // Note, we use the fact that the PAE flags are defined to the
 208                                 // same values.
 209                                 *paeEntry = *entry & (X86_PTE_PRESENT
 210                                         | X86_PTE_WRITABLE
 211                                         | X86_PTE_USER
 212                                         | X86_PTE_WRITE_THROUGH
 213                                         | X86_PTE_CACHING_DISABLED
 214                                         | X86_PTE_GLOBAL
 215                                         | X86_PTE_ADDRESS_MASK);
 216                         } else
 217                                 *paeEntry = 0;
 218                 }
 219
 220                 if (fFreeVirtualSlot / kPAEPageTableRange
 221                                 == virtualBase / kPAEPageTableRange) {
 222                         fFreeVirtualSlotPTE = paeTable
 223                                 + fFreeVirtualSlot / B_PAGE_SIZE % kPAEPageTableEntryCount;
 224                 }
 225         }
 226
 227         void _AllocateNeededPages(uint32 pagesNeeded)
 228         {
 229                 size_t virtualSize = ROUNDUP(pagesNeeded, 1024) * B_PAGE_SIZE;
 230                 addr_t virtualBase = vm_allocate_early(fKernelArgs, virtualSize, 0, 0,
 231                         kPageTableAlignment);
 232                 if (virtualBase == 0) {
 233                         panic("Failed to reserve virtual address space for the switch to "
 234                                 "PAE!");
 235                 }
 236
 237                 TRACE("virtual space: %#" B_PRIxADDR ", size: %#" B_PRIxSIZE "\n",
 238                         virtualBase, virtualSize);
 239
 240                 // allocate pages for the 32 bit page tables and prepare the tables
 241                 uint32 oldPageTableCount = virtualSize / B_PAGE_SIZE / 1024;
 242                 for (uint32 i = 0; i < oldPageTableCount; i++) {
 243                         // allocate a page
 244                         phys_addr_t physicalTable =_AllocatePage32Bit();
 245
 246                         // put the page into the page dir
 247                         page_directory_entry* entry = &fPageHolePageDir[
 248                                 virtualBase / B_PAGE_SIZE / 1024 + i];
 249                         X86PagingMethod32Bit::PutPageTableInPageDir(entry, physicalTable,
 250                                 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
 251
 252                         // clear the table
 253                         memset((void*)((addr_t)fPageHole
 254                                         + (virtualBase / B_PAGE_SIZE / 1024 + i) * B_PAGE_SIZE),
 255                                 0, B_PAGE_SIZE);
 256                 }
 257
 258                 // We don't need a physical page for the free virtual slot.
 259                 pagesNeeded--;
 260
 261                 // allocate and map the pages we need
 262                 for (uint32 i = 0; i < pagesNeeded; i++) {
 263                         // allocate a page
 264                         phys_addr_t physicalAddress =_AllocatePage32Bit();
 265
 266                         // put the page into the page table
 267                         page_table_entry* entry = fPageHole + virtualBase / B_PAGE_SIZE + i;
 268                         X86PagingMethod32Bit::PutPageTableEntryInTable(entry,
 269                                 physicalAddress, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0,
 270                                 true);
 271
 272                         // Write the page's physical address into the page itself, so we
 273                         // don't need to look it up later.
 274                         *(phys_addr_t*)(virtualBase + i * B_PAGE_SIZE) = physicalAddress;
 275                 }
 276
 277                 fAllocatedPages = (uint8*)virtualBase;
 278                 fAllocatedPagesCount = pagesNeeded;
 279                 fUsedPagesCount = 0;
 280                 fFreeVirtualSlot
 281                         = (addr_t)(fAllocatedPages + pagesNeeded * B_PAGE_SIZE);
 282         }
 283
 284         phys_addr_t _AllocatePage()
 285         {
 286                 phys_addr_t physicalAddress
 287                         = (phys_addr_t)vm_allocate_early_physical_page(fKernelArgs)
 288                                 * B_PAGE_SIZE;
 289                 if (physicalAddress == 0)
 290                         panic("Failed to allocate page for the switch to PAE!");
 291                 return physicalAddress;
 292         }
 293
 294         phys_addr_t _AllocatePage32Bit()
 295         {
 296                 phys_addr_t physicalAddress = _AllocatePage();
 297                 if (physicalAddress > 0xffffffff) {
 298                         panic("Failed to allocate 32 bit addressable page for the switch "
 299                                 "to PAE!");
 300                         return 0;
 301                 }
 302                 return physicalAddress;
 303         }
 304
 305         void* _NextPage(bool clearPage, phys_addr_t& _physicalAddress)
 306         {
 307                 if (fUsedPagesCount >= fAllocatedPagesCount) {
 308                         panic("X86PagingMethodPAE::ToPAESwitcher::_NextPage(): no more "
 309                                 "allocated pages!");
 310                         return NULL;
 311                 }
 312
 313                 void* page = fAllocatedPages + (fUsedPagesCount++) * B_PAGE_SIZE;
 314                 _physicalAddress = *((phys_addr_t*)page);
 315
 316                 if (clearPage)
 317                         memset(page, 0, B_PAGE_SIZE);
 318
 319                 return page;
 320         }
 321
 322         bool _IsVirtualAddressAllocated(addr_t address) const
 323         {
 324                 for (uint32 i = 0; i < fKernelArgs->num_virtual_allocated_ranges; i++) {
 325                         addr_t start = fKernelArgs->virtual_allocated_range[i].start;
 326                         addr_t end = start + fKernelArgs->virtual_allocated_range[i].size;
 327                         if (address < start)
 328                                 return false;
 329                         if (address <= end - 1)
 330                                 return true;
 331                 }
 332
 333                 return false;
 334         }
 335
 336 private:
 337         kernel_args*                            fKernelArgs;
 338         page_table_entry*                       fPageHole;
 339         page_directory_entry*           fPageHolePageDir;
 340         phys_addr_t                                     fPhysicalPageDir;
 341         uint8*                                          fAllocatedPages;
 342         uint32                                          fAllocatedPagesCount;
 343         uint32                                          fUsedPagesCount;
 344         addr_t                                          fFreeVirtualSlot;
 345         pae_page_table_entry*           fFreeVirtualSlotPTE;
 346         pae_page_directory_entry*       fPageDirs[4];
 347         phys_addr_t                                     fPhysicalPageDirs[4];
 348 };
 349
 350
 351 // #pragma mark - PhysicalPageSlotPool
 352
 353
 354 struct X86PagingMethodPAE::PhysicalPageSlotPool
 355         : X86LargePhysicalPageMapper::PhysicalPageSlotPool {
 356 public:
 357         virtual                                         ~PhysicalPageSlotPool();
 358
 359                         status_t                        InitInitial(X86PagingMethodPAE* method,
 360                                                                         kernel_args* args);
 361                         status_t                        InitInitialPostArea(kernel_args* args);
 362
 363                         void                            Init(area_id dataArea,
 364                                                                         pae_page_table_entry* pageTable,
 365                                                                         area_id virtualArea, addr_t virtualBase);
 366
 367         virtual status_t                        AllocatePool(
 368                                                                         X86LargePhysicalPageMapper
 369                                                                                 ::PhysicalPageSlotPool*& _pool);
 370         virtual void                            Map(phys_addr_t physicalAddress,
 371                                                                         addr_t virtualAddress);
 372
 373 public:
 374         static  PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS];
 375
 376 private:
 377                         area_id                         fDataArea;
 378                         area_id                         fVirtualArea;
 379                         addr_t                          fVirtualBase;
 380                         pae_page_table_entry* fPageTable;
 381 };
 382
 383
 384 X86PagingMethodPAE::PhysicalPageSlotPool
 385         X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool[
 386                 MAX_INITIAL_POOLS];
 387
 388
 389 X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool()
 390 {
 391 }
 392
 393
 394 status_t
 395 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(
 396         X86PagingMethodPAE* method, kernel_args* args)
 397 {
 398         // allocate a virtual address range for the pages to be mapped into
 399         addr_t virtualBase = vm_allocate_early(args, kPAEPageTableRange, 0, 0,
 400                 kPAEPageTableRange);
 401         if (virtualBase == 0) {
 402                 panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
 403                         "physical page pool space in virtual address space!");
 404                 return B_ERROR;
 405         }
 406
 407         // allocate memory for the page table and data
 408         size_t areaSize = B_PAGE_SIZE
 409                 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
 410         pae_page_table_entry* pageTable = (pae_page_table_entry*)vm_allocate_early(
 411                 args, areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
 412         if (pageTable == 0) {
 413                 panic("X86PagingMethodPAE::PhysicalPageSlotPool::InitInitial(): Failed "
 414                         "to allocate memory for page table!");
 415                 return B_ERROR;
 416         }
 417
 418         // clear the page table and put it in the page dir
 419         memset(pageTable, 0, B_PAGE_SIZE);
 420
 421         phys_addr_t physicalTable = 0;
 422         method->_EarlyQuery((addr_t)pageTable, &physicalTable);
 423
 424         pae_page_directory_entry* entry = PageDirEntryForAddress(
 425                 method->KernelVirtualPageDirs(), virtualBase);
 426         PutPageTableInPageDir(entry, physicalTable,
 427                 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
 428
 429         // init the pool structure and add the initial pool
 430         Init(-1, pageTable, -1, (addr_t)virtualBase);
 431
 432         return B_OK;
 433 }
 434
 435
 436 status_t
 437 X86PagingMethodPAE::PhysicalPageSlotPool::InitInitialPostArea(
 438         kernel_args* args)
 439 {
 440         // create an area for the (already allocated) data
 441         size_t areaSize = B_PAGE_SIZE
 442                 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
 443         void* temp = fPageTable;
 444         area_id area = create_area("physical page pool", &temp,
 445                 B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
 446                 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
 447         if (area < B_OK) {
 448                 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
 449                         "create area for physical page pool.");
 450                 return area;
 451         }
 452         fDataArea = area;
 453
 454         // create an area for the virtual address space
 455         temp = (void*)fVirtualBase;
 456         area = vm_create_null_area(VMAddressSpace::KernelID(),
 457                 "physical page pool space", &temp, B_EXACT_ADDRESS,
 458                 kPAEPageTableRange, 0);
 459         if (area < B_OK) {
 460                 panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
 461                         "create area for physical page pool space.");
 462                 return area;
 463         }
 464         fVirtualArea = area;
 465
 466         return B_OK;
 467 }
 468
 469
 470 void
 471 X86PagingMethodPAE::PhysicalPageSlotPool::Init(area_id dataArea,
 472         pae_page_table_entry* pageTable, area_id virtualArea, addr_t virtualBase)
 473 {
 474         fDataArea = dataArea;
 475         fVirtualArea = virtualArea;
 476         fVirtualBase = virtualBase;
 477         fPageTable = pageTable;
 478
 479         // init slot list
 480         fSlots = (PhysicalPageSlot*)(fPageTable + kPAEPageTableEntryCount);
 481         addr_t slotAddress = virtualBase;
 482         for (uint32 i = 0; i < kPAEPageTableEntryCount;
 483                         i++, slotAddress += B_PAGE_SIZE) {
 484                 PhysicalPageSlot* slot = &fSlots[i];
 485                 slot->next = slot + 1;
 486                 slot->pool = this;
 487                 slot->address = slotAddress;
 488         }
 489
 490         fSlots[kPAEPageTableEntryCount - 1].next = NULL;
 491                 // terminate list
 492 }
 493
 494
 495 void
 496 X86PagingMethodPAE::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
 497         addr_t virtualAddress)
 498 {
 499         pae_page_table_entry& pte = fPageTable[
 500                 (virtualAddress - fVirtualBase) / B_PAGE_SIZE];
 501         pte = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
 502                 | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL | X86_PAE_PTE_PRESENT;
 503
 504         invalidate_TLB(virtualAddress);
 505 }
 506
 507
 508 status_t
 509 X86PagingMethodPAE::PhysicalPageSlotPool::AllocatePool(
 510         X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool)
 511 {
 512         // create the pool structure
 513         PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool;
 514         if (pool == NULL)
 515                 return B_NO_MEMORY;
 516         ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);
 517
 518         // create an area that can contain the page table and the slot
 519         // structures
 520         size_t areaSize = B_PAGE_SIZE
 521                 + sizeof(PhysicalPageSlot[kPAEPageTableEntryCount]);
 522         void* data;
 523         virtual_address_restrictions virtualRestrictions = {};
 524         virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
 525         physical_address_restrictions physicalRestrictions = {};
 526         area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool",
 527                 PAGE_ALIGN(areaSize), B_FULL_LOCK,
 528                 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
 529                 &virtualRestrictions, &physicalRestrictions, &data);
 530         if (dataArea < 0)
 531                 return dataArea;
 532
 533         // create the null area for the virtual address space
 534         void* virtualBase;
 535         area_id virtualArea = vm_create_null_area(
 536                 VMAddressSpace::KernelID(), "physical page pool space",
 537                 &virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, kPAEPageTableRange,
 538                 CREATE_AREA_PRIORITY_VIP);
 539         if (virtualArea < 0) {
 540                 delete_area(dataArea);
 541                 return virtualArea;
 542         }
 543
 544         // prepare the page table
 545         memset(data, 0, B_PAGE_SIZE);
 546
 547         // get the page table's physical address
 548         phys_addr_t physicalTable;
 549         X86VMTranslationMapPAE* map = static_cast<X86VMTranslationMapPAE*>(
 550                 VMAddressSpace::Kernel()->TranslationMap());
 551         uint32 dummyFlags;
 552         cpu_status state = disable_interrupts();
 553         map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags);
 554         restore_interrupts(state);
 555
 556         // put the page table into the page directory
 557         pae_page_directory_entry* pageDirEntry
 558                 = X86PagingMethodPAE::PageDirEntryForAddress(
 559                         map->PagingStructuresPAE()->VirtualPageDirs(), (addr_t)virtualBase);
 560         PutPageTableInPageDir(pageDirEntry, physicalTable,
 561                 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
 562
 563         // init the pool structure
 564         pool->Init(dataArea, (pae_page_table_entry*)data, virtualArea,
 565                 (addr_t)virtualBase);
 566         poolDeleter.Detach();
 567         _pool = pool;
 568         return B_OK;
 569 }
 570
 571
 572 // #pragma mark - X86PagingMethodPAE
 573
 574
 575 X86PagingMethodPAE::X86PagingMethodPAE()
 576         :
 577         fPhysicalPageMapper(NULL),
 578         fKernelPhysicalPageMapper(NULL),
 579         fFreePages(NULL),
 580         fFreePagesCount(0)
 581 {
 582         mutex_init(&fFreePagesLock, "x86 PAE free pages");
 583 }
 584
 585
 586 X86PagingMethodPAE::~X86PagingMethodPAE()
 587 {
 588 }
 589
 590
 591 status_t
 592 X86PagingMethodPAE::Init(kernel_args* args,
 593         VMPhysicalPageMapper** _physicalPageMapper)
 594 {
 595         // switch to PAE
 596         ToPAESwitcher(args).Switch(fKernelVirtualPageDirPointerTable,
 597                 fKernelPhysicalPageDirPointerTable, fEarlyPageStructures,
 598                 fEarlyPageStructuresSize, fKernelVirtualPageDirs,
 599                 fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE);
 600
 601         // create the initial pools for the physical page mapper
 602         int32 poolCount = _GetInitialPoolCount();
 603         PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool;
 604
 605         for (int32 i = 0; i < poolCount; i++) {
 606                 new(&pool[i]) PhysicalPageSlotPool;
 607                 status_t error = pool[i].InitInitial(this, args);
 608                 if (error != B_OK) {
 609                         panic("X86PagingMethodPAE::Init(): Failed to create initial pool "
 610                                 "for physical page mapper!");
 611                         return error;
 612                 }
 613         }
 614
 615         // create physical page mapper
 616         large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool),
 617                 fPhysicalPageMapper, fKernelPhysicalPageMapper);
 618
 619         *_physicalPageMapper = fPhysicalPageMapper;
 620         return B_OK;
 621 }
 622
 623
 624 status_t
 625 X86PagingMethodPAE::InitPostArea(kernel_args* args)
 626 {
 627         // wrap the kernel paging structures in an area
 628         area_id area = create_area("kernel paging structs", &fEarlyPageStructures,
 629                 B_EXACT_ADDRESS, fEarlyPageStructuresSize, B_ALREADY_WIRED,
 630                 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
 631         if (area < B_OK)
 632                 return area;
 633
 634         // let the initial page pools create areas for its structures
 635         int32 poolCount = _GetInitialPoolCount();
 636         for (int32 i = 0; i < poolCount; i++) {
 637                 status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i]
 638                         .InitInitialPostArea(args);
 639                 if (error != B_OK)
 640                         return error;
 641         }
 642
 643         // The early physical page mapping mechanism is no longer needed. Unmap the
 644         // slot.
 645         *fFreeVirtualSlotPTE = 0;
 646         invalidate_TLB(fFreeVirtualSlot);
 647
 648         fFreeVirtualSlotPTE = NULL;
 649         fFreeVirtualSlot = 0;
 650
 651         return B_OK;
 652 }
 653
 654
 655 status_t
 656 X86PagingMethodPAE::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
 657 {
 658         X86VMTranslationMapPAE* map = new(std::nothrow) X86VMTranslationMapPAE;
 659         if (map == NULL)
 660                 return B_NO_MEMORY;
 661
 662         status_t error = map->Init(kernel);
 663         if (error != B_OK) {
 664                 delete map;
 665                 return error;
 666         }
 667
 668         *_map = map;
 669         return B_OK;
 670 }
 671
 672
 673 status_t
 674 X86PagingMethodPAE::MapEarly(kernel_args* args, addr_t virtualAddress,
 675         phys_addr_t physicalAddress, uint8 attributes,
 676         page_num_t (*get_free_page)(kernel_args*))
 677 {
 678         // check to see if a page table exists for this range
 679         pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
 680                 fKernelVirtualPageDirs, virtualAddress);
 681         pae_page_table_entry* pageTable;
 682         if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
 683                 // we need to allocate a page table
 684                 phys_addr_t physicalPageTable = get_free_page(args) * B_PAGE_SIZE;
 685
 686                 TRACE("X86PagingMethodPAE::MapEarly(): asked for free page for "
 687                         "page table: %#" B_PRIxPHYSADDR "\n", physicalPageTable);
 688
 689                 // put it in the page dir
 690                 PutPageTableInPageDir(pageDirEntry, physicalPageTable, attributes);
 691
 692                 // zero it out
 693                 pageTable = _EarlyGetPageTable(physicalPageTable);
 694                 memset(pageTable, 0, B_PAGE_SIZE);
 695         } else {
 696                 // table already exists -- map it
 697                 pageTable = _EarlyGetPageTable(
 698                         *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK);
 699         }
 700
 701         pae_page_table_entry* entry = pageTable
 702                 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;
 703
 704         ASSERT_PRINT(
 705                 (*entry & X86_PAE_PTE_PRESENT) == 0,
 706                 "virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx64
 707                 ", existing pte: %#" B_PRIx64, virtualAddress, *pageDirEntry, *entry);
 708
 709         // now, fill in the pentry
 710         PutPageTableEntryInTable(entry, physicalAddress, attributes, 0,
 711                 IS_KERNEL_ADDRESS(virtualAddress));
 712
 713         return B_OK;
 714 }
 715
 716
 717 bool
 718 X86PagingMethodPAE::IsKernelPageAccessible(addr_t virtualAddress,
 719         uint32 protection)
 720 {
 721         // we can't check much without the physical page mapper
 722         if (fPhysicalPageMapper == NULL)
 723                 return false;
 724
 725         // We only trust the kernel team's page directories. So switch to the
 726         // kernel PDPT first. Always set it to make sure the TLBs don't contain
 727         // obsolete data.
 728         uint32 physicalPDPT = x86_read_cr3();
 729         x86_write_cr3(fKernelPhysicalPageDirPointerTable);
 730
 731         // get the PDPT entry for the address
 732         pae_page_directory_pointer_table_entry pdptEntry = 0;
 733         if (physicalPDPT == fKernelPhysicalPageDirPointerTable) {
 734                 pdptEntry = fKernelVirtualPageDirPointerTable[
 735                         virtualAddress / kPAEPageDirRange];
 736         } else {
 737                 // map the original PDPT and get the entry
 738                 void* handle;
 739                 addr_t virtualPDPT;
 740                 status_t error = fPhysicalPageMapper->GetPageDebug(physicalPDPT,
 741                         &virtualPDPT, &handle);
 742                 if (error == B_OK) {
 743                         pdptEntry = ((pae_page_directory_pointer_table_entry*)
 744                                 virtualPDPT)[virtualAddress / kPAEPageDirRange];
 745                         fPhysicalPageMapper->PutPageDebug(virtualPDPT, handle);
 746                 }
 747         }
 748
 749         // map the page dir and get the entry
 750         pae_page_directory_entry pageDirEntry = 0;
 751         if ((pdptEntry & X86_PAE_PDPTE_PRESENT) != 0) {
 752                 void* handle;
 753                 addr_t virtualPageDir;
 754                 status_t error = fPhysicalPageMapper->GetPageDebug(
 755                         pdptEntry & X86_PAE_PDPTE_ADDRESS_MASK, &virtualPageDir, &handle);
 756                 if (error == B_OK) {
 757                         pageDirEntry = ((pae_page_directory_entry*)virtualPageDir)[
 758                                 virtualAddress / kPAEPageTableRange % kPAEPageDirEntryCount];
 759                         fPhysicalPageMapper->PutPageDebug(virtualPageDir, handle);
 760                 }
 761         }
 762
 763         // map the page table and get the entry
 764         pae_page_table_entry pageTableEntry = 0;
 765         if ((pageDirEntry & X86_PAE_PDE_PRESENT) != 0) {
 766                 void* handle;
 767                 addr_t virtualPageTable;
 768                 status_t error = fPhysicalPageMapper->GetPageDebug(
 769                         pageDirEntry & X86_PAE_PDE_ADDRESS_MASK, &virtualPageTable,
 770                         &handle);
 771                 if (error == B_OK) {
 772                         pageTableEntry = ((pae_page_table_entry*)virtualPageTable)[
 773                                 virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount];
 774                         fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle);
 775                 }
 776         }
 777
 778         // switch back to the original page directory
 779         if (physicalPDPT != fKernelPhysicalPageDirPointerTable)
 780                 x86_write_cr3(physicalPDPT);
 781
 782         if ((pageTableEntry & X86_PAE_PTE_PRESENT) == 0)
 783                 return false;
 784
 785         // present means kernel-readable, so check for writable
 786         return (protection & B_KERNEL_WRITE_AREA) == 0
 787                 || (pageTableEntry & X86_PAE_PTE_WRITABLE) != 0;
 788 }
 789
 790
 791 /*static*/ void
 792 X86PagingMethodPAE::PutPageTableInPageDir(pae_page_directory_entry* entry,
 793         phys_addr_t physicalTable, uint32 attributes)
 794 {
 795         *entry = (physicalTable & X86_PAE_PDE_ADDRESS_MASK)
 796                 | X86_PAE_PDE_PRESENT
 797                 | X86_PAE_PDE_WRITABLE
 798                 | X86_PAE_PDE_USER;
 799                 // TODO: We ignore the attributes of the page table -- for compatibility
 800                 // with BeOS we allow having user accessible areas in the kernel address
 801                 // space. This is currently being used by some drivers, mainly for the
 802                 // frame buffer. Our current real time data implementation makes use of
 803                 // this fact, too.
 804                 // We might want to get rid of this possibility one day, especially if
 805                 // we intend to port it to a platform that does not support this.
 806 }
 807
 808
 809 /*static*/ void
 810 X86PagingMethodPAE::PutPageTableEntryInTable(pae_page_table_entry* entry,
 811         phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
 812         bool globalPage)
 813 {
 814         pae_page_table_entry page = (physicalAddress & X86_PAE_PTE_ADDRESS_MASK)
 815                 | X86_PAE_PTE_PRESENT | (globalPage ? X86_PAE_PTE_GLOBAL : 0)
 816                 | MemoryTypeToPageTableEntryFlags(memoryType);
 817
 818         // if the page is user accessible, it's automatically
 819         // accessible in kernel space, too (but with the same
 820         // protection)
 821         if ((attributes & B_USER_PROTECTION) != 0) {
 822                 page |= X86_PAE_PTE_USER;
 823                 if ((attributes & B_WRITE_AREA) != 0)
 824                         page |= X86_PAE_PTE_WRITABLE;
 825                 if ((attributes & B_EXECUTE_AREA) == 0
 826                         && x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) {
 827                         page |= X86_PAE_PTE_NOT_EXECUTABLE;
 828                 }
 829         } else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
 830                 page |= X86_PAE_PTE_WRITABLE;
 831
 832         // put it in the page table
 833         *(volatile pae_page_table_entry*)entry = page;
 834 }
 835
 836
 837 void*
 838 X86PagingMethodPAE::Allocate32BitPage(phys_addr_t& _physicalAddress,
 839         void*& _handle)
 840 {
 841         // get a free page
 842         MutexLocker locker(fFreePagesLock);
 843         vm_page* page;
 844         if (fFreePages != NULL) {
 845                 page = fFreePages;
 846                 fFreePages = page->cache_next;
 847                 fFreePagesCount--;
 848                 locker.Unlock();
 849         } else {
 850                 // no pages -- allocate one
 851                 locker.Unlock();
 852
 853                 physical_address_restrictions restrictions = {};
 854                 restrictions.high_address = 0x100000000LL;
 855                 page = vm_page_allocate_page_run(PAGE_STATE_UNUSED, 1, &restrictions,
 856                         VM_PRIORITY_SYSTEM);
 857                 if (page == NULL)
 858                         return NULL;
 859
 860                 DEBUG_PAGE_ACCESS_END(page);
 861         }
 862
 863         // map the page
 864         phys_addr_t physicalAddress
 865                 = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
 866         addr_t virtualAddress;
 867         if (fPhysicalPageMapper->GetPage(physicalAddress, &virtualAddress, &_handle)
 868                         != B_OK) {
 869                 // mapping failed -- free page
 870                 locker.Lock();
 871                 page->cache_next = fFreePages;
 872                 fFreePages = page;
 873                 fFreePagesCount++;
 874                 return NULL;
 875         }
 876
 877         _physicalAddress = physicalAddress;
 878         return (void*)virtualAddress;
 879 }
 880
 881
 882 void
 883 X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress,
 884         void* handle)
 885 {
 886         // unmap the page
 887         fPhysicalPageMapper->PutPage((addr_t)address, handle);
 888
 889         // free it
 890         vm_page* page = vm_lookup_page(physicalAddress / B_PAGE_SIZE);
 891         MutexLocker locker(fFreePagesLock);
 892         if (fFreePagesCount < kMaxFree32BitPagesCount) {
 893                 // cache not full yet -- cache it
 894                 page->cache_next = fFreePages;
 895                 fFreePages = page;
 896                 fFreePagesCount++;
 897         } else {
 898                 // cache full -- free it
 899                 locker.Unlock();
 900                 DEBUG_PAGE_ACCESS_START(page);
 901                 vm_page_free(NULL, page);
 902         }
 903 }
 904
 905
 906 inline int32
 907 X86PagingMethodPAE::_GetInitialPoolCount()
 908 {
 909         int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU
 910                         + EXTRA_SLOTS;
 911         return (requiredSlots + kPAEPageTableEntryCount - 1)
 912                 / kPAEPageTableEntryCount;
 913 }
 914
 915
 916 bool
 917 X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress,
 918         phys_addr_t* _physicalAddress)
 919 {
 920         pae_page_directory_entry* pageDirEntry = PageDirEntryForAddress(
 921                 fKernelVirtualPageDirs, virtualAddress);
 922         if ((*pageDirEntry & X86_PAE_PDE_PRESENT) == 0) {
 923                 // no pagetable here
 924                 return false;
 925         }
 926
 927         pae_page_table_entry* entry = _EarlyGetPageTable(
 928                         *pageDirEntry & X86_PAE_PDE_ADDRESS_MASK)
 929                 + virtualAddress / B_PAGE_SIZE % kPAEPageTableEntryCount;
 930         if ((*entry & X86_PAE_PTE_PRESENT) == 0) {
 931                 // page mapping not valid
 932                 return false;
 933         }
 934
 935         *_physicalAddress = *entry & X86_PAE_PTE_ADDRESS_MASK;
 936         return true;
 937 }
 938
 939
 940 pae_page_table_entry*
 941 X86PagingMethodPAE::_EarlyGetPageTable(phys_addr_t address)
 942 {
 943         *fFreeVirtualSlotPTE = (address & X86_PAE_PTE_ADDRESS_MASK)
 944                 | X86_PAE_PTE_PRESENT | X86_PAE_PTE_WRITABLE | X86_PAE_PTE_GLOBAL;
 945
 946         invalidate_TLB(fFreeVirtualSlot);
 947
 948         return (pae_page_table_entry*)fFreeVirtualSlot;
 949 }
 950
 951
 952 #endif  // B_HAIKU_PHYSICAL_BITS == 64