arch/s390/mm/pgtable.c

   1 /*
   2  *    Copyright IBM Corp. 2007,2009
   3  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
   4  */
   5
   6 #include <linux/sched.h>
   7 #include <linux/kernel.h>
   8 #include <linux/errno.h>
   9 #include <linux/gfp.h>
  10 #include <linux/mm.h>
  11 #include <linux/swap.h>
  12 #include <linux/smp.h>
  13 #include <linux/highmem.h>
  14 #include <linux/pagemap.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/module.h>
  17 #include <linux/quicklist.h>
  18 #include <linux/rcupdate.h>
  19
  20 #include <asm/system.h>
  21 #include <asm/pgtable.h>
  22 #include <asm/pgalloc.h>
  23 #include <asm/tlb.h>
  24 #include <asm/tlbflush.h>
  25 #include <asm/mmu_context.h>
  26
  27 struct rcu_table_freelist {
  28         struct rcu_head rcu;
  29         struct mm_struct *mm;
  30         unsigned int pgt_index;
  31         unsigned int crst_index;
  32         unsigned long *table[0];
  33 };
  34
  35 #define RCU_FREELIST_SIZE \
  36         ((PAGE_SIZE - sizeof(struct rcu_table_freelist)) \
  37           / sizeof(unsigned long))
  38
  39 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
  40 static DEFINE_PER_CPU(struct rcu_table_freelist *, rcu_table_freelist);
  41
  42 static void __page_table_free(struct mm_struct *mm, unsigned long *table);
  43
  44 static struct rcu_table_freelist *rcu_table_freelist_get(struct mm_struct *mm)
  45 {
  46         struct rcu_table_freelist **batchp = &__get_cpu_var(rcu_table_freelist);
  47         struct rcu_table_freelist *batch = *batchp;
  48
  49         if (batch)
  50                 return batch;
  51         batch = (struct rcu_table_freelist *) __get_free_page(GFP_ATOMIC);
  52         if (batch) {
  53                 batch->mm = mm;
  54                 batch->pgt_index = 0;
  55                 batch->crst_index = RCU_FREELIST_SIZE;
  56                 *batchp = batch;
  57         }
  58         return batch;
  59 }
  60
  61 static void rcu_table_freelist_callback(struct rcu_head *head)
  62 {
  63         struct rcu_table_freelist *batch =
  64                 container_of(head, struct rcu_table_freelist, rcu);
  65
  66         while (batch->pgt_index > 0)
  67                 __page_table_free(batch->mm, batch->table[--batch->pgt_index]);
  68         while (batch->crst_index < RCU_FREELIST_SIZE)
  69                 crst_table_free(batch->mm, batch->table[batch->crst_index++]);
  70         free_page((unsigned long) batch);
  71 }
  72
  73 void rcu_table_freelist_finish(void)
  74 {
  75         struct rcu_table_freelist *batch = __get_cpu_var(rcu_table_freelist);
  76
  77         if (!batch)
  78                 return;
  79         call_rcu(&batch->rcu, rcu_table_freelist_callback);
  80         __get_cpu_var(rcu_table_freelist) = NULL;
  81 }
  82
  83 static void smp_sync(void *arg)
  84 {
  85 }
  86
  87 #ifndef CONFIG_64BIT
  88 #define ALLOC_ORDER     1
  89 #define TABLES_PER_PAGE 4
  90 #define FRAG_MASK       15UL
  91 #define SECOND_HALVES   10UL
  92
  93 void clear_table_pgstes(unsigned long *table)
  94 {
  95         clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/4);
  96         memset(table + 256, 0, PAGE_SIZE/4);
  97         clear_table(table + 512, _PAGE_TYPE_EMPTY, PAGE_SIZE/4);
  98         memset(table + 768, 0, PAGE_SIZE/4);
  99 }
 100
 101 #else
 102 #define ALLOC_ORDER     2
 103 #define TABLES_PER_PAGE 2
 104 #define FRAG_MASK       3UL
 105 #define SECOND_HALVES   2UL
 106
 107 void clear_table_pgstes(unsigned long *table)
 108 {
 109         clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2);
 110         memset(table + 256, 0, PAGE_SIZE/2);
 111 }
 112
 113 #endif
 114
 115 unsigned long VMALLOC_START = VMALLOC_END - VMALLOC_SIZE;
 116 EXPORT_SYMBOL(VMALLOC_START);
 117
 118 static int __init parse_vmalloc(char *arg)
 119 {
 120         if (!arg)
 121                 return -EINVAL;
 122         VMALLOC_START = (VMALLOC_END - memparse(arg, &arg)) & PAGE_MASK;
 123         return 0;
 124 }
 125 early_param("vmalloc", parse_vmalloc);
 126
 127 unsigned long *crst_table_alloc(struct mm_struct *mm)
 128 {
 129         struct page *page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
 130
 131         if (!page)
 132                 return NULL;
 133         return (unsigned long *) page_to_phys(page);
 134 }
 135
 136 void crst_table_free(struct mm_struct *mm, unsigned long *table)
 137 {
 138         free_pages((unsigned long) table, ALLOC_ORDER);
 139 }
 140
 141 void crst_table_free_rcu(struct mm_struct *mm, unsigned long *table)
 142 {
 143         struct rcu_table_freelist *batch;
 144
 145         if (atomic_read(&mm->mm_users) < 2 &&
 146             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) {
 147                 crst_table_free(mm, table);
 148                 return;
 149         }
 150         batch = rcu_table_freelist_get(mm);
 151         if (!batch) {
 152                 smp_call_function(smp_sync, NULL, 1);
 153                 crst_table_free(mm, table);
 154                 return;
 155         }
 156         batch->table[--batch->crst_index] = table;
 157         if (batch->pgt_index >= batch->crst_index)
 158                 rcu_table_freelist_finish();
 159 }
 160
 161 #ifdef CONFIG_64BIT
 162 int crst_table_upgrade(struct mm_struct *mm, unsigned long limit)
 163 {
 164         unsigned long *table, *pgd;
 165         unsigned long entry;
 166
 167         BUG_ON(limit > (1UL << 53));
 168 repeat:
 169         table = crst_table_alloc(mm);
 170         if (!table)
 171                 return -ENOMEM;
 172         spin_lock_bh(&mm->page_table_lock);
 173         if (mm->context.asce_limit < limit) {
 174                 pgd = (unsigned long *) mm->pgd;
 175                 if (mm->context.asce_limit <= (1UL << 31)) {
 176                         entry = _REGION3_ENTRY_EMPTY;
 177                         mm->context.asce_limit = 1UL << 42;
 178                         mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 179                                                 _ASCE_USER_BITS |
 180                                                 _ASCE_TYPE_REGION3;
 181                 } else {
 182                         entry = _REGION2_ENTRY_EMPTY;
 183                         mm->context.asce_limit = 1UL << 53;
 184                         mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 185                                                 _ASCE_USER_BITS |
 186                                                 _ASCE_TYPE_REGION2;
 187                 }
 188                 crst_table_init(table, entry);
 189                 pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd);
 190                 mm->pgd = (pgd_t *) table;
 191                 mm->task_size = mm->context.asce_limit;
 192                 table = NULL;
 193         }
 194         spin_unlock_bh(&mm->page_table_lock);
 195         if (table)
 196                 crst_table_free(mm, table);
 197         if (mm->context.asce_limit < limit)
 198                 goto repeat;
 199         update_mm(mm, current);
 200         return 0;
 201 }
 202
 203 void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
 204 {
 205         pgd_t *pgd;
 206
 207         if (mm->context.asce_limit <= limit)
 208                 return;
 209         __tlb_flush_mm(mm);
 210         while (mm->context.asce_limit > limit) {
 211                 pgd = mm->pgd;
 212                 switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) {
 213                 case _REGION_ENTRY_TYPE_R2:
 214                         mm->context.asce_limit = 1UL << 42;
 215                         mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 216                                                 _ASCE_USER_BITS |
 217                                                 _ASCE_TYPE_REGION3;
 218                         break;
 219                 case _REGION_ENTRY_TYPE_R3:
 220                         mm->context.asce_limit = 1UL << 31;
 221                         mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 222                                                 _ASCE_USER_BITS |
 223                                                 _ASCE_TYPE_SEGMENT;
 224                         break;
 225                 default:
 226                         BUG();
 227                 }
 228                 mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN);
 229                 mm->task_size = mm->context.asce_limit;
 230                 crst_table_free(mm, (unsigned long *) pgd);
 231         }
 232         update_mm(mm, current);
 233 }
 234 #endif
 235
 236 /*
 237  * page table entry allocation/free routines.
 238  */
 239 unsigned long *page_table_alloc(struct mm_struct *mm)
 240 {
 241         struct page *page;
 242         unsigned long *table;
 243         unsigned long bits;
 244
 245         bits = (mm->context.has_pgste) ? 3UL : 1UL;
 246         spin_lock_bh(&mm->context.list_lock);
 247         page = NULL;
 248         if (!list_empty(&mm->context.pgtable_list)) {
 249                 page = list_first_entry(&mm->context.pgtable_list,
 250                                         struct page, lru);
 251                 if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1))
 252                         page = NULL;
 253         }
 254         if (!page) {
 255                 spin_unlock_bh(&mm->context.list_lock);
 256                 page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
 257                 if (!page)
 258                         return NULL;
 259                 pgtable_page_ctor(page);
 260                 page->flags &= ~FRAG_MASK;
 261                 table = (unsigned long *) page_to_phys(page);
 262                 if (mm->context.has_pgste)
 263                         clear_table_pgstes(table);
 264                 else
 265                         clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE);
 266                 spin_lock_bh(&mm->context.list_lock);
 267                 list_add(&page->lru, &mm->context.pgtable_list);
 268         }
 269         table = (unsigned long *) page_to_phys(page);
 270         while (page->flags & bits) {
 271                 table += 256;
 272                 bits <<= 1;
 273         }
 274         page->flags |= bits;
 275         if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1))
 276                 list_move_tail(&page->lru, &mm->context.pgtable_list);
 277         spin_unlock_bh(&mm->context.list_lock);
 278         return table;
 279 }
 280
 281 static void __page_table_free(struct mm_struct *mm, unsigned long *table)
 282 {
 283         struct page *page;
 284         unsigned long bits;
 285
 286         bits = ((unsigned long) table) & 15;
 287         table = (unsigned long *)(((unsigned long) table) ^ bits);
 288         page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 289         page->flags ^= bits;
 290         if (!(page->flags & FRAG_MASK)) {
 291                 pgtable_page_dtor(page);
 292                 __free_page(page);
 293         }
 294 }
 295
 296 void page_table_free(struct mm_struct *mm, unsigned long *table)
 297 {
 298         struct page *page;
 299         unsigned long bits;
 300
 301         bits = (mm->context.has_pgste) ? 3UL : 1UL;
 302         bits <<= (__pa(table) & (PAGE_SIZE - 1)) / 256 / sizeof(unsigned long);
 303         page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 304         spin_lock_bh(&mm->context.list_lock);
 305         page->flags ^= bits;
 306         if (page->flags & FRAG_MASK) {
 307                 /* Page now has some free pgtable fragments. */
 308                 if (!list_empty(&page->lru))
 309                         list_move(&page->lru, &mm->context.pgtable_list);
 310                 page = NULL;
 311         } else
 312                 /* All fragments of the 4K page have been freed. */
 313                 list_del(&page->lru);
 314         spin_unlock_bh(&mm->context.list_lock);
 315         if (page) {
 316                 pgtable_page_dtor(page);
 317                 __free_page(page);
 318         }
 319 }
 320
 321 void page_table_free_rcu(struct mm_struct *mm, unsigned long *table)
 322 {
 323         struct rcu_table_freelist *batch;
 324         struct page *page;
 325         unsigned long bits;
 326
 327         if (atomic_read(&mm->mm_users) < 2 &&
 328             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) {
 329                 page_table_free(mm, table);
 330                 return;
 331         }
 332         batch = rcu_table_freelist_get(mm);
 333         if (!batch) {
 334                 smp_call_function(smp_sync, NULL, 1);
 335                 page_table_free(mm, table);
 336                 return;
 337         }
 338         bits = (mm->context.has_pgste) ? 3UL : 1UL;
 339         bits <<= (__pa(table) & (PAGE_SIZE - 1)) / 256 / sizeof(unsigned long);
 340         page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 341         spin_lock_bh(&mm->context.list_lock);
 342         /* Delayed freeing with rcu prevents reuse of pgtable fragments */
 343         list_del_init(&page->lru);
 344         spin_unlock_bh(&mm->context.list_lock);
 345         table = (unsigned long *)(((unsigned long) table) | bits);
 346         batch->table[batch->pgt_index++] = table;
 347         if (batch->pgt_index >= batch->crst_index)
 348                 rcu_table_freelist_finish();
 349 }
 350
 351 /*
 352  * switch on pgstes for its userspace process (for kvm)
 353  */
 354 int s390_enable_sie(void)
 355 {
 356         struct task_struct *tsk = current;
 357         struct mm_struct *mm, *old_mm;
 358
 359         /* Do we have switched amode? If no, we cannot do sie */
 360         if (user_mode == HOME_SPACE_MODE)
 361                 return -EINVAL;
 362
 363         /* Do we have pgstes? if yes, we are done */
 364         if (tsk->mm->context.has_pgste)
 365                 return 0;
 366
 367         /* lets check if we are allowed to replace the mm */
 368         task_lock(tsk);
 369         if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
 370 #ifdef CONFIG_AIO
 371             !hlist_empty(&tsk->mm->ioctx_list) ||
 372 #endif
 373             tsk->mm != tsk->active_mm) {
 374                 task_unlock(tsk);
 375                 return -EINVAL;
 376         }
 377         task_unlock(tsk);
 378
 379         /* we copy the mm and let dup_mm create the page tables with_pgstes */
 380         tsk->mm->context.alloc_pgste = 1;
 381         mm = dup_mm(tsk);
 382         tsk->mm->context.alloc_pgste = 0;
 383         if (!mm)
 384                 return -ENOMEM;
 385
 386         /* Now lets check again if something happened */
 387         task_lock(tsk);
 388         if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
 389 #ifdef CONFIG_AIO
 390             !hlist_empty(&tsk->mm->ioctx_list) ||
 391 #endif
 392             tsk->mm != tsk->active_mm) {
 393                 mmput(mm);
 394                 task_unlock(tsk);
 395                 return -EINVAL;
 396         }
 397
 398         /* ok, we are alone. No ptrace, no threads, etc. */
 399         old_mm = tsk->mm;
 400         tsk->mm = tsk->active_mm = mm;
 401         preempt_disable();
 402         update_mm(mm, tsk);
 403         atomic_inc(&mm->context.attach_count);
 404         atomic_dec(&old_mm->context.attach_count);
 405         cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
 406         preempt_enable();
 407         task_unlock(tsk);
 408         mmput(old_mm);
 409         return 0;
 410 }
 411 EXPORT_SYMBOL_GPL(s390_enable_sie);
 412
 413 #if defined(CONFIG_DEBUG_PAGEALLOC) && defined(CONFIG_HIBERNATION)
 414 bool kernel_page_present(struct page *page)
 415 {
 416         unsigned long addr;
 417         int cc;
 418
 419         addr = page_to_phys(page);
 420         asm volatile(
 421                 "       lra     %1,0(%1)\n"
 422                 "       ipm     %0\n"
 423                 "       srl     %0,28"
 424                 : "=d" (cc), "+a" (addr) : : "cc");
 425         return cc == 0;
 426 }
 427 #endif /* CONFIG_HIBERNATION && CONFIG_DEBUG_PAGEALLOC */