arch/powerpc/platforms/cell/beat_htab.c

   1 /*
   2  * "Cell Reference Set" HTAB support.
   3  *
   4  * (C) Copyright 2006-2007 TOSHIBA CORPORATION
   5  *
   6  * This code is based on arch/powerpc/platforms/pseries/lpar.c:
   7  *  Copyright (C) 2001 Todd Inglett, IBM Corporation
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write to the Free Software Foundation, Inc.,
  21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  22  */
  23
  24 #undef DEBUG_LOW
  25
  26 #include <linux/kernel.h>
  27 #include <linux/spinlock.h>
  28
  29 #include <asm/mmu.h>
  30 #include <asm/page.h>
  31 #include <asm/pgtable.h>
  32 #include <asm/machdep.h>
  33 #include <asm/udbg.h>
  34
  35 #include "beat_wrapper.h"
  36
  37 #ifdef DEBUG_LOW
  38 #define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
  39 #else
  40 #define DBG_LOW(fmt...) do { } while (0)
  41 #endif
  42
  43 static DEFINE_RAW_SPINLOCK(beat_htab_lock);
  44
  45 static inline unsigned int beat_read_mask(unsigned hpte_group)
  46 {
  47         unsigned long rmask = 0;
  48         u64 hpte_v[5];
  49
  50         beat_read_htab_entries(0, hpte_group + 0, hpte_v);
  51         if (!(hpte_v[0] & HPTE_V_BOLTED))
  52                 rmask |= 0x8000;
  53         if (!(hpte_v[1] & HPTE_V_BOLTED))
  54                 rmask |= 0x4000;
  55         if (!(hpte_v[2] & HPTE_V_BOLTED))
  56                 rmask |= 0x2000;
  57         if (!(hpte_v[3] & HPTE_V_BOLTED))
  58                 rmask |= 0x1000;
  59         beat_read_htab_entries(0, hpte_group + 4, hpte_v);
  60         if (!(hpte_v[0] & HPTE_V_BOLTED))
  61                 rmask |= 0x0800;
  62         if (!(hpte_v[1] & HPTE_V_BOLTED))
  63                 rmask |= 0x0400;
  64         if (!(hpte_v[2] & HPTE_V_BOLTED))
  65                 rmask |= 0x0200;
  66         if (!(hpte_v[3] & HPTE_V_BOLTED))
  67                 rmask |= 0x0100;
  68         hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP);
  69         beat_read_htab_entries(0, hpte_group + 0, hpte_v);
  70         if (!(hpte_v[0] & HPTE_V_BOLTED))
  71                 rmask |= 0x80;
  72         if (!(hpte_v[1] & HPTE_V_BOLTED))
  73                 rmask |= 0x40;
  74         if (!(hpte_v[2] & HPTE_V_BOLTED))
  75                 rmask |= 0x20;
  76         if (!(hpte_v[3] & HPTE_V_BOLTED))
  77                 rmask |= 0x10;
  78         beat_read_htab_entries(0, hpte_group + 4, hpte_v);
  79         if (!(hpte_v[0] & HPTE_V_BOLTED))
  80                 rmask |= 0x08;
  81         if (!(hpte_v[1] & HPTE_V_BOLTED))
  82                 rmask |= 0x04;
  83         if (!(hpte_v[2] & HPTE_V_BOLTED))
  84                 rmask |= 0x02;
  85         if (!(hpte_v[3] & HPTE_V_BOLTED))
  86                 rmask |= 0x01;
  87         return rmask;
  88 }
  89
  90 static long beat_lpar_hpte_insert(unsigned long hpte_group,
  91                                   unsigned long vpn, unsigned long pa,
  92                                   unsigned long rflags, unsigned long vflags,
  93                                   int psize, int apsize, int ssize)
  94 {
  95         unsigned long lpar_rc;
  96         u64 hpte_v, hpte_r, slot;
  97
  98         if (vflags & HPTE_V_SECONDARY)
  99                 return -1;
 100
 101         if (!(vflags & HPTE_V_BOLTED))
 102                 DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
 103                         "rflags=%lx, vflags=%lx, psize=%d)\n",
 104                 hpte_group, va, pa, rflags, vflags, psize);
 105
 106         hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
 107                 vflags | HPTE_V_VALID;
 108         hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
 109
 110         if (!(vflags & HPTE_V_BOLTED))
 111                 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
 112
 113         if (rflags & _PAGE_NO_CACHE)
 114                 hpte_r &= ~HPTE_R_M;
 115
 116         raw_spin_lock(&beat_htab_lock);
 117         lpar_rc = beat_read_mask(hpte_group);
 118         if (lpar_rc == 0) {
 119                 if (!(vflags & HPTE_V_BOLTED))
 120                         DBG_LOW(" full\n");
 121                 raw_spin_unlock(&beat_htab_lock);
 122                 return -1;
 123         }
 124
 125         lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
 126                 hpte_v, hpte_r, &slot);
 127         raw_spin_unlock(&beat_htab_lock);
 128
 129         /*
 130          * Since we try and ioremap PHBs we don't own, the pte insert
 131          * will fail. However we must catch the failure in hash_page
 132          * or we will loop forever, so return -2 in this case.
 133          */
 134         if (unlikely(lpar_rc != 0)) {
 135                 if (!(vflags & HPTE_V_BOLTED))
 136                         DBG_LOW(" lpar err %lx\n", lpar_rc);
 137                 return -2;
 138         }
 139         if (!(vflags & HPTE_V_BOLTED))
 140                 DBG_LOW(" -> slot: %lx\n", slot);
 141
 142         /* We have to pass down the secondary bucket bit here as well */
 143         return (slot ^ hpte_group) & 15;
 144 }
 145
 146 static long beat_lpar_hpte_remove(unsigned long hpte_group)
 147 {
 148         DBG_LOW("hpte_remove(group=%lx)\n", hpte_group);
 149         return -1;
 150 }
 151
 152 static unsigned long beat_lpar_hpte_getword0(unsigned long slot)
 153 {
 154         unsigned long dword0;
 155         unsigned long lpar_rc;
 156         u64 dword[5];
 157
 158         lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword);
 159
 160         dword0 = dword[slot&3];
 161
 162         BUG_ON(lpar_rc != 0);
 163
 164         return dword0;
 165 }
 166
 167 static void beat_lpar_hptab_clear(void)
 168 {
 169         unsigned long size_bytes = 1UL << ppc64_pft_size;
 170         unsigned long hpte_count = size_bytes >> 4;
 171         int i;
 172         u64 dummy0, dummy1;
 173
 174         /* TODO: Use bulk call */
 175         for (i = 0; i < hpte_count; i++)
 176                 beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1);
 177 }
 178
 179 /*
 180  * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 181  * the low 3 bits of flags happen to line up.  So no transform is needed.
 182  * We can probably optimize here and assume the high bits of newpp are
 183  * already zero.  For now I am paranoid.
 184  */
 185 static long beat_lpar_hpte_updatepp(unsigned long slot,
 186                                     unsigned long newpp,
 187                                     unsigned long vpn,
 188                                     int psize, int apsize,
 189                                     int ssize, int local)
 190 {
 191         unsigned long lpar_rc;
 192         u64 dummy0, dummy1;
 193         unsigned long want_v;
 194
 195         want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 196
 197         DBG_LOW("    update: "
 198                 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
 199                 want_v & HPTE_V_AVPN, slot, psize, newpp);
 200
 201         raw_spin_lock(&beat_htab_lock);
 202         dummy0 = beat_lpar_hpte_getword0(slot);
 203         if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
 204                 DBG_LOW("not found !\n");
 205                 raw_spin_unlock(&beat_htab_lock);
 206                 return -1;
 207         }
 208
 209         lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
 210                                         &dummy1);
 211         raw_spin_unlock(&beat_htab_lock);
 212         if (lpar_rc != 0 || dummy0 == 0) {
 213                 DBG_LOW("not found !\n");
 214                 return -1;
 215         }
 216
 217         DBG_LOW("ok %lx %lx\n", dummy0, dummy1);
 218
 219         BUG_ON(lpar_rc != 0);
 220
 221         return 0;
 222 }
 223
 224 static long beat_lpar_hpte_find(unsigned long vpn, int psize)
 225 {
 226         unsigned long hash;
 227         unsigned long i, j;
 228         long slot;
 229         unsigned long want_v, hpte_v;
 230
 231         hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
 232         want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 233
 234         for (j = 0; j < 2; j++) {
 235                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
 236                 for (i = 0; i < HPTES_PER_GROUP; i++) {
 237                         hpte_v = beat_lpar_hpte_getword0(slot);
 238
 239                         if (HPTE_V_COMPARE(hpte_v, want_v)
 240                             && (hpte_v & HPTE_V_VALID)
 241                             && (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
 242                                 /* HPTE matches */
 243                                 if (j)
 244                                         slot = -slot;
 245                                 return slot;
 246                         }
 247                         ++slot;
 248                 }
 249                 hash = ~hash;
 250         }
 251
 252         return -1;
 253 }
 254
 255 static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
 256                                           unsigned long ea,
 257                                           int psize, int ssize)
 258 {
 259         unsigned long vpn;
 260         unsigned long lpar_rc, slot, vsid;
 261         u64 dummy0, dummy1;
 262
 263         vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
 264         vpn = hpt_vpn(ea, vsid, MMU_SEGSIZE_256M);
 265
 266         raw_spin_lock(&beat_htab_lock);
 267         slot = beat_lpar_hpte_find(vpn, psize);
 268         BUG_ON(slot == -1);
 269
 270         lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7,
 271                 &dummy0, &dummy1);
 272         raw_spin_unlock(&beat_htab_lock);
 273
 274         BUG_ON(lpar_rc != 0);
 275 }
 276
 277 static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
 278                                       int psize, int apsize,
 279                                       int ssize, int local)
 280 {
 281         unsigned long want_v;
 282         unsigned long lpar_rc;
 283         u64 dummy1, dummy2;
 284         unsigned long flags;
 285
 286         DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
 287                 slot, va, psize, local);
 288         want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 289
 290         raw_spin_lock_irqsave(&beat_htab_lock, flags);
 291         dummy1 = beat_lpar_hpte_getword0(slot);
 292
 293         if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
 294                 DBG_LOW("not found !\n");
 295                 raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
 296                 return;
 297         }
 298
 299         lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
 300                 &dummy1, &dummy2);
 301         raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
 302
 303         BUG_ON(lpar_rc != 0);
 304 }
 305
 306 void __init hpte_init_beat(void)
 307 {
 308         ppc_md.hpte_invalidate  = beat_lpar_hpte_invalidate;
 309         ppc_md.hpte_updatepp    = beat_lpar_hpte_updatepp;
 310         ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
 311         ppc_md.hpte_insert      = beat_lpar_hpte_insert;
 312         ppc_md.hpte_remove      = beat_lpar_hpte_remove;
 313         ppc_md.hpte_clear_all   = beat_lpar_hptab_clear;
 314 }
 315
 316 static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
 317                                   unsigned long vpn, unsigned long pa,
 318                                   unsigned long rflags, unsigned long vflags,
 319                                   int psize, int apsize, int ssize)
 320 {
 321         unsigned long lpar_rc;
 322         u64 hpte_v, hpte_r, slot;
 323
 324         if (vflags & HPTE_V_SECONDARY)
 325                 return -1;
 326
 327         if (!(vflags & HPTE_V_BOLTED))
 328                 DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, "
 329                         "rflags=%lx, vflags=%lx, psize=%d)\n",
 330                 hpte_group, vpn, pa, rflags, vflags, psize);
 331
 332         hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
 333                 vflags | HPTE_V_VALID;
 334         hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
 335
 336         if (!(vflags & HPTE_V_BOLTED))
 337                 DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
 338
 339         if (rflags & _PAGE_NO_CACHE)
 340                 hpte_r &= ~HPTE_R_M;
 341
 342         /* insert into not-volted entry */
 343         lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
 344                 HPTE_V_BOLTED, 0, &slot);
 345         /*
 346          * Since we try and ioremap PHBs we don't own, the pte insert
 347          * will fail. However we must catch the failure in hash_page
 348          * or we will loop forever, so return -2 in this case.
 349          */
 350         if (unlikely(lpar_rc != 0)) {
 351                 if (!(vflags & HPTE_V_BOLTED))
 352                         DBG_LOW(" lpar err %lx\n", lpar_rc);
 353                 return -2;
 354         }
 355         if (!(vflags & HPTE_V_BOLTED))
 356                 DBG_LOW(" -> slot: %lx\n", slot);
 357
 358         /* We have to pass down the secondary bucket bit here as well */
 359         return (slot ^ hpte_group) & 15;
 360 }
 361
 362 /*
 363  * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 364  * the low 3 bits of flags happen to line up.  So no transform is needed.
 365  * We can probably optimize here and assume the high bits of newpp are
 366  * already zero.  For now I am paranoid.
 367  */
 368 static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
 369                                        unsigned long newpp,
 370                                        unsigned long vpn,
 371                                        int psize, int apsize,
 372                                        int ssize, int local)
 373 {
 374         unsigned long lpar_rc;
 375         unsigned long want_v;
 376         unsigned long pss;
 377
 378         want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 379         pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
 380
 381         DBG_LOW("    update: "
 382                 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
 383                 want_v & HPTE_V_AVPN, slot, psize, newpp);
 384
 385         lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);
 386
 387         if (lpar_rc == 0xfffffff7) {
 388                 DBG_LOW("not found !\n");
 389                 return -1;
 390         }
 391
 392         DBG_LOW("ok\n");
 393
 394         BUG_ON(lpar_rc != 0);
 395
 396         return 0;
 397 }
 398
 399 static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,
 400                                          int psize, int apsize,
 401                                          int ssize, int local)
 402 {
 403         unsigned long want_v;
 404         unsigned long lpar_rc;
 405         unsigned long pss;
 406
 407         DBG_LOW("    inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
 408                 slot, vpn, psize, local);
 409         want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
 410         pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
 411
 412         lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
 413
 414         /* E_busy can be valid output: page may be already replaced */
 415         BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
 416 }
 417
 418 static int64_t _beat_lpar_hptab_clear_v3(void)
 419 {
 420         return beat_clear_htab3(0);
 421 }
 422
 423 static void beat_lpar_hptab_clear_v3(void)
 424 {
 425         _beat_lpar_hptab_clear_v3();
 426 }
 427
 428 void __init hpte_init_beat_v3(void)
 429 {
 430         if (_beat_lpar_hptab_clear_v3() == 0) {
 431                 ppc_md.hpte_invalidate  = beat_lpar_hpte_invalidate_v3;
 432                 ppc_md.hpte_updatepp    = beat_lpar_hpte_updatepp_v3;
 433                 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
 434                 ppc_md.hpte_insert      = beat_lpar_hpte_insert_v3;
 435                 ppc_md.hpte_remove      = beat_lpar_hpte_remove;
 436                 ppc_md.hpte_clear_all   = beat_lpar_hptab_clear_v3;
 437         } else {
 438                 ppc_md.hpte_invalidate  = beat_lpar_hpte_invalidate;
 439                 ppc_md.hpte_updatepp    = beat_lpar_hpte_updatepp;
 440                 ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
 441                 ppc_md.hpte_insert      = beat_lpar_hpte_insert;
 442                 ppc_md.hpte_remove      = beat_lpar_hpte_remove;
 443                 ppc_md.hpte_clear_all   = beat_lpar_hptab_clear;
 444         }
 445 }