drivers/iommu/arm-smmu.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  */
   9
  10 #ifndef _ARM_SMMU_H
  11 #define _ARM_SMMU_H
  12
  13 #include <linux/atomic.h>
  14 #include <linux/bits.h>
  15 #include <linux/clk.h>
  16 #include <linux/device.h>
  17 #include <linux/io-64-nonatomic-hi-lo.h>
  18 #include <linux/io-pgtable.h>
  19 #include <linux/iommu.h>
  20 #include <linux/mutex.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/types.h>
  23
  24 /* Configuration registers */
  25 #define ARM_SMMU_GR0_sCR0               0x0
  26 #define sCR0_VMID16EN                   BIT(31)
  27 #define sCR0_BSU                        GENMASK(15, 14)
  28 #define sCR0_FB                         BIT(13)
  29 #define sCR0_PTM                        BIT(12)
  30 #define sCR0_VMIDPNE                    BIT(11)
  31 #define sCR0_USFCFG                     BIT(10)
  32 #define sCR0_GCFGFIE                    BIT(5)
  33 #define sCR0_GCFGFRE                    BIT(4)
  34 #define sCR0_EXIDENABLE                 BIT(3)
  35 #define sCR0_GFIE                       BIT(2)
  36 #define sCR0_GFRE                       BIT(1)
  37 #define sCR0_CLIENTPD                   BIT(0)
  38
  39 /* Auxiliary Configuration register */
  40 #define ARM_SMMU_GR0_sACR               0x10
  41
  42 /* Identification registers */
  43 #define ARM_SMMU_GR0_ID0                0x20
  44 #define ID0_S1TS                        BIT(30)
  45 #define ID0_S2TS                        BIT(29)
  46 #define ID0_NTS                         BIT(28)
  47 #define ID0_SMS                         BIT(27)
  48 #define ID0_ATOSNS                      BIT(26)
  49 #define ID0_PTFS_NO_AARCH32             BIT(25)
  50 #define ID0_PTFS_NO_AARCH32S            BIT(24)
  51 #define ID0_NUMIRPT                     GENMASK(23, 16)
  52 #define ID0_CTTW                        BIT(14)
  53 #define ID0_NUMSIDB                     GENMASK(12, 9)
  54 #define ID0_EXIDS                       BIT(8)
  55 #define ID0_NUMSMRG                     GENMASK(7, 0)
  56
  57 #define ARM_SMMU_GR0_ID1                0x24
  58 #define ID1_PAGESIZE                    BIT(31)
  59 #define ID1_NUMPAGENDXB                 GENMASK(30, 28)
  60 #define ID1_NUMS2CB                     GENMASK(23, 16)
  61 #define ID1_NUMCB                       GENMASK(7, 0)
  62
  63 #define ARM_SMMU_GR0_ID2                0x28
  64 #define ID2_VMID16                      BIT(15)
  65 #define ID2_PTFS_64K                    BIT(14)
  66 #define ID2_PTFS_16K                    BIT(13)
  67 #define ID2_PTFS_4K                     BIT(12)
  68 #define ID2_UBS                         GENMASK(11, 8)
  69 #define ID2_OAS                         GENMASK(7, 4)
  70 #define ID2_IAS                         GENMASK(3, 0)
  71
  72 #define ARM_SMMU_GR0_ID3                0x2c
  73 #define ARM_SMMU_GR0_ID4                0x30
  74 #define ARM_SMMU_GR0_ID5                0x34
  75 #define ARM_SMMU_GR0_ID6                0x38
  76
  77 #define ARM_SMMU_GR0_ID7                0x3c
  78 #define ID7_MAJOR                       GENMASK(7, 4)
  79 #define ID7_MINOR                       GENMASK(3, 0)
  80
  81 #define ARM_SMMU_GR0_sGFSR              0x48
  82 #define sGFSR_USF                       BIT(1)
  83
  84 #define ARM_SMMU_GR0_sGFSYNR0           0x50
  85 #define ARM_SMMU_GR0_sGFSYNR1           0x54
  86 #define ARM_SMMU_GR0_sGFSYNR2           0x58
  87
  88 /* Global TLB invalidation */
  89 #define ARM_SMMU_GR0_TLBIVMID           0x64
  90 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
  91 #define ARM_SMMU_GR0_TLBIALLH           0x6c
  92 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
  93
  94 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
  95 #define sTLBGSTATUS_GSACTIVE            BIT(0)
  96
  97 /* Stream mapping registers */
  98 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
  99 #define SMR_VALID                       BIT(31)
 100 #define SMR_MASK                        GENMASK(31, 16)
 101 #define SMR_ID                          GENMASK(15, 0)
 102
 103 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 104 #define S2CR_PRIVCFG                    GENMASK(25, 24)
 105 enum arm_smmu_s2cr_privcfg {
 106         S2CR_PRIVCFG_DEFAULT,
 107         S2CR_PRIVCFG_DIPAN,
 108         S2CR_PRIVCFG_UNPRIV,
 109         S2CR_PRIVCFG_PRIV,
 110 };
 111 #define S2CR_TYPE                       GENMASK(17, 16)
 112 enum arm_smmu_s2cr_type {
 113         S2CR_TYPE_TRANS,
 114         S2CR_TYPE_BYPASS,
 115         S2CR_TYPE_FAULT,
 116 };
 117 #define S2CR_EXIDVALID                  BIT(10)
 118 #define S2CR_CBNDX                      GENMASK(7, 0)
 119
 120 /* Context bank attribute registers */
 121 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 122 #define CBAR_IRPTNDX                    GENMASK(31, 24)
 123 #define CBAR_TYPE                       GENMASK(17, 16)
 124 enum arm_smmu_cbar_type {
 125         CBAR_TYPE_S2_TRANS,
 126         CBAR_TYPE_S1_TRANS_S2_BYPASS,
 127         CBAR_TYPE_S1_TRANS_S2_FAULT,
 128         CBAR_TYPE_S1_TRANS_S2_TRANS,
 129 };
 130 #define CBAR_S1_MEMATTR                 GENMASK(15, 12)
 131 #define CBAR_S1_MEMATTR_WB              0xf
 132 #define CBAR_S1_BPSHCFG                 GENMASK(9, 8)
 133 #define CBAR_S1_BPSHCFG_NSH             3
 134 #define CBAR_VMID                       GENMASK(7, 0)
 135
 136 #define ARM_SMMU_GR1_CBFRSYNRA(n)       (0x400 + ((n) << 2))
 137
 138 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 139 #define CBA2R_VMID16                    GENMASK(31, 16)
 140 #define CBA2R_VA64                      BIT(0)
 141
 142 #define ARM_SMMU_CB_SCTLR               0x0
 143 #define SCTLR_S1_ASIDPNE                BIT(12)
 144 #define SCTLR_CFCFG                     BIT(7)
 145 #define SCTLR_CFIE                      BIT(6)
 146 #define SCTLR_CFRE                      BIT(5)
 147 #define SCTLR_E                         BIT(4)
 148 #define SCTLR_AFE                       BIT(2)
 149 #define SCTLR_TRE                       BIT(1)
 150 #define SCTLR_M                         BIT(0)
 151
 152 #define ARM_SMMU_CB_ACTLR               0x4
 153
 154 #define ARM_SMMU_CB_RESUME              0x8
 155 #define RESUME_TERMINATE                BIT(0)
 156
 157 #define ARM_SMMU_CB_TCR2                0x10
 158 #define TCR2_SEP                        GENMASK(17, 15)
 159 #define TCR2_SEP_UPSTREAM               0x7
 160 #define TCR2_AS                         BIT(4)
 161
 162 #define ARM_SMMU_CB_TTBR0               0x20
 163 #define ARM_SMMU_CB_TTBR1               0x28
 164 #define TTBRn_ASID                      GENMASK_ULL(63, 48)
 165
 166 #define ARM_SMMU_CB_TCR                 0x30
 167 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 168 #define ARM_SMMU_CB_S1_MAIR0            0x38
 169 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 170
 171 #define ARM_SMMU_CB_PAR                 0x50
 172 #define CB_PAR_F                        BIT(0)
 173
 174 #define ARM_SMMU_CB_FSR                 0x58
 175 #define FSR_MULTI                       BIT(31)
 176 #define FSR_SS                          BIT(30)
 177 #define FSR_UUT                         BIT(8)
 178 #define FSR_ASF                         BIT(7)
 179 #define FSR_TLBLKF                      BIT(6)
 180 #define FSR_TLBMCF                      BIT(5)
 181 #define FSR_EF                          BIT(4)
 182 #define FSR_PF                          BIT(3)
 183 #define FSR_AFF                         BIT(2)
 184 #define FSR_TF                          BIT(1)
 185
 186 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 187                                          FSR_TLBMCF | FSR_TLBLKF)
 188 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 189                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 190
 191 #define ARM_SMMU_CB_FAR                 0x60
 192
 193 #define ARM_SMMU_CB_FSYNR0              0x68
 194 #define FSYNR0_WNR                      BIT(4)
 195
 196 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 197 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 198 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 199 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 200 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 201 #define ARM_SMMU_CB_TLBSYNC             0x7f0
 202 #define ARM_SMMU_CB_TLBSTATUS           0x7f4
 203 #define ARM_SMMU_CB_ATS1PR              0x800
 204
 205 #define ARM_SMMU_CB_ATSR                0x8f0
 206 #define ATSR_ACTIVE                     BIT(0)
 207
 208
 209 /* Maximum number of context banks per SMMU */
 210 #define ARM_SMMU_MAX_CBS                128
 211
 212
 213 /* Shared driver definitions */
 214 enum arm_smmu_arch_version {
 215         ARM_SMMU_V1,
 216         ARM_SMMU_V1_64K,
 217         ARM_SMMU_V2,
 218 };
 219
 220 enum arm_smmu_implementation {
 221         GENERIC_SMMU,
 222         ARM_MMU500,
 223         CAVIUM_SMMUV2,
 224         QCOM_SMMUV2,
 225 };
 226
 227 struct arm_smmu_device {
 228         struct device                   *dev;
 229
 230         void __iomem                    *base;
 231         unsigned int                    numpage;
 232         unsigned int                    pgshift;
 233
 234 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 235 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 236 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 237 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 238 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 239 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 240 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 241 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 242 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 243 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 244 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 245 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 246 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 247         u32                             features;
 248
 249         enum arm_smmu_arch_version      version;
 250         enum arm_smmu_implementation    model;
 251         const struct arm_smmu_impl      *impl;
 252
 253         u32                             num_context_banks;
 254         u32                             num_s2_context_banks;
 255         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 256         struct arm_smmu_cb              *cbs;
 257         atomic_t                        irptndx;
 258
 259         u32                             num_mapping_groups;
 260         u16                             streamid_mask;
 261         u16                             smr_mask_mask;
 262         struct arm_smmu_smr             *smrs;
 263         struct arm_smmu_s2cr            *s2crs;
 264         struct mutex                    stream_map_mutex;
 265
 266         unsigned long                   va_size;
 267         unsigned long                   ipa_size;
 268         unsigned long                   pa_size;
 269         unsigned long                   pgsize_bitmap;
 270
 271         u32                             num_global_irqs;
 272         u32                             num_context_irqs;
 273         unsigned int                    *irqs;
 274         struct clk_bulk_data            *clks;
 275         int                             num_clks;
 276
 277         spinlock_t                      global_sync_lock;
 278
 279         /* IOMMU core code handle */
 280         struct iommu_device             iommu;
 281 };
 282
 283 enum arm_smmu_context_fmt {
 284         ARM_SMMU_CTX_FMT_NONE,
 285         ARM_SMMU_CTX_FMT_AARCH64,
 286         ARM_SMMU_CTX_FMT_AARCH32_L,
 287         ARM_SMMU_CTX_FMT_AARCH32_S,
 288 };
 289
 290 struct arm_smmu_cfg {
 291         u8                              cbndx;
 292         u8                              irptndx;
 293         union {
 294                 u16                     asid;
 295                 u16                     vmid;
 296         };
 297         enum arm_smmu_cbar_type         cbar;
 298         enum arm_smmu_context_fmt       fmt;
 299 };
 300 #define INVALID_IRPTNDX                 0xff
 301
 302 enum arm_smmu_domain_stage {
 303         ARM_SMMU_DOMAIN_S1 = 0,
 304         ARM_SMMU_DOMAIN_S2,
 305         ARM_SMMU_DOMAIN_NESTED,
 306         ARM_SMMU_DOMAIN_BYPASS,
 307 };
 308
 309 struct arm_smmu_domain {
 310         struct arm_smmu_device          *smmu;
 311         struct io_pgtable_ops           *pgtbl_ops;
 312         const struct iommu_flush_ops    *flush_ops;
 313         struct arm_smmu_cfg             cfg;
 314         enum arm_smmu_domain_stage      stage;
 315         bool                            non_strict;
 316         struct mutex                    init_mutex; /* Protects smmu pointer */
 317         spinlock_t                      cb_lock; /* Serialises ATS1* ops and TLB syncs */
 318         struct iommu_domain             domain;
 319 };
 320
 321
 322 /* Implementation details, yay! */
 323 struct arm_smmu_impl {
 324         u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
 325         void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
 326                           u32 val);
 327         u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
 328         void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
 329                             u64 val);
 330         int (*cfg_probe)(struct arm_smmu_device *smmu);
 331         int (*reset)(struct arm_smmu_device *smmu);
 332         int (*init_context)(struct arm_smmu_domain *smmu_domain);
 333         void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
 334                          int status);
 335 };
 336
 337 static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
 338 {
 339         return smmu->base + (n << smmu->pgshift);
 340 }
 341
 342 static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
 343 {
 344         if (smmu->impl && unlikely(smmu->impl->read_reg))
 345                 return smmu->impl->read_reg(smmu, page, offset);
 346         return readl_relaxed(arm_smmu_page(smmu, page) + offset);
 347 }
 348
 349 static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
 350                                    int offset, u32 val)
 351 {
 352         if (smmu->impl && unlikely(smmu->impl->write_reg))
 353                 smmu->impl->write_reg(smmu, page, offset, val);
 354         else
 355                 writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
 356 }
 357
 358 static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
 359 {
 360         if (smmu->impl && unlikely(smmu->impl->read_reg64))
 361                 return smmu->impl->read_reg64(smmu, page, offset);
 362         return readq_relaxed(arm_smmu_page(smmu, page) + offset);
 363 }
 364
 365 static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
 366                                    int offset, u64 val)
 367 {
 368         if (smmu->impl && unlikely(smmu->impl->write_reg64))
 369                 smmu->impl->write_reg64(smmu, page, offset, val);
 370         else
 371                 writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
 372 }
 373
 374 #define ARM_SMMU_GR0            0
 375 #define ARM_SMMU_GR1            1
 376 #define ARM_SMMU_CB(s, n)       ((s)->numpage + (n))
 377
 378 #define arm_smmu_gr0_read(s, o)         \
 379         arm_smmu_readl((s), ARM_SMMU_GR0, (o))
 380 #define arm_smmu_gr0_write(s, o, v)     \
 381         arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
 382
 383 #define arm_smmu_gr1_read(s, o)         \
 384         arm_smmu_readl((s), ARM_SMMU_GR1, (o))
 385 #define arm_smmu_gr1_write(s, o, v)     \
 386         arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
 387
 388 #define arm_smmu_cb_read(s, n, o)       \
 389         arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
 390 #define arm_smmu_cb_write(s, n, o, v)   \
 391         arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
 392 #define arm_smmu_cb_readq(s, n, o)      \
 393         arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
 394 #define arm_smmu_cb_writeq(s, n, o, v)  \
 395         arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
 396
 397 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
 398 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
 399
 400 int arm_mmu500_reset(struct arm_smmu_device *smmu);
 401
 402 #endif /* _ARM_SMMU_H */