arch/arm64/kernel/pi/patch-scs.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2022 - Google LLC
   4  * Author: Ard Biesheuvel <ardb@google.com>
   5  */
   6
   7 #include <linux/errno.h>
   8 #include <linux/init.h>
   9 #include <linux/linkage.h>
  10 #include <linux/types.h>
  11
  12 #include <asm/scs.h>
  13
  14 #include "pi.h"
  15
  16 bool dynamic_scs_is_enabled;
  17
  18 //
  19 // This minimal DWARF CFI parser is partially based on the code in
  20 // arch/arc/kernel/unwind.c, and on the document below:
  21 // https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
  22 //
  23
  24 #define DW_CFA_nop                          0x00
  25 #define DW_CFA_set_loc                      0x01
  26 #define DW_CFA_advance_loc1                 0x02
  27 #define DW_CFA_advance_loc2                 0x03
  28 #define DW_CFA_advance_loc4                 0x04
  29 #define DW_CFA_offset_extended              0x05
  30 #define DW_CFA_restore_extended             0x06
  31 #define DW_CFA_undefined                    0x07
  32 #define DW_CFA_same_value                   0x08
  33 #define DW_CFA_register                     0x09
  34 #define DW_CFA_remember_state               0x0a
  35 #define DW_CFA_restore_state                0x0b
  36 #define DW_CFA_def_cfa                      0x0c
  37 #define DW_CFA_def_cfa_register             0x0d
  38 #define DW_CFA_def_cfa_offset               0x0e
  39 #define DW_CFA_def_cfa_expression           0x0f
  40 #define DW_CFA_expression                   0x10
  41 #define DW_CFA_offset_extended_sf           0x11
  42 #define DW_CFA_def_cfa_sf                   0x12
  43 #define DW_CFA_def_cfa_offset_sf            0x13
  44 #define DW_CFA_val_offset                   0x14
  45 #define DW_CFA_val_offset_sf                0x15
  46 #define DW_CFA_val_expression               0x16
  47 #define DW_CFA_lo_user                      0x1c
  48 #define DW_CFA_negate_ra_state              0x2d
  49 #define DW_CFA_GNU_args_size                0x2e
  50 #define DW_CFA_GNU_negative_offset_extended 0x2f
  51 #define DW_CFA_hi_user                      0x3f
  52
  53 #define DW_EH_PE_sdata4                     0x0b
  54 #define DW_EH_PE_sdata8                     0x0c
  55 #define DW_EH_PE_pcrel                      0x10
  56
  57 enum {
  58         PACIASP         = 0xd503233f,
  59         AUTIASP         = 0xd50323bf,
  60         SCS_PUSH        = 0xf800865e,
  61         SCS_POP         = 0xf85f8e5e,
  62 };
  63
  64 static void __always_inline scs_patch_loc(u64 loc)
  65 {
  66         u32 insn = le32_to_cpup((void *)loc);
  67
  68         switch (insn) {
  69         case PACIASP:
  70                 *(u32 *)loc = cpu_to_le32(SCS_PUSH);
  71                 break;
  72         case AUTIASP:
  73                 *(u32 *)loc = cpu_to_le32(SCS_POP);
  74                 break;
  75         default:
  76                 /*
  77                  * While the DW_CFA_negate_ra_state directive is guaranteed to
  78                  * appear right after a PACIASP/AUTIASP instruction, it may
  79                  * also appear after a DW_CFA_restore_state directive that
  80                  * restores a state that is only partially accurate, and is
  81                  * followed by DW_CFA_negate_ra_state directive to toggle the
  82                  * PAC bit again. So we permit other instructions here, and ignore
  83                  * them.
  84                  */
  85                 return;
  86         }
  87         if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_CLEAN_CACHE))
  88                 asm("dc civac, %0" :: "r"(loc));
  89         else
  90                 asm(ALTERNATIVE("dc cvau, %0", "nop", ARM64_HAS_CACHE_IDC)
  91                     :: "r"(loc));
  92 }
  93
  94 /*
  95  * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
  96  * except the last one have bit #7 set.
  97  */
  98 static int __always_inline skip_xleb128(const u8 **opcode, int size)
  99 {
 100         u8 c;
 101
 102         do {
 103                 c = *(*opcode)++;
 104                 size--;
 105         } while (c & BIT(7));
 106
 107         return size;
 108 }
 109
 110 struct eh_frame {
 111         /*
 112          * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
 113          */
 114         u32     size;
 115
 116         /*
 117          * The first frame is a Common Information Entry (CIE) frame, followed
 118          * by one or more Frame Description Entry (FDE) frames. In the former
 119          * case, this field is 0, otherwise it is the negated offset relative
 120          * to the associated CIE frame.
 121          */
 122         u32     cie_id_or_pointer;
 123
 124         union {
 125                 struct { // CIE
 126                         u8      version;
 127                         u8      augmentation_string[3];
 128                         u8      code_alignment_factor;
 129                         u8      data_alignment_factor;
 130                         u8      return_address_register;
 131                         u8      augmentation_data_size;
 132                         u8      fde_pointer_format;
 133                 };
 134
 135                 struct { // FDE
 136                         s32     initial_loc;
 137                         s32     range;
 138                         u8      opcodes[];
 139                 };
 140
 141                 struct { // FDE
 142                         s64     initial_loc64;
 143                         s64     range64;
 144                         u8      opcodes64[];
 145                 };
 146         };
 147 };
 148
 149 static int scs_handle_fde_frame(const struct eh_frame *frame,
 150                                 int code_alignment_factor,
 151                                 bool use_sdata8,
 152                                 bool dry_run)
 153 {
 154         int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
 155         u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
 156         const u8 *opcode = frame->opcodes;
 157         int l;
 158
 159         if (use_sdata8) {
 160                 loc = (u64)&frame->initial_loc64 + frame->initial_loc64;
 161                 opcode = frame->opcodes64;
 162                 size -= 8;
 163         }
 164
 165         // assume single byte uleb128_t for augmentation data size
 166         if (*opcode & BIT(7))
 167                 return EDYNSCS_INVALID_FDE_AUGM_DATA_SIZE;
 168
 169         l = *opcode++;
 170         opcode += l;
 171         size -= l + 1;
 172
 173         /*
 174          * Starting from 'loc', apply the CFA opcodes that advance the location
 175          * pointer, and identify the locations of the PAC instructions.
 176          */
 177         while (size-- > 0) {
 178                 switch (*opcode++) {
 179                 case DW_CFA_nop:
 180                 case DW_CFA_remember_state:
 181                 case DW_CFA_restore_state:
 182                         break;
 183
 184                 case DW_CFA_advance_loc1:
 185                         loc += *opcode++ * code_alignment_factor;
 186                         size--;
 187                         break;
 188
 189                 case DW_CFA_advance_loc2:
 190                         loc += *opcode++ * code_alignment_factor;
 191                         loc += (*opcode++ << 8) * code_alignment_factor;
 192                         size -= 2;
 193                         break;
 194
 195                 case DW_CFA_def_cfa:
 196                 case DW_CFA_offset_extended:
 197                         size = skip_xleb128(&opcode, size);
 198                         fallthrough;
 199                 case DW_CFA_def_cfa_offset:
 200                 case DW_CFA_def_cfa_offset_sf:
 201                 case DW_CFA_def_cfa_register:
 202                 case DW_CFA_same_value:
 203                 case DW_CFA_restore_extended:
 204                 case 0x80 ... 0xbf:
 205                         size = skip_xleb128(&opcode, size);
 206                         break;
 207
 208                 case DW_CFA_negate_ra_state:
 209                         if (!dry_run)
 210                                 scs_patch_loc(loc - 4);
 211                         break;
 212
 213                 case 0x40 ... 0x7f:
 214                         // advance loc
 215                         loc += (opcode[-1] & 0x3f) * code_alignment_factor;
 216                         break;
 217
 218                 case 0xc0 ... 0xff:
 219                         break;
 220
 221                 default:
 222                         return EDYNSCS_INVALID_CFA_OPCODE;
 223                 }
 224         }
 225         return 0;
 226 }
 227
 228 int scs_patch(const u8 eh_frame[], int size)
 229 {
 230         int code_alignment_factor = 1;
 231         bool fde_use_sdata8 = false;
 232         const u8 *p = eh_frame;
 233
 234         while (size > 4) {
 235                 const struct eh_frame *frame = (const void *)p;
 236                 int ret;
 237
 238                 if (frame->size == 0 ||
 239                     frame->size == U32_MAX ||
 240                     frame->size > size)
 241                         break;
 242
 243                 if (frame->cie_id_or_pointer == 0) {
 244                         /*
 245                          * Require presence of augmentation data (z) with a
 246                          * specifier for the size of the FDE initial_loc and
 247                          * range fields (R), and nothing else.
 248                          */
 249                         if (strcmp(frame->augmentation_string, "zR"))
 250                                 return EDYNSCS_INVALID_CIE_HEADER;
 251
 252                         /*
 253                          * The code alignment factor is a uleb128 encoded field
 254                          * but given that the only sensible values are 1 or 4,
 255                          * there is no point in decoding the whole thing.  Also
 256                          * sanity check the size of the data alignment factor
 257                          * field, and the values of the return address register
 258                          * and augmentation data size fields.
 259                          */
 260                         if ((frame->code_alignment_factor & BIT(7)) ||
 261                             (frame->data_alignment_factor & BIT(7)) ||
 262                             frame->return_address_register != 30 ||
 263                             frame->augmentation_data_size != 1)
 264                                 return EDYNSCS_INVALID_CIE_HEADER;
 265
 266                         code_alignment_factor = frame->code_alignment_factor;
 267
 268                         switch (frame->fde_pointer_format) {
 269                         case DW_EH_PE_pcrel | DW_EH_PE_sdata4:
 270                                 fde_use_sdata8 = false;
 271                                 break;
 272                         case DW_EH_PE_pcrel | DW_EH_PE_sdata8:
 273                                 fde_use_sdata8 = true;
 274                                 break;
 275                         default:
 276                                 return EDYNSCS_INVALID_CIE_SDATA_SIZE;
 277                         }
 278                 } else {
 279                         ret = scs_handle_fde_frame(frame, code_alignment_factor,
 280                                                    fde_use_sdata8, true);
 281                         if (ret)
 282                                 return ret;
 283                         scs_handle_fde_frame(frame, code_alignment_factor,
 284                                              fde_use_sdata8, false);
 285                 }
 286
 287                 p += sizeof(frame->size) + frame->size;
 288                 size -= sizeof(frame->size) + frame->size;
 289         }
 290         return 0;
 291 }