lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp

   1 //===-- NativeRegisterContextLinux_arm64.cpp ------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #if defined(__arm64__) || defined(__aarch64__)
  10
  11 #include "NativeRegisterContextLinux_arm.h"
  12 #include "NativeRegisterContextLinux_arm64.h"
  13
  14 #include "lldb/Host/HostInfo.h"
  15 #include "lldb/Host/common/NativeProcessProtocol.h"
  16 #include "lldb/Host/linux/Ptrace.h"
  17 #include "lldb/Utility/DataBufferHeap.h"
  18 #include "lldb/Utility/Log.h"
  19 #include "lldb/Utility/RegisterValue.h"
  20 #include "lldb/Utility/Status.h"
  21
  22 #include "Plugins/Process/Linux/NativeProcessLinux.h"
  23 #include "Plugins/Process/Linux/Procfs.h"
  24 #include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
  25 #include "Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h"
  26 #include "Plugins/Process/Utility/RegisterFlagsDetector_arm64.h"
  27 #include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
  28
  29 // System includes - They have to be included after framework includes because
  30 // they define some macros which collide with variable names in other modules
  31 #include <sys/uio.h>
  32 // NT_PRSTATUS and NT_FPREGSET definition
  33 #include <elf.h>
  34 #include <mutex>
  35 #include <optional>
  36
  37 #ifndef NT_ARM_SVE
  38 #define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */
  39 #endif
  40
  41 #ifndef NT_ARM_SSVE
  42 #define NT_ARM_SSVE                                                            \
  43   0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */
  44 #endif
  45
  46 #ifndef NT_ARM_ZA
  47 #define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */
  48 #endif
  49
  50 #ifndef NT_ARM_ZT
  51 #define NT_ARM_ZT                                                              \
  52   0x40d /* ARM Scalable Matrix Extension 2, lookup table register */
  53 #endif
  54
  55 #ifndef NT_ARM_PAC_MASK
  56 #define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */
  57 #endif
  58
  59 #ifndef NT_ARM_TAGGED_ADDR_CTRL
  60 #define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */
  61 #endif
  62
  63 #ifndef NT_ARM_FPMR
  64 #define NT_ARM_FPMR 0x40e /* Floating point mode register */
  65 #endif
  66
  67 #define HWCAP_PACA (1 << 30)
  68
  69 #define HWCAP2_MTE (1 << 18)
  70
  71 #define HWCAP2_FPMR (1UL << 48)
  72
  73 using namespace lldb;
  74 using namespace lldb_private;
  75 using namespace lldb_private::process_linux;
  76
  77 // A NativeRegisterContext is constructed per thread, but all threads' registers
  78 // will contain the same fields. Therefore this mutex prevents each instance
  79 // competing with the other, and subsequent instances from having to detect the
  80 // fields all over again.
  81 static std::mutex g_register_flags_detector_mutex;
  82 static Arm64RegisterFlagsDetector g_register_flags_detector;
  83
  84 std::unique_ptr<NativeRegisterContextLinux>
  85 NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
  86     const ArchSpec &target_arch, NativeThreadLinux &native_thread) {
  87   switch (target_arch.GetMachine()) {
  88   case llvm::Triple::arm:
  89     return std::make_unique<NativeRegisterContextLinux_arm>(target_arch,
  90                                                             native_thread);
  91   case llvm::Triple::aarch64: {
  92     // Configure register sets supported by this AArch64 target.
  93     // Read SVE header to check for SVE support.
  94     struct sve::user_sve_header sve_header;
  95     struct iovec ioVec;
  96     ioVec.iov_base = &sve_header;
  97     ioVec.iov_len = sizeof(sve_header);
  98     unsigned int regset = NT_ARM_SVE;
  99
 100     Flags opt_regsets;
 101     if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
 102                                           native_thread.GetID(), &regset,
 103                                           &ioVec, sizeof(sve_header))
 104             .Success()) {
 105       opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSVE);
 106
 107       // We may also have the Scalable Matrix Extension (SME) which adds a
 108       // streaming SVE mode.
 109       ioVec.iov_len = sizeof(sve_header);
 110       regset = NT_ARM_SSVE;
 111       if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
 112                                             native_thread.GetID(), &regset,
 113                                             &ioVec, sizeof(sve_header))
 114               .Success())
 115         opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE);
 116     }
 117
 118     sve::user_za_header za_header;
 119     ioVec.iov_base = &za_header;
 120     ioVec.iov_len = sizeof(za_header);
 121     regset = NT_ARM_ZA;
 122     if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
 123                                           native_thread.GetID(), &regset,
 124                                           &ioVec, sizeof(za_header))
 125             .Success())
 126       opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZA);
 127
 128     // SME's ZT0 is a 512 bit register.
 129     std::array<uint8_t, 64> zt_reg;
 130     ioVec.iov_base = zt_reg.data();
 131     ioVec.iov_len = zt_reg.size();
 132     regset = NT_ARM_ZT;
 133     if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,
 134                                           native_thread.GetID(), &regset,
 135                                           &ioVec, zt_reg.size())
 136             .Success())
 137       opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZT);
 138
 139     NativeProcessLinux &process = native_thread.GetProcess();
 140
 141     std::optional<uint64_t> auxv_at_hwcap =
 142         process.GetAuxValue(AuxVector::AUXV_AT_HWCAP);
 143     if (auxv_at_hwcap && (*auxv_at_hwcap & HWCAP_PACA))
 144       opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskPAuth);
 145
 146     std::optional<uint64_t> auxv_at_hwcap2 =
 147         process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2);
 148     if (auxv_at_hwcap2) {
 149       if (*auxv_at_hwcap2 & HWCAP2_MTE)
 150         opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskMTE);
 151       if (*auxv_at_hwcap2 & HWCAP2_FPMR)
 152         opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskFPMR);
 153     }
 154
 155     opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskTLS);
 156
 157     std::lock_guard<std::mutex> lock(g_register_flags_detector_mutex);
 158     if (!g_register_flags_detector.HasDetected())
 159       g_register_flags_detector.DetectFields(auxv_at_hwcap.value_or(0),
 160                                              auxv_at_hwcap2.value_or(0));
 161
 162     auto register_info_up =
 163         std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets);
 164     return std::make_unique<NativeRegisterContextLinux_arm64>(
 165         target_arch, native_thread, std::move(register_info_up));
 166   }
 167   default:
 168     llvm_unreachable("have no register context for architecture");
 169   }
 170 }
 171
 172 llvm::Expected<ArchSpec>
 173 NativeRegisterContextLinux::DetermineArchitecture(lldb::tid_t tid) {
 174   return DetermineArchitectureViaGPR(
 175       tid, RegisterInfoPOSIX_arm64::GetGPRSizeStatic());
 176 }
 177
 178 NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
 179     const ArchSpec &target_arch, NativeThreadProtocol &native_thread,
 180     std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up)
 181     : NativeRegisterContextRegisterInfo(native_thread,
 182                                         register_info_up.release()),
 183       NativeRegisterContextLinux(native_thread) {
 184   g_register_flags_detector.UpdateRegisterInfo(
 185       GetRegisterInfoInterface().GetRegisterInfo(),
 186       GetRegisterInfoInterface().GetRegisterCount());
 187
 188   ::memset(&m_fpr, 0, sizeof(m_fpr));
 189   ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64));
 190   ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs));
 191   ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs));
 192   ::memset(&m_sve_header, 0, sizeof(m_sve_header));
 193   ::memset(&m_pac_mask, 0, sizeof(m_pac_mask));
 194   ::memset(&m_tls_regs, 0, sizeof(m_tls_regs));
 195   ::memset(&m_sme_pseudo_regs, 0, sizeof(m_sme_pseudo_regs));
 196   std::fill(m_zt_reg.begin(), m_zt_reg.end(), 0);
 197
 198   m_mte_ctrl_reg = 0;
 199   m_fpmr_reg = 0;
 200
 201   // 16 is just a maximum value, query hardware for actual watchpoint count
 202   m_max_hwp_supported = 16;
 203   m_max_hbp_supported = 16;
 204
 205   m_refresh_hwdebug_info = true;
 206
 207   m_gpr_is_valid = false;
 208   m_fpu_is_valid = false;
 209   m_sve_buffer_is_valid = false;
 210   m_sve_header_is_valid = false;
 211   m_pac_mask_is_valid = false;
 212   m_mte_ctrl_is_valid = false;
 213   m_tls_is_valid = false;
 214   m_zt_buffer_is_valid = false;
 215   m_fpmr_is_valid = false;
 216
 217   // SME adds the tpidr2 register
 218   m_tls_size = GetRegisterInfo().IsSSVEPresent() ? sizeof(m_tls_regs)
 219                                                  : sizeof(m_tls_regs.tpidr_reg);
 220
 221   if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent())
 222     m_sve_state = SVEState::Unknown;
 223   else
 224     m_sve_state = SVEState::Disabled;
 225 }
 226
 227 RegisterInfoPOSIX_arm64 &
 228 NativeRegisterContextLinux_arm64::GetRegisterInfo() const {
 229   return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up);
 230 }
 231
 232 uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const {
 233   return GetRegisterInfo().GetRegisterSetCount();
 234 }
 235
 236 const RegisterSet *
 237 NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const {
 238   return GetRegisterInfo().GetRegisterSet(set_index);
 239 }
 240
 241 uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const {
 242   uint32_t count = 0;
 243   for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index)
 244     count += GetRegisterSet(set_index)->num_registers;
 245   return count;
 246 }
 247
 248 Status
 249 NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info,
 250                                                RegisterValue &reg_value) {
 251   Status error;
 252
 253   if (!reg_info) {
 254     error = Status::FromErrorString("reg_info NULL");
 255     return error;
 256   }
 257
 258   const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
 259
 260   if (reg == LLDB_INVALID_REGNUM)
 261     return Status::FromErrorStringWithFormat(
 262         "no lldb regnum for %s",
 263         reg_info && reg_info->name ? reg_info->name : "<unknown register>");
 264
 265   uint8_t *src;
 266   uint32_t offset = LLDB_INVALID_INDEX32;
 267   uint64_t sve_vg;
 268   std::vector<uint8_t> sve_reg_non_live;
 269
 270   if (IsGPR(reg)) {
 271     error = ReadGPR();
 272     if (error.Fail())
 273       return error;
 274
 275     offset = reg_info->byte_offset;
 276     assert(offset < GetGPRSize());
 277     src = (uint8_t *)GetGPRBuffer() + offset;
 278
 279   } else if (IsFPR(reg)) {
 280     if (m_sve_state == SVEState::Disabled) {
 281       // SVE is disabled take legacy route for FPU register access
 282       error = ReadFPR();
 283       if (error.Fail())
 284         return error;
 285
 286       offset = CalculateFprOffset(reg_info);
 287       assert(offset < GetFPRSize());
 288       src = (uint8_t *)GetFPRBuffer() + offset;
 289     } else {
 290       // SVE or SSVE enabled, we will read and cache SVE ptrace data.
 291       // In SIMD or Full mode, the data comes from the SVE regset. In streaming
 292       // mode it comes from the streaming SVE regset.
 293       error = ReadAllSVE();
 294       if (error.Fail())
 295         return error;
 296
 297       // FPSR and FPCR will be located right after Z registers in
 298       // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
 299       // will be located at the end of register data after an alignment
 300       // correction based on currently selected vector length.
 301       uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
 302       if (reg == GetRegisterInfo().GetRegNumFPSR()) {
 303         sve_reg_num = reg;
 304         if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
 305           offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
 306         else if (m_sve_state == SVEState::FPSIMD)
 307           offset = sve::ptrace_fpsimd_offset + (32 * 16);
 308       } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
 309         sve_reg_num = reg;
 310         if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
 311           offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
 312         else if (m_sve_state == SVEState::FPSIMD)
 313           offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
 314       } else {
 315         // Extract SVE Z register value register number for this reg_info
 316         if (reg_info->value_regs &&
 317             reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
 318           sve_reg_num = reg_info->value_regs[0];
 319         offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
 320       }
 321
 322       assert(offset < GetSVEBufferSize());
 323       src = (uint8_t *)GetSVEBuffer() + offset;
 324     }
 325   } else if (IsTLS(reg)) {
 326     error = ReadTLS();
 327     if (error.Fail())
 328       return error;
 329
 330     offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
 331     assert(offset < GetTLSBufferSize());
 332     src = (uint8_t *)GetTLSBuffer() + offset;
 333   } else if (IsSVE(reg)) {
 334     if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
 335       return Status::FromErrorString("SVE disabled or not supported");
 336
 337     if (GetRegisterInfo().IsSVERegVG(reg)) {
 338       sve_vg = GetSVERegVG();
 339       src = (uint8_t *)&sve_vg;
 340     } else {
 341       // SVE enabled, we will read and cache SVE ptrace data
 342       error = ReadAllSVE();
 343       if (error.Fail())
 344         return error;
 345
 346       if (m_sve_state == SVEState::FPSIMD) {
 347         // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so
 348         // just copy 16 bytes of v register to the start of z register. All
 349         // other SVE register will be set to zero.
 350         sve_reg_non_live.resize(reg_info->byte_size, 0);
 351         src = sve_reg_non_live.data();
 352
 353         if (GetRegisterInfo().IsSVEZReg(reg)) {
 354           offset = CalculateSVEOffset(reg_info);
 355           assert(offset < GetSVEBufferSize());
 356           ::memcpy(sve_reg_non_live.data(), (uint8_t *)GetSVEBuffer() + offset,
 357                    16);
 358         }
 359       } else {
 360         offset = CalculateSVEOffset(reg_info);
 361         assert(offset < GetSVEBufferSize());
 362         src = (uint8_t *)GetSVEBuffer() + offset;
 363       }
 364     }
 365   } else if (IsPAuth(reg)) {
 366     error = ReadPAuthMask();
 367     if (error.Fail())
 368       return error;
 369
 370     offset = reg_info->byte_offset - GetRegisterInfo().GetPAuthOffset();
 371     assert(offset < GetPACMaskSize());
 372     src = (uint8_t *)GetPACMask() + offset;
 373   } else if (IsMTE(reg)) {
 374     error = ReadMTEControl();
 375     if (error.Fail())
 376       return error;
 377
 378     offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
 379     assert(offset < GetMTEControlSize());
 380     src = (uint8_t *)GetMTEControl() + offset;
 381   } else if (IsSME(reg)) {
 382     if (GetRegisterInfo().IsSMERegZA(reg)) {
 383       error = ReadZAHeader();
 384       if (error.Fail())
 385         return error;
 386
 387       // If there is only a header and no registers, ZA is inactive. Read as 0
 388       // in this case.
 389       if (m_za_header.size == sizeof(m_za_header)) {
 390         // This will get reconfigured/reset later, so we are safe to use it.
 391         // ZA is a square of VL * VL and the ptrace buffer also includes the
 392         // header itself.
 393         m_za_ptrace_payload.resize(((m_za_header.vl) * (m_za_header.vl)) +
 394                                    GetZAHeaderSize());
 395         std::fill(m_za_ptrace_payload.begin(), m_za_ptrace_payload.end(), 0);
 396       } else {
 397         // ZA is active, read the real register.
 398         error = ReadZA();
 399         if (error.Fail())
 400           return error;
 401       }
 402
 403       // ZA is part of the SME set but uses a separate member buffer for
 404       // storage. Therefore its effective byte offset is always 0 even if it
 405       // isn't 0 within the SME register set.
 406       src = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
 407     } else if (GetRegisterInfo().IsSMERegZT(reg)) {
 408       // Unlike ZA, the kernel will return register data for ZT0 when ZA is not
 409       // enabled. This data will be all 0s so we don't have to invent anything
 410       // like we did for ZA.
 411       error = ReadZT();
 412       if (error.Fail())
 413         return error;
 414
 415       src = (uint8_t *)GetZTBuffer();
 416     } else {
 417       error = ReadSMESVG();
 418       if (error.Fail())
 419         return error;
 420
 421       // This is a psuedo so it never fails.
 422       ReadSMEControl();
 423
 424       offset = reg_info->byte_offset - GetRegisterInfo().GetSMEOffset();
 425       assert(offset < GetSMEPseudoBufferSize());
 426       src = (uint8_t *)GetSMEPseudoBuffer() + offset;
 427     }
 428   } else if (IsFPMR(reg)) {
 429     error = ReadFPMR();
 430     if (error.Fail())
 431       return error;
 432
 433     offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
 434     assert(offset < GetFPMRBufferSize());
 435     src = (uint8_t *)GetFPMRBuffer() + offset;
 436   } else
 437     return Status::FromErrorString(
 438         "failed - register wasn't recognized to be a GPR or an FPR, "
 439         "write strategy unknown");
 440
 441   reg_value.SetFromMemoryData(*reg_info, src, reg_info->byte_size,
 442                               eByteOrderLittle, error);
 443
 444   return error;
 445 }
 446
 447 Status NativeRegisterContextLinux_arm64::WriteRegister(
 448     const RegisterInfo *reg_info, const RegisterValue &reg_value) {
 449   Status error;
 450
 451   if (!reg_info)
 452     return Status::FromErrorString("reg_info NULL");
 453
 454   const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
 455
 456   if (reg == LLDB_INVALID_REGNUM)
 457     return Status::FromErrorStringWithFormat(
 458         "no lldb regnum for %s",
 459         reg_info && reg_info->name ? reg_info->name : "<unknown register>");
 460
 461   uint8_t *dst;
 462   uint32_t offset = LLDB_INVALID_INDEX32;
 463   std::vector<uint8_t> sve_reg_non_live;
 464
 465   if (IsGPR(reg)) {
 466     error = ReadGPR();
 467     if (error.Fail())
 468       return error;
 469
 470     assert(reg_info->byte_offset < GetGPRSize());
 471     dst = (uint8_t *)GetGPRBuffer() + reg_info->byte_offset;
 472     ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 473
 474     return WriteGPR();
 475   } else if (IsFPR(reg)) {
 476     if (m_sve_state == SVEState::Disabled) {
 477       // SVE is disabled take legacy route for FPU register access
 478       error = ReadFPR();
 479       if (error.Fail())
 480         return error;
 481
 482       offset = CalculateFprOffset(reg_info);
 483       assert(offset < GetFPRSize());
 484       dst = (uint8_t *)GetFPRBuffer() + offset;
 485       ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 486
 487       return WriteFPR();
 488     } else {
 489       // SVE enabled, we will read and cache SVE ptrace data.
 490       error = ReadAllSVE();
 491       if (error.Fail())
 492         return error;
 493
 494       // FPSR and FPCR will be located right after Z registers in
 495       // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they
 496       // will be located at the end of register data after an alignment
 497       // correction based on currently selected vector length.
 498       uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
 499       if (reg == GetRegisterInfo().GetRegNumFPSR()) {
 500         sve_reg_num = reg;
 501         if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
 502           offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));
 503         else if (m_sve_state == SVEState::FPSIMD)
 504           offset = sve::ptrace_fpsimd_offset + (32 * 16);
 505       } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {
 506         sve_reg_num = reg;
 507         if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)
 508           offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));
 509         else if (m_sve_state == SVEState::FPSIMD)
 510           offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;
 511       } else {
 512         // Extract SVE Z register value register number for this reg_info
 513         if (reg_info->value_regs &&
 514             reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
 515           sve_reg_num = reg_info->value_regs[0];
 516         offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
 517       }
 518
 519       assert(offset < GetSVEBufferSize());
 520       dst = (uint8_t *)GetSVEBuffer() + offset;
 521       ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 522       return WriteAllSVE();
 523     }
 524   } else if (IsSVE(reg)) {
 525     if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
 526       return Status::FromErrorString("SVE disabled or not supported");
 527     else {
 528       // Target has SVE enabled, we will read and cache SVE ptrace data
 529       error = ReadAllSVE();
 530       if (error.Fail())
 531         return error;
 532
 533       if (GetRegisterInfo().IsSVERegVG(reg)) {
 534         uint64_t vg_value = reg_value.GetAsUInt64();
 535
 536         if (sve::vl_valid(vg_value * 8)) {
 537           if (m_sve_header_is_valid && vg_value == GetSVERegVG())
 538             return error;
 539
 540           SetSVERegVG(vg_value);
 541
 542           error = WriteSVEHeader();
 543           if (error.Success()) {
 544             // Changing VG during streaming mode also changes the size of ZA.
 545             if (m_sve_state == SVEState::Streaming)
 546               m_za_header_is_valid = false;
 547             ConfigureRegisterContext();
 548           }
 549
 550           if (m_sve_header_is_valid && vg_value == GetSVERegVG())
 551             return error;
 552         }
 553
 554         return Status::FromErrorString("SVE vector length update failed.");
 555       }
 556
 557       // If target supports SVE but currently in FPSIMD mode.
 558       if (m_sve_state == SVEState::FPSIMD) {
 559         // Here we will check if writing this SVE register enables
 560         // SVEState::Full
 561         bool set_sve_state_full = false;
 562         const uint8_t *reg_bytes = (const uint8_t *)reg_value.GetBytes();
 563         if (GetRegisterInfo().IsSVEZReg(reg)) {
 564           for (uint32_t i = 16; i < reg_info->byte_size; i++) {
 565             if (reg_bytes[i]) {
 566               set_sve_state_full = true;
 567               break;
 568             }
 569           }
 570         } else if (GetRegisterInfo().IsSVEPReg(reg) ||
 571                    reg == GetRegisterInfo().GetRegNumSVEFFR()) {
 572           for (uint32_t i = 0; i < reg_info->byte_size; i++) {
 573             if (reg_bytes[i]) {
 574               set_sve_state_full = true;
 575               break;
 576             }
 577           }
 578         }
 579
 580         if (!set_sve_state_full && GetRegisterInfo().IsSVEZReg(reg)) {
 581           // We are writing a Z register which is zero beyond 16 bytes so copy
 582           // first 16 bytes only as SVE payload mirrors legacy fpsimd structure
 583           offset = CalculateSVEOffset(reg_info);
 584           assert(offset < GetSVEBufferSize());
 585           dst = (uint8_t *)GetSVEBuffer() + offset;
 586           ::memcpy(dst, reg_value.GetBytes(), 16);
 587
 588           return WriteAllSVE();
 589         } else
 590           return Status::FromErrorString(
 591               "SVE state change operation not supported");
 592       } else {
 593         offset = CalculateSVEOffset(reg_info);
 594         assert(offset < GetSVEBufferSize());
 595         dst = (uint8_t *)GetSVEBuffer() + offset;
 596         ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 597         return WriteAllSVE();
 598       }
 599     }
 600   } else if (IsMTE(reg)) {
 601     error = ReadMTEControl();
 602     if (error.Fail())
 603       return error;
 604
 605     offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();
 606     assert(offset < GetMTEControlSize());
 607     dst = (uint8_t *)GetMTEControl() + offset;
 608     ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 609
 610     return WriteMTEControl();
 611   } else if (IsTLS(reg)) {
 612     error = ReadTLS();
 613     if (error.Fail())
 614       return error;
 615
 616     offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
 617     assert(offset < GetTLSBufferSize());
 618     dst = (uint8_t *)GetTLSBuffer() + offset;
 619     ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 620
 621     return WriteTLS();
 622   } else if (IsSME(reg)) {
 623     if (GetRegisterInfo().IsSMERegZA(reg)) {
 624       error = ReadZA();
 625       if (error.Fail())
 626         return error;
 627
 628       // ZA is part of the SME set but not stored with the other SME registers.
 629       // So its byte offset is effectively always 0.
 630       dst = (uint8_t *)GetZABuffer() + GetZAHeaderSize();
 631       ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 632
 633       // While this is writing a header that contains a vector length, the only
 634       // way to change that is via the vg register. So here we assume the length
 635       // will always be the current length and no reconfigure is needed.
 636       return WriteZA();
 637     } else if (GetRegisterInfo().IsSMERegZT(reg)) {
 638       error = ReadZT();
 639       if (error.Fail())
 640         return error;
 641
 642       dst = (uint8_t *)GetZTBuffer();
 643       ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 644
 645       return WriteZT();
 646     } else
 647       return Status::FromErrorString(
 648           "Writing to SVG or SVCR is not supported.");
 649   } else if (IsFPMR(reg)) {
 650     error = ReadFPMR();
 651     if (error.Fail())
 652       return error;
 653
 654     offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();
 655     assert(offset < GetFPMRBufferSize());
 656     dst = (uint8_t *)GetFPMRBuffer() + offset;
 657     ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 658
 659     return WriteFPMR();
 660   }
 661
 662   return Status::FromErrorString("Failed to write register value");
 663 }
 664
 665 enum RegisterSetType : uint32_t {
 666   GPR,
 667   SVE, // Used for SVE and SSVE.
 668   FPR, // When there is no SVE, or SVE in FPSIMD mode.
 669   // Pointer authentication registers are read only, so not included here.
 670   MTE,
 671   TLS,
 672   SME,  // ZA only, because SVCR and SVG are pseudo registers.
 673   SME2, // ZT only.
 674   FPMR,
 675 };
 676
 677 static uint8_t *AddRegisterSetType(uint8_t *dst,
 678                                    RegisterSetType register_set_type) {
 679   *(reinterpret_cast<uint32_t *>(dst)) = register_set_type;
 680   return dst + sizeof(uint32_t);
 681 }
 682
 683 static uint8_t *AddSavedRegistersData(uint8_t *dst, void *src, size_t size) {
 684   ::memcpy(dst, src, size);
 685   return dst + size;
 686 }
 687
 688 static uint8_t *AddSavedRegisters(uint8_t *dst,
 689                                   enum RegisterSetType register_set_type,
 690                                   void *src, size_t size) {
 691   dst = AddRegisterSetType(dst, register_set_type);
 692   return AddSavedRegistersData(dst, src, size);
 693 }
 694
 695 Status
 696 NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) {
 697   Status error;
 698   cached_size = sizeof(RegisterSetType) + GetGPRBufferSize();
 699   error = ReadGPR();
 700   if (error.Fail())
 701     return error;
 702
 703   if (GetRegisterInfo().IsZAPresent()) {
 704     error = ReadZAHeader();
 705     if (error.Fail())
 706       return error;
 707     // Use header size here because the buffer may contain fake data when ZA is
 708     // disabled. We do not want to write this fake data (all 0s) because this
 709     // would tell the kernel that we want ZA to become active. Which is the
 710     // opposite of what we want in the case where it is currently inactive.
 711     cached_size += sizeof(RegisterSetType) + m_za_header.size;
 712     // For the same reason, we need to force it to be re-read so that it will
 713     // always contain the real header.
 714     m_za_buffer_is_valid = false;
 715     error = ReadZA();
 716     if (error.Fail())
 717       return error;
 718
 719     // We will only be restoring ZT data if ZA is active. As writing to an
 720     // inactive ZT enables ZA, which may not be desireable.
 721     if (
 722         // If we have ZT0, or in other words, if we have SME2.
 723         GetRegisterInfo().IsZTPresent() &&
 724         // And ZA is active, which means that ZT0 is also active.
 725         m_za_header.size > sizeof(m_za_header)) {
 726       cached_size += sizeof(RegisterSetType) + GetZTBufferSize();
 727       // The kernel handles an inactive ZT0 for us, and it will read as 0s if
 728       // inactive (unlike ZA where we fake that behaviour).
 729       error = ReadZT();
 730       if (error.Fail())
 731         return error;
 732     }
 733   }
 734
 735   // If SVE is enabled we need not copy FPR separately.
 736   if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
 737     // Store mode and register data.
 738     cached_size +=
 739         sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize();
 740     error = ReadAllSVE();
 741   } else {
 742     cached_size += sizeof(RegisterSetType) + GetFPRSize();
 743     error = ReadFPR();
 744   }
 745   if (error.Fail())
 746     return error;
 747
 748   if (GetRegisterInfo().IsMTEPresent()) {
 749     cached_size += sizeof(RegisterSetType) + GetMTEControlSize();
 750     error = ReadMTEControl();
 751     if (error.Fail())
 752       return error;
 753   }
 754
 755   if (GetRegisterInfo().IsFPMRPresent()) {
 756     cached_size += sizeof(RegisterSetType) + GetFPMRBufferSize();
 757     error = ReadFPMR();
 758     if (error.Fail())
 759       return error;
 760   }
 761
 762   // tpidr is always present but tpidr2 depends on SME.
 763   cached_size += sizeof(RegisterSetType) + GetTLSBufferSize();
 764   error = ReadTLS();
 765
 766   return error;
 767 }
 768
 769 Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(
 770     lldb::WritableDataBufferSP &data_sp) {
 771   // AArch64 register data must contain GPRs and either FPR or SVE registers.
 772   // SVE registers can be non-streaming (aka SVE) or streaming (aka SSVE).
 773   // Finally an optional MTE register. Pointer Authentication (PAC) registers
 774   // are read-only and will be skipped.
 775
 776   // In order to create register data checkpoint we first read all register
 777   // values if not done already and calculate total size of register set data.
 778   // We store all register values in data_sp by copying full PTrace data that
 779   // corresponds to register sets enabled by current register context.
 780
 781   uint32_t reg_data_byte_size = 0;
 782   Status error = CacheAllRegisters(reg_data_byte_size);
 783   if (error.Fail())
 784     return error;
 785
 786   data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0));
 787   uint8_t *dst = data_sp->GetBytes();
 788
 789   dst = AddSavedRegisters(dst, RegisterSetType::GPR, GetGPRBuffer(),
 790                           GetGPRBufferSize());
 791
 792   // Streaming SVE and the ZA register both use the streaming vector length.
 793   // When you change this, the kernel will invalidate parts of the process
 794   // state. Therefore we need a specific order of restoration for each mode, if
 795   // we also have ZA to restore.
 796   //
 797   // Streaming mode enabled, ZA enabled:
 798   // * Write streaming registers. This sets SVCR.SM and clears SVCR.ZA.
 799   // * Write ZA, this set SVCR.ZA. The register data we provide is written to
 800   // ZA.
 801   // * Result is SVCR.SM and SVCR.ZA set, with the expected data in both
 802   //   register sets.
 803   //
 804   // Streaming mode disabled, ZA enabled:
 805   // * Write ZA. This sets SVCR.ZA, and the ZA content. In the majority of cases
 806   //   the streaming vector length is changing, so the thread is converted into
 807   //   an FPSIMD thread if it is not already one. This also clears SVCR.SM.
 808   // * Write SVE registers, which also clears SVCR.SM but most importantly, puts
 809   //   us into full SVE mode instead of FPSIMD mode (where the registers are
 810   //   actually the 128 bit Neon registers).
 811   // * Result is we have SVCR.SM = 0, SVCR.ZA = 1 and the expected register
 812   //   state.
 813   //
 814   // Restoring in different orders leads to things like the SVE registers being
 815   // truncated due to the FPSIMD mode and ZA being disabled or filled with 0s
 816   // (disabled and 0s looks the same from inside lldb since we fake the value
 817   // when it's disabled).
 818   //
 819   // For more information on this, look up the uses of the relevant NT_ARM_
 820   // constants and the functions vec_set_vector_length, sve_set_common and
 821   // za_set in the Linux Kernel.
 822
 823   if ((m_sve_state != SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
 824     // Use the header size not the buffer size, as we may be using the buffer
 825     // for fake data, which we do not want to write out.
 826     assert(m_za_header.size <= GetZABufferSize());
 827     dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
 828                             m_za_header.size);
 829   }
 830
 831   if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {
 832     dst = AddRegisterSetType(dst, RegisterSetType::SVE);
 833     *(reinterpret_cast<SVEState *>(dst)) = m_sve_state;
 834     dst += sizeof(m_sve_state);
 835     dst = AddSavedRegistersData(dst, GetSVEBuffer(), GetSVEBufferSize());
 836   } else {
 837     dst = AddSavedRegisters(dst, RegisterSetType::FPR, GetFPRBuffer(),
 838                             GetFPRSize());
 839   }
 840
 841   if ((m_sve_state == SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {
 842     assert(m_za_header.size <= GetZABufferSize());
 843     dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),
 844                             m_za_header.size);
 845   }
 846
 847   // If ZT0 is present and we are going to be restoring an active ZA (which
 848   // implies an active ZT0), then restore ZT0 after ZA has been set. This
 849   // prevents us enabling ZA accidentally after the restore of ZA disabled it.
 850   // If we leave ZA/ZT0 inactive and read ZT0, the kernel returns 0s. Therefore
 851   // there's nothing for us to restore if ZA was originally inactive.
 852   if (
 853       // If we have SME2 and therefore ZT0.
 854       GetRegisterInfo().IsZTPresent() &&
 855       // And ZA is enabled.
 856       m_za_header.size > sizeof(m_za_header))
 857     dst = AddSavedRegisters(dst, RegisterSetType::SME2, GetZTBuffer(),
 858                             GetZTBufferSize());
 859
 860   if (GetRegisterInfo().IsMTEPresent()) {
 861     dst = AddSavedRegisters(dst, RegisterSetType::MTE, GetMTEControl(),
 862                             GetMTEControlSize());
 863   }
 864
 865   if (GetRegisterInfo().IsFPMRPresent()) {
 866     dst = AddSavedRegisters(dst, RegisterSetType::FPMR, GetFPMRBuffer(),
 867                             GetFPMRBufferSize());
 868   }
 869
 870   dst = AddSavedRegisters(dst, RegisterSetType::TLS, GetTLSBuffer(),
 871                           GetTLSBufferSize());
 872
 873   return error;
 874 }
 875
 876 static Status RestoreRegisters(void *buffer, const uint8_t **src, size_t len,
 877                                bool &is_valid, std::function<Status()> writer) {
 878   ::memcpy(buffer, *src, len);
 879   is_valid = true;
 880   *src += len;
 881   return writer();
 882 }
 883
 884 Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
 885     const lldb::DataBufferSP &data_sp) {
 886   // AArch64 register data must contain GPRs, either FPR or SVE registers
 887   // (which can be streaming or non-streaming) and optional MTE register.
 888   // Pointer Authentication (PAC) registers are read-only and will be skipped.
 889
 890   // We store all register values in data_sp by copying full PTrace data that
 891   // corresponds to register sets enabled by current register context. In order
 892   // to restore from register data checkpoint we will first restore GPRs, based
 893   // on size of remaining register data either SVE or FPRs should be restored
 894   // next. SVE is not enabled if we have register data size less than or equal
 895   // to size of GPR + FPR + MTE.
 896
 897   Status error;
 898   if (!data_sp) {
 899     error = Status::FromErrorStringWithFormat(
 900         "NativeRegisterContextLinux_arm64::%s invalid data_sp provided",
 901         __FUNCTION__);
 902     return error;
 903   }
 904
 905   const uint8_t *src = data_sp->GetBytes();
 906   if (src == nullptr) {
 907     error = Status::FromErrorStringWithFormat(
 908         "NativeRegisterContextLinux_arm64::%s "
 909         "DataBuffer::GetBytes() returned a null "
 910         "pointer",
 911         __FUNCTION__);
 912     return error;
 913   }
 914
 915   uint64_t reg_data_min_size =
 916       GetGPRBufferSize() + GetFPRSize() + 2 * (sizeof(RegisterSetType));
 917   if (data_sp->GetByteSize() < reg_data_min_size) {
 918     error = Status::FromErrorStringWithFormat(
 919         "NativeRegisterContextLinux_arm64::%s data_sp contained insufficient "
 920         "register data bytes, expected at least %" PRIu64 ", actual %" PRIu64,
 921         __FUNCTION__, reg_data_min_size, data_sp->GetByteSize());
 922     return error;
 923   }
 924
 925   const uint8_t *end = src + data_sp->GetByteSize();
 926   while (src < end) {
 927     const RegisterSetType kind =
 928         *reinterpret_cast<const RegisterSetType *>(src);
 929     src += sizeof(RegisterSetType);
 930
 931     switch (kind) {
 932     case RegisterSetType::GPR:
 933       error = RestoreRegisters(
 934           GetGPRBuffer(), &src, GetGPRBufferSize(), m_gpr_is_valid,
 935           std::bind(&NativeRegisterContextLinux_arm64::WriteGPR, this));
 936       break;
 937     case RegisterSetType::SVE:
 938       // Restore to the correct mode, streaming or not.
 939       m_sve_state = static_cast<SVEState>(*src);
 940       src += sizeof(m_sve_state);
 941
 942       // First write SVE header. We do not use RestoreRegisters because we do
 943       // not want src to be modified yet.
 944       ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize());
 945       if (!sve::vl_valid(m_sve_header.vl)) {
 946         m_sve_header_is_valid = false;
 947         error = Status::FromErrorStringWithFormat(
 948             "NativeRegisterContextLinux_arm64::%s "
 949             "Invalid SVE header in data_sp",
 950             __FUNCTION__);
 951         return error;
 952       }
 953       m_sve_header_is_valid = true;
 954       error = WriteSVEHeader();
 955       if (error.Fail())
 956         return error;
 957
 958       // SVE header has been written configure SVE vector length if needed.
 959       // This could change ZA data too, but that will be restored again later
 960       // anyway.
 961       ConfigureRegisterContext();
 962
 963       // Write header and register data, incrementing src this time.
 964       error = RestoreRegisters(
 965           GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid,
 966           std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this));
 967       break;
 968     case RegisterSetType::FPR:
 969       error = RestoreRegisters(
 970           GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid,
 971           std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this));
 972       break;
 973     case RegisterSetType::MTE:
 974       error = RestoreRegisters(
 975           GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid,
 976           std::bind(&NativeRegisterContextLinux_arm64::WriteMTEControl, this));
 977       break;
 978     case RegisterSetType::TLS:
 979       error = RestoreRegisters(
 980           GetTLSBuffer(), &src, GetTLSBufferSize(), m_tls_is_valid,
 981           std::bind(&NativeRegisterContextLinux_arm64::WriteTLS, this));
 982       break;
 983     case RegisterSetType::SME:
 984       // To enable or disable ZA you write the regset with or without register
 985       // data. The kernel detects this by looking at the ioVec's length, not the
 986       // ZA header size you pass in. Therefore we must write header and register
 987       // data (if present) in one go every time. Read the header only first just
 988       // to get the size.
 989       ::memcpy(GetZAHeader(), src, GetZAHeaderSize());
 990       // Read the header and register data. Can't use the buffer size here, it
 991       // may be incorrect due to being filled with dummy data previously. Resize
 992       // this so WriteZA uses the correct size.
 993       m_za_ptrace_payload.resize(m_za_header.size);
 994       ::memcpy(GetZABuffer(), src, GetZABufferSize());
 995       m_za_buffer_is_valid = true;
 996
 997       error = WriteZA();
 998       if (error.Fail())
 999         return error;
1000
1001       // Update size of ZA, which resizes the ptrace payload potentially
1002       // trashing our copy of the data we just wrote.
1003       ConfigureRegisterContext();
1004
1005       // ZA buffer now has proper size, read back the data we wrote above, from
1006       // ptrace.
1007       error = ReadZA();
1008       src += GetZABufferSize();
1009       break;
1010     case RegisterSetType::SME2:
1011       // Doing this would activate an inactive ZA, however we will only get here
1012       // if the state we are restoring had an active ZA. Restoring ZT0 will
1013       // always come after restoring ZA.
1014       error = RestoreRegisters(
1015           GetZTBuffer(), &src, GetZTBufferSize(), m_zt_buffer_is_valid,
1016           std::bind(&NativeRegisterContextLinux_arm64::WriteZT, this));
1017       break;
1018     case RegisterSetType::FPMR:
1019       error = RestoreRegisters(
1020           GetFPMRBuffer(), &src, GetFPMRBufferSize(), m_fpmr_is_valid,
1021           std::bind(&NativeRegisterContextLinux_arm64::WriteFPMR, this));
1022       break;
1023     }
1024
1025     if (error.Fail())
1026       return error;
1027   }
1028
1029   return error;
1030 }
1031
1032 bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const {
1033   if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1034       RegisterInfoPOSIX_arm64::GPRegSet)
1035     return true;
1036   return false;
1037 }
1038
1039 bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const {
1040   if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==
1041       RegisterInfoPOSIX_arm64::FPRegSet)
1042     return true;
1043   return false;
1044 }
1045
1046 bool NativeRegisterContextLinux_arm64::IsSVE(unsigned reg) const {
1047   return GetRegisterInfo().IsSVEReg(reg);
1048 }
1049
1050 bool NativeRegisterContextLinux_arm64::IsSME(unsigned reg) const {
1051   return GetRegisterInfo().IsSMEReg(reg);
1052 }
1053
1054 bool NativeRegisterContextLinux_arm64::IsPAuth(unsigned reg) const {
1055   return GetRegisterInfo().IsPAuthReg(reg);
1056 }
1057
1058 bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const {
1059   return GetRegisterInfo().IsMTEReg(reg);
1060 }
1061
1062 bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const {
1063   return GetRegisterInfo().IsTLSReg(reg);
1064 }
1065
1066 bool NativeRegisterContextLinux_arm64::IsFPMR(unsigned reg) const {
1067   return GetRegisterInfo().IsFPMRReg(reg);
1068 }
1069
1070 llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() {
1071   if (!m_refresh_hwdebug_info) {
1072     return llvm::Error::success();
1073   }
1074
1075   ::pid_t tid = m_thread.GetID();
1076
1077   int regset = NT_ARM_HW_WATCH;
1078   struct iovec ioVec;
1079   struct user_hwdebug_state dreg_state;
1080   Status error;
1081
1082   ioVec.iov_base = &dreg_state;
1083   ioVec.iov_len = sizeof(dreg_state);
1084   error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1085                                             &ioVec, ioVec.iov_len);
1086
1087   if (error.Fail())
1088     return error.ToError();
1089
1090   m_max_hwp_supported = dreg_state.dbg_info & 0xff;
1091
1092   regset = NT_ARM_HW_BREAK;
1093   error = NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET, tid, &regset,
1094                                             &ioVec, ioVec.iov_len);
1095
1096   if (error.Fail())
1097     return error.ToError();
1098
1099   m_max_hbp_supported = dreg_state.dbg_info & 0xff;
1100   m_refresh_hwdebug_info = false;
1101
1102   return llvm::Error::success();
1103 }
1104
1105 llvm::Error
1106 NativeRegisterContextLinux_arm64::WriteHardwareDebugRegs(DREGType hwbType) {
1107   struct iovec ioVec;
1108   struct user_hwdebug_state dreg_state;
1109   int regset;
1110
1111   memset(&dreg_state, 0, sizeof(dreg_state));
1112   ioVec.iov_base = &dreg_state;
1113
1114   switch (hwbType) {
1115   case eDREGTypeWATCH:
1116     regset = NT_ARM_HW_WATCH;
1117     ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1118                     (sizeof(dreg_state.dbg_regs[0]) * m_max_hwp_supported);
1119
1120     for (uint32_t i = 0; i < m_max_hwp_supported; i++) {
1121       dreg_state.dbg_regs[i].addr = m_hwp_regs[i].address;
1122       dreg_state.dbg_regs[i].ctrl = m_hwp_regs[i].control;
1123     }
1124     break;
1125   case eDREGTypeBREAK:
1126     regset = NT_ARM_HW_BREAK;
1127     ioVec.iov_len = sizeof(dreg_state.dbg_info) + sizeof(dreg_state.pad) +
1128                     (sizeof(dreg_state.dbg_regs[0]) * m_max_hbp_supported);
1129
1130     for (uint32_t i = 0; i < m_max_hbp_supported; i++) {
1131       dreg_state.dbg_regs[i].addr = m_hbp_regs[i].address;
1132       dreg_state.dbg_regs[i].ctrl = m_hbp_regs[i].control;
1133     }
1134     break;
1135   }
1136
1137   return NativeProcessLinux::PtraceWrapper(PTRACE_SETREGSET, m_thread.GetID(),
1138                                            &regset, &ioVec, ioVec.iov_len)
1139       .ToError();
1140 }
1141
1142 Status NativeRegisterContextLinux_arm64::ReadGPR() {
1143   Status error;
1144
1145   if (m_gpr_is_valid)
1146     return error;
1147
1148   struct iovec ioVec;
1149   ioVec.iov_base = GetGPRBuffer();
1150   ioVec.iov_len = GetGPRBufferSize();
1151
1152   error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1153
1154   if (error.Success())
1155     m_gpr_is_valid = true;
1156
1157   return error;
1158 }
1159
1160 Status NativeRegisterContextLinux_arm64::WriteGPR() {
1161   Status error = ReadGPR();
1162   if (error.Fail())
1163     return error;
1164
1165   struct iovec ioVec;
1166   ioVec.iov_base = GetGPRBuffer();
1167   ioVec.iov_len = GetGPRBufferSize();
1168
1169   m_gpr_is_valid = false;
1170
1171   return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
1172 }
1173
1174 Status NativeRegisterContextLinux_arm64::ReadFPR() {
1175   Status error;
1176
1177   if (m_fpu_is_valid)
1178     return error;
1179
1180   struct iovec ioVec;
1181   ioVec.iov_base = GetFPRBuffer();
1182   ioVec.iov_len = GetFPRSize();
1183
1184   error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1185
1186   if (error.Success())
1187     m_fpu_is_valid = true;
1188
1189   return error;
1190 }
1191
1192 Status NativeRegisterContextLinux_arm64::WriteFPR() {
1193   Status error = ReadFPR();
1194   if (error.Fail())
1195     return error;
1196
1197   struct iovec ioVec;
1198   ioVec.iov_base = GetFPRBuffer();
1199   ioVec.iov_len = GetFPRSize();
1200
1201   m_fpu_is_valid = false;
1202
1203   return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);
1204 }
1205
1206 void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
1207   m_gpr_is_valid = false;
1208   m_fpu_is_valid = false;
1209   m_sve_buffer_is_valid = false;
1210   m_sve_header_is_valid = false;
1211   m_za_buffer_is_valid = false;
1212   m_za_header_is_valid = false;
1213   m_pac_mask_is_valid = false;
1214   m_mte_ctrl_is_valid = false;
1215   m_tls_is_valid = false;
1216   m_zt_buffer_is_valid = false;
1217   m_fpmr_is_valid = false;
1218
1219   // Update SVE and ZA registers in case there is change in configuration.
1220   ConfigureRegisterContext();
1221 }
1222
1223 unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() {
1224   return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE;
1225 }
1226
1227 Status NativeRegisterContextLinux_arm64::ReadSVEHeader() {
1228   Status error;
1229
1230   if (m_sve_header_is_valid)
1231     return error;
1232
1233   struct iovec ioVec;
1234   ioVec.iov_base = GetSVEHeader();
1235   ioVec.iov_len = GetSVEHeaderSize();
1236
1237   error = ReadRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1238
1239   if (error.Success())
1240     m_sve_header_is_valid = true;
1241
1242   return error;
1243 }
1244
1245 Status NativeRegisterContextLinux_arm64::ReadPAuthMask() {
1246   Status error;
1247
1248   if (m_pac_mask_is_valid)
1249     return error;
1250
1251   struct iovec ioVec;
1252   ioVec.iov_base = GetPACMask();
1253   ioVec.iov_len = GetPACMaskSize();
1254
1255   error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK);
1256
1257   if (error.Success())
1258     m_pac_mask_is_valid = true;
1259
1260   return error;
1261 }
1262
1263 Status NativeRegisterContextLinux_arm64::WriteSVEHeader() {
1264   Status error;
1265
1266   error = ReadSVEHeader();
1267   if (error.Fail())
1268     return error;
1269
1270   struct iovec ioVec;
1271   ioVec.iov_base = GetSVEHeader();
1272   ioVec.iov_len = GetSVEHeaderSize();
1273
1274   m_sve_buffer_is_valid = false;
1275   m_sve_header_is_valid = false;
1276   m_fpu_is_valid = false;
1277
1278   return WriteRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());
1279 }
1280
1281 Status NativeRegisterContextLinux_arm64::ReadAllSVE() {
1282   Status error;
1283   if (m_sve_buffer_is_valid)
1284     return error;
1285
1286   struct iovec ioVec;
1287   ioVec.iov_base = GetSVEBuffer();
1288   ioVec.iov_len = GetSVEBufferSize();
1289
1290   error = ReadRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1291
1292   if (error.Success())
1293     m_sve_buffer_is_valid = true;
1294
1295   return error;
1296 }
1297
1298 Status NativeRegisterContextLinux_arm64::WriteAllSVE() {
1299   Status error;
1300
1301   error = ReadAllSVE();
1302   if (error.Fail())
1303     return error;
1304
1305   struct iovec ioVec;
1306
1307   ioVec.iov_base = GetSVEBuffer();
1308   ioVec.iov_len = GetSVEBufferSize();
1309
1310   m_sve_buffer_is_valid = false;
1311   m_sve_header_is_valid = false;
1312   m_fpu_is_valid = false;
1313
1314   return WriteRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());
1315 }
1316
1317 Status NativeRegisterContextLinux_arm64::ReadSMEControl() {
1318   // The real register is SVCR and is accessible from EL0. However we don't want
1319   // to have to JIT code into the target process so we'll just recreate it using
1320   // what we know from ptrace.
1321
1322   // Bit 0 indicates whether streaming mode is active.
1323   m_sme_pseudo_regs.ctrl_reg = m_sve_state == SVEState::Streaming;
1324
1325   // Bit 1 indicates whether the array storage is active.
1326   // It is active if we can read the header and the size field tells us that
1327   // there is register data following it.
1328   Status error = ReadZAHeader();
1329   if (error.Success() && (m_za_header.size > sizeof(m_za_header)))
1330     m_sme_pseudo_regs.ctrl_reg |= 2;
1331
1332   return error;
1333 }
1334
1335 Status NativeRegisterContextLinux_arm64::ReadMTEControl() {
1336   Status error;
1337
1338   if (m_mte_ctrl_is_valid)
1339     return error;
1340
1341   struct iovec ioVec;
1342   ioVec.iov_base = GetMTEControl();
1343   ioVec.iov_len = GetMTEControlSize();
1344
1345   error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1346
1347   if (error.Success())
1348     m_mte_ctrl_is_valid = true;
1349
1350   return error;
1351 }
1352
1353 Status NativeRegisterContextLinux_arm64::WriteMTEControl() {
1354   Status error;
1355
1356   error = ReadMTEControl();
1357   if (error.Fail())
1358     return error;
1359
1360   struct iovec ioVec;
1361   ioVec.iov_base = GetMTEControl();
1362   ioVec.iov_len = GetMTEControlSize();
1363
1364   m_mte_ctrl_is_valid = false;
1365
1366   return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
1367 }
1368
1369 Status NativeRegisterContextLinux_arm64::ReadTLS() {
1370   Status error;
1371
1372   if (m_tls_is_valid)
1373     return error;
1374
1375   struct iovec ioVec;
1376   ioVec.iov_base = GetTLSBuffer();
1377   ioVec.iov_len = GetTLSBufferSize();
1378
1379   error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1380
1381   if (error.Success())
1382     m_tls_is_valid = true;
1383
1384   return error;
1385 }
1386
1387 Status NativeRegisterContextLinux_arm64::WriteTLS() {
1388   Status error;
1389
1390   error = ReadTLS();
1391   if (error.Fail())
1392     return error;
1393
1394   struct iovec ioVec;
1395   ioVec.iov_base = GetTLSBuffer();
1396   ioVec.iov_len = GetTLSBufferSize();
1397
1398   m_tls_is_valid = false;
1399
1400   return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);
1401 }
1402
1403 Status NativeRegisterContextLinux_arm64::ReadZAHeader() {
1404   Status error;
1405
1406   if (m_za_header_is_valid)
1407     return error;
1408
1409   struct iovec ioVec;
1410   ioVec.iov_base = GetZAHeader();
1411   ioVec.iov_len = GetZAHeaderSize();
1412
1413   error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA);
1414
1415   if (error.Success())
1416     m_za_header_is_valid = true;
1417
1418   return error;
1419 }
1420
1421 Status NativeRegisterContextLinux_arm64::ReadZA() {
1422   Status error;
1423
1424   if (m_za_buffer_is_valid)
1425     return error;
1426
1427   struct iovec ioVec;
1428   ioVec.iov_base = GetZABuffer();
1429   ioVec.iov_len = GetZABufferSize();
1430
1431   error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1432
1433   if (error.Success())
1434     m_za_buffer_is_valid = true;
1435
1436   return error;
1437 }
1438
1439 Status NativeRegisterContextLinux_arm64::WriteZA() {
1440   // Note that because the ZA ptrace payload contains the header also, this
1441   // method will write both. This is done because writing only the header
1442   // will disable ZA, even if .size in the header is correct for an enabled ZA.
1443   Status error;
1444
1445   error = ReadZA();
1446   if (error.Fail())
1447     return error;
1448
1449   struct iovec ioVec;
1450   ioVec.iov_base = GetZABuffer();
1451   ioVec.iov_len = GetZABufferSize();
1452
1453   m_za_buffer_is_valid = false;
1454   m_za_header_is_valid = false;
1455   // Writing to ZA may enable ZA, which means ZT0 may change too.
1456   m_zt_buffer_is_valid = false;
1457
1458   return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);
1459 }
1460
1461 Status NativeRegisterContextLinux_arm64::ReadZT() {
1462   Status error;
1463
1464   if (m_zt_buffer_is_valid)
1465     return error;
1466
1467   struct iovec ioVec;
1468   ioVec.iov_base = GetZTBuffer();
1469   ioVec.iov_len = GetZTBufferSize();
1470
1471   error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1472   m_zt_buffer_is_valid = error.Success();
1473
1474   return error;
1475 }
1476
1477 Status NativeRegisterContextLinux_arm64::WriteZT() {
1478   Status error;
1479
1480   error = ReadZT();
1481   if (error.Fail())
1482     return error;
1483
1484   struct iovec ioVec;
1485   ioVec.iov_base = GetZTBuffer();
1486   ioVec.iov_len = GetZTBufferSize();
1487
1488   m_zt_buffer_is_valid = false;
1489   // Writing to an inactive ZT0 will enable ZA as well, which invalidates our
1490   // current copy of it.
1491   m_za_buffer_is_valid = false;
1492   m_za_header_is_valid = false;
1493
1494   return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);
1495 }
1496
1497 Status NativeRegisterContextLinux_arm64::ReadFPMR() {
1498   Status error;
1499
1500   if (m_fpmr_is_valid)
1501     return error;
1502
1503   struct iovec ioVec;
1504   ioVec.iov_base = GetFPMRBuffer();
1505   ioVec.iov_len = GetFPMRBufferSize();
1506
1507   error = ReadRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1508
1509   if (error.Success())
1510     m_fpmr_is_valid = true;
1511
1512   return error;
1513 }
1514
1515 Status NativeRegisterContextLinux_arm64::WriteFPMR() {
1516   Status error;
1517
1518   error = ReadFPMR();
1519   if (error.Fail())
1520     return error;
1521
1522   struct iovec ioVec;
1523   ioVec.iov_base = GetFPMRBuffer();
1524   ioVec.iov_len = GetFPMRBufferSize();
1525
1526   m_fpmr_is_valid = false;
1527
1528   return WriteRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);
1529 }
1530
1531 void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
1532   // ConfigureRegisterContext gets called from InvalidateAllRegisters
1533   // on every stop and configures SVE vector length and whether we are in
1534   // streaming SVE mode.
1535   // If m_sve_state is set to SVEState::Disabled on first stop, code below will
1536   // be deemed non operational for the lifetime of current process.
1537   if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) {
1538     // If we have SVE we may also have the SVE streaming mode that SME added.
1539     // We can read the header of either mode, but only the active mode will
1540     // have valid register data.
1541
1542     // Check whether SME is present and the streaming SVE mode is active.
1543     m_sve_header_is_valid = false;
1544     m_sve_buffer_is_valid = false;
1545     m_sve_state = SVEState::Streaming;
1546     Status error = ReadSVEHeader();
1547
1548     // Streaming mode is active if the header has the SVE active flag set.
1549     if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1550                               sve::ptrace_regs_sve))) {
1551       // Non-streaming might be active instead.
1552       m_sve_header_is_valid = false;
1553       m_sve_buffer_is_valid = false;
1554       m_sve_state = SVEState::Full;
1555       error = ReadSVEHeader();
1556       if (error.Success()) {
1557         // If SVE is enabled thread can switch between SVEState::FPSIMD and
1558         // SVEState::Full on every stop.
1559         if ((m_sve_header.flags & sve::ptrace_regs_mask) ==
1560             sve::ptrace_regs_fpsimd)
1561           m_sve_state = SVEState::FPSIMD;
1562         // Else we are in SVEState::Full.
1563       } else {
1564         m_sve_state = SVEState::Disabled;
1565       }
1566     }
1567
1568     if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD ||
1569         m_sve_state == SVEState::Streaming) {
1570       // On every stop we configure SVE vector length by calling
1571       // ConfigureVectorLengthSVE regardless of current SVEState of this thread.
1572       uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1573       if (sve::vl_valid(m_sve_header.vl))
1574         vq = sve::vq_from_vl(m_sve_header.vl);
1575
1576       GetRegisterInfo().ConfigureVectorLengthSVE(vq);
1577       m_sve_ptrace_payload.resize(sve::PTraceSize(vq, sve::ptrace_regs_sve));
1578     }
1579   }
1580
1581   if (!m_za_header_is_valid) {
1582     Status error = ReadZAHeader();
1583     if (error.Success()) {
1584       uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
1585       if (sve::vl_valid(m_za_header.vl))
1586         vq = sve::vq_from_vl(m_za_header.vl);
1587
1588       GetRegisterInfo().ConfigureVectorLengthZA(vq);
1589       m_za_ptrace_payload.resize(m_za_header.size);
1590       m_za_buffer_is_valid = false;
1591     }
1592   }
1593 }
1594
1595 uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset(
1596     const RegisterInfo *reg_info) const {
1597   return reg_info->byte_offset - GetGPRSize();
1598 }
1599
1600 uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset(
1601     const RegisterInfo *reg_info) const {
1602   // Start of Z0 data is after GPRs plus 8 bytes of vg register
1603   uint32_t sve_reg_offset = LLDB_INVALID_INDEX32;
1604   if (m_sve_state == SVEState::FPSIMD) {
1605     const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
1606     sve_reg_offset = sve::ptrace_fpsimd_offset +
1607                      (reg - GetRegisterInfo().GetRegNumSVEZ0()) * 16;
1608     // Between non-streaming and streaming mode, the layout is identical.
1609   } else if (m_sve_state == SVEState::Full ||
1610              m_sve_state == SVEState::Streaming) {
1611     uint32_t sve_z0_offset = GetGPRSize() + 16;
1612     sve_reg_offset =
1613         sve::SigRegsOffset() + reg_info->byte_offset - sve_z0_offset;
1614   }
1615   return sve_reg_offset;
1616 }
1617
1618 Status NativeRegisterContextLinux_arm64::ReadSMESVG() {
1619   // This register is the streaming vector length, so we will get it from
1620   // NT_ARM_ZA regardless of the current streaming mode.
1621   Status error = ReadZAHeader();
1622   if (error.Success())
1623     m_sme_pseudo_regs.svg_reg = m_za_header.vl / 8;
1624
1625   return error;
1626 }
1627
1628 std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters(
1629     ExpeditedRegs expType) const {
1630   std::vector<uint32_t> expedited_reg_nums =
1631       NativeRegisterContext::GetExpeditedRegisters(expType);
1632   // SVE, non-streaming vector length.
1633   if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full)
1634     expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG());
1635   // SME, streaming vector length. This is used by the ZA register which is
1636   // present even when streaming mode is not enabled.
1637   if (GetRegisterInfo().IsSSVEPresent())
1638     expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSMESVG());
1639
1640   return expedited_reg_nums;
1641 }
1642
1643 llvm::Expected<NativeRegisterContextLinux::MemoryTaggingDetails>
1644 NativeRegisterContextLinux_arm64::GetMemoryTaggingDetails(int32_t type) {
1645   if (type == MemoryTagManagerAArch64MTE::eMTE_allocation) {
1646     return MemoryTaggingDetails{std::make_unique<MemoryTagManagerAArch64MTE>(),
1647                                 PTRACE_PEEKMTETAGS, PTRACE_POKEMTETAGS};
1648   }
1649
1650   return llvm::createStringError(llvm::inconvertibleErrorCode(),
1651                                  "Unknown AArch64 memory tag type %d", type);
1652 }
1653
1654 lldb::addr_t NativeRegisterContextLinux_arm64::FixWatchpointHitAddress(
1655     lldb::addr_t hit_addr) {
1656   // Linux configures user-space virtual addresses with top byte ignored.
1657   // We set default value of mask such that top byte is masked out.
1658   lldb::addr_t mask = ~((1ULL << 56) - 1);
1659
1660   // Try to read pointer authentication data_mask register and calculate a
1661   // consolidated data address mask after ignoring the top byte.
1662   if (ReadPAuthMask().Success())
1663     mask |= m_pac_mask.data_mask;
1664
1665   return hit_addr & ~mask;
1666   ;
1667 }
1668
1669 #endif // defined (__arm64__) || defined (__aarch64__)