hw/nvme/nvme.h

   1 /*
   2  * QEMU NVM Express
   3  *
   4  * Copyright (c) 2012 Intel Corporation
   5  * Copyright (c) 2021 Minwoo Im
   6  * Copyright (c) 2021 Samsung Electronics Co., Ltd.
   7  *
   8  * Authors:
   9  *   Keith Busch            <kbusch@kernel.org>
  10  *   Klaus Jensen           <k.jensen@samsung.com>
  11  *   Gollu Appalanaidu      <anaidu.gollu@samsung.com>
  12  *   Dmitry Fomichev        <dmitry.fomichev@wdc.com>
  13  *   Minwoo Im              <minwoo.im.dev@gmail.com>
  14  *
  15  * This code is licensed under the GNU GPL v2 or later.
  16  */
  17
  18 #ifndef HW_NVME_NVME_H
  19 #define HW_NVME_NVME_H
  20
  21 #include "qemu/uuid.h"
  22 #include "hw/pci/pci_device.h"
  23 #include "hw/block/block.h"
  24
  25 #include "block/nvme.h"
  26
  27 #define NVME_MAX_CONTROLLERS 256
  28 #define NVME_MAX_NAMESPACES  256
  29 #define NVME_EUI64_DEFAULT ((uint64_t)0x5254000000000000)
  30 #define NVME_FDP_MAX_EVENTS 63
  31 #define NVME_FDP_MAXPIDS 128
  32
  33 /*
  34  * The controller only supports Submission and Completion Queue Entry Sizes of
  35  * 64 and 16 bytes respectively.
  36  */
  37 #define NVME_SQES 6
  38 #define NVME_CQES 4
  39
  40 QEMU_BUILD_BUG_ON(NVME_MAX_NAMESPACES > NVME_NSID_BROADCAST - 1);
  41
  42 typedef struct NvmeCtrl NvmeCtrl;
  43 typedef struct NvmeNamespace NvmeNamespace;
  44
  45 #define TYPE_NVME_BUS "nvme-bus"
  46 OBJECT_DECLARE_SIMPLE_TYPE(NvmeBus, NVME_BUS)
  47
  48 typedef struct NvmeBus {
  49     BusState parent_bus;
  50 } NvmeBus;
  51
  52 #define TYPE_NVME_SUBSYS "nvme-subsys"
  53 #define NVME_SUBSYS(obj) \
  54     OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS)
  55 #define SUBSYS_SLOT_RSVD (void *)0xFFFF
  56
  57 typedef struct NvmeReclaimUnit {
  58     uint64_t ruamw;
  59 } NvmeReclaimUnit;
  60
  61 typedef struct NvmeRuHandle {
  62     uint8_t  ruht;
  63     uint8_t  ruha;
  64     uint64_t event_filter;
  65     uint8_t  lbafi;
  66     uint64_t ruamw;
  67
  68     /* reclaim units indexed by reclaim group */
  69     NvmeReclaimUnit *rus;
  70 } NvmeRuHandle;
  71
  72 typedef struct NvmeFdpEventBuffer {
  73     NvmeFdpEvent     events[NVME_FDP_MAX_EVENTS];
  74     unsigned int     nelems;
  75     unsigned int     start;
  76     unsigned int     next;
  77 } NvmeFdpEventBuffer;
  78
  79 typedef struct NvmeEnduranceGroup {
  80     uint8_t event_conf;
  81
  82     struct {
  83         NvmeFdpEventBuffer host_events, ctrl_events;
  84
  85         uint16_t nruh;
  86         uint16_t nrg;
  87         uint8_t  rgif;
  88         uint64_t runs;
  89
  90         uint64_t hbmw;
  91         uint64_t mbmw;
  92         uint64_t mbe;
  93
  94         bool enabled;
  95
  96         NvmeRuHandle *ruhs;
  97     } fdp;
  98 } NvmeEnduranceGroup;
  99
 100 typedef struct NvmeSubsystem {
 101     DeviceState parent_obj;
 102     NvmeBus     bus;
 103     uint8_t     subnqn[256];
 104     char        *serial;
 105
 106     NvmeCtrl           *ctrls[NVME_MAX_CONTROLLERS];
 107     NvmeNamespace      *namespaces[NVME_MAX_NAMESPACES + 1];
 108     NvmeEnduranceGroup endgrp;
 109
 110     struct {
 111         char *nqn;
 112
 113         struct {
 114             bool     enabled;
 115             uint64_t runs;
 116             uint16_t nruh;
 117             uint32_t nrg;
 118         } fdp;
 119     } params;
 120 } NvmeSubsystem;
 121
 122 int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp);
 123 void nvme_subsys_unregister_ctrl(NvmeSubsystem *subsys, NvmeCtrl *n);
 124
 125 static inline NvmeCtrl *nvme_subsys_ctrl(NvmeSubsystem *subsys,
 126                                          uint32_t cntlid)
 127 {
 128     if (!subsys || cntlid >= NVME_MAX_CONTROLLERS) {
 129         return NULL;
 130     }
 131
 132     if (subsys->ctrls[cntlid] == SUBSYS_SLOT_RSVD) {
 133         return NULL;
 134     }
 135
 136     return subsys->ctrls[cntlid];
 137 }
 138
 139 static inline NvmeNamespace *nvme_subsys_ns(NvmeSubsystem *subsys,
 140                                             uint32_t nsid)
 141 {
 142     if (!subsys || !nsid || nsid > NVME_MAX_NAMESPACES) {
 143         return NULL;
 144     }
 145
 146     return subsys->namespaces[nsid];
 147 }
 148
 149 #define TYPE_NVME_NS "nvme-ns"
 150 #define NVME_NS(obj) \
 151     OBJECT_CHECK(NvmeNamespace, (obj), TYPE_NVME_NS)
 152
 153 typedef struct NvmeZone {
 154     NvmeZoneDescr   d;
 155     uint64_t        w_ptr;
 156     QTAILQ_ENTRY(NvmeZone) entry;
 157 } NvmeZone;
 158
 159 #define FDP_EVT_MAX 0xff
 160 #define NVME_FDP_MAX_NS_RUHS 32u
 161 #define FDPVSS 0
 162
 163 static const uint8_t nvme_fdp_evf_shifts[FDP_EVT_MAX] = {
 164     /* Host events */
 165     [FDP_EVT_RU_NOT_FULLY_WRITTEN]      = 0,
 166     [FDP_EVT_RU_ATL_EXCEEDED]           = 1,
 167     [FDP_EVT_CTRL_RESET_RUH]            = 2,
 168     [FDP_EVT_INVALID_PID]               = 3,
 169     /* CTRL events */
 170     [FDP_EVT_MEDIA_REALLOC]             = 32,
 171     [FDP_EVT_RUH_IMPLICIT_RU_CHANGE]    = 33,
 172 };
 173
 174 #define NGUID_LEN 16
 175
 176 typedef struct {
 177     uint8_t data[NGUID_LEN];
 178 } NvmeNGUID;
 179
 180 bool nvme_nguid_is_null(const NvmeNGUID *nguid);
 181
 182 extern const PropertyInfo qdev_prop_nguid;
 183
 184 #define DEFINE_PROP_NGUID_NODEFAULT(_name, _state, _field) \
 185     DEFINE_PROP(_name, _state, _field, qdev_prop_nguid, NvmeNGUID)
 186
 187 typedef struct NvmeNamespaceParams {
 188     bool      detached;
 189     bool      shared;
 190     uint32_t  nsid;
 191     QemuUUID  uuid;
 192     NvmeNGUID nguid;
 193     uint64_t  eui64;
 194     bool      eui64_default;
 195
 196     uint16_t ms;
 197     uint8_t  mset;
 198     uint8_t  pi;
 199     uint8_t  pil;
 200     uint8_t  pif;
 201
 202     uint16_t mssrl;
 203     uint32_t mcl;
 204     uint8_t  msrc;
 205
 206     bool     zoned;
 207     bool     cross_zone_read;
 208     uint64_t zone_size_bs;
 209     uint64_t zone_cap_bs;
 210     uint32_t max_active_zones;
 211     uint32_t max_open_zones;
 212     uint32_t zd_extension_size;
 213
 214     uint32_t numzrwa;
 215     uint64_t zrwas;
 216     uint64_t zrwafg;
 217
 218     struct {
 219         char *ruhs;
 220     } fdp;
 221 } NvmeNamespaceParams;
 222
 223 typedef struct NvmeAtomic {
 224     uint32_t    atomic_max_write_size;
 225     bool        atomic_writes;
 226 } NvmeAtomic;
 227
 228 typedef struct NvmeNamespace {
 229     DeviceState  parent_obj;
 230     BlockConf    blkconf;
 231     int32_t      bootindex;
 232     int64_t      size;
 233     int64_t      moff;
 234     NvmeIdNs     id_ns;
 235     NvmeIdNsNvm  id_ns_nvm;
 236     NvmeLBAF     lbaf;
 237     unsigned int nlbaf;
 238     size_t       lbasz;
 239     const uint32_t *iocs;
 240     uint8_t      csi;
 241     uint16_t     status;
 242     int          attached;
 243     uint8_t      pif;
 244
 245     struct {
 246         uint16_t zrwas;
 247         uint16_t zrwafg;
 248         uint32_t numzrwa;
 249     } zns;
 250
 251     QTAILQ_ENTRY(NvmeNamespace) entry;
 252
 253     NvmeIdNsZoned   *id_ns_zoned;
 254     NvmeZone        *zone_array;
 255     QTAILQ_HEAD(, NvmeZone) exp_open_zones;
 256     QTAILQ_HEAD(, NvmeZone) imp_open_zones;
 257     QTAILQ_HEAD(, NvmeZone) closed_zones;
 258     QTAILQ_HEAD(, NvmeZone) full_zones;
 259     uint32_t        num_zones;
 260     uint64_t        zone_size;
 261     uint64_t        zone_capacity;
 262     uint32_t        zone_size_log2;
 263     uint8_t         *zd_extensions;
 264     int32_t         nr_open_zones;
 265     int32_t         nr_active_zones;
 266
 267     NvmeNamespaceParams params;
 268     NvmeSubsystem *subsys;
 269     NvmeEnduranceGroup *endgrp;
 270
 271     struct {
 272         uint32_t err_rec;
 273     } features;
 274
 275     struct {
 276         uint16_t nphs;
 277         /* reclaim unit handle identifiers indexed by placement handle */
 278         uint16_t *phs;
 279     } fdp;
 280 } NvmeNamespace;
 281
 282 static inline uint32_t nvme_nsid(NvmeNamespace *ns)
 283 {
 284     if (ns) {
 285         return ns->params.nsid;
 286     }
 287
 288     return 0;
 289 }
 290
 291 static inline size_t nvme_l2b(NvmeNamespace *ns, uint64_t lba)
 292 {
 293     return lba << ns->lbaf.ds;
 294 }
 295
 296 static inline size_t nvme_m2b(NvmeNamespace *ns, uint64_t lba)
 297 {
 298     return ns->lbaf.ms * lba;
 299 }
 300
 301 static inline int64_t nvme_moff(NvmeNamespace *ns, uint64_t lba)
 302 {
 303     return ns->moff + nvme_m2b(ns, lba);
 304 }
 305
 306 static inline bool nvme_ns_ext(NvmeNamespace *ns)
 307 {
 308     return !!NVME_ID_NS_FLBAS_EXTENDED(ns->id_ns.flbas);
 309 }
 310
 311 static inline NvmeZoneState nvme_get_zone_state(NvmeZone *zone)
 312 {
 313     return zone->d.zs >> 4;
 314 }
 315
 316 static inline void nvme_set_zone_state(NvmeZone *zone, NvmeZoneState state)
 317 {
 318     zone->d.zs = state << 4;
 319 }
 320
 321 static inline uint64_t nvme_zone_rd_boundary(NvmeNamespace *ns, NvmeZone *zone)
 322 {
 323     return zone->d.zslba + ns->zone_size;
 324 }
 325
 326 static inline uint64_t nvme_zone_wr_boundary(NvmeZone *zone)
 327 {
 328     return zone->d.zslba + zone->d.zcap;
 329 }
 330
 331 static inline bool nvme_wp_is_valid(NvmeZone *zone)
 332 {
 333     uint8_t st = nvme_get_zone_state(zone);
 334
 335     return st != NVME_ZONE_STATE_FULL &&
 336            st != NVME_ZONE_STATE_READ_ONLY &&
 337            st != NVME_ZONE_STATE_OFFLINE;
 338 }
 339
 340 static inline uint8_t *nvme_get_zd_extension(NvmeNamespace *ns,
 341                                              uint32_t zone_idx)
 342 {
 343     return &ns->zd_extensions[zone_idx * ns->params.zd_extension_size];
 344 }
 345
 346 static inline void nvme_aor_inc_open(NvmeNamespace *ns)
 347 {
 348     assert(ns->nr_open_zones >= 0);
 349     if (ns->params.max_open_zones) {
 350         ns->nr_open_zones++;
 351         assert(ns->nr_open_zones <= ns->params.max_open_zones);
 352     }
 353 }
 354
 355 static inline void nvme_aor_dec_open(NvmeNamespace *ns)
 356 {
 357     if (ns->params.max_open_zones) {
 358         assert(ns->nr_open_zones > 0);
 359         ns->nr_open_zones--;
 360     }
 361     assert(ns->nr_open_zones >= 0);
 362 }
 363
 364 static inline void nvme_aor_inc_active(NvmeNamespace *ns)
 365 {
 366     assert(ns->nr_active_zones >= 0);
 367     if (ns->params.max_active_zones) {
 368         ns->nr_active_zones++;
 369         assert(ns->nr_active_zones <= ns->params.max_active_zones);
 370     }
 371 }
 372
 373 static inline void nvme_aor_dec_active(NvmeNamespace *ns)
 374 {
 375     if (ns->params.max_active_zones) {
 376         assert(ns->nr_active_zones > 0);
 377         ns->nr_active_zones--;
 378         assert(ns->nr_active_zones >= ns->nr_open_zones);
 379     }
 380     assert(ns->nr_active_zones >= 0);
 381 }
 382
 383 static inline void nvme_fdp_stat_inc(uint64_t *a, uint64_t b)
 384 {
 385     uint64_t ret = *a + b;
 386     *a = ret < *a ? UINT64_MAX : ret;
 387 }
 388
 389 void nvme_ns_init_format(NvmeNamespace *ns);
 390 int nvme_ns_setup(NvmeNamespace *ns, Error **errp);
 391 void nvme_ns_drain(NvmeNamespace *ns);
 392 void nvme_ns_shutdown(NvmeNamespace *ns);
 393 void nvme_ns_cleanup(NvmeNamespace *ns);
 394
 395 typedef struct NvmeAsyncEvent {
 396     QTAILQ_ENTRY(NvmeAsyncEvent) entry;
 397     NvmeAerResult result;
 398 } NvmeAsyncEvent;
 399
 400 enum {
 401     NVME_SG_ALLOC = 1 << 0,
 402     NVME_SG_DMA   = 1 << 1,
 403 };
 404
 405 typedef struct NvmeSg {
 406     int flags;
 407
 408     union {
 409         QEMUSGList   qsg;
 410         QEMUIOVector iov;
 411     };
 412 } NvmeSg;
 413
 414 typedef enum NvmeTxDirection {
 415     NVME_TX_DIRECTION_TO_DEVICE   = 0,
 416     NVME_TX_DIRECTION_FROM_DEVICE = 1,
 417 } NvmeTxDirection;
 418
 419 typedef struct NvmeRequest {
 420     struct NvmeSQueue       *sq;
 421     struct NvmeNamespace    *ns;
 422     BlockAIOCB              *aiocb;
 423     uint16_t                status;
 424     void                    *opaque;
 425     NvmeCqe                 cqe;
 426     NvmeCmd                 cmd;
 427     BlockAcctCookie         acct;
 428     NvmeSg                  sg;
 429     bool                    atomic_write;
 430     QTAILQ_ENTRY(NvmeRequest)entry;
 431 } NvmeRequest;
 432
 433 typedef struct NvmeBounceContext {
 434     NvmeRequest *req;
 435
 436     struct {
 437         QEMUIOVector iov;
 438         uint8_t *bounce;
 439     } data, mdata;
 440 } NvmeBounceContext;
 441
 442 static inline const char *nvme_adm_opc_str(uint8_t opc)
 443 {
 444     switch (opc) {
 445     case NVME_ADM_CMD_DELETE_SQ:        return "NVME_ADM_CMD_DELETE_SQ";
 446     case NVME_ADM_CMD_CREATE_SQ:        return "NVME_ADM_CMD_CREATE_SQ";
 447     case NVME_ADM_CMD_GET_LOG_PAGE:     return "NVME_ADM_CMD_GET_LOG_PAGE";
 448     case NVME_ADM_CMD_DELETE_CQ:        return "NVME_ADM_CMD_DELETE_CQ";
 449     case NVME_ADM_CMD_CREATE_CQ:        return "NVME_ADM_CMD_CREATE_CQ";
 450     case NVME_ADM_CMD_IDENTIFY:         return "NVME_ADM_CMD_IDENTIFY";
 451     case NVME_ADM_CMD_ABORT:            return "NVME_ADM_CMD_ABORT";
 452     case NVME_ADM_CMD_SET_FEATURES:     return "NVME_ADM_CMD_SET_FEATURES";
 453     case NVME_ADM_CMD_GET_FEATURES:     return "NVME_ADM_CMD_GET_FEATURES";
 454     case NVME_ADM_CMD_ASYNC_EV_REQ:     return "NVME_ADM_CMD_ASYNC_EV_REQ";
 455     case NVME_ADM_CMD_NS_ATTACHMENT:    return "NVME_ADM_CMD_NS_ATTACHMENT";
 456     case NVME_ADM_CMD_DIRECTIVE_SEND:   return "NVME_ADM_CMD_DIRECTIVE_SEND";
 457     case NVME_ADM_CMD_VIRT_MNGMT:       return "NVME_ADM_CMD_VIRT_MNGMT";
 458     case NVME_ADM_CMD_DIRECTIVE_RECV:   return "NVME_ADM_CMD_DIRECTIVE_RECV";
 459     case NVME_ADM_CMD_DBBUF_CONFIG:     return "NVME_ADM_CMD_DBBUF_CONFIG";
 460     case NVME_ADM_CMD_FORMAT_NVM:       return "NVME_ADM_CMD_FORMAT_NVM";
 461     default:                            return "NVME_ADM_CMD_UNKNOWN";
 462     }
 463 }
 464
 465 static inline const char *nvme_io_opc_str(uint8_t opc)
 466 {
 467     switch (opc) {
 468     case NVME_CMD_FLUSH:            return "NVME_NVM_CMD_FLUSH";
 469     case NVME_CMD_WRITE:            return "NVME_NVM_CMD_WRITE";
 470     case NVME_CMD_READ:             return "NVME_NVM_CMD_READ";
 471     case NVME_CMD_COMPARE:          return "NVME_NVM_CMD_COMPARE";
 472     case NVME_CMD_WRITE_ZEROES:     return "NVME_NVM_CMD_WRITE_ZEROES";
 473     case NVME_CMD_DSM:              return "NVME_NVM_CMD_DSM";
 474     case NVME_CMD_VERIFY:           return "NVME_NVM_CMD_VERIFY";
 475     case NVME_CMD_COPY:             return "NVME_NVM_CMD_COPY";
 476     case NVME_CMD_ZONE_MGMT_SEND:   return "NVME_ZONED_CMD_MGMT_SEND";
 477     case NVME_CMD_ZONE_MGMT_RECV:   return "NVME_ZONED_CMD_MGMT_RECV";
 478     case NVME_CMD_ZONE_APPEND:      return "NVME_ZONED_CMD_ZONE_APPEND";
 479     default:                        return "NVME_NVM_CMD_UNKNOWN";
 480     }
 481 }
 482
 483 typedef struct NvmeSQueue {
 484     struct NvmeCtrl *ctrl;
 485     uint16_t    sqid;
 486     uint16_t    cqid;
 487     uint32_t    head;
 488     uint32_t    tail;
 489     uint32_t    size;
 490     uint64_t    dma_addr;
 491     uint64_t    db_addr;
 492     uint64_t    ei_addr;
 493     QEMUBH      *bh;
 494     EventNotifier notifier;
 495     bool        ioeventfd_enabled;
 496     NvmeRequest *io_req;
 497     QTAILQ_HEAD(, NvmeRequest) req_list;
 498     QTAILQ_HEAD(, NvmeRequest) out_req_list;
 499     QTAILQ_ENTRY(NvmeSQueue) entry;
 500 } NvmeSQueue;
 501
 502 typedef struct NvmeCQueue {
 503     struct NvmeCtrl *ctrl;
 504     uint8_t     phase;
 505     uint16_t    cqid;
 506     uint16_t    irq_enabled;
 507     uint32_t    head;
 508     uint32_t    tail;
 509     uint32_t    vector;
 510     uint32_t    size;
 511     uint64_t    dma_addr;
 512     uint64_t    db_addr;
 513     uint64_t    ei_addr;
 514     QEMUBH      *bh;
 515     EventNotifier notifier;
 516     bool        ioeventfd_enabled;
 517     QTAILQ_HEAD(, NvmeSQueue) sq_list;
 518     QTAILQ_HEAD(, NvmeRequest) req_list;
 519 } NvmeCQueue;
 520
 521 #define TYPE_NVME "nvme"
 522 #define NVME(obj) \
 523         OBJECT_CHECK(NvmeCtrl, (obj), TYPE_NVME)
 524
 525 typedef struct NvmeParams {
 526     char     *serial;
 527     uint32_t num_queues; /* deprecated since 5.1 */
 528     uint32_t max_ioqpairs;
 529     uint16_t msix_qsize;
 530     uint16_t mqes;
 531     uint32_t cmb_size_mb;
 532     uint8_t  aerl;
 533     uint32_t aer_max_queued;
 534     uint8_t  mdts;
 535     uint8_t  vsl;
 536     bool     use_intel_id;
 537     uint8_t  zasl;
 538     bool     auto_transition_zones;
 539     bool     legacy_cmb;
 540     bool     ioeventfd;
 541     uint16_t  sriov_max_vfs;
 542     uint16_t sriov_vq_flexible;
 543     uint16_t sriov_vi_flexible;
 544     uint32_t  sriov_max_vq_per_vf;
 545     uint32_t  sriov_max_vi_per_vf;
 546     bool     msix_exclusive_bar;
 547
 548     struct {
 549         bool mem;
 550     } ctratt;
 551
 552     uint16_t atomic_awun;
 553     uint16_t atomic_awupf;
 554     bool     atomic_dn;
 555 } NvmeParams;
 556
 557 typedef struct NvmeCtrl {
 558     PCIDevice    parent_obj;
 559     MemoryRegion bar0;
 560     MemoryRegion iomem;
 561     NvmeBar      bar;
 562     NvmeParams   params;
 563     NvmeBus      bus;
 564
 565     uint16_t    cntlid;
 566     bool        qs_created;
 567     uint32_t    page_size;
 568     uint16_t    page_bits;
 569     uint16_t    max_prp_ents;
 570     uint32_t    max_q_ents;
 571     uint8_t     outstanding_aers;
 572     uint32_t    irq_status;
 573     int         cq_pending;
 574     uint64_t    host_timestamp;                 /* Timestamp sent by the host */
 575     uint64_t    timestamp_set_qemu_clock_ms;    /* QEMU clock time */
 576     uint64_t    starttime_ms;
 577     uint16_t    temperature;
 578     uint8_t     smart_critical_warning;
 579     uint32_t    conf_msix_qsize;
 580     uint32_t    conf_ioqpairs;
 581     uint64_t    dbbuf_dbs;
 582     uint64_t    dbbuf_eis;
 583     bool        dbbuf_enabled;
 584
 585     struct {
 586         MemoryRegion mem;
 587         uint8_t      *buf;
 588         bool         cmse;
 589         hwaddr       cba;
 590     } cmb;
 591
 592     struct {
 593         HostMemoryBackend *dev;
 594         bool              cmse;
 595         hwaddr            cba;
 596     } pmr;
 597
 598     uint8_t     aer_mask;
 599     NvmeRequest **aer_reqs;
 600     QTAILQ_HEAD(, NvmeAsyncEvent) aer_queue;
 601     int         aer_queued;
 602
 603     uint32_t    dmrsl;
 604
 605     /* Namespace ID is started with 1 so bitmap should be 1-based */
 606 #define NVME_CHANGED_NSID_SIZE  (NVME_MAX_NAMESPACES + 1)
 607     DECLARE_BITMAP(changed_nsids, NVME_CHANGED_NSID_SIZE);
 608
 609     NvmeSubsystem   *subsys;
 610
 611     NvmeNamespace   namespace;
 612     NvmeNamespace   *namespaces[NVME_MAX_NAMESPACES + 1];
 613     NvmeSQueue      **sq;
 614     NvmeCQueue      **cq;
 615     NvmeSQueue      admin_sq;
 616     NvmeCQueue      admin_cq;
 617     NvmeIdCtrl      id_ctrl;
 618
 619     struct {
 620         struct {
 621             uint16_t temp_thresh_hi;
 622             uint16_t temp_thresh_low;
 623         };
 624
 625         uint32_t                async_config;
 626         NvmeHostBehaviorSupport hbs;
 627     } features;
 628
 629     NvmePriCtrlCap  pri_ctrl_cap;
 630     uint32_t nr_sec_ctrls;
 631     NvmeSecCtrlEntry *sec_ctrl_list;
 632     struct {
 633         uint16_t    vqrfap;
 634         uint16_t    virfap;
 635     } next_pri_ctrl_cap;    /* These override pri_ctrl_cap after reset */
 636     uint32_t    dn; /* Disable Normal */
 637     NvmeAtomic  atomic;
 638 } NvmeCtrl;
 639
 640 typedef enum NvmeResetType {
 641     NVME_RESET_FUNCTION   = 0,
 642     NVME_RESET_CONTROLLER = 1,
 643 } NvmeResetType;
 644
 645 static inline NvmeNamespace *nvme_ns(NvmeCtrl *n, uint32_t nsid)
 646 {
 647     if (!nsid || nsid > NVME_MAX_NAMESPACES) {
 648         return NULL;
 649     }
 650
 651     return n->namespaces[nsid];
 652 }
 653
 654 static inline NvmeCQueue *nvme_cq(NvmeRequest *req)
 655 {
 656     NvmeSQueue *sq = req->sq;
 657     NvmeCtrl *n = sq->ctrl;
 658
 659     return n->cq[sq->cqid];
 660 }
 661
 662 static inline NvmeCtrl *nvme_ctrl(NvmeRequest *req)
 663 {
 664     NvmeSQueue *sq = req->sq;
 665     return sq->ctrl;
 666 }
 667
 668 static inline uint16_t nvme_cid(NvmeRequest *req)
 669 {
 670     if (!req) {
 671         return 0xffff;
 672     }
 673
 674     return le16_to_cpu(req->cqe.cid);
 675 }
 676
 677 static inline NvmeSecCtrlEntry *nvme_sctrl(NvmeCtrl *n)
 678 {
 679     PCIDevice *pci_dev = &n->parent_obj;
 680     NvmeCtrl *pf = NVME(pcie_sriov_get_pf(pci_dev));
 681
 682     if (pci_is_vf(pci_dev)) {
 683         return &pf->sec_ctrl_list[pcie_sriov_vf_number(pci_dev)];
 684     }
 685
 686     return NULL;
 687 }
 688
 689 static inline NvmeSecCtrlEntry *nvme_sctrl_for_cntlid(NvmeCtrl *n,
 690                                                       uint16_t cntlid)
 691 {
 692     NvmeSecCtrlEntry *list = n->sec_ctrl_list;
 693     uint8_t i;
 694
 695     for (i = 0; i < n->nr_sec_ctrls; i++) {
 696         if (le16_to_cpu(list[i].scid) == cntlid) {
 697             return &list[i];
 698         }
 699     }
 700
 701     return NULL;
 702 }
 703
 704 void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns);
 705 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
 706                           NvmeTxDirection dir, NvmeRequest *req);
 707 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
 708                            NvmeTxDirection dir, NvmeRequest *req);
 709 void nvme_rw_complete_cb(void *opaque, int ret);
 710 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
 711                        NvmeCmd *cmd);
 712
 713 #endif /* HW_NVME_NVME_H */