drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2021, HiSilicon Ltd.
   4  */
   5
   6 #include <linux/device.h>
   7 #include <linux/eventfd.h>
   8 #include <linux/file.h>
   9 #include <linux/hisi_acc_qm.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/module.h>
  12 #include <linux/pci.h>
  13 #include <linux/vfio.h>
  14 #include <linux/vfio_pci_core.h>
  15 #include <linux/anon_inodes.h>
  16
  17 #include "hisi_acc_vfio_pci.h"
  18
  19 /* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */
  20 static int qm_wait_dev_not_ready(struct hisi_qm *qm)
  21 {
  22         u32 val;
  23
  24         return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE,
  25                                 val, !(val & 0x1), MB_POLL_PERIOD_US,
  26                                 MB_POLL_TIMEOUT_US);
  27 }
  28
  29 /*
  30  * Each state Reg is checked 100 times,
  31  * with a delay of 100 microseconds after each check
  32  */
  33 static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs)
  34 {
  35         int check_times = 0;
  36         u32 state;
  37
  38         state = readl(qm->io_base + regs);
  39         while (state && check_times < ERROR_CHECK_TIMEOUT) {
  40                 udelay(CHECK_DELAY_TIME);
  41                 state = readl(qm->io_base + regs);
  42                 check_times++;
  43         }
  44
  45         return state;
  46 }
  47
  48 static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr,
  49                         u32 *data, u8 nums)
  50 {
  51         int i;
  52
  53         if (nums < 1 || nums > QM_REGS_MAX_LEN)
  54                 return -EINVAL;
  55
  56         for (i = 0; i < nums; i++) {
  57                 data[i] = readl(qm->io_base + reg_addr);
  58                 reg_addr += QM_REG_ADDR_OFFSET;
  59         }
  60
  61         return 0;
  62 }
  63
  64 static int qm_write_regs(struct hisi_qm *qm, u32 reg,
  65                          u32 *data, u8 nums)
  66 {
  67         int i;
  68
  69         if (nums < 1 || nums > QM_REGS_MAX_LEN)
  70                 return -EINVAL;
  71
  72         for (i = 0; i < nums; i++)
  73                 writel(data[i], qm->io_base + reg + i * QM_REG_ADDR_OFFSET);
  74
  75         return 0;
  76 }
  77
  78 static int qm_get_vft(struct hisi_qm *qm, u32 *base)
  79 {
  80         u64 sqc_vft;
  81         u32 qp_num;
  82         int ret;
  83
  84         ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
  85         if (ret)
  86                 return ret;
  87
  88         sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
  89                   ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
  90                   QM_XQC_ADDR_OFFSET);
  91         *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
  92         qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
  93                   (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
  94
  95         return qp_num;
  96 }
  97
  98 static int qm_get_sqc(struct hisi_qm *qm, u64 *addr)
  99 {
 100         int ret;
 101
 102         ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, 0, 0, 1);
 103         if (ret)
 104                 return ret;
 105
 106         *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
 107                   ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
 108                   QM_XQC_ADDR_OFFSET);
 109
 110         return 0;
 111 }
 112
 113 static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)
 114 {
 115         int ret;
 116
 117         ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, 0, 0, 1);
 118         if (ret)
 119                 return ret;
 120
 121         *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
 122                   ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
 123                   QM_XQC_ADDR_OFFSET);
 124
 125         return 0;
 126 }
 127
 128 static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 129 {
 130         struct device *dev = &qm->pdev->dev;
 131         int ret;
 132
 133         ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);
 134         if (ret) {
 135                 dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n");
 136                 return ret;
 137         }
 138
 139         ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);
 140         if (ret) {
 141                 dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n");
 142                 return ret;
 143         }
 144
 145         ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V,
 146                            &vf_data->ifc_int_source, 1);
 147         if (ret) {
 148                 dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n");
 149                 return ret;
 150         }
 151
 152         ret = qm_read_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);
 153         if (ret) {
 154                 dev_err(dev, "failed to read QM_IFC_INT_MASK\n");
 155                 return ret;
 156         }
 157
 158         ret = qm_read_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);
 159         if (ret) {
 160                 dev_err(dev, "failed to read QM_IFC_INT_SET_V\n");
 161                 return ret;
 162         }
 163
 164         ret = qm_read_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);
 165         if (ret) {
 166                 dev_err(dev, "failed to read QM_PAGE_SIZE\n");
 167                 return ret;
 168         }
 169
 170         /* QM_EQC_DW has 7 regs */
 171         ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
 172         if (ret) {
 173                 dev_err(dev, "failed to read QM_EQC_DW\n");
 174                 return ret;
 175         }
 176
 177         /* QM_AEQC_DW has 7 regs */
 178         ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
 179         if (ret) {
 180                 dev_err(dev, "failed to read QM_AEQC_DW\n");
 181                 return ret;
 182         }
 183
 184         return 0;
 185 }
 186
 187 static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
 188 {
 189         struct device *dev = &qm->pdev->dev;
 190         int ret;
 191
 192         /* Check VF state */
 193         if (unlikely(hisi_qm_wait_mb_ready(qm))) {
 194                 dev_err(&qm->pdev->dev, "QM device is not ready to write\n");
 195                 return -EBUSY;
 196         }
 197
 198         ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);
 199         if (ret) {
 200                 dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n");
 201                 return ret;
 202         }
 203
 204         ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);
 205         if (ret) {
 206                 dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n");
 207                 return ret;
 208         }
 209
 210         ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V,
 211                             &vf_data->ifc_int_source, 1);
 212         if (ret) {
 213                 dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n");
 214                 return ret;
 215         }
 216
 217         ret = qm_write_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);
 218         if (ret) {
 219                 dev_err(dev, "failed to write QM_IFC_INT_MASK\n");
 220                 return ret;
 221         }
 222
 223         ret = qm_write_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);
 224         if (ret) {
 225                 dev_err(dev, "failed to write QM_IFC_INT_SET_V\n");
 226                 return ret;
 227         }
 228
 229         ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);
 230         if (ret) {
 231                 dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n");
 232                 return ret;
 233         }
 234
 235         ret = qm_write_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);
 236         if (ret) {
 237                 dev_err(dev, "failed to write QM_PAGE_SIZE\n");
 238                 return ret;
 239         }
 240
 241         /* QM_EQC_DW has 7 regs */
 242         ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);
 243         if (ret) {
 244                 dev_err(dev, "failed to write QM_EQC_DW\n");
 245                 return ret;
 246         }
 247
 248         /* QM_AEQC_DW has 7 regs */
 249         ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);
 250         if (ret) {
 251                 dev_err(dev, "failed to write QM_AEQC_DW\n");
 252                 return ret;
 253         }
 254
 255         return 0;
 256 }
 257
 258 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd,
 259                   u16 index, u8 priority)
 260 {
 261         u64 doorbell;
 262         u64 dbase;
 263         u16 randata = 0;
 264
 265         if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
 266                 dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
 267         else
 268                 dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
 269
 270         doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
 271                    ((u64)randata << QM_DB_RAND_SHIFT_V2) |
 272                    ((u64)index << QM_DB_INDEX_SHIFT_V2)  |
 273                    ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
 274
 275         writeq(doorbell, qm->io_base + dbase);
 276 }
 277
 278 static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase)
 279 {
 280         unsigned int val;
 281         u64 sqc_vft;
 282         u32 qp_num;
 283         int ret;
 284
 285         ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 286                                          val & BIT(0), MB_POLL_PERIOD_US,
 287                                          MB_POLL_TIMEOUT_US);
 288         if (ret)
 289                 return ret;
 290
 291         writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);
 292         /* 0 mean SQC VFT */
 293         writel(0x0, qm->io_base + QM_VFT_CFG_TYPE);
 294         writel(vf_id, qm->io_base + QM_VFT_CFG);
 295
 296         writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
 297         writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
 298
 299         ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 300                                          val & BIT(0), MB_POLL_PERIOD_US,
 301                                          MB_POLL_TIMEOUT_US);
 302         if (ret)
 303                 return ret;
 304
 305         sqc_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |
 306                   ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) <<
 307                   QM_XQC_ADDR_OFFSET);
 308         *rbase = QM_SQC_VFT_BASE_MASK_V2 &
 309                   (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
 310         qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
 311                   (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
 312
 313         return qp_num;
 314 }
 315
 316 static void qm_dev_cmd_init(struct hisi_qm *qm)
 317 {
 318         /* Clear VF communication status registers. */
 319         writel(0x1, qm->io_base + QM_IFC_INT_SOURCE_V);
 320
 321         /* Enable pf and vf communication. */
 322         writel(0x0, qm->io_base + QM_IFC_INT_MASK);
 323 }
 324
 325 static int vf_qm_cache_wb(struct hisi_qm *qm)
 326 {
 327         unsigned int val;
 328
 329         writel(0x1, qm->io_base + QM_CACHE_WB_START);
 330         if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
 331                                        val, val & BIT(0), MB_POLL_PERIOD_US,
 332                                        MB_POLL_TIMEOUT_US)) {
 333                 dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n");
 334                 return -EINVAL;
 335         }
 336
 337         return 0;
 338 }
 339
 340 static void vf_qm_fun_reset(struct hisi_qm *qm)
 341 {
 342         int i;
 343
 344         for (i = 0; i < qm->qp_num; i++)
 345                 qm_db(qm, i, QM_DOORBELL_CMD_SQ, 0, 1);
 346 }
 347
 348 static int vf_qm_func_stop(struct hisi_qm *qm)
 349 {
 350         return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, 0, 0, 0);
 351 }
 352
 353 static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 354                              struct hisi_acc_vf_migration_file *migf)
 355 {
 356         struct acc_vf_data *vf_data = &migf->vf_data;
 357         struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
 358         struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
 359         struct device *dev = &vf_qm->pdev->dev;
 360         u32 que_iso_state;
 361         int ret;
 362
 363         if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done)
 364                 return 0;
 365
 366         if (vf_data->acc_magic != ACC_DEV_MAGIC) {
 367                 dev_err(dev, "failed to match ACC_DEV_MAGIC\n");
 368                 return -EINVAL;
 369         }
 370
 371         if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) {
 372                 dev_err(dev, "failed to match VF devices\n");
 373                 return -EINVAL;
 374         }
 375
 376         /* VF qp num check */
 377         ret = qm_get_vft(vf_qm, &vf_qm->qp_base);
 378         if (ret <= 0) {
 379                 dev_err(dev, "failed to get vft qp nums\n");
 380                 return -EINVAL;
 381         }
 382
 383         if (ret != vf_data->qp_num) {
 384                 dev_err(dev, "failed to match VF qp num\n");
 385                 return -EINVAL;
 386         }
 387
 388         vf_qm->qp_num = ret;
 389
 390         /* VF isolation state check */
 391         ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &que_iso_state, 1);
 392         if (ret) {
 393                 dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n");
 394                 return ret;
 395         }
 396
 397         if (vf_data->que_iso_cfg != que_iso_state) {
 398                 dev_err(dev, "failed to match isolation state\n");
 399                 return -EINVAL;
 400         }
 401
 402         ret = qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1);
 403         if (ret) {
 404                 dev_err(dev, "failed to write QM_VF_STATE\n");
 405                 return ret;
 406         }
 407
 408         hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
 409         hisi_acc_vdev->match_done = true;
 410         return 0;
 411 }
 412
 413 static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 414                                 struct acc_vf_data *vf_data)
 415 {
 416         struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
 417         struct device *dev = &pf_qm->pdev->dev;
 418         int vf_id = hisi_acc_vdev->vf_id;
 419         int ret;
 420
 421         vf_data->acc_magic = ACC_DEV_MAGIC;
 422         /* Save device id */
 423         vf_data->dev_id = hisi_acc_vdev->vf_dev->device;
 424
 425         /* VF qp num save from PF */
 426         ret = pf_qm_get_qp_num(pf_qm, vf_id, &vf_data->qp_base);
 427         if (ret <= 0) {
 428                 dev_err(dev, "failed to get vft qp nums!\n");
 429                 return -EINVAL;
 430         }
 431
 432         vf_data->qp_num = ret;
 433
 434         /* VF isolation state save from PF */
 435         ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);
 436         if (ret) {
 437                 dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n");
 438                 return ret;
 439         }
 440
 441         return 0;
 442 }
 443
 444 static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 445                            struct hisi_acc_vf_migration_file *migf)
 446 {
 447         struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
 448         struct device *dev = &qm->pdev->dev;
 449         struct acc_vf_data *vf_data = &migf->vf_data;
 450         int ret;
 451
 452         /* Return if only match data was transferred */
 453         if (migf->total_length == QM_MATCH_SIZE)
 454                 return 0;
 455
 456         if (migf->total_length < sizeof(struct acc_vf_data))
 457                 return -EINVAL;
 458
 459         qm->eqe_dma = vf_data->eqe_dma;
 460         qm->aeqe_dma = vf_data->aeqe_dma;
 461         qm->sqc_dma = vf_data->sqc_dma;
 462         qm->cqc_dma = vf_data->cqc_dma;
 463
 464         qm->qp_base = vf_data->qp_base;
 465         qm->qp_num = vf_data->qp_num;
 466
 467         ret = qm_set_regs(qm, vf_data);
 468         if (ret) {
 469                 dev_err(dev, "set VF regs failed\n");
 470                 return ret;
 471         }
 472
 473         ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
 474         if (ret) {
 475                 dev_err(dev, "set sqc failed\n");
 476                 return ret;
 477         }
 478
 479         ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
 480         if (ret) {
 481                 dev_err(dev, "set cqc failed\n");
 482                 return ret;
 483         }
 484
 485         qm_dev_cmd_init(qm);
 486         return 0;
 487 }
 488
 489 static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 490                             struct hisi_acc_vf_migration_file *migf)
 491 {
 492         struct acc_vf_data *vf_data = &migf->vf_data;
 493         struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
 494         struct device *dev = &vf_qm->pdev->dev;
 495         int ret;
 496
 497         if (unlikely(qm_wait_dev_not_ready(vf_qm))) {
 498                 /* Update state and return with match data */
 499                 vf_data->vf_qm_state = QM_NOT_READY;
 500                 hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
 501                 migf->total_length = QM_MATCH_SIZE;
 502                 return 0;
 503         }
 504
 505         vf_data->vf_qm_state = QM_READY;
 506         hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
 507
 508         ret = vf_qm_cache_wb(vf_qm);
 509         if (ret) {
 510                 dev_err(dev, "failed to writeback QM Cache!\n");
 511                 return ret;
 512         }
 513
 514         ret = qm_get_regs(vf_qm, vf_data);
 515         if (ret)
 516                 return -EINVAL;
 517
 518         /* Every reg is 32 bit, the dma address is 64 bit. */
 519         vf_data->eqe_dma = vf_data->qm_eqc_dw[1];
 520         vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET;
 521         vf_data->eqe_dma |= vf_data->qm_eqc_dw[0];
 522         vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1];
 523         vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET;
 524         vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0];
 525
 526         /* Through SQC_BT/CQC_BT to get sqc and cqc address */
 527         ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma);
 528         if (ret) {
 529                 dev_err(dev, "failed to read SQC addr!\n");
 530                 return -EINVAL;
 531         }
 532
 533         ret = qm_get_cqc(vf_qm, &vf_data->cqc_dma);
 534         if (ret) {
 535                 dev_err(dev, "failed to read CQC addr!\n");
 536                 return -EINVAL;
 537         }
 538
 539         migf->total_length = sizeof(struct acc_vf_data);
 540         return 0;
 541 }
 542
 543 static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev)
 544 {
 545         struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
 546
 547         return container_of(core_device, struct hisi_acc_vf_core_device,
 548                             core_device);
 549 }
 550
 551 /* Check the PF's RAS state and Function INT state */
 552 static int
 553 hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 554 {
 555         struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm;
 556         struct hisi_qm *qm = hisi_acc_vdev->pf_qm;
 557         struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev;
 558         struct device *dev = &qm->pdev->dev;
 559         u32 state;
 560
 561         /* Check RAS state */
 562         state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS);
 563         if (state) {
 564                 dev_err(dev, "failed to check QM RAS state!\n");
 565                 return -EBUSY;
 566         }
 567
 568         /* Check Function Communication state between PF and VF */
 569         state = qm_check_reg_state(vfqm, QM_IFC_INT_STATUS);
 570         if (state) {
 571                 dev_err(dev, "failed to check QM IFC INT state!\n");
 572                 return -EBUSY;
 573         }
 574         state = qm_check_reg_state(vfqm, QM_IFC_INT_SET_V);
 575         if (state) {
 576                 dev_err(dev, "failed to check QM IFC INT SET state!\n");
 577                 return -EBUSY;
 578         }
 579
 580         /* Check submodule task state */
 581         switch (vf_pdev->device) {
 582         case PCI_DEVICE_ID_HUAWEI_SEC_VF:
 583                 state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS);
 584                 if (state) {
 585                         dev_err(dev, "failed to check QM SEC Core INT state!\n");
 586                         return -EBUSY;
 587                 }
 588                 return 0;
 589         case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
 590                 state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS);
 591                 if (state) {
 592                         dev_err(dev, "failed to check QM HPRE HAC INT state!\n");
 593                         return -EBUSY;
 594                 }
 595                 return 0;
 596         case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
 597                 state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS);
 598                 if (state) {
 599                         dev_err(dev, "failed to check QM ZIP Core INT state!\n");
 600                         return -EBUSY;
 601                 }
 602                 return 0;
 603         default:
 604                 dev_err(dev, "failed to detect acc module type!\n");
 605                 return -EINVAL;
 606         }
 607 }
 608
 609 static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf)
 610 {
 611         mutex_lock(&migf->lock);
 612         migf->disabled = true;
 613         migf->total_length = 0;
 614         migf->filp->f_pos = 0;
 615         mutex_unlock(&migf->lock);
 616 }
 617
 618 static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 619 {
 620         if (hisi_acc_vdev->resuming_migf) {
 621                 hisi_acc_vf_disable_fd(hisi_acc_vdev->resuming_migf);
 622                 fput(hisi_acc_vdev->resuming_migf->filp);
 623                 hisi_acc_vdev->resuming_migf = NULL;
 624         }
 625
 626         if (hisi_acc_vdev->saving_migf) {
 627                 hisi_acc_vf_disable_fd(hisi_acc_vdev->saving_migf);
 628                 fput(hisi_acc_vdev->saving_migf->filp);
 629                 hisi_acc_vdev->saving_migf = NULL;
 630         }
 631 }
 632
 633 static void hisi_acc_vf_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 634 {
 635         hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
 636         hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
 637         hisi_acc_vf_disable_fds(hisi_acc_vdev);
 638 }
 639
 640 static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 641 {
 642         struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
 643
 644         if (hisi_acc_vdev->vf_qm_state != QM_READY)
 645                 return;
 646
 647         /* Make sure the device is enabled */
 648         qm_dev_cmd_init(vf_qm);
 649
 650         vf_qm_fun_reset(vf_qm);
 651 }
 652
 653 static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 654 {
 655         struct device *dev = &hisi_acc_vdev->vf_dev->dev;
 656         struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf;
 657         int ret;
 658
 659         /* Recover data to VF */
 660         ret = vf_qm_load_data(hisi_acc_vdev, migf);
 661         if (ret) {
 662                 dev_err(dev, "failed to recover the VF!\n");
 663                 return ret;
 664         }
 665
 666         return 0;
 667 }
 668
 669 static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp)
 670 {
 671         struct hisi_acc_vf_migration_file *migf = filp->private_data;
 672
 673         hisi_acc_vf_disable_fd(migf);
 674         mutex_destroy(&migf->lock);
 675         kfree(migf);
 676         return 0;
 677 }
 678
 679 static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf,
 680                                         size_t len, loff_t *pos)
 681 {
 682         struct hisi_acc_vf_migration_file *migf = filp->private_data;
 683         u8 *vf_data = (u8 *)&migf->vf_data;
 684         loff_t requested_length;
 685         ssize_t done = 0;
 686         int ret;
 687
 688         if (pos)
 689                 return -ESPIPE;
 690         pos = &filp->f_pos;
 691
 692         if (*pos < 0 ||
 693             check_add_overflow((loff_t)len, *pos, &requested_length))
 694                 return -EINVAL;
 695
 696         if (requested_length > sizeof(struct acc_vf_data))
 697                 return -ENOMEM;
 698
 699         mutex_lock(&migf->lock);
 700         if (migf->disabled) {
 701                 done = -ENODEV;
 702                 goto out_unlock;
 703         }
 704
 705         ret = copy_from_user(vf_data + *pos, buf, len);
 706         if (ret) {
 707                 done = -EFAULT;
 708                 goto out_unlock;
 709         }
 710         *pos += len;
 711         done = len;
 712         migf->total_length += len;
 713
 714         ret = vf_qm_check_match(migf->hisi_acc_vdev, migf);
 715         if (ret)
 716                 done = -EFAULT;
 717 out_unlock:
 718         mutex_unlock(&migf->lock);
 719         return done;
 720 }
 721
 722 static const struct file_operations hisi_acc_vf_resume_fops = {
 723         .owner = THIS_MODULE,
 724         .write = hisi_acc_vf_resume_write,
 725         .release = hisi_acc_vf_release_file,
 726 };
 727
 728 static struct hisi_acc_vf_migration_file *
 729 hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 730 {
 731         struct hisi_acc_vf_migration_file *migf;
 732
 733         migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
 734         if (!migf)
 735                 return ERR_PTR(-ENOMEM);
 736
 737         migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_resume_fops, migf,
 738                                         O_WRONLY);
 739         if (IS_ERR(migf->filp)) {
 740                 int err = PTR_ERR(migf->filp);
 741
 742                 kfree(migf);
 743                 return ERR_PTR(err);
 744         }
 745
 746         stream_open(migf->filp->f_inode, migf->filp);
 747         mutex_init(&migf->lock);
 748         migf->hisi_acc_vdev = hisi_acc_vdev;
 749         return migf;
 750 }
 751
 752 static long hisi_acc_vf_precopy_ioctl(struct file *filp,
 753                                       unsigned int cmd, unsigned long arg)
 754 {
 755         struct hisi_acc_vf_migration_file *migf = filp->private_data;
 756         struct hisi_acc_vf_core_device *hisi_acc_vdev = migf->hisi_acc_vdev;
 757         loff_t *pos = &filp->f_pos;
 758         struct vfio_precopy_info info;
 759         unsigned long minsz;
 760         int ret;
 761
 762         if (cmd != VFIO_MIG_GET_PRECOPY_INFO)
 763                 return -ENOTTY;
 764
 765         minsz = offsetofend(struct vfio_precopy_info, dirty_bytes);
 766
 767         if (copy_from_user(&info, (void __user *)arg, minsz))
 768                 return -EFAULT;
 769         if (info.argsz < minsz)
 770                 return -EINVAL;
 771
 772         mutex_lock(&hisi_acc_vdev->state_mutex);
 773         if (hisi_acc_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY) {
 774                 mutex_unlock(&hisi_acc_vdev->state_mutex);
 775                 return -EINVAL;
 776         }
 777
 778         mutex_lock(&migf->lock);
 779
 780         if (migf->disabled) {
 781                 ret = -ENODEV;
 782                 goto out;
 783         }
 784
 785         if (*pos > migf->total_length) {
 786                 ret = -EINVAL;
 787                 goto out;
 788         }
 789
 790         info.dirty_bytes = 0;
 791         info.initial_bytes = migf->total_length - *pos;
 792         mutex_unlock(&migf->lock);
 793         mutex_unlock(&hisi_acc_vdev->state_mutex);
 794
 795         return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
 796 out:
 797         mutex_unlock(&migf->lock);
 798         mutex_unlock(&hisi_acc_vdev->state_mutex);
 799         return ret;
 800 }
 801
 802 static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len,
 803                                      loff_t *pos)
 804 {
 805         struct hisi_acc_vf_migration_file *migf = filp->private_data;
 806         ssize_t done = 0;
 807         int ret;
 808
 809         if (pos)
 810                 return -ESPIPE;
 811         pos = &filp->f_pos;
 812
 813         mutex_lock(&migf->lock);
 814         if (*pos > migf->total_length) {
 815                 done = -EINVAL;
 816                 goto out_unlock;
 817         }
 818
 819         if (migf->disabled) {
 820                 done = -ENODEV;
 821                 goto out_unlock;
 822         }
 823
 824         len = min_t(size_t, migf->total_length - *pos, len);
 825         if (len) {
 826                 u8 *vf_data = (u8 *)&migf->vf_data;
 827
 828                 ret = copy_to_user(buf, vf_data + *pos, len);
 829                 if (ret) {
 830                         done = -EFAULT;
 831                         goto out_unlock;
 832                 }
 833                 *pos += len;
 834                 done = len;
 835         }
 836 out_unlock:
 837         mutex_unlock(&migf->lock);
 838         return done;
 839 }
 840
 841 static const struct file_operations hisi_acc_vf_save_fops = {
 842         .owner = THIS_MODULE,
 843         .read = hisi_acc_vf_save_read,
 844         .unlocked_ioctl = hisi_acc_vf_precopy_ioctl,
 845         .compat_ioctl = compat_ptr_ioctl,
 846         .release = hisi_acc_vf_release_file,
 847 };
 848
 849 static struct hisi_acc_vf_migration_file *
 850 hisi_acc_open_saving_migf(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 851 {
 852         struct hisi_acc_vf_migration_file *migf;
 853         int ret;
 854
 855         migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
 856         if (!migf)
 857                 return ERR_PTR(-ENOMEM);
 858
 859         migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_save_fops, migf,
 860                                         O_RDONLY);
 861         if (IS_ERR(migf->filp)) {
 862                 int err = PTR_ERR(migf->filp);
 863
 864                 kfree(migf);
 865                 return ERR_PTR(err);
 866         }
 867
 868         stream_open(migf->filp->f_inode, migf->filp);
 869         mutex_init(&migf->lock);
 870         migf->hisi_acc_vdev = hisi_acc_vdev;
 871
 872         ret = vf_qm_get_match_data(hisi_acc_vdev, &migf->vf_data);
 873         if (ret) {
 874                 fput(migf->filp);
 875                 return ERR_PTR(ret);
 876         }
 877
 878         return migf;
 879 }
 880
 881 static struct hisi_acc_vf_migration_file *
 882 hisi_acc_vf_pre_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 883 {
 884         struct hisi_acc_vf_migration_file *migf;
 885
 886         migf = hisi_acc_open_saving_migf(hisi_acc_vdev);
 887         if (IS_ERR(migf))
 888                 return migf;
 889
 890         migf->total_length = QM_MATCH_SIZE;
 891         return migf;
 892 }
 893
 894 static struct hisi_acc_vf_migration_file *
 895 hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev, bool open)
 896 {
 897         int ret;
 898         struct hisi_acc_vf_migration_file *migf = NULL;
 899
 900         if (open) {
 901                 /*
 902                  * Userspace didn't use PRECOPY support. Hence saving_migf
 903                  * is not opened yet.
 904                  */
 905                 migf = hisi_acc_open_saving_migf(hisi_acc_vdev);
 906                 if (IS_ERR(migf))
 907                         return migf;
 908         } else {
 909                 migf = hisi_acc_vdev->saving_migf;
 910         }
 911
 912         ret = vf_qm_state_save(hisi_acc_vdev, migf);
 913         if (ret)
 914                 return ERR_PTR(ret);
 915
 916         return open ? migf : NULL;
 917 }
 918
 919 static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
 920 {
 921         struct device *dev = &hisi_acc_vdev->vf_dev->dev;
 922         struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
 923         int ret;
 924
 925         ret = vf_qm_func_stop(vf_qm);
 926         if (ret) {
 927                 dev_err(dev, "failed to stop QM VF function!\n");
 928                 return ret;
 929         }
 930
 931         ret = hisi_acc_check_int_state(hisi_acc_vdev);
 932         if (ret) {
 933                 dev_err(dev, "failed to check QM INT state!\n");
 934                 return ret;
 935         }
 936         return 0;
 937 }
 938
 939 static struct file *
 940 hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev,
 941                              u32 new)
 942 {
 943         u32 cur = hisi_acc_vdev->mig_state;
 944         int ret;
 945
 946         if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) {
 947                 struct hisi_acc_vf_migration_file *migf;
 948
 949                 migf = hisi_acc_vf_pre_copy(hisi_acc_vdev);
 950                 if (IS_ERR(migf))
 951                         return ERR_CAST(migf);
 952                 get_file(migf->filp);
 953                 hisi_acc_vdev->saving_migf = migf;
 954                 return migf->filp;
 955         }
 956
 957         if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_STOP_COPY) {
 958                 struct hisi_acc_vf_migration_file *migf;
 959
 960                 ret = hisi_acc_vf_stop_device(hisi_acc_vdev);
 961                 if (ret)
 962                         return ERR_PTR(ret);
 963
 964                 migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, false);
 965                 if (IS_ERR(migf))
 966                         return ERR_CAST(migf);
 967
 968                 return NULL;
 969         }
 970
 971         if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) {
 972                 ret = hisi_acc_vf_stop_device(hisi_acc_vdev);
 973                 if (ret)
 974                         return ERR_PTR(ret);
 975                 return NULL;
 976         }
 977
 978         if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) {
 979                 struct hisi_acc_vf_migration_file *migf;
 980
 981                 migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, true);
 982                 if (IS_ERR(migf))
 983                         return ERR_CAST(migf);
 984                 get_file(migf->filp);
 985                 hisi_acc_vdev->saving_migf = migf;
 986                 return migf->filp;
 987         }
 988
 989         if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) {
 990                 hisi_acc_vf_disable_fds(hisi_acc_vdev);
 991                 return NULL;
 992         }
 993
 994         if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) {
 995                 struct hisi_acc_vf_migration_file *migf;
 996
 997                 migf = hisi_acc_vf_pci_resume(hisi_acc_vdev);
 998                 if (IS_ERR(migf))
 999                         return ERR_CAST(migf);
1000                 get_file(migf->filp);
1001                 hisi_acc_vdev->resuming_migf = migf;
1002                 return migf->filp;
1003         }
1004
1005         if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) {
1006                 ret = hisi_acc_vf_load_state(hisi_acc_vdev);
1007                 if (ret)
1008                         return ERR_PTR(ret);
1009                 hisi_acc_vf_disable_fds(hisi_acc_vdev);
1010                 return NULL;
1011         }
1012
1013         if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) {
1014                 hisi_acc_vf_disable_fds(hisi_acc_vdev);
1015                 return NULL;
1016         }
1017
1018         if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) {
1019                 hisi_acc_vf_start_device(hisi_acc_vdev);
1020                 return NULL;
1021         }
1022
1023         /*
1024          * vfio_mig_get_next_state() does not use arcs other than the above
1025          */
1026         WARN_ON(true);
1027         return ERR_PTR(-EINVAL);
1028 }
1029
1030 static struct file *
1031 hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev,
1032                                    enum vfio_device_mig_state new_state)
1033 {
1034         struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,
1035                         struct hisi_acc_vf_core_device, core_device.vdev);
1036         enum vfio_device_mig_state next_state;
1037         struct file *res = NULL;
1038         int ret;
1039
1040         mutex_lock(&hisi_acc_vdev->state_mutex);
1041         while (new_state != hisi_acc_vdev->mig_state) {
1042                 ret = vfio_mig_get_next_state(vdev,
1043                                               hisi_acc_vdev->mig_state,
1044                                               new_state, &next_state);
1045                 if (ret) {
1046                         res = ERR_PTR(-EINVAL);
1047                         break;
1048                 }
1049
1050                 res = hisi_acc_vf_set_device_state(hisi_acc_vdev, next_state);
1051                 if (IS_ERR(res))
1052                         break;
1053                 hisi_acc_vdev->mig_state = next_state;
1054                 if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) {
1055                         fput(res);
1056                         res = ERR_PTR(-EINVAL);
1057                         break;
1058                 }
1059         }
1060         mutex_unlock(&hisi_acc_vdev->state_mutex);
1061         return res;
1062 }
1063
1064 static int
1065 hisi_acc_vfio_pci_get_data_size(struct vfio_device *vdev,
1066                                 unsigned long *stop_copy_length)
1067 {
1068         *stop_copy_length = sizeof(struct acc_vf_data);
1069         return 0;
1070 }
1071
1072 static int
1073 hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,
1074                                    enum vfio_device_mig_state *curr_state)
1075 {
1076         struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,
1077                         struct hisi_acc_vf_core_device, core_device.vdev);
1078
1079         mutex_lock(&hisi_acc_vdev->state_mutex);
1080         *curr_state = hisi_acc_vdev->mig_state;
1081         mutex_unlock(&hisi_acc_vdev->state_mutex);
1082         return 0;
1083 }
1084
1085 static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
1086 {
1087         struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
1088
1089         if (hisi_acc_vdev->core_device.vdev.migration_flags !=
1090                                 VFIO_MIGRATION_STOP_COPY)
1091                 return;
1092
1093         mutex_lock(&hisi_acc_vdev->state_mutex);
1094         hisi_acc_vf_reset(hisi_acc_vdev);
1095         mutex_unlock(&hisi_acc_vdev->state_mutex);
1096 }
1097
1098 static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
1099 {
1100         struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
1101         struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1102         struct pci_dev *vf_dev = vdev->pdev;
1103
1104         /*
1105          * ACC VF dev BAR2 region consists of both functional register space
1106          * and migration control register space. For migration to work, we
1107          * need access to both. Hence, we map the entire BAR2 region here.
1108          * But unnecessarily exposing the migration BAR region to the Guest
1109          * has the potential to prevent/corrupt the Guest migration. Hence,
1110          * we restrict access to the migration control space from
1111          * Guest(Please see mmap/ioctl/read/write override functions).
1112          *
1113          * Please note that it is OK to expose the entire VF BAR if migration
1114          * is not supported or required as this cannot affect the ACC PF
1115          * configurations.
1116          *
1117          * Also the HiSilicon ACC VF devices supported by this driver on
1118          * HiSilicon hardware platforms are integrated end point devices
1119          * and the platform lacks the capability to perform any PCIe P2P
1120          * between these devices.
1121          */
1122
1123         vf_qm->io_base =
1124                 ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
1125                         pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
1126         if (!vf_qm->io_base)
1127                 return -EIO;
1128
1129         vf_qm->fun_type = QM_HW_VF;
1130         vf_qm->pdev = vf_dev;
1131         mutex_init(&vf_qm->mailbox_lock);
1132
1133         return 0;
1134 }
1135
1136 static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev)
1137 {
1138         struct hisi_qm  *pf_qm;
1139         struct pci_driver *pf_driver;
1140
1141         if (!pdev->is_virtfn)
1142                 return NULL;
1143
1144         switch (pdev->device) {
1145         case PCI_DEVICE_ID_HUAWEI_SEC_VF:
1146                 pf_driver = hisi_sec_get_pf_driver();
1147                 break;
1148         case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
1149                 pf_driver = hisi_hpre_get_pf_driver();
1150                 break;
1151         case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
1152                 pf_driver = hisi_zip_get_pf_driver();
1153                 break;
1154         default:
1155                 return NULL;
1156         }
1157
1158         if (!pf_driver)
1159                 return NULL;
1160
1161         pf_qm = pci_iov_get_pf_drvdata(pdev, pf_driver);
1162
1163         return !IS_ERR(pf_qm) ? pf_qm : NULL;
1164 }
1165
1166 static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
1167                                         size_t count, loff_t *ppos,
1168                                         size_t *new_count)
1169 {
1170         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1171         struct vfio_pci_core_device *vdev =
1172                 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1173
1174         if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1175                 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
1176                 resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
1177
1178                 /* Check if access is for migration control region */
1179                 if (pos >= end)
1180                         return -EINVAL;
1181
1182                 *new_count = min(count, (size_t)(end - pos));
1183         }
1184
1185         return 0;
1186 }
1187
1188 static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev,
1189                                   struct vm_area_struct *vma)
1190 {
1191         struct vfio_pci_core_device *vdev =
1192                 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1193         unsigned int index;
1194
1195         index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1196         if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1197                 u64 req_len, pgoff, req_start;
1198                 resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
1199
1200                 req_len = vma->vm_end - vma->vm_start;
1201                 pgoff = vma->vm_pgoff &
1202                         ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1203                 req_start = pgoff << PAGE_SHIFT;
1204
1205                 if (req_start + req_len > end)
1206                         return -EINVAL;
1207         }
1208
1209         return vfio_pci_core_mmap(core_vdev, vma);
1210 }
1211
1212 static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev,
1213                                        const char __user *buf, size_t count,
1214                                        loff_t *ppos)
1215 {
1216         size_t new_count = count;
1217         int ret;
1218
1219         ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);
1220         if (ret)
1221                 return ret;
1222
1223         return vfio_pci_core_write(core_vdev, buf, new_count, ppos);
1224 }
1225
1226 static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev,
1227                                       char __user *buf, size_t count,
1228                                       loff_t *ppos)
1229 {
1230         size_t new_count = count;
1231         int ret;
1232
1233         ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);
1234         if (ret)
1235                 return ret;
1236
1237         return vfio_pci_core_read(core_vdev, buf, new_count, ppos);
1238 }
1239
1240 static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
1241                                     unsigned long arg)
1242 {
1243         if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1244                 struct vfio_pci_core_device *vdev =
1245                         container_of(core_vdev, struct vfio_pci_core_device, vdev);
1246                 struct pci_dev *pdev = vdev->pdev;
1247                 struct vfio_region_info info;
1248                 unsigned long minsz;
1249
1250                 minsz = offsetofend(struct vfio_region_info, offset);
1251
1252                 if (copy_from_user(&info, (void __user *)arg, minsz))
1253                         return -EFAULT;
1254
1255                 if (info.argsz < minsz)
1256                         return -EINVAL;
1257
1258                 if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
1259                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1260
1261                         /*
1262                          * ACC VF dev BAR2 region consists of both functional
1263                          * register space and migration control register space.
1264                          * Report only the functional region to Guest.
1265                          */
1266                         info.size = pci_resource_len(pdev, info.index) / 2;
1267
1268                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1269                                         VFIO_REGION_INFO_FLAG_WRITE |
1270                                         VFIO_REGION_INFO_FLAG_MMAP;
1271
1272                         return copy_to_user((void __user *)arg, &info, minsz) ?
1273                                             -EFAULT : 0;
1274                 }
1275         }
1276         return vfio_pci_core_ioctl(core_vdev, cmd, arg);
1277 }
1278
1279 static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev)
1280 {
1281         struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1282                         struct hisi_acc_vf_core_device, core_device.vdev);
1283         struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
1284         int ret;
1285
1286         ret = vfio_pci_core_enable(vdev);
1287         if (ret)
1288                 return ret;
1289
1290         if (core_vdev->mig_ops) {
1291                 ret = hisi_acc_vf_qm_init(hisi_acc_vdev);
1292                 if (ret) {
1293                         vfio_pci_core_disable(vdev);
1294                         return ret;
1295                 }
1296                 hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
1297         }
1298
1299         vfio_pci_core_finish_enable(vdev);
1300         return 0;
1301 }
1302
1303 static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)
1304 {
1305         struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1306                         struct hisi_acc_vf_core_device, core_device.vdev);
1307         struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1308
1309         iounmap(vf_qm->io_base);
1310         vfio_pci_core_close_device(core_vdev);
1311 }
1312
1313 static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = {
1314         .migration_set_state = hisi_acc_vfio_pci_set_device_state,
1315         .migration_get_state = hisi_acc_vfio_pci_get_device_state,
1316         .migration_get_data_size = hisi_acc_vfio_pci_get_data_size,
1317 };
1318
1319 static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev)
1320 {
1321         struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1322                         struct hisi_acc_vf_core_device, core_device.vdev);
1323         struct pci_dev *pdev = to_pci_dev(core_vdev->dev);
1324         struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev);
1325
1326         hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1;
1327         hisi_acc_vdev->pf_qm = pf_qm;
1328         hisi_acc_vdev->vf_dev = pdev;
1329         mutex_init(&hisi_acc_vdev->state_mutex);
1330
1331         core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY;
1332         core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops;
1333
1334         return vfio_pci_core_init_dev(core_vdev);
1335 }
1336
1337 static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = {
1338         .name = "hisi-acc-vfio-pci-migration",
1339         .init = hisi_acc_vfio_pci_migrn_init_dev,
1340         .release = vfio_pci_core_release_dev,
1341         .open_device = hisi_acc_vfio_pci_open_device,
1342         .close_device = hisi_acc_vfio_pci_close_device,
1343         .ioctl = hisi_acc_vfio_pci_ioctl,
1344         .device_feature = vfio_pci_core_ioctl_feature,
1345         .read = hisi_acc_vfio_pci_read,
1346         .write = hisi_acc_vfio_pci_write,
1347         .mmap = hisi_acc_vfio_pci_mmap,
1348         .request = vfio_pci_core_request,
1349         .match = vfio_pci_core_match,
1350         .bind_iommufd = vfio_iommufd_physical_bind,
1351         .unbind_iommufd = vfio_iommufd_physical_unbind,
1352         .attach_ioas = vfio_iommufd_physical_attach_ioas,
1353         .detach_ioas = vfio_iommufd_physical_detach_ioas,
1354 };
1355
1356 static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {
1357         .name = "hisi-acc-vfio-pci",
1358         .init = vfio_pci_core_init_dev,
1359         .release = vfio_pci_core_release_dev,
1360         .open_device = hisi_acc_vfio_pci_open_device,
1361         .close_device = vfio_pci_core_close_device,
1362         .ioctl = vfio_pci_core_ioctl,
1363         .device_feature = vfio_pci_core_ioctl_feature,
1364         .read = vfio_pci_core_read,
1365         .write = vfio_pci_core_write,
1366         .mmap = vfio_pci_core_mmap,
1367         .request = vfio_pci_core_request,
1368         .match = vfio_pci_core_match,
1369         .bind_iommufd = vfio_iommufd_physical_bind,
1370         .unbind_iommufd = vfio_iommufd_physical_unbind,
1371         .attach_ioas = vfio_iommufd_physical_attach_ioas,
1372         .detach_ioas = vfio_iommufd_physical_detach_ioas,
1373 };
1374
1375 static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1376 {
1377         struct hisi_acc_vf_core_device *hisi_acc_vdev;
1378         const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops;
1379         struct hisi_qm *pf_qm;
1380         int vf_id;
1381         int ret;
1382
1383         pf_qm = hisi_acc_get_pf_qm(pdev);
1384         if (pf_qm && pf_qm->ver >= QM_HW_V3) {
1385                 vf_id = pci_iov_vf_id(pdev);
1386                 if (vf_id >= 0)
1387                         ops = &hisi_acc_vfio_pci_migrn_ops;
1388                 else
1389                         pci_warn(pdev, "migration support failed, continue with generic interface\n");
1390         }
1391
1392         hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device,
1393                                           core_device.vdev, &pdev->dev, ops);
1394         if (IS_ERR(hisi_acc_vdev))
1395                 return PTR_ERR(hisi_acc_vdev);
1396
1397         dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device);
1398         ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device);
1399         if (ret)
1400                 goto out_put_vdev;
1401         return 0;
1402
1403 out_put_vdev:
1404         vfio_put_device(&hisi_acc_vdev->core_device.vdev);
1405         return ret;
1406 }
1407
1408 static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev)
1409 {
1410         struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
1411
1412         vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device);
1413         vfio_put_device(&hisi_acc_vdev->core_device.vdev);
1414 }
1415
1416 static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
1417         { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) },
1418         { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) },
1419         { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },
1420         { }
1421 };
1422
1423 MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);
1424
1425 static const struct pci_error_handlers hisi_acc_vf_err_handlers = {
1426         .reset_done = hisi_acc_vf_pci_aer_reset_done,
1427         .error_detected = vfio_pci_core_aer_err_detected,
1428 };
1429
1430 static struct pci_driver hisi_acc_vfio_pci_driver = {
1431         .name = KBUILD_MODNAME,
1432         .id_table = hisi_acc_vfio_pci_table,
1433         .probe = hisi_acc_vfio_pci_probe,
1434         .remove = hisi_acc_vfio_pci_remove,
1435         .err_handler = &hisi_acc_vf_err_handlers,
1436         .driver_managed_dma = true,
1437 };
1438
1439 module_pci_driver(hisi_acc_vfio_pci_driver);
1440
1441 MODULE_LICENSE("GPL v2");
1442 MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>");
1443 MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>");
1444 MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family");