drivers/misc/habanalabs/habanalabs_ioctl.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/fs.h>
  12 #include <linux/uaccess.h>
  13 #include <linux/slab.h>
  14
  15 static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
  16         [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
  17         [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
  18         [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
  19         [HL_DEBUG_OP_FUNNEL] = 0,
  20         [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
  21         [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
  22         [HL_DEBUG_OP_TIMESTAMP] = 0
  23
  24 };
  25
  26 static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
  27 {
  28         struct hl_info_device_status dev_stat = {0};
  29         u32 size = args->return_size;
  30         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  31
  32         if ((!size) || (!out))
  33                 return -EINVAL;
  34
  35         dev_stat.status = hl_device_status(hdev);
  36
  37         return copy_to_user(out, &dev_stat,
  38                         min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
  39 }
  40
  41 static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
  42 {
  43         struct hl_info_hw_ip_info hw_ip = {0};
  44         u32 size = args->return_size;
  45         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  46         struct asic_fixed_properties *prop = &hdev->asic_prop;
  47         u64 sram_kmd_size, dram_kmd_size;
  48
  49         if ((!size) || (!out))
  50                 return -EINVAL;
  51
  52         sram_kmd_size = (prop->sram_user_base_address -
  53                                 prop->sram_base_address);
  54         dram_kmd_size = (prop->dram_user_base_address -
  55                                 prop->dram_base_address);
  56
  57         hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
  58         hw_ip.sram_base_address = prop->sram_user_base_address;
  59         hw_ip.dram_base_address = prop->dram_user_base_address;
  60         hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
  61         hw_ip.sram_size = prop->sram_size - sram_kmd_size;
  62         hw_ip.dram_size = prop->dram_size - dram_kmd_size;
  63         if (hw_ip.dram_size > PAGE_SIZE)
  64                 hw_ip.dram_enabled = 1;
  65         hw_ip.num_of_events = prop->num_of_events;
  66
  67         memcpy(hw_ip.armcp_version, prop->armcp_info.armcp_version,
  68                 min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN));
  69
  70         memcpy(hw_ip.card_name, prop->armcp_info.card_name,
  71                 min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
  72
  73         hw_ip.armcp_cpld_version = le32_to_cpu(prop->armcp_info.cpld_version);
  74         hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
  75         hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
  76         hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
  77         hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
  78
  79         return copy_to_user(out, &hw_ip,
  80                 min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0;
  81 }
  82
  83 static int hw_events_info(struct hl_device *hdev, bool aggregate,
  84                         struct hl_info_args *args)
  85 {
  86         u32 size, max_size = args->return_size;
  87         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  88         void *arr;
  89
  90         if ((!max_size) || (!out))
  91                 return -EINVAL;
  92
  93         arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size);
  94
  95         return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
  96 }
  97
  98 static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
  99 {
 100         struct hl_device *hdev = hpriv->hdev;
 101         struct hl_info_dram_usage dram_usage = {0};
 102         u32 max_size = args->return_size;
 103         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 104         struct asic_fixed_properties *prop = &hdev->asic_prop;
 105         u64 dram_kmd_size;
 106
 107         if ((!max_size) || (!out))
 108                 return -EINVAL;
 109
 110         dram_kmd_size = (prop->dram_user_base_address -
 111                                 prop->dram_base_address);
 112         dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
 113                                         atomic64_read(&hdev->dram_used_mem);
 114         if (hpriv->ctx)
 115                 dram_usage.ctx_dram_mem =
 116                         atomic64_read(&hpriv->ctx->dram_phys_mem);
 117
 118         return copy_to_user(out, &dram_usage,
 119                 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
 120 }
 121
 122 static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
 123 {
 124         struct hl_info_hw_idle hw_idle = {0};
 125         u32 max_size = args->return_size;
 126         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 127
 128         if ((!max_size) || (!out))
 129                 return -EINVAL;
 130
 131         hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
 132                                         &hw_idle.busy_engines_mask, NULL);
 133
 134         return copy_to_user(out, &hw_idle,
 135                 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
 136 }
 137
 138 static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
 139 {
 140         struct hl_debug_params *params;
 141         void *input = NULL, *output = NULL;
 142         int rc;
 143
 144         params = kzalloc(sizeof(*params), GFP_KERNEL);
 145         if (!params)
 146                 return -ENOMEM;
 147
 148         params->reg_idx = args->reg_idx;
 149         params->enable = args->enable;
 150         params->op = args->op;
 151
 152         if (args->input_ptr && args->input_size) {
 153                 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL);
 154                 if (!input) {
 155                         rc = -ENOMEM;
 156                         goto out;
 157                 }
 158
 159                 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr),
 160                                         args->input_size)) {
 161                         rc = -EFAULT;
 162                         dev_err(hdev->dev, "failed to copy input debug data\n");
 163                         goto out;
 164                 }
 165
 166                 params->input = input;
 167         }
 168
 169         if (args->output_ptr && args->output_size) {
 170                 output = kzalloc(args->output_size, GFP_KERNEL);
 171                 if (!output) {
 172                         rc = -ENOMEM;
 173                         goto out;
 174                 }
 175
 176                 params->output = output;
 177                 params->output_size = args->output_size;
 178         }
 179
 180         rc = hdev->asic_funcs->debug_coresight(hdev, params);
 181         if (rc) {
 182                 dev_err(hdev->dev,
 183                         "debug coresight operation failed %d\n", rc);
 184                 goto out;
 185         }
 186
 187         if (output && copy_to_user((void __user *) (uintptr_t) args->output_ptr,
 188                                         output, args->output_size)) {
 189                 dev_err(hdev->dev, "copy to user failed in debug ioctl\n");
 190                 rc = -EFAULT;
 191                 goto out;
 192         }
 193
 194
 195 out:
 196         kfree(params);
 197         kfree(output);
 198         kfree(input);
 199
 200         return rc;
 201 }
 202
 203 static int device_utilization(struct hl_device *hdev, struct hl_info_args *args)
 204 {
 205         struct hl_info_device_utilization device_util = {0};
 206         u32 max_size = args->return_size;
 207         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 208
 209         if ((!max_size) || (!out))
 210                 return -EINVAL;
 211
 212         if ((args->period_ms < 100) || (args->period_ms > 1000) ||
 213                 (args->period_ms % 100)) {
 214                 dev_err(hdev->dev,
 215                         "period %u must be between 100 - 1000 and must be divisible by 100\n",
 216                         args->period_ms);
 217                 return -EINVAL;
 218         }
 219
 220         device_util.utilization = hl_device_utilization(hdev, args->period_ms);
 221
 222         return copy_to_user(out, &device_util,
 223                 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
 224 }
 225
 226 static int get_clk_rate(struct hl_device *hdev, struct hl_info_args *args)
 227 {
 228         struct hl_info_clk_rate clk_rate = {0};
 229         u32 max_size = args->return_size;
 230         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 231         int rc;
 232
 233         if ((!max_size) || (!out))
 234                 return -EINVAL;
 235
 236         rc = hdev->asic_funcs->get_clk_rate(hdev, &clk_rate.cur_clk_rate_mhz,
 237                                                 &clk_rate.max_clk_rate_mhz);
 238         if (rc)
 239                 return rc;
 240
 241         return copy_to_user(out, &clk_rate,
 242                 min((size_t) max_size, sizeof(clk_rate))) ? -EFAULT : 0;
 243 }
 244
 245 static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args)
 246 {
 247         struct hl_info_reset_count reset_count = {0};
 248         u32 max_size = args->return_size;
 249         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 250
 251         if ((!max_size) || (!out))
 252                 return -EINVAL;
 253
 254         reset_count.hard_reset_cnt = hdev->hard_reset_cnt;
 255         reset_count.soft_reset_cnt = hdev->soft_reset_cnt;
 256
 257         return copy_to_user(out, &reset_count,
 258                 min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0;
 259 }
 260
 261 static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 262                                 struct device *dev)
 263 {
 264         struct hl_info_args *args = data;
 265         struct hl_device *hdev = hpriv->hdev;
 266         int rc;
 267
 268         /*
 269          * Information is returned for the following opcodes even if the device
 270          * is disabled or in reset.
 271          */
 272         switch (args->op) {
 273         case HL_INFO_HW_IP_INFO:
 274                 return hw_ip_info(hdev, args);
 275
 276         case HL_INFO_DEVICE_STATUS:
 277                 return device_status_info(hdev, args);
 278
 279         case HL_INFO_RESET_COUNT:
 280                 return get_reset_count(hdev, args);
 281
 282         default:
 283                 break;
 284         }
 285
 286         if (hl_device_disabled_or_in_reset(hdev)) {
 287                 dev_warn_ratelimited(dev,
 288                         "Device is %s. Can't execute INFO IOCTL\n",
 289                         atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
 290                 return -EBUSY;
 291         }
 292
 293         switch (args->op) {
 294         case HL_INFO_HW_EVENTS:
 295                 rc = hw_events_info(hdev, false, args);
 296                 break;
 297
 298         case HL_INFO_DRAM_USAGE:
 299                 rc = dram_usage_info(hpriv, args);
 300                 break;
 301
 302         case HL_INFO_HW_IDLE:
 303                 rc = hw_idle(hdev, args);
 304                 break;
 305
 306         case HL_INFO_DEVICE_UTILIZATION:
 307                 rc = device_utilization(hdev, args);
 308                 break;
 309
 310         case HL_INFO_HW_EVENTS_AGGREGATE:
 311                 rc = hw_events_info(hdev, true, args);
 312                 break;
 313
 314         case HL_INFO_CLK_RATE:
 315                 rc = get_clk_rate(hdev, args);
 316                 break;
 317
 318         default:
 319                 dev_err(dev, "Invalid request %d\n", args->op);
 320                 rc = -ENOTTY;
 321                 break;
 322         }
 323
 324         return rc;
 325 }
 326
 327 static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
 328 {
 329         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
 330 }
 331
 332 static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
 333 {
 334         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
 335 }
 336
 337 static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
 338 {
 339         struct hl_debug_args *args = data;
 340         struct hl_device *hdev = hpriv->hdev;
 341         int rc = 0;
 342
 343         if (hl_device_disabled_or_in_reset(hdev)) {
 344                 dev_warn_ratelimited(hdev->dev,
 345                         "Device is %s. Can't execute DEBUG IOCTL\n",
 346                         atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
 347                 return -EBUSY;
 348         }
 349
 350         switch (args->op) {
 351         case HL_DEBUG_OP_ETR:
 352         case HL_DEBUG_OP_ETF:
 353         case HL_DEBUG_OP_STM:
 354         case HL_DEBUG_OP_FUNNEL:
 355         case HL_DEBUG_OP_BMON:
 356         case HL_DEBUG_OP_SPMU:
 357         case HL_DEBUG_OP_TIMESTAMP:
 358                 if (!hdev->in_debug) {
 359                         dev_err_ratelimited(hdev->dev,
 360                                 "Rejecting debug configuration request because device not in debug mode\n");
 361                         return -EFAULT;
 362                 }
 363                 args->input_size =
 364                         min(args->input_size, hl_debug_struct_size[args->op]);
 365                 rc = debug_coresight(hdev, args);
 366                 break;
 367         case HL_DEBUG_OP_SET_MODE:
 368                 rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
 369                 break;
 370         default:
 371                 dev_err(hdev->dev, "Invalid request %d\n", args->op);
 372                 rc = -ENOTTY;
 373                 break;
 374         }
 375
 376         return rc;
 377 }
 378
 379 #define HL_IOCTL_DEF(ioctl, _func) \
 380         [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
 381
 382 static const struct hl_ioctl_desc hl_ioctls[] = {
 383         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
 384         HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
 385         HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
 386         HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
 387         HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
 388         HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
 389 };
 390
 391 static const struct hl_ioctl_desc hl_ioctls_control[] = {
 392         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
 393 };
 394
 395 static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
 396                 const struct hl_ioctl_desc *ioctl, struct device *dev)
 397 {
 398         struct hl_fpriv *hpriv = filep->private_data;
 399         struct hl_device *hdev = hpriv->hdev;
 400         unsigned int nr = _IOC_NR(cmd);
 401         char stack_kdata[128] = {0};
 402         char *kdata = NULL;
 403         unsigned int usize, asize;
 404         hl_ioctl_t *func;
 405         u32 hl_size;
 406         int retcode;
 407
 408         if (hdev->hard_reset_pending) {
 409                 dev_crit_ratelimited(hdev->dev_ctrl,
 410                         "Device HARD reset pending! Please close FD\n");
 411                 return -ENODEV;
 412         }
 413
 414         /* Do not trust userspace, use our own definition */
 415         func = ioctl->func;
 416
 417         if (unlikely(!func)) {
 418                 dev_dbg(dev, "no function\n");
 419                 retcode = -ENOTTY;
 420                 goto out_err;
 421         }
 422
 423         hl_size = _IOC_SIZE(ioctl->cmd);
 424         usize = asize = _IOC_SIZE(cmd);
 425         if (hl_size > asize)
 426                 asize = hl_size;
 427
 428         cmd = ioctl->cmd;
 429
 430         if (cmd & (IOC_IN | IOC_OUT)) {
 431                 if (asize <= sizeof(stack_kdata)) {
 432                         kdata = stack_kdata;
 433                 } else {
 434                         kdata = kzalloc(asize, GFP_KERNEL);
 435                         if (!kdata) {
 436                                 retcode = -ENOMEM;
 437                                 goto out_err;
 438                         }
 439                 }
 440         }
 441
 442         if (cmd & IOC_IN) {
 443                 if (copy_from_user(kdata, (void __user *)arg, usize)) {
 444                         retcode = -EFAULT;
 445                         goto out_err;
 446                 }
 447         } else if (cmd & IOC_OUT) {
 448                 memset(kdata, 0, usize);
 449         }
 450
 451         retcode = func(hpriv, kdata);
 452
 453         if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, kdata, usize))
 454                 retcode = -EFAULT;
 455
 456 out_err:
 457         if (retcode)
 458                 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
 459                           task_pid_nr(current), cmd, nr);
 460
 461         if (kdata != stack_kdata)
 462                 kfree(kdata);
 463
 464         return retcode;
 465 }
 466
 467 long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 468 {
 469         struct hl_fpriv *hpriv = filep->private_data;
 470         struct hl_device *hdev = hpriv->hdev;
 471         const struct hl_ioctl_desc *ioctl = NULL;
 472         unsigned int nr = _IOC_NR(cmd);
 473
 474         if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
 475                 ioctl = &hl_ioctls[nr];
 476         } else {
 477                 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
 478                         task_pid_nr(current), nr);
 479                 return -ENOTTY;
 480         }
 481
 482         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
 483 }
 484
 485 long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
 486 {
 487         struct hl_fpriv *hpriv = filep->private_data;
 488         struct hl_device *hdev = hpriv->hdev;
 489         const struct hl_ioctl_desc *ioctl = NULL;
 490         unsigned int nr = _IOC_NR(cmd);
 491
 492         if (nr == _IOC_NR(HL_IOCTL_INFO)) {
 493                 ioctl = &hl_ioctls_control[nr];
 494         } else {
 495                 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
 496                         task_pid_nr(current), nr);
 497                 return -ENOTTY;
 498         }
 499
 500         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);
 501 }