arch/tile/gxio/mpipe.c

   1 /*
   2  * Copyright 2012 Tilera Corporation. All Rights Reserved.
   3  *
   4  *   This program is free software; you can redistribute it and/or
   5  *   modify it under the terms of the GNU General Public License
   6  *   as published by the Free Software Foundation, version 2.
   7  *
   8  *   This program is distributed in the hope that it will be useful, but
   9  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  10  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  11  *   NON INFRINGEMENT.  See the GNU General Public License for
  12  *   more details.
  13  */
  14
  15 /*
  16  * Implementation of mpipe gxio calls.
  17  */
  18
  19 #include <linux/errno.h>
  20 #include <linux/io.h>
  21 #include <linux/module.h>
  22
  23 #include <gxio/iorpc_globals.h>
  24 #include <gxio/iorpc_mpipe.h>
  25 #include <gxio/iorpc_mpipe_info.h>
  26 #include <gxio/kiorpc.h>
  27 #include <gxio/mpipe.h>
  28
  29 /* HACK: Avoid pointless "shadow" warnings. */
  30 #define link link_shadow
  31
  32 /**
  33  * strscpy - Copy a C-string into a sized buffer, but only if it fits
  34  * @dest: Where to copy the string to
  35  * @src: Where to copy the string from
  36  * @size: size of destination buffer
  37  *
  38  * Use this routine to avoid copying too-long strings.
  39  * The routine returns the total number of bytes copied
  40  * (including the trailing NUL) or zero if the buffer wasn't
  41  * big enough.  To ensure that programmers pay attention
  42  * to the return code, the destination has a single NUL
  43  * written at the front (if size is non-zero) when the
  44  * buffer is not big enough.
  45  */
  46 static size_t strscpy(char *dest, const char *src, size_t size)
  47 {
  48         size_t len = strnlen(src, size) + 1;
  49         if (len > size) {
  50                 if (size)
  51                         dest[0] = '\0';
  52                 return 0;
  53         }
  54         memcpy(dest, src, len);
  55         return len;
  56 }
  57
  58 int gxio_mpipe_init(gxio_mpipe_context_t *context, unsigned int mpipe_index)
  59 {
  60         char file[32];
  61
  62         int fd;
  63         int i;
  64
  65         if (mpipe_index >= GXIO_MPIPE_INSTANCE_MAX)
  66                 return -EINVAL;
  67
  68         snprintf(file, sizeof(file), "mpipe/%d/iorpc", mpipe_index);
  69         fd = hv_dev_open((HV_VirtAddr) file, 0);
  70
  71         context->fd = fd;
  72
  73         if (fd < 0) {
  74                 if (fd >= GXIO_ERR_MIN && fd <= GXIO_ERR_MAX)
  75                         return fd;
  76                 else
  77                         return -ENODEV;
  78         }
  79
  80         /* Map in the MMIO space. */
  81         context->mmio_cfg_base = (void __force *)
  82                 iorpc_ioremap(fd, HV_MPIPE_CONFIG_MMIO_OFFSET,
  83                               HV_MPIPE_CONFIG_MMIO_SIZE);
  84         if (context->mmio_cfg_base == NULL)
  85                 goto cfg_failed;
  86
  87         context->mmio_fast_base = (void __force *)
  88                 iorpc_ioremap(fd, HV_MPIPE_FAST_MMIO_OFFSET,
  89                               HV_MPIPE_FAST_MMIO_SIZE);
  90         if (context->mmio_fast_base == NULL)
  91                 goto fast_failed;
  92
  93         /* Initialize the stacks. */
  94         for (i = 0; i < 8; i++)
  95                 context->__stacks.stacks[i] = 255;
  96
  97         context->instance = mpipe_index;
  98
  99         return 0;
 100
 101       fast_failed:
 102         iounmap((void __force __iomem *)(context->mmio_cfg_base));
 103       cfg_failed:
 104         hv_dev_close(context->fd);
 105         context->fd = -1;
 106         return -ENODEV;
 107 }
 108
 109 EXPORT_SYMBOL_GPL(gxio_mpipe_init);
 110
 111 int gxio_mpipe_destroy(gxio_mpipe_context_t *context)
 112 {
 113         iounmap((void __force __iomem *)(context->mmio_cfg_base));
 114         iounmap((void __force __iomem *)(context->mmio_fast_base));
 115         return hv_dev_close(context->fd);
 116 }
 117
 118 EXPORT_SYMBOL_GPL(gxio_mpipe_destroy);
 119
 120 static int16_t gxio_mpipe_buffer_sizes[8] =
 121         { 128, 256, 512, 1024, 1664, 4096, 10368, 16384 };
 122
 123 gxio_mpipe_buffer_size_enum_t gxio_mpipe_buffer_size_to_buffer_size_enum(size_t
 124                                                                          size)
 125 {
 126         int i;
 127         for (i = 0; i < 7; i++)
 128                 if (size <= gxio_mpipe_buffer_sizes[i])
 129                         break;
 130         return i;
 131 }
 132
 133 EXPORT_SYMBOL_GPL(gxio_mpipe_buffer_size_to_buffer_size_enum);
 134
 135 size_t gxio_mpipe_buffer_size_enum_to_buffer_size(gxio_mpipe_buffer_size_enum_t
 136                                                   buffer_size_enum)
 137 {
 138         if (buffer_size_enum > 7)
 139                 buffer_size_enum = 7;
 140
 141         return gxio_mpipe_buffer_sizes[buffer_size_enum];
 142 }
 143
 144 EXPORT_SYMBOL_GPL(gxio_mpipe_buffer_size_enum_to_buffer_size);
 145
 146 size_t gxio_mpipe_calc_buffer_stack_bytes(unsigned long buffers)
 147 {
 148         const int BUFFERS_PER_LINE = 12;
 149
 150         /* Count the number of cachlines. */
 151         unsigned long lines =
 152                 (buffers + BUFFERS_PER_LINE - 1) / BUFFERS_PER_LINE;
 153
 154         /* Convert to bytes. */
 155         return lines * CHIP_L2_LINE_SIZE();
 156 }
 157
 158 EXPORT_SYMBOL_GPL(gxio_mpipe_calc_buffer_stack_bytes);
 159
 160 int gxio_mpipe_init_buffer_stack(gxio_mpipe_context_t *context,
 161                                  unsigned int stack,
 162                                  gxio_mpipe_buffer_size_enum_t
 163                                  buffer_size_enum, void *mem, size_t mem_size,
 164                                  unsigned int mem_flags)
 165 {
 166         int result;
 167
 168         memset(mem, 0, mem_size);
 169
 170         result = gxio_mpipe_init_buffer_stack_aux(context, mem, mem_size,
 171                                                   mem_flags, stack,
 172                                                   buffer_size_enum);
 173         if (result < 0)
 174                 return result;
 175
 176         /* Save the stack. */
 177         context->__stacks.stacks[buffer_size_enum] = stack;
 178
 179         return 0;
 180 }
 181
 182 EXPORT_SYMBOL_GPL(gxio_mpipe_init_buffer_stack);
 183
 184 int gxio_mpipe_init_notif_ring(gxio_mpipe_context_t *context,
 185                                unsigned int ring,
 186                                void *mem, size_t mem_size,
 187                                unsigned int mem_flags)
 188 {
 189         return gxio_mpipe_init_notif_ring_aux(context, mem, mem_size,
 190                                               mem_flags, ring);
 191 }
 192
 193 EXPORT_SYMBOL_GPL(gxio_mpipe_init_notif_ring);
 194
 195 int gxio_mpipe_init_notif_group_and_buckets(gxio_mpipe_context_t *context,
 196                                             unsigned int group,
 197                                             unsigned int ring,
 198                                             unsigned int num_rings,
 199                                             unsigned int bucket,
 200                                             unsigned int num_buckets,
 201                                             gxio_mpipe_bucket_mode_t mode)
 202 {
 203         int i;
 204         int result;
 205
 206         gxio_mpipe_bucket_info_t bucket_info = { {
 207                                                   .group = group,
 208                                                   .mode = mode,
 209                                                   }
 210         };
 211
 212         gxio_mpipe_notif_group_bits_t bits = { {0} };
 213
 214         for (i = 0; i < num_rings; i++)
 215                 gxio_mpipe_notif_group_add_ring(&bits, ring + i);
 216
 217         result = gxio_mpipe_init_notif_group(context, group, bits);
 218         if (result != 0)
 219                 return result;
 220
 221         for (i = 0; i < num_buckets; i++) {
 222                 bucket_info.notifring = ring + (i % num_rings);
 223
 224                 result = gxio_mpipe_init_bucket(context, bucket + i,
 225                                                 bucket_info);
 226                 if (result != 0)
 227                         return result;
 228         }
 229
 230         return 0;
 231 }
 232
 233 EXPORT_SYMBOL_GPL(gxio_mpipe_init_notif_group_and_buckets);
 234
 235 int gxio_mpipe_init_edma_ring(gxio_mpipe_context_t *context,
 236                               unsigned int ring, unsigned int channel,
 237                               void *mem, size_t mem_size,
 238                               unsigned int mem_flags)
 239 {
 240         memset(mem, 0, mem_size);
 241
 242         return gxio_mpipe_init_edma_ring_aux(context, mem, mem_size, mem_flags,
 243                                              ring, channel);
 244 }
 245
 246 EXPORT_SYMBOL_GPL(gxio_mpipe_init_edma_ring);
 247
 248 void gxio_mpipe_rules_init(gxio_mpipe_rules_t *rules,
 249                            gxio_mpipe_context_t *context)
 250 {
 251         rules->context = context;
 252         memset(&rules->list, 0, sizeof(rules->list));
 253 }
 254
 255 EXPORT_SYMBOL_GPL(gxio_mpipe_rules_init);
 256
 257 int gxio_mpipe_rules_begin(gxio_mpipe_rules_t *rules,
 258                            unsigned int bucket, unsigned int num_buckets,
 259                            gxio_mpipe_rules_stacks_t *stacks)
 260 {
 261         int i;
 262         int stack = 255;
 263
 264         gxio_mpipe_rules_list_t *list = &rules->list;
 265
 266         /* Current rule. */
 267         gxio_mpipe_rules_rule_t *rule =
 268                 (gxio_mpipe_rules_rule_t *) (list->rules + list->head);
 269
 270         unsigned int head = list->tail;
 271
 272         /*
 273          * Align next rule properly.
 274          *Note that "dmacs_and_vlans" will also be aligned.
 275          */
 276         unsigned int pad = 0;
 277         while (((head + pad) % __alignof__(gxio_mpipe_rules_rule_t)) != 0)
 278                 pad++;
 279
 280         /*
 281          * Verify room.
 282          * ISSUE: Mark rules as broken on error?
 283          */
 284         if (head + pad + sizeof(*rule) >= sizeof(list->rules))
 285                 return GXIO_MPIPE_ERR_RULES_FULL;
 286
 287         /* Verify num_buckets is a power of 2. */
 288         if (__builtin_popcount(num_buckets) != 1)
 289                 return GXIO_MPIPE_ERR_RULES_INVALID;
 290
 291         /* Add padding to previous rule. */
 292         rule->size += pad;
 293
 294         /* Start a new rule. */
 295         list->head = head + pad;
 296
 297         rule = (gxio_mpipe_rules_rule_t *) (list->rules + list->head);
 298
 299         /* Default some values. */
 300         rule->headroom = 2;
 301         rule->tailroom = 0;
 302         rule->capacity = 16384;
 303
 304         /* Save the bucket info. */
 305         rule->bucket_mask = num_buckets - 1;
 306         rule->bucket_first = bucket;
 307
 308         for (i = 8 - 1; i >= 0; i--) {
 309                 int maybe =
 310                         stacks ? stacks->stacks[i] : rules->context->__stacks.
 311                         stacks[i];
 312                 if (maybe != 255)
 313                         stack = maybe;
 314                 rule->stacks.stacks[i] = stack;
 315         }
 316
 317         if (stack == 255)
 318                 return GXIO_MPIPE_ERR_RULES_INVALID;
 319
 320         /* NOTE: Only entries at the end of the array can be 255. */
 321         for (i = 8 - 1; i > 0; i--) {
 322                 if (rule->stacks.stacks[i] == 255) {
 323                         rule->stacks.stacks[i] = stack;
 324                         rule->capacity =
 325                                 gxio_mpipe_buffer_size_enum_to_buffer_size(i -
 326                                                                            1);
 327                 }
 328         }
 329
 330         rule->size = sizeof(*rule);
 331         list->tail = list->head + rule->size;
 332
 333         return 0;
 334 }
 335
 336 EXPORT_SYMBOL_GPL(gxio_mpipe_rules_begin);
 337
 338 int gxio_mpipe_rules_add_channel(gxio_mpipe_rules_t *rules,
 339                                  unsigned int channel)
 340 {
 341         gxio_mpipe_rules_list_t *list = &rules->list;
 342
 343         gxio_mpipe_rules_rule_t *rule =
 344                 (gxio_mpipe_rules_rule_t *) (list->rules + list->head);
 345
 346         /* Verify channel. */
 347         if (channel >= 32)
 348                 return GXIO_MPIPE_ERR_RULES_INVALID;
 349
 350         /* Verify begun. */
 351         if (list->tail == 0)
 352                 return GXIO_MPIPE_ERR_RULES_EMPTY;
 353
 354         rule->channel_bits |= (1UL << channel);
 355
 356         return 0;
 357 }
 358
 359 EXPORT_SYMBOL_GPL(gxio_mpipe_rules_add_channel);
 360
 361 int gxio_mpipe_rules_set_headroom(gxio_mpipe_rules_t *rules, uint8_t headroom)
 362 {
 363         gxio_mpipe_rules_list_t *list = &rules->list;
 364
 365         gxio_mpipe_rules_rule_t *rule =
 366                 (gxio_mpipe_rules_rule_t *) (list->rules + list->head);
 367
 368         /* Verify begun. */
 369         if (list->tail == 0)
 370                 return GXIO_MPIPE_ERR_RULES_EMPTY;
 371
 372         rule->headroom = headroom;
 373
 374         return 0;
 375 }
 376
 377 EXPORT_SYMBOL_GPL(gxio_mpipe_rules_set_headroom);
 378
 379 int gxio_mpipe_rules_commit(gxio_mpipe_rules_t *rules)
 380 {
 381         gxio_mpipe_rules_list_t *list = &rules->list;
 382         unsigned int size =
 383                 offsetof(gxio_mpipe_rules_list_t, rules) + list->tail;
 384         return gxio_mpipe_commit_rules(rules->context, list, size);
 385 }
 386
 387 EXPORT_SYMBOL_GPL(gxio_mpipe_rules_commit);
 388
 389 int gxio_mpipe_iqueue_init(gxio_mpipe_iqueue_t *iqueue,
 390                            gxio_mpipe_context_t *context,
 391                            unsigned int ring,
 392                            void *mem, size_t mem_size, unsigned int mem_flags)
 393 {
 394         /* The init call below will verify that "mem_size" is legal. */
 395         unsigned int num_entries = mem_size / sizeof(gxio_mpipe_idesc_t);
 396
 397         iqueue->context = context;
 398         iqueue->idescs = (gxio_mpipe_idesc_t *)mem;
 399         iqueue->ring = ring;
 400         iqueue->num_entries = num_entries;
 401         iqueue->mask_num_entries = num_entries - 1;
 402         iqueue->log2_num_entries = __builtin_ctz(num_entries);
 403         iqueue->head = 1;
 404 #ifdef __BIG_ENDIAN__
 405         iqueue->swapped = 0;
 406 #endif
 407
 408         /* Initialize the "tail". */
 409         __gxio_mmio_write(mem, iqueue->head);
 410
 411         return gxio_mpipe_init_notif_ring(context, ring, mem, mem_size,
 412                                           mem_flags);
 413 }
 414
 415 EXPORT_SYMBOL_GPL(gxio_mpipe_iqueue_init);
 416
 417 int gxio_mpipe_equeue_init(gxio_mpipe_equeue_t *equeue,
 418                            gxio_mpipe_context_t *context,
 419                            unsigned int ering,
 420                            unsigned int channel,
 421                            void *mem, unsigned int mem_size,
 422                            unsigned int mem_flags)
 423 {
 424         /* The init call below will verify that "mem_size" is legal. */
 425         unsigned int num_entries = mem_size / sizeof(gxio_mpipe_edesc_t);
 426
 427         /* Offset used to read number of completed commands. */
 428         MPIPE_EDMA_POST_REGION_ADDR_t offset;
 429
 430         int result = gxio_mpipe_init_edma_ring(context, ering, channel,
 431                                                mem, mem_size, mem_flags);
 432         if (result < 0)
 433                 return result;
 434
 435         memset(equeue, 0, sizeof(*equeue));
 436
 437         offset.word = 0;
 438         offset.region =
 439                 MPIPE_MMIO_ADDR__REGION_VAL_EDMA -
 440                 MPIPE_MMIO_ADDR__REGION_VAL_IDMA;
 441         offset.ring = ering;
 442
 443         __gxio_dma_queue_init(&equeue->dma_queue,
 444                               context->mmio_fast_base + offset.word,
 445                               num_entries);
 446         equeue->edescs = mem;
 447         equeue->mask_num_entries = num_entries - 1;
 448         equeue->log2_num_entries = __builtin_ctz(num_entries);
 449         equeue->context = context;
 450         equeue->ering = ering;
 451         equeue->channel = channel;
 452
 453         return 0;
 454 }
 455
 456 EXPORT_SYMBOL_GPL(gxio_mpipe_equeue_init);
 457
 458 int gxio_mpipe_set_timestamp(gxio_mpipe_context_t *context,
 459                              const struct timespec64 *ts)
 460 {
 461         cycles_t cycles = get_cycles();
 462         return gxio_mpipe_set_timestamp_aux(context, (uint64_t)ts->tv_sec,
 463                                             (uint64_t)ts->tv_nsec,
 464                                             (uint64_t)cycles);
 465 }
 466 EXPORT_SYMBOL_GPL(gxio_mpipe_set_timestamp);
 467
 468 int gxio_mpipe_get_timestamp(gxio_mpipe_context_t *context,
 469                              struct timespec64 *ts)
 470 {
 471         int ret;
 472         cycles_t cycles_prev, cycles_now, clock_rate;
 473         cycles_prev = get_cycles();
 474         ret = gxio_mpipe_get_timestamp_aux(context, (uint64_t *)&ts->tv_sec,
 475                                            (uint64_t *)&ts->tv_nsec,
 476                                            (uint64_t *)&cycles_now);
 477         if (ret < 0) {
 478                 return ret;
 479         }
 480
 481         clock_rate = get_clock_rate();
 482         ts->tv_nsec -= (cycles_now - cycles_prev) * 1000000000LL / clock_rate;
 483         if (ts->tv_nsec < 0) {
 484                 ts->tv_nsec += 1000000000LL;
 485                 ts->tv_sec -= 1;
 486         }
 487         return ret;
 488 }
 489 EXPORT_SYMBOL_GPL(gxio_mpipe_get_timestamp);
 490
 491 int gxio_mpipe_adjust_timestamp(gxio_mpipe_context_t *context, int64_t delta)
 492 {
 493         return gxio_mpipe_adjust_timestamp_aux(context, delta);
 494 }
 495 EXPORT_SYMBOL_GPL(gxio_mpipe_adjust_timestamp);
 496
 497 /* Get our internal context used for link name access.  This context is
 498  *  special in that it is not associated with an mPIPE service domain.
 499  */
 500 static gxio_mpipe_context_t *_gxio_get_link_context(void)
 501 {
 502         static gxio_mpipe_context_t context;
 503         static gxio_mpipe_context_t *contextp;
 504         static int tried_open = 0;
 505         static DEFINE_MUTEX(mutex);
 506
 507         mutex_lock(&mutex);
 508
 509         if (!tried_open) {
 510                 int i = 0;
 511                 tried_open = 1;
 512
 513                 /*
 514                  * "4" here is the maximum possible number of mPIPE shims; it's
 515                  * an exaggeration but we shouldn't ever go beyond 2 anyway.
 516                  */
 517                 for (i = 0; i < 4; i++) {
 518                         char file[80];
 519
 520                         snprintf(file, sizeof(file), "mpipe/%d/iorpc_info", i);
 521                         context.fd = hv_dev_open((HV_VirtAddr) file, 0);
 522                         if (context.fd < 0)
 523                                 continue;
 524
 525                         contextp = &context;
 526                         break;
 527                 }
 528         }
 529
 530         mutex_unlock(&mutex);
 531
 532         return contextp;
 533 }
 534
 535 int gxio_mpipe_link_instance(const char *link_name)
 536 {
 537         _gxio_mpipe_link_name_t name;
 538         gxio_mpipe_context_t *context = _gxio_get_link_context();
 539
 540         if (!context)
 541                 return GXIO_ERR_NO_DEVICE;
 542
 543         if (strscpy(name.name, link_name, sizeof(name.name)) == 0)
 544                 return GXIO_ERR_NO_DEVICE;
 545
 546         return gxio_mpipe_info_instance_aux(context, name);
 547 }
 548 EXPORT_SYMBOL_GPL(gxio_mpipe_link_instance);
 549
 550 int gxio_mpipe_link_enumerate_mac(int idx, char *link_name, uint8_t *link_mac)
 551 {
 552         int rv;
 553         _gxio_mpipe_link_name_t name;
 554         _gxio_mpipe_link_mac_t mac;
 555
 556         gxio_mpipe_context_t *context = _gxio_get_link_context();
 557         if (!context)
 558                 return GXIO_ERR_NO_DEVICE;
 559
 560         rv = gxio_mpipe_info_enumerate_aux(context, idx, &name, &mac);
 561         if (rv >= 0) {
 562                 if (strscpy(link_name, name.name, sizeof(name.name)) == 0)
 563                         return GXIO_ERR_INVAL_MEMORY_SIZE;
 564                 memcpy(link_mac, mac.mac, sizeof(mac.mac));
 565         }
 566
 567         return rv;
 568 }
 569
 570 EXPORT_SYMBOL_GPL(gxio_mpipe_link_enumerate_mac);
 571
 572 int gxio_mpipe_link_open(gxio_mpipe_link_t *link,
 573                          gxio_mpipe_context_t *context, const char *link_name,
 574                          unsigned int flags)
 575 {
 576         _gxio_mpipe_link_name_t name;
 577         int rv;
 578
 579         if (strscpy(name.name, link_name, sizeof(name.name)) == 0)
 580                 return GXIO_ERR_NO_DEVICE;
 581
 582         rv = gxio_mpipe_link_open_aux(context, name, flags);
 583         if (rv < 0)
 584                 return rv;
 585
 586         link->context = context;
 587         link->channel = rv >> 8;
 588         link->mac = rv & 0xFF;
 589
 590         return 0;
 591 }
 592
 593 EXPORT_SYMBOL_GPL(gxio_mpipe_link_open);
 594
 595 int gxio_mpipe_link_close(gxio_mpipe_link_t *link)
 596 {
 597         return gxio_mpipe_link_close_aux(link->context, link->mac);
 598 }
 599
 600 EXPORT_SYMBOL_GPL(gxio_mpipe_link_close);
 601
 602 int gxio_mpipe_link_set_attr(gxio_mpipe_link_t *link, uint32_t attr,
 603                              int64_t val)
 604 {
 605         return gxio_mpipe_link_set_attr_aux(link->context, link->mac, attr,
 606                                             val);
 607 }
 608
 609 EXPORT_SYMBOL_GPL(gxio_mpipe_link_set_attr);